fix(eve): dynamic tools - reject lossy input schemas

.changeset/dynamic-tool-schema-replay.md

@@ -0,0 +1,5 @@
+---
+"eve": patch
+---
+
+Reject dynamic tool input schemas containing Zod transformations or custom validation that JSON Schema replay would silently discard. Errors now identify the tool and explain how to move runtime validation into its durable executor; opaque Standard Schema validators must provide JSON Schema explicitly.

docs/guides/dynamic-capabilities.md

@@ -247,6 +247,49 @@ export default defineDynamic({
 });
 ```
 
+### Input schemas and replay
+
+Dynamic tools persist `inputSchema` as JSON Schema and rebuild validation from
+that snapshot. Use Zod 3 or Zod 4 constraints that JSON Schema can express, such
+as `.min()`, `.max()`, `.regex()`, and `.optional()`, or provide plain JSON Schema.
+
+eve rejects Zod transformations, including `.trim()`, case conversion,
+`.transform()`, preprocessing, coercion, custom refinements, `.pipe()`, and
+`.catch()`, because JSON Schema cannot preserve their runtime behavior. Other
+Standard Schema validators are opaque to this check; provide their JSON Schema
+explicitly and perform custom validation in `execute()`. A rejected input
+schema logs an error with the tool name and omits the resolver's complete
+result.
+
+Keep normalization and custom validation inside a replayable executor. For
+example, this tool rejects whitespace-only input and trims valid input every
+time it executes, including after replay:
+
+```ts title="agent/tools/normalize.ts"
+import { defineDynamic, defineTool } from "eve/tools";
+import { z } from "zod";
+
+const runtimeInput = z.object({ value: z.string().trim().min(1) });
+
+export default defineDynamic({
+  events: {
+    "session.started": () =>
+      defineTool({
+        description: "Normalize a nonempty value.",
+        inputSchema: z.object({ value: z.string().min(1).regex(/\S/) }),
+        execute(input) {
+          return runtimeInput.parse(input);
+        },
+      }),
+  },
+});
+```
+
+The module-level validator remains available to the durable callback without
+being captured in session state. This restriction applies only to dynamic tool
+inputs. Sessions that already persisted a lossy schema must be restarted or re-resolved by a new
+deployment; the missing transformation cannot be recovered from JSON Schema.
+
 ### Author replayable callbacks
 
 Write callback properties as inline function expressions, arrows, method shorthand, or module-level function references. eve transforms authored modules that import `defineTool`, including helper modules outside `agent/tools/`, and stores each callback's referenced closure values independently.

docs/tools/overview.mdx

@@ -32,6 +32,10 @@ A tool definition needs:
 
 When a tool returns structured data, add an optional `outputSchema`. With Zod or Standard Schema it also types the `execute` return.
 
+Tools returned by `defineDynamic` persist their input schemas as JSON Schema.
+See [Input schemas and replay](/docs/guides/dynamic-capabilities#input-schemas-and-replay)
+for supported validators and where to put transformations or custom validation.
+
 ### Stream preliminary tool results
 
 An async generator lets a long-running tool stream complete output snapshots

e2e/fixtures/agent-tools/agent/agent.ts

@@ -1,7 +1,9 @@
 import { e2eAgentConfig } from "@eve-e2e/config";
 import { defineAgent } from "eve";
 
+import { respond } from "./lib/schema-replay-responder";
+
 export default defineAgent({
-  ...e2eAgentConfig(),
+  ...e2eAgentConfig({ mock: respond }),
   reasoning: "high",
 });

e2e/fixtures/agent-tools/agent/lib/schema-replay-responder.ts

@@ -0,0 +1,14 @@
+import type { MockModelRequest, MockModelResponse } from "eve/evals";
+
+export function respond(request: MockModelRequest): MockModelResponse | string {
+  const message = request.lastUserMessage ?? "";
+  if (!message.includes("schema replay regression")) return `Mock reply: ${message}`;
+  if (request.tools.some((tool) => tool.name === "invalid_dynamic_schema")) {
+    throw new Error("A dynamic schema with an unsupported transformation reached the model.");
+  }
+
+  const value = message.includes("whitespace") ? " " : "  accepted  ";
+  const id = `schema-replay-${request.userMessageCount}`;
+  if (request.toolResults.some((result) => result.id === id)) return "Schema replay checked.";
+  return { toolCalls: [{ id, name: "normalize_dynamic", input: { value } }] };
+}

e2e/fixtures/agent-tools/agent/tools/dynamic-schema-replay.ts

@@ -0,0 +1,30 @@
+import { defineDynamic, defineTool } from "eve/tools";
+import { z } from "zod";
+
+const runtimeInput = z.object({ value: z.string().trim().min(1) });
+
+export default defineDynamic({
+  events: {
+    "session.started": () => ({
+      normalize_dynamic: defineTool({
+        description: "Call only when asked to normalize_dynamic for the schema replay regression.",
+        inputSchema: z.object({ value: z.string().min(1) }),
+        execute(input) {
+          return runtimeInput.parse(input);
+        },
+      }),
+    }),
+    "turn.started": (_event, ctx) => {
+      if (!JSON.stringify(ctx.messages).includes("schema replay regression")) return null;
+      return {
+        invalid_dynamic_schema: defineTool({
+          description: "Must be rejected before the model sees it.",
+          inputSchema: z.object({ value: z.string().trim().min(1).optional() }),
+          execute(input) {
+            return input;
+          },
+        }),
+      };
+    },
+  },
+});

e2e/fixtures/agent-tools/evals/dynamic-tools/schema-replay.eval.ts

@@ -0,0 +1,29 @@
+import { defineEval } from "eve/evals";
+
+export default defineEval({
+  description:
+    "Dynamic schemas reject lossy validation and durable executors retain normalization.",
+  async test(t) {
+    if (process.env.EVE_E2E_MODEL !== "mock") {
+      t.skip(
+        "Requires the deterministic mock model to inspect available tools and send exact whitespace.",
+      );
+      return;
+    }
+
+    const first = await t.send("Run the schema replay regression with accepted input.");
+    first.expectOk();
+    first.calledTool("normalize_dynamic", { output: { value: "accepted" } });
+
+    const replayed = await t.send("Run the schema replay regression again with accepted input.");
+    replayed.expectOk();
+    replayed.calledTool("normalize_dynamic", { output: { value: "accepted" } });
+
+    const rejected = await t.send("Run the schema replay regression with whitespace input.");
+    rejected.expectOk();
+    rejected.calledTool("normalize_dynamic", { status: "failed" });
+
+    t.notCalledTool("invalid_dynamic_schema");
+    t.succeeded();
+  },
+});

packages/eve/src/context/dynamic-tool-lifecycle.test.ts

@@ -768,6 +768,36 @@ describe("dispatchDynamicToolEvent", () => {
     expect(ctx.get(SessionDynamicToolRuntimeRevisionKey)).toBe("deployment:dpl_new");
   });
 
+  it.each(["session.started", "turn.started", "step.started"] as const)(
+    "omits a resolver with a lossy input schema at %s, including after durable replay",
+    async (eventType) => {
+      const ctx = createCtx();
+      const execute = vi.fn(async () => ({ ok: true }));
+      const resolver = createResolver("normalize", [eventType], () => ({
+        unsafe: stampTestTool(
+          defineTool({
+            description: "Must not accept whitespace without trimming it first",
+            inputSchema: z.object({ value: z.string().trim().min(1).optional() }),
+            execute,
+          }),
+        ),
+        sibling: createReplayableTool(),
+      }));
+
+      await dispatchDynamicToolEvent({
+        ctx,
+        resolvers: [resolver],
+        messages: [],
+        event: makeEvent(eventType),
+      });
+
+      expect(buildDynamicTools(ctx)).toEqual([]);
+      const replayed = await deserializeContext(JSON.parse(JSON.stringify(serializeContext(ctx))));
+      expect(buildDynamicTools(replayed)).toEqual([]);
+      expect(execute).not.toHaveBeenCalled();
+    },
+  );
+
   it("leaves unsupported connection input schemas for the MCP server to validate", async () => {
     const ctx = createCtx();
     const inputSchema: JsonObject = {

packages/eve/src/context/dynamic-tool-lifecycle.ts

@@ -36,7 +36,8 @@ import {
 } from "#tools/durable-callbacks.js";
 import { toErrorMessage } from "#shared/errors.js";
 import { parseJsonObject } from "#shared/json.js";
-import { serializeInputSchema, serializeOutputSchema } from "#tools/schema.js";
+import { serializeOutputSchema } from "#tools/schema.js";
+import { serializeDynamicToolInputSchema } from "#context/dynamic-tool-schema.js";
 import type { ResolvedDynamicToolResolver } from "#runtime/types.js";
 
 const log = createLogger("dynamic-tools");
@@ -227,12 +228,13 @@ function createMetadata(input: {
   readonly name: string;
   readonly resolver: ResolvedDynamicToolResolver;
 }): CurrentDynamicToolMetadata {
+  const inputSchema = serializeDynamicToolInputSchema(input.name, input.entry.inputSchema);
   return {
     callbacks: validateDurableDynamicToolCallbacks(input.name, input.entry),
     description: input.entry.description,
     execution: input.entry.execution === "background" ? "background" : undefined,
     entryKey: input.entryKey,
-    inputSchema: serializeInputSchema(input.entry.inputSchema),
+    inputSchema,
     name: input.name,
     outputSchema: serializeOutputSchema(input.entry.outputSchema),
     resolverSlug: input.resolver.slug,

packages/eve/src/context/dynamic-tool-schema.test.ts

@@ -0,0 +1,108 @@
+import { asSchema } from "ai";
+import { describe, expect, it } from "vitest";
+import { z } from "zod";
+import { z as z3 } from "zod/v3";
+
+import { serializeDynamicToolInputSchema } from "#context/dynamic-tool-schema.js";
+import { toInputSchema } from "#tools/schema.js";
+
+describe("dynamic tool input schemas", () => {
+  it.each([
+    ["Zod 4 trim", z.object({ value: z.string().trim().min(1).optional() })],
+    ["Zod 3 trim", z3.object({ value: z3.string().trim().min(1).optional() })],
+  ])("rejects %s before replay can accept whitespace", async (_label, schema) => {
+    expect(await schema["~standard"].validate({ value: " " })).toHaveProperty("issues");
+    expect(() => serializeDynamicToolInputSchema("normalize", schema)).toThrow(
+      /Dynamic tool "normalize" inputSchema contains Zod .*cannot be preserved.*move normalization or custom validation into execute\(\)/,
+    );
+  });
+
+  it.each([
+    ["transform", z.string().transform((value) => value.length)],
+    ["overwrite", z.string().overwrite((value) => value.trim())],
+    ["lowercase", z.string().toLowerCase()],
+    ["uppercase", z.string().toUpperCase()],
+    ["preprocess", z.preprocess(String, z.string())],
+    ["coercion", z.coerce.number()],
+    ["refinement", z.string().refine((value) => value !== "forbidden")],
+    ["super refinement", z.string().superRefine(() => {})],
+    ["custom", z.custom(() => true)],
+    ["pipe", z.string().pipe(z.string().min(3))],
+    ["catch", z.string().catch("fallback")],
+    ["Zod 3 transform", z3.string().transform((value) => value.length)],
+    ["Zod 3 refinement", z3.string().refine((value) => value !== "forbidden")],
+    ["Zod 3 preprocess", z3.preprocess(String, z3.string())],
+    ["Zod 3 lowercase", z3.string().toLowerCase()],
+    ["Zod 3 uppercase", z3.string().toUpperCase()],
+    ["Zod 3 coercion", z3.coerce.number()],
+    ["Zod 3 pipe", z3.string().pipe(z3.string().min(3))],
+    ["Zod 3 catch", z3.string().catch("fallback")],
+  ])("rejects %s", (_label, schema) => {
+    expect(() => serializeDynamicToolInputSchema("tool", schema)).toThrow(
+      "cannot be preserved in JSON Schema during replay",
+    );
+  });
+
+  it.each([
+    ["array", z.array(z.string().trim())],
+    ["tuple rest", z.tuple([]).rest(z.string().trim())],
+    ["union", z.union([z.number(), z.string().trim()])],
+    ["intersection", z.intersection(z.object({}), z.object({ value: z.string().trim() }))],
+    ["record key", z.record(z.string().trim(), z.string())],
+    ["record value", z.record(z.string(), z.string().trim())],
+    ["catchall", z.object({}).catchall(z.string().trim())],
+    ["nullable", z.string().trim().nullable()],
+    ["default", z.string().trim().default("value")],
+    ["lazy", z.lazy(() => z.string().trim())],
+    ["Zod 3 array", z3.array(z3.string().trim())],
+    ["Zod 3 lazy", z3.lazy(() => z3.string().trim())],
+    [
+      "schema-like property names",
+      z.object({ _def: z.string(), _zod: z.string(), value: z.string().trim() }),
+    ],
+  ])("finds transformations inside %s", (_label, schema) => {
+    expect(() => serializeDynamicToolInputSchema("nested", schema)).toThrow(
+      "cannot be preserved in JSON Schema during replay",
+    );
+  });
+
+  it.each([
+    ["Zod 4", z.object({ value: z.string().min(1).regex(/\S/).optional() })],
+    ["Zod 3", z3.object({ value: z3.string().min(1).regex(/\S/).optional() })],
+  ])("replays supported %s validation after a JSON round trip", async (_label, source) => {
+    const serialized = serializeDynamicToolInputSchema("validate", source);
+    const validate = asSchema(toInputSchema(JSON.parse(JSON.stringify(serialized)))).validate;
+    await expect(validate?.({ value: " " })).resolves.toMatchObject({ success: false });
+    await expect(validate?.({ value: "valid" })).resolves.toMatchObject({ success: true });
+    await expect(validate?.({})).resolves.toMatchObject({ success: true });
+  });
+
+  it("handles recursive schemas without revisiting them", () => {
+    const schema = z.object({
+      value: z.string(),
+      get children() {
+        return z.array(schema);
+      },
+    });
+    expect(serializeDynamicToolInputSchema("tree", schema)).toMatchObject({ type: "object" });
+  });
+
+  it("requires opaque Standard Schema validators to declare their replay contract explicitly", () => {
+    const schema = {
+      "~standard": {
+        version: 1 as const,
+        vendor: "custom",
+        validate: (value: unknown) => ({ value }),
+        jsonSchema: { input: () => ({ type: "string" }), output: () => ({ type: "string" }) },
+      },
+    };
+    expect(() => serializeDynamicToolInputSchema("custom", schema)).toThrow(
+      "an opaque Standard Schema validator",
+    );
+  });
+
+  it("preserves plain JSON Schema without interpreting its data as Zod internals", () => {
+    const schema = { type: "object", properties: { _def: { const: { type: "transform" } } } };
+    expect(serializeDynamicToolInputSchema("remote", schema)).toEqual(schema);
+  });
+});