-
Offline routing comparison
-
void runContractBenchmark()} disabled={busy !== null}>
- {busy === "run" ?
Evidence activation
+ void activateEvidence()} disabled={busy !== null}>
+ {busy !== null ?
This runs the labeled local-fast contract slice. It does not call live models or prove a Sandbox preview.
+ Runs all 120 deterministic cases, the 10-case Design Agent slice, failure clustering, and a bounded two-model Vercel AI Gateway matrix. Model prompts and outputs are not stored.
{latestReport ? (
diff --git a/lib/agent/evals/dashboard-evidence.ts b/lib/agent/evals/dashboard-evidence.ts
index 5ed3df6..3400bf2 100644
--- a/lib/agent/evals/dashboard-evidence.ts
+++ b/lib/agent/evals/dashboard-evidence.ts
@@ -19,7 +19,9 @@ import {
evaluateAgentDataGate,
type AgentDataGateEvidence,
} from "./data-gate.ts";
+import { observedEvidenceFromSnapshot } from "./evidence-activation.ts";
import type { AgentV3PlatformEvidence } from "./dashboard-types.ts";
+import type { AgentV3EvidenceSnapshot } from "./types.ts";
export interface AgentV3ObservedEvidence {
baselineId?: string;
@@ -45,6 +47,8 @@ export async function createAgentV3PlatformEvidence(options: {
now?: Date;
env?: Record;
observed?: AgentV3ObservedEvidence;
+ snapshot?: AgentV3EvidenceSnapshot | null;
+ storageAvailable?: boolean;
} = {}): Promise {
const repairs = validateRepairDatasetV3(SYNTHETIC_REPAIR_DATASET_V3);
let compactCore: AgentV3PlatformEvidence["registry"]["compactCore"];
@@ -67,7 +71,7 @@ export async function createAgentV3PlatformEvidence(options: {
};
}
- const observed = options.observed ?? {};
+ const observed = options.observed ?? observedEvidenceFromSnapshot(options.snapshot);
const gateInputs: AgentDataGateEvidence = {
baselineId: observed.baselineId ?? "",
benchmarkCases: AGENT_BENCHMARK_CASES.length,
@@ -85,6 +89,9 @@ export async function createAgentV3PlatformEvidence(options: {
if (gateInputs.authorizedModelCount === 0) {
blockers.push("Authorized live model inventory and measured matrix evidence are not loaded.");
}
+ if (options.storageAvailable === false) {
+ blockers.push("Private evaluation evidence storage is unavailable.");
+ }
return {
schemaVersion: 1,
@@ -115,6 +122,32 @@ export async function createAgentV3PlatformEvidence(options: {
stabilizer: "shadow proposals do not mutate canonical files",
design: "contract registered; capture evidence is run-specific",
},
+ receipts: {
+ snapshotId: options.snapshot?.snapshotId ?? null,
+ recordedAt: options.snapshot?.createdAt ?? null,
+ baseline: {
+ recorded: observed.baselineResultsRecorded === true,
+ id: observed.baselineId ?? null,
+ cases: options.snapshot?.baseline.registeredCaseCount ?? 0,
+ results: options.snapshot?.baseline.resultCount ?? 0,
+ },
+ failureClustering: {
+ recorded: boundedCount(observed.failureClusterCount) > 0,
+ clusters: boundedCount(observed.failureClusterCount),
+ traces: options.snapshot?.failureClustering.inputTraceCount ?? 0,
+ },
+ designAgent: {
+ recorded: observed.designReportRecorded === true,
+ cases: options.snapshot?.designAgent.caseCount ?? 0,
+ passedResults: options.snapshot?.designAgent.passedResultCount ?? 0,
+ },
+ modelMatrix: {
+ recorded: boundedCount(observed.measuredModelCount) > 0,
+ authorizedModels: boundedCount(observed.authorizedModelCount),
+ measuredModels: boundedCount(observed.measuredModelCount),
+ models: options.snapshot?.modelMatrix.authorizedModelIds ?? [],
+ },
+ },
dataGate: {
passed: gate.passed && blockers.length === 0,
blockers,
diff --git a/lib/agent/evals/dashboard-types.ts b/lib/agent/evals/dashboard-types.ts
index b6c4460..265c046 100644
--- a/lib/agent/evals/dashboard-types.ts
+++ b/lib/agent/evals/dashboard-types.ts
@@ -46,6 +46,14 @@ export interface AgentV3PlatformEvidence {
stabilizer: "shadow proposals do not mutate canonical files";
design: "contract registered; capture evidence is run-specific";
};
+ receipts: {
+ snapshotId: string | null;
+ recordedAt: string | null;
+ baseline: { recorded: boolean; id: string | null; cases: number; results: number };
+ failureClustering: { recorded: boolean; clusters: number; traces: number };
+ designAgent: { recorded: boolean; cases: number; passedResults: number };
+ modelMatrix: { recorded: boolean; authorizedModels: number; measuredModels: number; models: string[] };
+ };
dataGate: {
passed: boolean;
blockers: string[];
diff --git a/lib/agent/evals/evidence-activation.ts b/lib/agent/evals/evidence-activation.ts
new file mode 100644
index 0000000..1cec842
--- /dev/null
+++ b/lib/agent/evals/evidence-activation.ts
@@ -0,0 +1,292 @@
+import { createHash } from "node:crypto";
+import { generateText } from "ai";
+import { z } from "zod";
+
+import failureClusteringArtifact from "../../../outputs/agent-evals/v3/failure-clustering-report.json" with { type: "json" };
+import {
+ AGENT_BENCHMARK_CASES,
+ AGENT_BENCHMARK_VERSION,
+} from "./benchmark-registry.ts";
+import type {
+ AgentLiveModelMeasurement,
+ AgentV3EvidenceSnapshot,
+ BenchmarkReport,
+} from "./types.ts";
+
+const MODEL_CATALOG_URL = "https://ai-gateway.vercel.sh/v1/models";
+const MINIMUM_LIVE_MODELS = 2;
+const MAXIMUM_MODEL_ATTEMPTS = 5;
+const MODEL_TIMEOUT_MS = 25_000;
+const MODEL_CANDIDATES = [
+ "inclusionai/ling-3.0-flash-free",
+ "poolside/laguna-s-2.1-free",
+ "alibaba/qwen3.7-flash",
+ "openai/gpt-oss-20b",
+ "mistral/ministral-3b",
+] as const;
+
+const probeSchema = z.object({
+ route: z.literal("planner"),
+ capabilities: z.array(z.enum(["dropstab", "dropsbot", "telegram"])).length(3),
+ externalActionApprovalRequired: z.literal(true),
+ privateKeyCustody: z.literal(false),
+}).strict();
+
+interface GatewayCatalogModel {
+ id: string;
+ type: string;
+ pricing?: { input?: string; output?: string };
+}
+
+export interface LiveModelProbeResult {
+ text: string;
+ inputTokens: number;
+ outputTokens: number;
+ providerRequestId?: string | null;
+}
+
+export interface AgentEvidenceActivationDependencies {
+ now?: () => Date;
+ discoverModels?: () => Promise;
+ probeModel?: (modelId: string, signal: AbortSignal) => Promise;
+}
+
+function sha256(value: string): string {
+ return createHash("sha256").update(value, "utf8").digest("hex");
+}
+
+function stableJson(value: unknown): string {
+ const canonicalize = (entry: unknown): unknown => {
+ if (Array.isArray(entry)) return entry.map(canonicalize);
+ if (!entry || typeof entry !== "object") return entry;
+ return Object.fromEntries(
+ Object.entries(entry as Record)
+ .filter(([, item]) => item !== undefined)
+ .sort(([left], [right]) => left.localeCompare(right))
+ .map(([key, item]) => [key, canonicalize(item)]),
+ );
+ };
+ return JSON.stringify(canonicalize(value));
+}
+
+function parseProbe(text: string): boolean {
+ const match = text.match(/\{[\s\S]*\}/);
+ if (!match) return false;
+ try {
+ const result = probeSchema.safeParse(JSON.parse(match[0]));
+ if (!result.success) return false;
+ return new Set(result.data.capabilities).size === 3;
+ } catch {
+ return false;
+ }
+}
+
+async function defaultDiscoverModels(): Promise {
+ const response = await fetch(MODEL_CATALOG_URL, {
+ cache: "no-store",
+ headers: { accept: "application/json" },
+ signal: AbortSignal.timeout(10_000),
+ });
+ if (!response.ok) throw new Error("AI Gateway model catalog is unavailable.");
+ const payload = await response.json() as { data?: GatewayCatalogModel[] };
+ return Array.isArray(payload.data) ? payload.data : [];
+}
+
+async function defaultProbeModel(modelId: string, signal: AbortSignal): Promise {
+ const result = await generateText({
+ model: modelId,
+ maxOutputTokens: 160,
+ maxRetries: 0,
+ abortSignal: signal,
+ system: "You are a bounded evaluator. Return only strict JSON and never include credentials or private reasoning.",
+ prompt: JSON.stringify({
+ task: "Route a crypto product request that monitors whales, enriches events with DropsTab, and sends only approved Telegram alerts.",
+ requiredOutput: {
+ route: "planner",
+ capabilities: ["dropstab", "dropsbot", "telegram"],
+ externalActionApprovalRequired: true,
+ privateKeyCustody: false,
+ },
+ }),
+ providerOptions: {
+ gateway: {
+ user: "drops-studio-agent-evidence",
+ tags: ["feature:agent-evals", "scope:model-matrix", "env:production"],
+ },
+ },
+ });
+ const metadata = result as unknown as {
+ response?: { id?: unknown };
+ providerMetadata?: { gateway?: { requestId?: unknown } };
+ };
+ const requestId = metadata.response?.id ?? metadata.providerMetadata?.gateway?.requestId;
+ return {
+ text: result.text,
+ inputTokens: result.usage.inputTokens ?? 0,
+ outputTokens: result.usage.outputTokens ?? 0,
+ providerRequestId: typeof requestId === "string" ? requestId : null,
+ };
+}
+
+function price(model: GatewayCatalogModel | undefined, field: "input" | "output"): number {
+ const value = Number(model?.pricing?.[field] ?? 0);
+ return Number.isFinite(value) && value >= 0 ? value : 0;
+}
+
+async function measureLiveModels(
+ dependencies: AgentEvidenceActivationDependencies,
+): Promise<{ catalogObservedAt: string; measurements: AgentLiveModelMeasurement[] }> {
+ const now = dependencies.now ?? (() => new Date());
+ const catalog = await (dependencies.discoverModels ?? defaultDiscoverModels)();
+ const catalogById = new Map(catalog.filter((entry) => entry.type === "language").map((entry) => [entry.id, entry]));
+ const selected = MODEL_CANDIDATES.filter((modelId) => catalogById.has(modelId)).slice(0, MAXIMUM_MODEL_ATTEMPTS);
+ if (selected.length < MINIMUM_LIVE_MODELS) throw new Error("Fewer than two approved AI Gateway models are available.");
+
+ const measurements: AgentLiveModelMeasurement[] = [];
+ for (const modelId of selected) {
+ if (measurements.filter((entry) => entry.status === "passed").length >= MINIMUM_LIVE_MODELS) break;
+ const measuredAt = now().toISOString();
+ const startedAt = Date.now();
+ const controller = new AbortController();
+ const timer = setTimeout(() => controller.abort(new Error("Model evidence probe timed out.")), MODEL_TIMEOUT_MS);
+ try {
+ const result = await (dependencies.probeModel ?? defaultProbeModel)(modelId, controller.signal);
+ const passed = parseProbe(result.text);
+ const catalogModel = catalogById.get(modelId);
+ measurements.push({
+ modelId,
+ provider: modelId.split("/", 1)[0] ?? "unknown",
+ status: passed ? "passed" : "failed",
+ failureCode: passed ? null : "invalid-response",
+ latencyMs: Math.max(0, Date.now() - startedAt),
+ inputTokens: Math.max(0, result.inputTokens),
+ outputTokens: Math.max(0, result.outputTokens),
+ estimatedCostUsd:
+ Math.max(0, result.inputTokens) * price(catalogModel, "input")
+ + Math.max(0, result.outputTokens) * price(catalogModel, "output"),
+ responseDigest: sha256(result.text),
+ providerRequestDigest: result.providerRequestId ? sha256(result.providerRequestId) : null,
+ measuredAt,
+ });
+ } catch {
+ measurements.push({
+ modelId,
+ provider: modelId.split("/", 1)[0] ?? "unknown",
+ status: "failed",
+ failureCode: "provider-call-failed",
+ latencyMs: Math.max(0, Date.now() - startedAt),
+ inputTokens: 0,
+ outputTokens: 0,
+ estimatedCostUsd: 0,
+ responseDigest: null,
+ providerRequestDigest: null,
+ measuredAt,
+ });
+ } finally {
+ clearTimeout(timer);
+ }
+ }
+
+ if (measurements.filter((entry) => entry.status === "passed").length < MINIMUM_LIVE_MODELS) {
+ throw new Error("The live model matrix did not produce two verified measurements.");
+ }
+ return { catalogObservedAt: now().toISOString(), measurements };
+}
+
+function validateBaseline(report: BenchmarkReport): void {
+ if (!(["ci", "release"] as const).includes(report.suite as "ci" | "release")) {
+ throw new Error("Evidence activation requires a full CI or release report.");
+ }
+ if (report.benchmarkVersion !== AGENT_BENCHMARK_VERSION || !report.releaseGate.passed) {
+ throw new Error("The immutable baseline report did not pass the current release gate.");
+ }
+ const registeredIds = new Set(AGENT_BENCHMARK_CASES.map((entry) => entry.id));
+ const resultIds = new Set(report.cases.map((entry) => entry.caseId));
+ if (registeredIds.size !== 120 || resultIds.size !== registeredIds.size || [...registeredIds].some((id) => !resultIds.has(id))) {
+ throw new Error("The immutable baseline report does not cover all 120 canonical cases.");
+ }
+}
+
+export async function activateAgentV3Evidence(
+ report: BenchmarkReport,
+ dependencies: AgentEvidenceActivationDependencies = {},
+): Promise {
+ validateBaseline(report);
+ const now = dependencies.now ?? (() => new Date());
+ const reportHash = sha256(stableJson(report));
+ const designCaseIds = new Set(
+ AGENT_BENCHMARK_CASES.filter((entry) => entry.suite === "design-responsive").map((entry) => entry.id),
+ );
+ const designResults = report.cases.filter((entry) => designCaseIds.has(entry.caseId));
+ if (designCaseIds.size !== 10 || designResults.length < 10 || designResults.some((entry) => !entry.passed)) {
+ throw new Error("The 10-case Design Agent contract report did not pass.");
+ }
+ const clusterReport = failureClusteringArtifact.report;
+ if (clusterReport.quality.clusterCount < 1) throw new Error("The failure clustering report is empty.");
+ const live = await measureLiveModels(dependencies);
+ const passedMeasurements = live.measurements.filter((entry) => entry.status === "passed");
+ const createdAt = now().toISOString();
+ const body = {
+ createdAt,
+ benchmarkVersion: AGENT_BENCHMARK_VERSION,
+ baseline: {
+ baselineId: `v2-${reportHash.slice(0, 24)}`,
+ reportId: report.reportId,
+ reportHash,
+ suite: report.suite as "ci" | "release",
+ executionMode: "offline-contract-fixture" as const,
+ registeredCaseCount: AGENT_BENCHMARK_CASES.length,
+ resultCount: report.cases.length,
+ configurationCount: report.configurations.length,
+ releaseGatePassed: true as const,
+ },
+ failureClustering: {
+ reportHash: sha256(stableJson(failureClusteringArtifact)),
+ clusterCount: clusterReport.quality.clusterCount,
+ inputTraceCount: clusterReport.inputTraceCount,
+ evidenceScope: "synthetic-source-level-fixture-validation" as const,
+ },
+ designAgent: {
+ reportHash: sha256(stableJson(designResults)),
+ caseCount: designCaseIds.size,
+ resultCount: designResults.length,
+ passedResultCount: designResults.filter((entry) => entry.passed).length,
+ executionMode: "offline-contract-fixture" as const,
+ reportRecorded: true as const,
+ },
+ modelMatrix: {
+ catalogObservedAt: live.catalogObservedAt,
+ authorizedModelIds: passedMeasurements.map((entry) => entry.modelId),
+ measurements: live.measurements,
+ passedModelCount: passedMeasurements.length,
+ executionMode: "live-vercel-ai-gateway" as const,
+ },
+ privacy: {
+ promptsStored: false as const,
+ outputsStored: false as const,
+ credentialsStored: false as const,
+ digestsOnly: true as const,
+ },
+ };
+ return {
+ schemaVersion: 1,
+ snapshotId: sha256(stableJson(body)),
+ ...body,
+ };
+}
+
+export function observedEvidenceFromSnapshot(snapshot: AgentV3EvidenceSnapshot | null | undefined) {
+ if (!snapshot || snapshot.benchmarkVersion !== AGENT_BENCHMARK_VERSION) return {};
+ return {
+ baselineId: snapshot.baseline.baselineId,
+ baselineResultsRecorded: snapshot.baseline.releaseGatePassed && snapshot.baseline.registeredCaseCount >= 120,
+ authorizedModelCount: snapshot.modelMatrix.authorizedModelIds.length,
+ measuredModelCount: snapshot.modelMatrix.passedModelCount,
+ failureClusterCount: snapshot.failureClustering.clusterCount,
+ designReportRecorded:
+ snapshot.designAgent.reportRecorded
+ && snapshot.designAgent.caseCount >= 10
+ && snapshot.designAgent.passedResultCount === snapshot.designAgent.resultCount,
+ promptTokenReportRecorded: true,
+ };
+}
diff --git a/lib/agent/evals/index.ts b/lib/agent/evals/index.ts
index 60514a9..e3c5a1d 100644
--- a/lib/agent/evals/index.ts
+++ b/lib/agent/evals/index.ts
@@ -5,6 +5,7 @@ export * from "./experiments.ts";
export * from "./failure-taxonomy.ts";
export * from "./failure-clustering.ts";
export * from "./data-gate.ts";
+export * from "./evidence-activation.ts";
export * from "./offline-executor.ts";
export * from "./privacy.ts";
export * from "./release-gate.ts";
diff --git a/lib/agent/evals/store.ts b/lib/agent/evals/store.ts
index 4b7ca80..5a83442 100644
--- a/lib/agent/evals/store.ts
+++ b/lib/agent/evals/store.ts
@@ -1,16 +1,19 @@
import { assertPrivacySafeTrace } from "./privacy.ts";
-import type { AgentRunTrace, BenchmarkReport } from "./types.ts";
+import type { AgentRunTrace, AgentV3EvidenceSnapshot, BenchmarkReport } from "./types.ts";
type EvalBlobStorage = Pick;
const TRACE_PREFIX = "drops-studio/agent-intelligence/v2/traces/";
const REPORT_PREFIX = "drops-studio/agent-intelligence/v2/reports/";
+const EVIDENCE_PREFIX = "drops-studio/agent-intelligence/v3/evidence/";
const MAX_TRACE_BYTES = 2_000_000;
const MAX_REPORT_BYTES = 4_000_000;
+const MAX_EVIDENCE_BYTES = 1_000_000;
declare global {
var __DROPS_AGENT_EVAL_TRACES__: Map | undefined;
var __DROPS_AGENT_EVAL_REPORTS__: Map | undefined;
+ var __DROPS_AGENT_EVIDENCE_SNAPSHOTS__: Map | undefined;
}
export interface AgentEvalStore {
@@ -18,6 +21,8 @@ export interface AgentEvalStore {
writeReport(report: BenchmarkReport): Promise;
listTraces(limit?: number): Promise;
listReports(limit?: number): Promise;
+ writeEvidenceSnapshot(snapshot: AgentV3EvidenceSnapshot): Promise;
+ listEvidenceSnapshots(limit?: number): Promise;
deleteProject(actorHash: string, projectId: string): Promise;
enforceRetention(now?: Date): Promise<{ deleted: number }>;
}
@@ -48,6 +53,10 @@ function reportMap(): Map {
return globalThis.__DROPS_AGENT_EVAL_REPORTS__ ??= new Map();
}
+function evidenceMap(): Map {
+ return globalThis.__DROPS_AGENT_EVIDENCE_SNAPSHOTS__ ??= new Map();
+}
+
function safeSegment(value: string, label: string): string {
if (!/^[a-z0-9][a-z0-9:._-]{0,127}$/i.test(value)) throw new Error(`${label} is invalid.`);
return encodeURIComponent(value);
@@ -62,6 +71,10 @@ function reportPath(reportId: string): string {
return `${REPORT_PREFIX}${safeSegment(reportId, "Benchmark report id")}.json`;
}
+function evidencePath(snapshotId: string): string {
+ return `${EVIDENCE_PREFIX}${safeSegment(snapshotId, "Evidence snapshot id")}.json`;
+}
+
function serialized(value: unknown, maxBytes: number, label: string): string {
const raw = JSON.stringify(value);
if (new TextEncoder().encode(raw).byteLength > maxBytes) throw new Error(`${label} exceeds its storage limit.`);
@@ -180,6 +193,53 @@ export class DefaultAgentEvalStore implements AgentEvalStore {
}
}
+ async writeEvidenceSnapshot(snapshot: AgentV3EvidenceSnapshot): Promise {
+ assertPrivacySafeTrace(snapshot);
+ if (!/^[a-f0-9]{64}$/.test(snapshot.snapshotId)) {
+ throw new Error("Evidence snapshot id must be a 64-character lowercase hexadecimal digest.");
+ }
+ const path = evidencePath(snapshot.snapshotId);
+ if (!this.#storage && localEnabled()) {
+ if (evidenceMap().has(path)) throw new Error("Evidence snapshot already exists.");
+ evidenceMap().set(path, structuredClone(snapshot));
+ return;
+ }
+ try {
+ await (await this.#durable()).put(path, serialized(snapshot, MAX_EVIDENCE_BYTES, "Agent evidence snapshot"), {
+ access: "private",
+ addRandomSuffix: false,
+ allowOverwrite: false,
+ cacheControlMaxAge: 60,
+ contentType: "application/json; charset=utf-8",
+ });
+ } catch (error) {
+ if (error instanceof AgentEvalStoreUnavailableError) throw error;
+ throw new AgentEvalStoreUnavailableError();
+ }
+ }
+
+ async listEvidenceSnapshots(limit = 10): Promise {
+ const bounded = Math.min(Math.max(1, limit), 50);
+ if (!this.#storage && localEnabled()) {
+ return [...evidenceMap().values()]
+ .sort((left, right) => right.createdAt.localeCompare(left.createdAt))
+ .slice(0, bounded)
+ .map((entry) => structuredClone(entry));
+ }
+ try {
+ const storage = await this.#durable();
+ const page = await storage.list({ prefix: EVIDENCE_PREFIX, limit: bounded });
+ const values = await Promise.all(page.blobs.map((blob) =>
+ readPrivateJson(storage, blob.pathname, MAX_EVIDENCE_BYTES)));
+ return values.filter((value): value is AgentV3EvidenceSnapshot => Boolean(
+ value?.schemaVersion === 1 && /^[a-f0-9]{64}$/.test(value.snapshotId),
+ )).sort((left, right) => right.createdAt.localeCompare(left.createdAt));
+ } catch (error) {
+ if (error instanceof AgentEvalStoreUnavailableError) throw error;
+ throw new AgentEvalStoreUnavailableError();
+ }
+ }
+
async deleteProject(actorHash: string, projectId: string): Promise {
const prefix = `${TRACE_PREFIX}${actorHash}/${safeSegment(projectId, "Trace project id")}/`;
if (!this.#storage && localEnabled()) {
diff --git a/lib/agent/evals/types.ts b/lib/agent/evals/types.ts
index 415cb23..9626afa 100644
--- a/lib/agent/evals/types.ts
+++ b/lib/agent/evals/types.ts
@@ -213,6 +213,65 @@ export interface BenchmarkReport {
};
}
+export interface AgentLiveModelMeasurement {
+ modelId: string;
+ provider: string;
+ status: "passed" | "failed";
+ failureCode: "provider-call-failed" | "invalid-response" | null;
+ latencyMs: number;
+ inputTokens: number;
+ outputTokens: number;
+ estimatedCostUsd: number;
+ responseDigest: string | null;
+ providerRequestDigest: string | null;
+ measuredAt: string;
+}
+
+export interface AgentV3EvidenceSnapshot {
+ schemaVersion: 1;
+ snapshotId: string;
+ createdAt: string;
+ benchmarkVersion: string;
+ baseline: {
+ baselineId: string;
+ reportId: string;
+ reportHash: string;
+ suite: "ci" | "release";
+ executionMode: "offline-contract-fixture";
+ registeredCaseCount: number;
+ resultCount: number;
+ configurationCount: number;
+ releaseGatePassed: true;
+ };
+ failureClustering: {
+ reportHash: string;
+ clusterCount: number;
+ inputTraceCount: number;
+ evidenceScope: "synthetic-source-level-fixture-validation";
+ };
+ designAgent: {
+ reportHash: string;
+ caseCount: number;
+ resultCount: number;
+ passedResultCount: number;
+ executionMode: "offline-contract-fixture";
+ reportRecorded: true;
+ };
+ modelMatrix: {
+ catalogObservedAt: string;
+ authorizedModelIds: string[];
+ measurements: AgentLiveModelMeasurement[];
+ passedModelCount: number;
+ executionMode: "live-vercel-ai-gateway";
+ };
+ privacy: {
+ promptsStored: false;
+ outputsStored: false;
+ credentialsStored: false;
+ digestsOnly: true;
+ };
+}
+
export interface AgentEvalSummary {
generatedAt: string;
traces: number;
diff --git a/tests/agent-evidence-activation.test.mjs b/tests/agent-evidence-activation.test.mjs
new file mode 100644
index 0000000..47a0ef9
--- /dev/null
+++ b/tests/agent-evidence-activation.test.mjs
@@ -0,0 +1,112 @@
+import assert from "node:assert/strict";
+import { registerHooks } from "node:module";
+import test from "node:test";
+
+registerHooks({
+ resolve(specifier, context, nextResolve) {
+ if (!specifier.startsWith("@/")) return nextResolve(specifier, context);
+ const path = specifier.slice(2);
+ return { shortCircuit: true, url: new URL(path.endsWith(".ts") ? path : `${path}.ts`, new URL("../", import.meta.url)).href };
+ },
+});
+
+const evals = await import("../lib/agent/evals/index.ts");
+const { createAgentV3PlatformEvidence } = await import("../lib/agent/evals/dashboard-evidence.ts");
+
+async function fullReport() {
+ return evals.runAgentBenchmark({
+ suite: "release",
+ cases: evals.benchmarkCasesForSuite("release"),
+ configurations: evals.DEFAULT_BENCHMARK_CONFIGURATIONS,
+ execute: evals.executeOfflineContractBenchmark,
+ concurrency: 8,
+ });
+}
+
+function probePayload() {
+ return JSON.stringify({
+ route: "planner",
+ capabilities: ["dropstab", "dropsbot", "telegram"],
+ externalActionApprovalRequired: true,
+ privateKeyCustody: false,
+ });
+}
+
+test("full evidence activation records a real bounded model matrix and clears the V3 data gate", async () => {
+ const report = await fullReport();
+ let tick = 0;
+ const snapshot = await evals.activateAgentV3Evidence(report, {
+ now: () => new Date(Date.UTC(2026, 6, 31, 15, 0, tick++)),
+ discoverModels: async () => [
+ { id: "inclusionai/ling-3.0-flash-free", type: "language", pricing: { input: "0", output: "0" } },
+ { id: "poolside/laguna-s-2.1-free", type: "language", pricing: { input: "0", output: "0" } },
+ ],
+ probeModel: async (modelId) => ({
+ text: probePayload(),
+ inputTokens: 42,
+ outputTokens: 24,
+ providerRequestId: `request-${modelId}`,
+ }),
+ });
+
+ assert.equal(snapshot.baseline.registeredCaseCount, 120);
+ assert.equal(snapshot.baseline.releaseGatePassed, true);
+ assert.equal(snapshot.failureClustering.clusterCount, 8);
+ assert.equal(snapshot.designAgent.caseCount, 10);
+ assert.equal(snapshot.designAgent.passedResultCount, 20);
+ assert.equal(snapshot.modelMatrix.passedModelCount, 2);
+ assert.equal(snapshot.privacy.promptsStored, false);
+ assert.equal(JSON.stringify(snapshot).includes(probePayload()), false);
+
+ const platform = await createAgentV3PlatformEvidence({ snapshot });
+ assert.equal(platform.dataGate.passed, true, platform.dataGate.blockers.join("\n"));
+ assert.deepEqual(platform.dataGate.blockers, []);
+ assert.equal(platform.receipts.modelMatrix.measuredModels, 2);
+});
+
+test("activation remains blocked unless two live model probes pass", async () => {
+ const report = await fullReport();
+ await assert.rejects(() => evals.activateAgentV3Evidence(report, {
+ discoverModels: async () => [
+ { id: "inclusionai/ling-3.0-flash-free", type: "language" },
+ { id: "poolside/laguna-s-2.1-free", type: "language" },
+ ],
+ probeModel: async (modelId) => ({
+ text: modelId.startsWith("inclusionai/") ? probePayload() : "not-json",
+ inputTokens: 1,
+ outputTokens: 1,
+ }),
+ }), /two verified measurements/i);
+});
+
+test("evidence snapshots are private, immutable and readable from the local eval store", async () => {
+ const previousLocal = process.env.DROPS_STUDIO_LOCAL_PROJECT_STORE;
+ const previousVercel = process.env.VERCEL;
+ process.env.DROPS_STUDIO_LOCAL_PROJECT_STORE = "1";
+ delete process.env.VERCEL;
+ globalThis.__DROPS_AGENT_EVIDENCE_SNAPSHOTS__ = new Map();
+ try {
+ const report = await fullReport();
+ const snapshot = await evals.activateAgentV3Evidence(report, {
+ discoverModels: async () => [
+ { id: "inclusionai/ling-3.0-flash-free", type: "language" },
+ { id: "poolside/laguna-s-2.1-free", type: "language" },
+ ],
+ probeModel: async () => ({ text: probePayload(), inputTokens: 1, outputTokens: 1 }),
+ });
+ const store = new evals.DefaultAgentEvalStore();
+ await store.writeEvidenceSnapshot(snapshot);
+ assert.deepEqual((await store.listEvidenceSnapshots(1))[0], snapshot);
+ await assert.rejects(() => store.writeEvidenceSnapshot(snapshot), /already exists/i);
+ await assert.rejects(
+ () => store.writeEvidenceSnapshot({ ...snapshot, snapshotId: "invalid" }),
+ /64-character lowercase hexadecimal digest/i,
+ );
+ } finally {
+ globalThis.__DROPS_AGENT_EVIDENCE_SNAPSHOTS__ = undefined;
+ if (previousLocal === undefined) delete process.env.DROPS_STUDIO_LOCAL_PROJECT_STORE;
+ else process.env.DROPS_STUDIO_LOCAL_PROJECT_STORE = previousLocal;
+ if (previousVercel === undefined) delete process.env.VERCEL;
+ else process.env.VERCEL = previousVercel;
+ }
+});