diff --git a/lib/mcp-server/server.ts b/lib/mcp-server/server.ts index d25b47e4..2129f5cb 100644 --- a/lib/mcp-server/server.ts +++ b/lib/mcp-server/server.ts @@ -2,6 +2,7 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { listCapabilities, ListCapabilitiesInput } from "./tools/list-capabilities"; import { getPricing, GetPricingInput } from "./tools/get-pricing"; import { describeCapability, DescribeCapabilityInput } from "./tools/describe-capability"; +import { runCapability, RunCapabilityInput } from "./tools/run-capability"; import { capDeprecate, CapDeprecateInput, @@ -440,6 +441,18 @@ export function buildStoryboardMcpServer( async (args: unknown) => describeCapability(args as DescribeCapabilityInput, auth), ); + registerTool( + server, + "run_capability", + { + title: "Run any capability (output-kind generic)", + description: + "Dispatch ANY capability through the SDK — the escape hatch for caps whose output isn't media, e.g. a video-understanding / analysis cap that returns text or JSON. Media caps return a hosted URL; text/json caps return the payload inline. For ordinary image/video/audio generation prefer create_media (richer: auto-injection, quality gates). Use describe_capability first — it tells you which verb a capability wants.", + inputSchema: RunCapabilityInput, + }, + async (args: unknown) => runCapability(args as RunCapabilityInput, auth), + ); + registerTool( server, "cap_deprecate", diff --git a/lib/mcp-server/tools/describe-capability.ts b/lib/mcp-server/tools/describe-capability.ts index 536af926..52b8cec9 100644 --- a/lib/mcp-server/tools/describe-capability.ts +++ b/lib/mcp-server/tools/describe-capability.ts @@ -33,8 +33,23 @@ const OUTPUT_TO_ACTIONS: Record = { tool: ["generate"], }; -/** A ready-to-run create_media invocation for this cap (the stable verb). */ +// §3.5 / PR-4 — media caps invoke via create_media; everything else (text / +// json analysis caps) via the generic run_capability verb. Used only when the +// cap has no authored usage.invoke_via. +const MEDIA_OUTPUT_KINDS = new Set(["image", "video", "audio", "3d"]); +function defaultInvokeVia(outputKind: string | undefined): string { + return MEDIA_OUTPUT_KINDS.has(outputKind ?? "image") ? "create_media" : "run_capability"; +} + +/** A ready-to-run invocation for this cap via the verb its output kind wants. */ function buildExample(entry: RegistryEntry, action: string) { + // Non-media (text/json analysis) caps go through the generic run_capability. + if (!MEDIA_OUTPUT_KINDS.has(entry.output_kind ?? "image")) { + return { + tool: "run_capability", + args: { capability: entry.name, prompt: "" } as Record, + }; + } const args: Record = { action, model_override: entry.name }; if (entry.output_kind === "video" && action === "animate") { args.source_url = "https://…/first-frame.png"; @@ -73,7 +88,7 @@ export async function describeCapability(input: DescribeCapabilityInput, auth: A // Prefer the authored usage block (cap-std §8.10) when present; else synthesize. const usage = entry?.usage; const actions = usage?.actions ?? OUTPUT_TO_ACTIONS[entry?.output_kind ?? "image"] ?? ["generate"]; - const invokeVia = usage?.invoke_via ?? "create_media"; + const invokeVia = usage?.invoke_via ?? defaultInvokeVia(entry?.output_kind); const example = usage?.example ?? (entry ? buildExample(entry, actions[0]) : undefined); const card = { @@ -111,7 +126,9 @@ export async function describeCapability(input: DescribeCapabilityInput, auth: A usage?.avoid_for ? `avoid for: ${usage.avoid_for}` : null, entry?.display_price_usd != null ? `price: from $${entry.display_price_usd}/${entry.unit_kind ?? "unit"}` : null, entry?.aliases?.length ? `aliases: ${entry.aliases.join(", ")}` : null, - `invoke: ${invokeVia} { action: "${actions[0]}", model_override: "${name}", … }`, + invokeVia === "create_media" + ? `invoke: create_media { action: "${actions[0]}", model_override: "${name}", … }` + : `invoke: ${invokeVia} { capability: "${name}", … }`, availability === "registered" ? `⚠︎ registered but no live capacity yet — not currently routable.` : null, ].filter(Boolean); diff --git a/lib/mcp-server/tools/run-capability.ts b/lib/mcp-server/tools/run-capability.ts new file mode 100644 index 00000000..f734b2f8 --- /dev/null +++ b/lib/mcp-server/tools/run-capability.ts @@ -0,0 +1,125 @@ +/** + * run_capability — zero-code onboarding PR-4 (plan §4, gap G4). + * + * The output-kind-generic invocation verb. create_media is media-in/media-out; + * run_capability is the escape hatch for ANY capability — including text/json + * output (e.g. a video-understanding / screen-recording analyzer) — dispatched + * through the SAME transparent SDK /inference envelope, so it inherits PR-2's + * descriptor-driven routing (a new Live Runner app is reachable here with no + * client code). Media caps return a hosted URL; text/json caps return the + * payload inline. This is the verb describe_capability names for non-media caps. + * + * Deliberately THIN: it does not re-implement create_media's auto-injection / + * quality gates / project wiring. Spend-report integration is a documented + * follow-up (a generic escape hatch, not the primary media path). + */ +import { z } from "zod"; +import { extractMediaUrl, extractFalError } from "@livepeer/creative-kit"; +import { sdkPost, timeoutForCapability } from "../sdk-call"; +import type { AuthContext } from "../auth"; +import { classifyUpstreamError } from "./create-media"; +import { getRegistrySnapshot } from "@/lib/capabilities/registry-store"; +import { capabilityOutputKind } from "@/lib/sdk/capabilities"; + +export const RunCapabilityInput = z.object({ + capability: z + .string() + .describe("The exact capability name to run (from list_capabilities / describe_capability)."), + prompt: z.string().optional().describe("Text prompt / instruction, if the capability takes one."), + source_url: z + .string() + .optional() + .describe("Source media URL for image-to-X / edit / analysis capabilities."), + inputs: z + .record(z.string(), z.any()) + .optional() + .describe("Extra params merged into the inference payload (capability-specific keys)."), + timeout: z.number().int().positive().optional().describe("Override timeout in seconds."), +}); +export type RunCapabilityInput = z.infer; + +const MEDIA_KINDS = new Set(["image", "video", "audio", "3d"]); + +/** Registry-first output kind (post-PR-1/3), falling back to the static map, else text. */ +export function outputKindFor(cap: string): string { + const entry = getRegistrySnapshot().find((e) => e.name === cap); + return entry?.output_kind ?? capabilityOutputKind(cap) ?? "text"; +} + +type ToolResult = { + content: Array<{ type: "text"; text: string }>; + structuredContent: Record; + isError?: true; +}; + +function errorResult(cap: string, raw: string): ToolResult { + const { hint } = classifyUpstreamError(cap, raw); + return { + isError: true, + content: [{ type: "text", text: `run_capability(${cap}) failed: ${raw}${hint ? ` — ${hint}` : ""}` }], + structuredContent: { ok: false, capability: cap, error: raw }, + }; +} + +/** + * PURE — shape the SDK /inference result into a tool response by output kind: + * an upstream error → error; a media cap → hosted URL; text/json/tool → the + * runner payload inline. Unit-tested without a network. + */ +export function formatRunResult( + cap: string, + result: Record, + outputKind: string, +): ToolResult { + const topLevelError = typeof result.error === "string" ? (result.error as string) : undefined; + const falError = extractFalError(result); + if (topLevelError || falError) return errorResult(cap, topLevelError || falError!); + + if (MEDIA_KINDS.has(outputKind)) { + const url = extractMediaUrl(result); + if (!url) { + return { + isError: true, + content: [{ type: "text", text: `run_capability(${cap}): no media URL in the response.` }], + structuredContent: { ok: false, capability: cap, raw: result }, + }; + } + return { + content: [{ type: "text", text: `${cap} → ${url}` }], + structuredContent: { ok: true, capability: cap, output_kind: outputKind, url }, + }; + } + + // text / json / tool — return the runner payload inline. + const data = (result.data ?? result) as unknown; + const text = typeof data === "string" ? data : JSON.stringify(data, null, 2); + return { + content: [{ type: "text", text }], + structuredContent: { ok: true, capability: cap, output_kind: outputKind, result: data }, + }; +} + +export async function runCapability(input: RunCapabilityInput, auth: AuthContext): Promise { + const cap = input.capability; + const params: Record = { ...(input.inputs ?? {}) }; + if (input.source_url) params.source_url = input.source_url; + + const timeoutMs = timeoutForCapability(cap); + let result: Record; + try { + result = await sdkPost>( + "/inference", + { + capability: cap, + prompt: input.prompt ?? "", + params, + timeout: input.timeout ?? Math.ceil(timeoutMs / 1000), + }, + { ...auth, timeoutMs }, + ); + } catch (e) { + return errorResult(cap, e instanceof Error ? e.message : String(e)); + } + + return formatRunResult(cap, result, outputKindFor(cap)); +} diff --git a/tests/unit/run-capability.test.ts b/tests/unit/run-capability.test.ts new file mode 100644 index 00000000..9d1caf26 --- /dev/null +++ b/tests/unit/run-capability.test.ts @@ -0,0 +1,47 @@ +import { describe, it, expect } from "vitest"; +import { formatRunResult } from "@/lib/mcp-server/tools/run-capability"; + +describe("run_capability formatRunResult (PR-4)", () => { + it("media cap → hosted URL", () => { + const r = formatRunResult("flux-dev", { url: "https://x/a.png" }, "image"); + expect(r.isError).toBeUndefined(); + expect(r.structuredContent.ok).toBe(true); + expect(r.structuredContent.url).toBe("https://x/a.png"); + expect(r.content[0].text).toContain("https://x/a.png"); + }); + + it("media cap with NO url → error", () => { + const r = formatRunResult("flux-dev", { foo: 1 }, "image"); + expect(r.isError).toBe(true); + expect(r.structuredContent.ok).toBe(false); + }); + + it("text cap → payload inline (from result.data)", () => { + const r = formatRunResult( + "screen-agent", + { data: { report_markdown: "# Bug\n- step 1", summary: "one bug" } }, + "text", + ); + expect(r.isError).toBeUndefined(); + expect(r.structuredContent.ok).toBe(true); + expect(r.content[0].text).toContain("report_markdown"); + expect((r.structuredContent.result as Record).summary).toBe("one bug"); + }); + + it("text cap with a bare string payload → returned verbatim", () => { + const r = formatRunResult("some-text-cap", { data: "hello world" }, "text"); + expect(r.content[0].text).toBe("hello world"); + }); + + it("upstream error (top-level error string) → error with the raw message", () => { + const r = formatRunResult("flux-dev", { error: "no orchestrator available" }, "image"); + expect(r.isError).toBe(true); + expect(r.structuredContent.error).toBe("no orchestrator available"); + }); + + it("json/tool output kind falls through to the inline payload path (not media)", () => { + const r = formatRunResult("yolo-detect", { data: { detections: [] } }, "json"); + expect(r.structuredContent.ok).toBe(true); + expect(r.structuredContent.output_kind).toBe("json"); + }); +});