Skip to content

Commit 17687c6

Browse files
committed
feat(studio): compare checkpoint inference runs
1 parent 4e15583 commit 17687c6

5 files changed

Lines changed: 177 additions & 23 deletions

File tree

‎apps/studio/src-tauri/src/lib.rs‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ struct IgnitionProjectTraceFile {
6363
struct IgnitionProjectCheckpointFile {
6464
path: String,
6565
entry: Value,
66+
payload: Option<Value>,
6667
}
6768

6869
#[derive(Debug, Serialize)]
@@ -410,7 +411,7 @@ fn run_checkpoint_inference_path(
410411
let checkpoint_index_path = source_run_dir
411412
.join("checkpoints")
412413
.join(CHECKPOINT_INDEX_FILE);
413-
let (checkpoints, _) = read_checkpoint_index(&checkpoint_index_path)?;
414+
let (checkpoints, _) = read_checkpoint_index(&root, &checkpoint_index_path)?;
414415
let checkpoint = checkpoints
415416
.into_iter()
416417
.find(|entry| entry.entry.get("id").and_then(Value::as_str) == Some(checkpoint_id.as_str()))
@@ -612,8 +613,10 @@ fn read_project_runs(
612613

613614
let (metrics, metric_bytes) = read_metrics(&run_dir.join(METRICS_FILE))?;
614615
let (traces, trace_bytes) = read_trace_files(root, &run_dir.join("traces"))?;
615-
let (checkpoints, checkpoint_bytes) =
616-
read_checkpoint_index(&run_dir.join("checkpoints").join(CHECKPOINT_INDEX_FILE))?;
616+
let (checkpoints, checkpoint_bytes) = read_checkpoint_index(
617+
root,
618+
&run_dir.join("checkpoints").join(CHECKPOINT_INDEX_FILE),
619+
)?;
617620

618621
byte_length += metric_bytes + trace_bytes + checkpoint_bytes;
619622

@@ -683,7 +686,10 @@ fn read_trace_files(
683686
Ok((traces, byte_length))
684687
}
685688

686-
fn read_checkpoint_index(path: &Path) -> Result<(Vec<IgnitionProjectCheckpointFile>, u64), String> {
689+
fn read_checkpoint_index(
690+
root: &Path,
691+
path: &Path,
692+
) -> Result<(Vec<IgnitionProjectCheckpointFile>, u64), String> {
687693
if !path.exists() {
688694
return Ok((Vec::new(), 0));
689695
}
@@ -696,9 +702,15 @@ fn read_checkpoint_index(path: &Path) -> Result<(Vec<IgnitionProjectCheckpointFi
696702

697703
for entry in entries {
698704
validate_checkpoint_entry(entry)?;
705+
let payload =
706+
project_relative_json_path(root, json_string(entry, "path")?, "checkpoint payload")
707+
.ok()
708+
.and_then(|payload_path| read_json_value(&payload_path, "checkpoint payload").ok())
709+
.map(|(payload, _)| payload);
699710
checkpoints.push(IgnitionProjectCheckpointFile {
700711
path: json_string(entry, "path")?.to_string(),
701712
entry: entry.clone(),
713+
payload,
702714
});
703715
}
704716

‎apps/studio/src/App.test.tsx‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -360,10 +360,27 @@ describe("Studio app shell", () => {
360360
expect(inferenceCalls).toBe(1);
361361
expect(text).toContain("studio-checkpoint-inference-final");
362362
expect(text).toContain("checkpoint_inference");
363+
expect(text).toContain("Payload Preview");
364+
expect(text).toContain("Linked Inference Runs");
365+
expect(text).toContain("inference replay");
363366
expect(text).toContain("Frame Detail");
364367
expect(text).toContain("Reward Debugger");
365368
expect(text).not.toContain("No replay loaded.");
366369

370+
const baselineSelect = renderer?.root.findByProps({
371+
"aria-label": "Compare against run",
372+
});
373+
374+
act(() => {
375+
baselineSelect?.props.onChange({ target: { value: "native-run" } });
376+
});
377+
378+
const comparisonText = JSON.stringify(renderer?.toJSON());
379+
380+
expect(comparisonText).toContain("Compare against");
381+
expect(comparisonText).toContain("inference replay");
382+
expect(comparisonText).toContain("training run");
383+
367384
act(() => renderer?.unmount());
368385
});
369386

@@ -1036,6 +1053,34 @@ function chartPointLabel(renderer: ReactTestRenderer | undefined): string {
10361053
.join("") ?? "";
10371054
}
10381055

1056+
function nativeCheckpointPayload() {
1057+
return {
1058+
version: 1,
1059+
algorithm: "tabular-q-learning",
1060+
envId: "Target2D-v0",
1061+
observationShape: [4],
1062+
actions: ["right"],
1063+
config: {
1064+
learningRate: 0.1,
1065+
discount: 0.95,
1066+
epsilon: 0,
1067+
initialQ: 0,
1068+
observationPrecision: 2,
1069+
seed: 7,
1070+
},
1071+
metrics: {
1072+
states: 1,
1073+
transitions: 1,
1074+
episodes: 1,
1075+
lastTdError: 0,
1076+
},
1077+
qTable: {
1078+
"0,0,1,0": [1],
1079+
},
1080+
createdAt: "2026-05-28T00:01:00.000Z",
1081+
};
1082+
}
1083+
10391084
function nativeProjectDirectory(): NativeIgnitionProjectDirectory {
10401085
const createdAt = "2026-05-28T00:00:00.000Z";
10411086
const runUpdatedAt = "2026-05-28T00:01:00.000Z";
@@ -1132,6 +1177,7 @@ function nativeProjectDirectory(): NativeIgnitionProjectDirectory {
11321177
path: "runs/native-run/checkpoints/final.json",
11331178
createdAt: runUpdatedAt,
11341179
},
1180+
payload: nativeCheckpointPayload(),
11351181
}],
11361182
}],
11371183
artifacts: [{

‎apps/studio/src/App.tsx‎

Lines changed: 47 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,10 @@ import {
2828
type NativeStudioApi,
2929
type NativeTrainingMetricEvent,
3030
} from "./native";
31-
import { studioWorkspaceFromNativeProjectDirectory } from "./native-project";
31+
import {
32+
studioCheckpointViewsFromNativeProjectDirectory,
33+
studioWorkspaceFromNativeProjectDirectory,
34+
} from "./native-project";
3235
import { sampleWorkspace } from "./sample-workspace";
3336
import { StudioViewportPanel } from "./StudioViewportPanel";
3437
import "./styles.css";
@@ -301,6 +304,20 @@ export function App({
301304
setLoadError(undefined);
302305
}
303306

307+
function applyNativeProjectDirectory(
308+
directory: NativeCheckpointInferenceRunResult["project"],
309+
options: { readonly runId?: string; readonly episodeId?: string } = {},
310+
): StudioWorkspaceView {
311+
const nextWorkspace = studioWorkspaceFromNativeProjectDirectory(directory, options);
312+
313+
applyWorkspaceView(nextWorkspace);
314+
setCheckpointViews(studioCheckpointViewsFromNativeProjectDirectory(directory, nextWorkspace));
315+
setLoadedProjectPath(directory.path);
316+
setLoadedSource(directory.directoryName);
317+
318+
return nextWorkspace;
319+
}
320+
304321
function applyNativeTrainingResult(result: NativeLocalTrainingRunResult) {
305322
const trainingSession = result.trainingSession;
306323

@@ -309,12 +326,8 @@ export function App({
309326
}
310327

311328
const runId = trainingSession.run?.id;
312-
const nextWorkspace = studioWorkspaceFromNativeProjectDirectory(
313-
result.project,
314-
runId === undefined ? {} : { runId },
315-
);
316329

317-
applyWorkspaceView(nextWorkspace);
330+
applyNativeProjectDirectory(result.project, runId === undefined ? {} : { runId });
318331
setTrainingSessionViews((views) => upsertTrainingSessionView(views, trainingSession));
319332
setSelectedTrainingSessionId(trainingSession.id);
320333

@@ -325,26 +338,20 @@ export function App({
325338
if (runId !== undefined) {
326339
setSelectedRunId(runId);
327340
}
328-
329-
setLoadedProjectPath(result.project.path);
330-
setLoadedSource(result.project.directoryName);
331341
}
332342

333343
function applyNativeCheckpointInferenceResult(
334344
result: NativeCheckpointInferenceRunResult,
335345
checkpoint: StoredCheckpoint,
336346
) {
337-
const nextWorkspace = studioWorkspaceFromNativeProjectDirectory(result.project, {
347+
applyNativeProjectDirectory(result.project, {
338348
runId: result.runId,
339349
episodeId: result.episodeId,
340350
});
341351

342-
applyWorkspaceView(nextWorkspace);
343352
setSelectedRunId(result.runId);
344353
setSelectedEpisodeId(result.episodeId);
345354
setSelectedCheckpointKey(checkpointKey(checkpoint));
346-
setLoadedProjectPath(result.project.path);
347-
setLoadedSource(result.project.directoryName);
348355
}
349356

350357
function ingestTrainingMetricEvent(event: NativeTrainingMetricEvent) {
@@ -395,9 +402,7 @@ export function App({
395402
return;
396403
}
397404

398-
applyStudioArtifact(studioWorkspaceFromNativeProjectDirectory(directory));
399-
setLoadedSource(directory.directoryName);
400-
setLoadedProjectPath(directory.path);
405+
applyNativeProjectDirectory(directory);
401406
} catch (error) {
402407
setLoadError(error instanceof Error ? error.message : String(error));
403408
}
@@ -613,7 +618,10 @@ export function App({
613618
<div className="panel-heading">
614619
<div>
615620
<h2>{selectedHistoryRow?.runId ?? "No run selected"}</h2>
616-
<p>{selectedHistoryRow?.algorithm ?? "No algorithm"}</p>
621+
<p>
622+
{selectedHistoryRow?.algorithm ?? "No algorithm"}
623+
{selectedHistoryRow === undefined ? "" : ` · ${runModeLabel(selectedHistoryRow)}`}
624+
</p>
617625
</div>
618626
<span className={`status-pill ${selectedHistoryRow?.status ?? "created"}`}>
619627
{selectedHistoryRow?.status ?? "empty"}
@@ -740,7 +748,7 @@ function RunComparisonPanel(props: {
740748
<div className="comparison-selected-run">
741749
<span>Focus run</span>
742750
<strong>{selectedRun.runId}</strong>
743-
<small>{selectedRun.algorithm}</small>
751+
<small>{selectedRun.algorithm} · {runModeLabel(selectedRun)}</small>
744752
</div>
745753
<label className="comparison-select-group">
746754
<span>Compare against</span>
@@ -797,6 +805,7 @@ function RunComparisonPanel(props: {
797805
<span className="comparison-run-main">
798806
<strong>{row.runId}</strong>
799807
<small>{row.algorithm}</small>
808+
<small>{runModeLabel(row)}</small>
800809
</span>
801810
<span className="comparison-score-cell">
802811
<strong>{formatNumber(row.score ?? row.bestReward)}</strong>
@@ -808,6 +817,7 @@ function RunComparisonPanel(props: {
808817
<small>{formatPercent(row.successRate)} success</small>
809818
<small>{formatNumber(row.totalSteps)} steps</small>
810819
<small>{row.checkpointCount} ckpt</small>
820+
<small>{runModeLabel(row)}</small>
811821
</span>
812822
</button>
813823
))}
@@ -1950,7 +1960,25 @@ function comparisonOptionLabel(row: StudioRunHistoryRow): string {
19501960
const rank = row.rank === undefined ? "unranked" : `rank ${row.rank}`;
19511961
const score = formatNumber(row.score ?? row.bestReward);
19521962

1953-
return `${rank} / ${score}`;
1963+
return `${rank} / ${score} / ${runModeLabel(row)}`;
1964+
}
1965+
1966+
function runModeLabel(row: StudioRunHistoryRow | undefined): string {
1967+
if (row === undefined) {
1968+
return "run";
1969+
}
1970+
1971+
const algorithm = row.algorithm.toLowerCase();
1972+
1973+
if (algorithm.includes("inference")) {
1974+
return "inference replay";
1975+
}
1976+
1977+
if (row.status === "running" || row.status === "created") {
1978+
return "training run";
1979+
}
1980+
1981+
return "training run";
19541982
}
19551983

19561984
function firstCheckpointKey(workspace: StudioWorkspaceView, environmentId?: string): string | undefined {

‎apps/studio/src/native-project.ts‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import type {
99
StoredCheckpoint,
1010
StoredExportedArtifact,
1111
StoredExportedArtifactKind,
12+
StudioCheckpointView,
1213
StudioProjectRunSnapshot,
1314
StudioProjectTraceSnapshot,
1415
StudioWorkspaceViewOptions,
@@ -78,6 +79,52 @@ export function studioWorkspaceFromNativeProjectDirectory(
7879
}, options);
7980
}
8081

82+
export function studioCheckpointViewsFromNativeProjectDirectory(
83+
directory: NativeIgnitionProjectDirectory,
84+
workspace: StudioWorkspaceView,
85+
): StudioCheckpointView[] {
86+
return directory.runs.flatMap((runDirectory) =>
87+
runDirectory.checkpoints.flatMap((checkpointFile) => {
88+
if (checkpointFile.payload === undefined) {
89+
return [];
90+
}
91+
92+
const checkpoint = asStoredCheckpoint(checkpointFile.entry);
93+
const sourceRun = workspace.projectView.history.find((row) => row.runId === checkpoint.runId);
94+
95+
if (sourceRun === undefined) {
96+
return [];
97+
}
98+
99+
const inferenceRunIds = new Set(
100+
directory.runs
101+
.filter((candidate) => isCheckpointInferenceRun(candidate.run, checkpoint))
102+
.map((candidate) => candidate.id),
103+
);
104+
const inferenceRuns = workspace.projectView.history.filter((row) => inferenceRunIds.has(row.runId));
105+
106+
return [{
107+
kind: "ignitionrl.studio-checkpoint-view" as const,
108+
generatedAt: workspace.generatedAt,
109+
project: workspace.projectView.project,
110+
checkpoint,
111+
sourceRun,
112+
inferenceRuns,
113+
comparison: workspace.projectView.comparison,
114+
metricNames: workspace.projectView.metricNames,
115+
payload: checkpointFile.payload,
116+
counts: {
117+
inferenceRuns: inferenceRuns.length,
118+
sourceRunMetrics: sourceRun.metricCount,
119+
sourceRunTraces: sourceRun.traceCount,
120+
sourceRunCheckpoints: sourceRun.checkpointCount,
121+
payloadIncluded: true,
122+
},
123+
}];
124+
})
125+
);
126+
}
127+
81128
function asProjectManifest(value: unknown): IgnitionProjectManifest {
82129
const record = expectRecord(value, "project manifest");
83130

@@ -186,6 +233,26 @@ function asStoredExportedArtifact(file: NativeIgnitionProjectArtifactFile): Stor
186233
};
187234
}
188235

236+
function isCheckpointInferenceRun(run: unknown, checkpoint: StoredCheckpoint): boolean {
237+
const record = maybeRecord(run);
238+
const config = maybeRecord(record?.config);
239+
const metadata = maybeRecord(record?.metadata);
240+
const sourceRunId = stringFromUnknown(config?.sourceRunId) ?? stringFromUnknown(metadata?.sourceRunId);
241+
const checkpointId = stringFromUnknown(config?.checkpointId) ?? stringFromUnknown(metadata?.checkpointId);
242+
243+
return sourceRunId === checkpoint.runId && checkpointId === checkpoint.id;
244+
}
245+
246+
function maybeRecord(value: unknown): Record<string, unknown> | undefined {
247+
return typeof value === "object" && value !== null && !Array.isArray(value)
248+
? value as Record<string, unknown>
249+
: undefined;
250+
}
251+
252+
function stringFromUnknown(value: unknown): string | undefined {
253+
return typeof value === "string" && value.trim().length > 0 ? value : undefined;
254+
}
255+
189256
function expectRecord(value: unknown, label: string): Record<string, unknown> {
190257
if (typeof value !== "object" || value === null || Array.isArray(value)) {
191258
throw new Error(`Selected project has an invalid ${label}.`);

‎apps/studio/src/native.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ export type NativeIgnitionProjectTraceFile = {
1414
export type NativeIgnitionProjectCheckpointFile = {
1515
readonly path: string;
1616
readonly entry: unknown;
17+
readonly payload?: unknown;
1718
};
1819

1920
export type NativeIgnitionProjectRunDirectory = {

0 commit comments

Comments
 (0)