From fcf1d4cd7c8ff494047d88722401e75939c6136c Mon Sep 17 00:00:00 2001 From: salimlaimeche Date: Fri, 29 May 2026 03:04:55 +0200 Subject: [PATCH] feat(studio): stream live training metrics --- apps/studio/src-tauri/src/lib.rs | 116 ++++++- apps/studio/src/App.test.tsx | 153 ++++++++- apps/studio/src/App.tsx | 539 ++++++++++++++++++++++++++++++- apps/studio/src/native.ts | 34 ++ 4 files changed, 835 insertions(+), 7 deletions(-) diff --git a/apps/studio/src-tauri/src/lib.rs b/apps/studio/src-tauri/src/lib.rs index c8805c6..c8410f0 100644 --- a/apps/studio/src-tauri/src/lib.rs +++ b/apps/studio/src-tauri/src/lib.rs @@ -6,6 +6,7 @@ use std::{ path::{Path, PathBuf}, time::{SystemTime, UNIX_EPOCH}, }; +use tauri::Emitter; const MAX_STUDIO_ARTIFACT_BYTES: u64 = 25 * 1024 * 1024; const PROJECT_MANIFEST_FILE: &str = "ignitionrl.project.json"; @@ -16,6 +17,7 @@ const CHECKPOINT_INDEX_FILE: &str = "index.json"; const STUDIO_LOCAL_TRAINING_ALGORITHM: &str = "studio-local-headless"; const DEFAULT_LOCAL_TRAINING_EPISODES: u64 = 4; const DEFAULT_LOCAL_TRAINING_MAX_STEPS: u64 = 300; +const TRAINING_METRIC_EVENT: &str = "ignitionrl://training-metric"; #[derive(Debug, PartialEq, Eq, Serialize)] #[serde(rename_all = "camelCase")] @@ -93,25 +95,35 @@ fn open_ignition_project_directory(path: String) -> Result, max_steps: Option, ) -> Result { - start_local_training_path( + let result = start_local_training_path( PathBuf::from(path), env_id, episodes.unwrap_or(DEFAULT_LOCAL_TRAINING_EPISODES), max_steps.unwrap_or(DEFAULT_LOCAL_TRAINING_MAX_STEPS), - ) + )?; + + emit_training_metric_event(&app, &result.project.path, &result.training_session)?; + + Ok(result) } #[tauri::command] fn cancel_local_training_run( + app: tauri::AppHandle, path: String, run_id: String, ) -> Result { - cancel_local_training_path(PathBuf::from(path), run_id) + let result = cancel_local_training_path(PathBuf::from(path), run_id)?; + + emit_training_metric_event(&app, &result.project.path, &result.training_session)?; + + Ok(result) } fn read_studio_artifact_path(path: PathBuf) -> Result { @@ -905,6 +917,74 @@ fn training_session_view( Ok(session) } +fn emit_training_metric_event( + app: &tauri::AppHandle, + project_path: &str, + training_session: &Value, +) -> Result<(), String> { + let event = training_metric_event(project_path, training_session)?; + + app.emit(TRAINING_METRIC_EVENT, event) + .map_err(|error| format!("Unable to emit local training metric event: {error}")) +} + +fn training_metric_event(project_path: &str, training_session: &Value) -> Result { + let run = training_session + .get("run") + .ok_or_else(|| "Training session metric event requires a run.".to_string())?; + let completed_episodes = training_session + .get("completedEpisodes") + .and_then(Value::as_u64) + .unwrap_or(0); + let updated_at = training_session + .get("updatedAt") + .and_then(Value::as_str) + .or_else(|| run.get("updatedAt").and_then(Value::as_str)) + .unwrap_or_else(|| json_string(run, "createdAt").unwrap_or("unknown")); + let values = training_session + .get("latestEpisode") + .and_then(|episode| episode.get("metrics")) + .cloned() + .unwrap_or_else(|| json!({})); + let mut event = json!({ + "projectPath": project_path, + "sessionId": json_string(training_session, "id")?, + "runId": json_string(run, "id")?, + "envId": json_string(run, "envId")?, + "algorithm": json_string(run, "algorithm")?, + "status": json_string(training_session, "status")?, + "requestedEpisodes": training_session + .get("requestedEpisodes") + .and_then(Value::as_u64) + .unwrap_or(DEFAULT_LOCAL_TRAINING_EPISODES), + "completedEpisodes": completed_episodes, + "progress": training_session + .get("progress") + .and_then(Value::as_f64) + .unwrap_or(0.0), + "t": completed_episodes, + "values": values, + "createdAt": updated_at, + }); + let event_object = event + .as_object_mut() + .ok_or_else(|| "Training metric event must be a JSON object.".to_string())?; + + if let Some(latest_episode) = training_session.get("latestEpisode") { + event_object.insert("latestEpisode".to_string(), latest_episode.clone()); + } + + if let Some(summary) = training_session.get("summary") { + event_object.insert("summary".to_string(), summary.clone()); + } + + if let Some(error) = training_session.get("error") { + event_object.insert("error".to_string(), error.clone()); + } + + Ok(event) +} + fn json_string<'a>(value: &'a Value, key: &str) -> Result<&'a str, String> { value .get(key) @@ -1245,6 +1325,21 @@ mod tests { .and_then(Value::as_bool), Some(true), ); + let start_event = + training_metric_event(&started.project.path, &started.training_session).unwrap(); + + assert_eq!( + start_event.get("status").and_then(Value::as_str), + Some("running"), + ); + assert_eq!( + start_event + .get("values") + .and_then(Value::as_object) + .unwrap() + .len(), + 0, + ); let duplicate = start_local_training_path(root.clone(), "Target2D-v0".to_string(), 4, 300).unwrap_err(); @@ -1287,6 +1382,21 @@ mod tests { .and_then(Value::as_bool), Some(false), ); + let cancel_event = + training_metric_event(&cancelled.project.path, &cancelled.training_session).unwrap(); + + assert_eq!( + cancel_event.get("status").and_then(Value::as_str), + Some("cancelled"), + ); + assert_eq!( + cancel_event + .get("values") + .and_then(|values| values.get("episodeLength")) + .and_then(Value::as_u64), + Some(1), + ); + assert!(cancel_event.get("latestEpisode").is_some()); let run_file = root .join("runs") diff --git a/apps/studio/src/App.test.tsx b/apps/studio/src/App.test.tsx index 2b0da95..0eeb7e5 100644 --- a/apps/studio/src/App.test.tsx +++ b/apps/studio/src/App.test.tsx @@ -13,7 +13,12 @@ import type { StudioWorkspaceView, } from "@ignitionrl/sdk"; import { App } from "./App"; -import type { NativeIgnitionProjectDirectory, NativeIgnitionProjectRunDirectory, NativeStudioApi } from "./native"; +import type { + NativeIgnitionProjectDirectory, + NativeIgnitionProjectRunDirectory, + NativeStudioApi, + NativeTrainingMetricEvent, +} from "./native"; import { sampleWorkspace } from "./sample-workspace"; (globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true; @@ -442,6 +447,122 @@ describe("Studio app shell", () => { act(() => renderer?.unmount()); }); + test("ingests live training metrics and preserves sparse terminal updates", async () => { + let renderer: ReactTestRenderer | undefined; + let currentProject = nativeProjectDirectory(); + let metricListener: ((event: NativeTrainingMetricEvent) => void) | undefined; + const nativeApi: NativeStudioApi = { + isAvailable: () => true, + openIgnitionProjectDirectory: async () => currentProject, + openStudioArtifactFile: async () => undefined, + startLocalTrainingRun: async () => { + currentProject = nativeProjectWithLocalTraining("running"); + + return { + project: currentProject, + trainingSession: nativeTrainingSession("running"), + }; + }, + cancelLocalTrainingRun: async () => { + throw new Error("Unexpected local training cancellation."); + }, + listenToTrainingMetricEvents: async (listener) => { + metricListener = listener; + + return () => { + metricListener = undefined; + }; + }, + }; + + act(() => { + renderer = create(React.createElement(App, { + initialWorkspace: sampleWorkspace, + nativeApi, + })); + }); + + await act(async () => {}); + + await act(async () => { + await renderer?.root.findByProps({ + "aria-label": "Open native project directory", + }).props.onClick(); + }); + + await act(async () => { + await renderer?.root.findByProps({ + "aria-label": "Start local training", + }).props.onClick(); + }); + + expect(metricListener).toBeDefined(); + + await act(async () => { + metricListener?.(liveMetricEvent({ + status: "running", + completedEpisodes: 1, + progress: 0.25, + t: 1, + values: { + totalReward: 3.5, + success: 0, + }, + latestEpisode: { + episode: 0, + episodeId: "studio-local-run:episode:0", + trace: { + episodeId: "studio-local-run:episode:0", + path: "runs/studio-local-run/traces/studio-local-run_episode_0.json", + }, + totalReward: 3.5, + length: 7, + success: false, + terminated: false, + truncated: false, + metrics: { + totalReward: 3.5, + success: 0, + }, + }, + })); + }); + + let text = JSON.stringify(renderer?.toJSON()); + expect(renderer?.root.findByProps({ className: "training-progress-label" }) + .findByType("span").children.join("")).toBe("1 / 4 episodes"); + expect(text).toContain("3.5"); + expect(text).toContain("studio-local-run:episode:0"); + expect(text).toContain("totalReward"); + expect(chartPointLabel(renderer)).toBe("2 pts"); + expect(text).not.toContain("No metric points."); + + await act(async () => { + metricListener?.(liveMetricEvent({ + status: "cancelled", + completedEpisodes: 1, + progress: 0.25, + t: 2, + values: {}, + createdAt: "2026-05-28T00:03:00.000Z", + })); + }); + + text = JSON.stringify(renderer?.toJSON()); + + expect(text).toContain("cancelled"); + expect(text).toContain("3.5"); + expect(renderer?.root.findByProps({ className: "training-progress-label" }) + .findByType("span").children.join("")).toBe("1 / 4 episodes"); + expect(renderer?.root.findByProps({ + "aria-label": "Cancel local training", + }).props.disabled).toBe(true); + expect(chartPointLabel(renderer)).toBe("2 pts"); + expect(text).not.toContain("No metric points."); + + act(() => renderer?.unmount()); + }); + test("loads another run episode artifact and switches replay plus reward views", async () => { let renderer: ReactTestRenderer | undefined; const episodeOneView = secondEpisodeRunView(); @@ -780,6 +901,36 @@ describe("Studio app shell", () => { }); }); +function liveMetricEvent( + overrides: Partial, +): NativeTrainingMetricEvent { + return { + projectPath: "/tmp/project.ignitionrl", + sessionId: "studio-local-run:session", + runId: "studio-local-run", + envId: "Target2D-v0", + algorithm: "studio-local-headless", + status: "running", + requestedEpisodes: 4, + completedEpisodes: 0, + progress: 0, + t: 0, + values: {}, + createdAt: "2026-05-28T00:02:30.000Z", + ...overrides, + }; +} + +function chartPointLabel(renderer: ReactTestRenderer | undefined): string { + const chartPanel = renderer?.root.findByProps({ className: "chart-panel" }); + + return chartPanel + ?.findByProps({ className: "panel-heading compact" }) + .findByType("span") + .children + .join("") ?? ""; +} + function nativeProjectDirectory(): NativeIgnitionProjectDirectory { const createdAt = "2026-05-28T00:00:00.000Z"; const runUpdatedAt = "2026-05-28T00:01:00.000Z"; diff --git a/apps/studio/src/App.tsx b/apps/studio/src/App.tsx index 1b32e9e..299514d 100644 --- a/apps/studio/src/App.tsx +++ b/apps/studio/src/App.tsx @@ -2,20 +2,31 @@ import { useEffect, useMemo, useState, type ChangeEvent } from "react"; import { FolderOpen, Play, Square, Upload } from "lucide-react"; import type { EpisodeReplayFrame, + MetricRecord, RewardDebuggerTerm, RewardTimelinePoint, RunComparisonDirection, + RunManifest, + RunSummary, RunStatus, StudioCheckpointView, + StudioMetricChartPoint, + StudioMetricChartRun, StudioEnvironmentRow, StudioMetricChart, + StudioTrainingSessionEpisode, StudioRunHistoryRow, StudioRunView, StudioTrainingSessionView, StudioWorkspaceView, StoredCheckpoint, } from "@ignitionrl/sdk"; -import { defaultNativeStudioApi, type NativeLocalTrainingRunResult, type NativeStudioApi } from "./native"; +import { + defaultNativeStudioApi, + type NativeLocalTrainingRunResult, + type NativeStudioApi, + type NativeTrainingMetricEvent, +} from "./native"; import { studioWorkspaceFromNativeProjectDirectory } from "./native-project"; import { sampleWorkspace } from "./sample-workspace"; import { StudioViewportPanel } from "./StudioViewportPanel"; @@ -38,6 +49,7 @@ type RunSummaryLike = { type RunSortMode = "rank" | "updated" | "reward"; type RunStatusFilter = "all" | RunStatus; type ComparisonMetricKind = "number" | "percent"; +type MetricChartSeriesPoint = StudioMetricChartRun["points"][number]; export function App({ initialWorkspace = sampleWorkspace, @@ -73,6 +85,7 @@ export function App({ const [loadError, setLoadError] = useState(); const [loadedSource, setLoadedSource] = useState(); const [loadedProjectPath, setLoadedProjectPath] = useState(); + const [liveMetricEvents, setLiveMetricEvents] = useState([]); const history = workspace.projectView.history; const isNativeRuntime = nativeApi.isAvailable(); const selectedEnvironment = workspace.projectView.environments.find((environment) => @@ -121,9 +134,13 @@ export function App({ && loadedProjectPath !== undefined && selectedTrainingSession?.canCancel === true && selectedTrainingSession.run?.id !== undefined; + const metricCharts = useMemo( + () => mergeLiveMetricCharts(workspace.metricCharts, liveMetricEvents), + [workspace.metricCharts, liveMetricEvents], + ); const selectedMetricChart = useMemo( - () => bestMetricChart(workspace.metricCharts), - [workspace.metricCharts], + () => bestMetricChart(metricCharts), + [metricCharts], ); useEffect(() => { @@ -140,6 +157,39 @@ export function App({ }); }, [availableRunViews]); + useEffect(() => { + if (!isNativeRuntime || nativeApi.listenToTrainingMetricEvents === undefined) { + return; + } + + let disposed = false; + let unlisten: (() => void) | undefined; + + void nativeApi.listenToTrainingMetricEvents((event) => { + if (disposed) { + return; + } + + ingestTrainingMetricEvent(event); + }).then((nextUnlisten) => { + if (disposed) { + nextUnlisten(); + return; + } + + unlisten = nextUnlisten; + }).catch((error) => { + if (!disposed) { + setLoadError(error instanceof Error ? error.message : String(error)); + } + }); + + return () => { + disposed = true; + unlisten?.(); + }; + }, [isNativeRuntime, nativeApi, workspace.projectView.project.id]); + function selectRun(runId: string) { const row = history.find((entry) => entry.runId === runId); @@ -238,6 +288,7 @@ export function App({ setCheckpointViews([]); setRunViews([]); setTrainingSessionViews([]); + setLiveMetricEvents([]); setSelectedEpisodeId(nextWorkspace.selection.episodeId ?? runViewEpisodeId(nextWorkspace.selectedRun)); setSelectedTrainingSessionId(undefined); setRunQuery(""); @@ -275,6 +326,30 @@ export function App({ setLoadedSource(result.project.directoryName); } + function ingestTrainingMetricEvent(event: NativeTrainingMetricEvent) { + if ( + event.projectPath !== undefined + && loadedProjectPath !== undefined + && event.projectPath !== loadedProjectPath + ) { + return; + } + + setLiveMetricEvents((events) => [...events, event]); + setTrainingSessionViews((views) => { + const sessionId = event.sessionId ?? `${event.runId}:session`; + const previous = views.find((view) => view.id === sessionId); + + return upsertTrainingSessionView( + views, + trainingSessionViewFromMetricEvent(event, previous, workspace.projectView.project.id), + ); + }); + setSelectedTrainingSessionId(event.sessionId ?? `${event.runId}:session`); + setSelectedEnvironmentId(event.envId); + setSelectedRunId(event.runId); + } + async function openNativeWorkspaceFile() { try { const file = await nativeApi.openStudioArtifactFile(); @@ -1884,6 +1959,464 @@ function upsertTrainingSessionView( return views.map((view, index) => index === existingIndex ? nextView : view); } +function trainingSessionViewFromMetricEvent( + event: NativeTrainingMetricEvent, + previous: StudioTrainingSessionView | undefined, + projectId: string, +): StudioTrainingSessionView { + const sessionId = event.sessionId ?? `${event.runId}:session`; + const requestedEpisodes = positiveInteger( + event.requestedEpisodes, + previous?.requestedEpisodes ?? 1, + ); + const nextLatestEpisode = studioTrainingEpisodeFromUnknown(event.latestEpisode); + const episodes = nextLatestEpisode === undefined + ? [...(previous?.episodes ?? [])] + : upsertTrainingSessionEpisode(previous?.episodes ?? [], nextLatestEpisode); + const completedEpisodes = Math.max( + event.completedEpisodes ?? 0, + previous?.completedEpisodes ?? 0, + episodes.length, + ); + const derivedSummary = runSummaryFromEpisodes(episodes); + const summary = runSummaryFromUnknown(event.summary) ?? derivedSummary ?? previous?.summary; + const updatedAt = event.createdAt; + const run = runManifestFromMetricEvent(event, previous?.run, projectId, summary, updatedAt); + const progress = clampProgress( + event.progress ?? (requestedEpisodes === 0 ? 0 : completedEpisodes / requestedEpisodes), + ); + const latestEpisode = nextLatestEpisode ?? previous?.latestEpisode; + const isTerminal = isTerminalTrainingStatus(event.status); + const error = event.error ?? previous?.error; + + return { + kind: "ignitionrl.studio-training-session-view", + generatedAt: previous?.generatedAt ?? updatedAt, + id: sessionId, + sessionKind: previous?.sessionKind ?? "learner", + status: event.status, + requestedEpisodes, + completedEpisodes, + remainingEpisodes: Math.max(0, requestedEpisodes - completedEpisodes), + progress, + cancelled: event.status === "cancelled", + canStart: false, + canCancel: event.status === "running", + isTerminal, + updatedAt, + startedAt: previous?.startedAt ?? run.createdAt, + run, + ...(summary !== undefined ? { summary } : {}), + ...(latestEpisode !== undefined ? { latestEpisode } : {}), + episodes, + ...(error !== undefined ? { error } : {}), + counts: trainingEpisodeCounts(episodes), + }; +} + +function runManifestFromMetricEvent( + event: NativeTrainingMetricEvent, + previous: RunManifest | undefined, + projectId: string, + summary: RunSummary | undefined, + updatedAt: string, +): RunManifest { + const error = event.error ?? previous?.error; + + return { + schemaVersion: 1, + id: event.runId, + projectId: previous?.projectId ?? projectId, + envId: event.envId, + algorithm: event.algorithm ?? previous?.algorithm ?? "studio-local-headless", + status: event.status, + createdAt: previous?.createdAt ?? event.createdAt, + updatedAt, + ...(previous?.seed !== undefined ? { seed: previous.seed } : {}), + ...(previous?.config !== undefined ? { config: previous.config } : {}), + ...(summary !== undefined ? { summary } : {}), + ...(error !== undefined ? { error } : {}), + ...(previous?.metadata !== undefined ? { metadata: previous.metadata } : {}), + }; +} + +function trainingEpisodeCounts(episodes: readonly StudioTrainingSessionEpisode[]): StudioTrainingSessionView["counts"] { + return { + episodes: episodes.length, + successfulEpisodes: episodes.filter((episode) => episode.success).length, + terminatedEpisodes: episodes.filter((episode) => episode.terminated).length, + truncatedEpisodes: episodes.filter((episode) => episode.truncated).length, + totalSteps: episodes.reduce((sum, episode) => sum + episode.length, 0), + totalReward: episodes.reduce((sum, episode) => sum + episode.totalReward, 0), + }; +} + +function upsertTrainingSessionEpisode( + episodes: readonly StudioTrainingSessionEpisode[], + nextEpisode: StudioTrainingSessionEpisode, +): StudioTrainingSessionEpisode[] { + const existingIndex = episodes.findIndex((episode) => episode.episodeId === nextEpisode.episodeId); + + if (existingIndex === -1) { + return [...episodes, nextEpisode]; + } + + return episodes.map((episode, index) => index === existingIndex ? nextEpisode : episode); +} + +function studioTrainingEpisodeFromUnknown(value: unknown): StudioTrainingSessionEpisode | undefined { + if (!isRecord(value) || !isRecord(value.trace)) { + return undefined; + } + + const episode = numberFromUnknown(value.episode); + const totalReward = numberFromUnknown(value.totalReward); + const length = numberFromUnknown(value.length); + const metrics = metricValuesFromUnknown(value.metrics); + + if ( + episode === undefined + || typeof value.episodeId !== "string" + || typeof value.trace.episodeId !== "string" + || typeof value.trace.path !== "string" + || totalReward === undefined + || length === undefined + || typeof value.success !== "boolean" + || typeof value.terminated !== "boolean" + || typeof value.truncated !== "boolean" + ) { + return undefined; + } + + return { + episode, + episodeId: value.episodeId, + trace: { + episodeId: value.trace.episodeId, + path: value.trace.path, + }, + totalReward, + length, + success: value.success, + terminated: value.terminated, + truncated: value.truncated, + ...(typeof value.reason === "string" ? { reason: value.reason } : {}), + ...(metrics !== undefined ? { metrics } : {}), + }; +} + +function runSummaryFromUnknown(value: unknown): RunSummary | undefined { + if (!isRecord(value)) { + return undefined; + } + + const summary: RunSummaryLike = { + episodes: numberFromUnknown(value.episodes), + totalSteps: numberFromUnknown(value.totalSteps), + totalReward: numberFromUnknown(value.totalReward), + successRate: numberFromUnknown(value.successRate), + bestReward: numberFromUnknown(value.bestReward), + lastReward: numberFromUnknown(value.lastReward), + }; + + if ( + summary.episodes === undefined + || summary.totalSteps === undefined + || summary.totalReward === undefined + || summary.successRate === undefined + ) { + return undefined; + } + + return { + episodes: summary.episodes, + totalSteps: summary.totalSteps, + totalReward: summary.totalReward, + successRate: summary.successRate, + ...(summary.bestReward !== undefined ? { bestReward: summary.bestReward } : {}), + ...(summary.lastReward !== undefined ? { lastReward: summary.lastReward } : {}), + }; +} + +function runSummaryFromEpisodes( + episodes: readonly StudioTrainingSessionEpisode[], +): RunSummary | undefined { + if (episodes.length === 0) { + return undefined; + } + + const rewards = episodes.map((episode) => episode.totalReward); + + return { + episodes: episodes.length, + totalSteps: episodes.reduce((sum, episode) => sum + episode.length, 0), + totalReward: rewards.reduce((sum, rewardValue) => sum + rewardValue, 0), + successRate: episodes.filter((episode) => episode.success).length / episodes.length, + bestReward: Math.max(...rewards), + lastReward: rewards[rewards.length - 1], + }; +} + +function mergeLiveMetricCharts( + baseCharts: readonly StudioMetricChart[], + events: readonly NativeTrainingMetricEvent[], +): StudioMetricChart[] { + if (events.length === 0) { + return [...baseCharts]; + } + + const byName = new Map(baseCharts.map((chart) => [chart.name, chart])); + + for (const liveChart of liveMetricCharts(events)) { + byName.set(liveChart.name, mergeMetricChart(byName.get(liveChart.name), liveChart)); + } + + return [...byName.values()]; +} + +function liveMetricCharts(events: readonly NativeTrainingMetricEvent[]): StudioMetricChart[] { + const byMetric = new Map>(); + + for (const event of events) { + const metric = metricRecordFromTrainingMetricEvent(event); + + for (const [name, value] of Object.entries(metric.values)) { + let runs = byMetric.get(name); + + if (runs === undefined) { + runs = new Map(); + byMetric.set(name, runs); + } + + const bucket = runs.get(event.runId) ?? { + envId: event.envId, + algorithm: event.algorithm ?? "studio-local-headless", + status: event.status, + points: [], + }; + + bucket.envId = event.envId; + bucket.algorithm = event.algorithm ?? bucket.algorithm; + bucket.status = event.status; + bucket.points.push({ + t: metric.t, + value, + ...(metric.episode !== undefined ? { episode: metric.episode } : {}), + ...(metric.step !== undefined ? { step: metric.step } : {}), + createdAt: metric.createdAt, + }); + runs.set(event.runId, bucket); + } + } + + return [...byMetric.entries()].flatMap(([name, runsById]) => { + const runs = [...runsById.entries()].flatMap(([runId, bucket]) => { + const run = metricChartRunFromPoints({ + runId, + envId: bucket.envId, + algorithm: bucket.algorithm, + status: bucket.status, + points: bucket.points, + }); + + return run === undefined ? [] : [run]; + }); + + return runs.length === 0 ? [] : [metricChartFromRuns(name, runs)]; + }); +} + +function mergeMetricChart( + baseChart: StudioMetricChart | undefined, + liveChart: StudioMetricChart, +): StudioMetricChart { + if (baseChart === undefined) { + return liveChart; + } + + const liveRunsById = new Map(liveChart.runs.map((run) => [run.runId, run])); + const mergedRuns = baseChart.runs.map((baseRun) => { + const liveRun = liveRunsById.get(baseRun.runId); + + if (liveRun === undefined) { + return baseRun; + } + + liveRunsById.delete(baseRun.runId); + + return metricChartRunFromPoints({ + runId: baseRun.runId, + envId: liveRun.envId, + algorithm: liveRun.algorithm, + status: liveRun.status, + points: dedupeMetricPoints([...baseRun.points, ...liveRun.points]), + }) ?? liveRun; + }); + + return metricChartFromRuns(baseChart.name, [...mergedRuns, ...liveRunsById.values()]); +} + +function metricChartRunFromPoints(input: { + readonly runId: string; + readonly envId: string; + readonly algorithm: string; + readonly status: RunStatus; + readonly points: readonly MetricChartSeriesPoint[]; +}): StudioMetricChartRun | undefined { + const points = dedupeMetricPoints(input.points).sort(compareMetricPoints); + + if (points.length === 0) { + return undefined; + } + + const values = points.map((point) => point.value); + const first = values[0] as number; + const last = values[values.length - 1] as number; + + return { + runId: input.runId, + envId: input.envId, + algorithm: input.algorithm, + status: input.status, + pointCount: points.length, + points, + first, + last, + min: Math.min(...values), + max: Math.max(...values), + mean: values.reduce((sum, value) => sum + value, 0) / values.length, + delta: last - first, + }; +} + +function metricChartFromRuns( + name: string, + runs: readonly StudioMetricChartRun[], +): StudioMetricChart { + const allPoints: StudioMetricChartPoint[] = runs.flatMap((run) => + run.points.map((point) => ({ + ...point, + runId: run.runId, + envId: run.envId, + algorithm: run.algorithm, + status: run.status, + })) + ); + const values = allPoints.map((point) => point.value); + const latestPoint = latestMetricChartPoint(allPoints); + + return { + name, + pointCount: allPoints.length, + runs, + ...(values.length > 0 ? { min: Math.min(...values), max: Math.max(...values) } : {}), + ...(latestPoint !== undefined ? { latestPoint } : {}), + }; +} + +function latestMetricChartPoint( + points: readonly StudioMetricChartPoint[], +): StudioMetricChartPoint | undefined { + return [...points].sort((left, right) => + right.createdAt.localeCompare(left.createdAt) + || right.t - left.t + || right.runId.localeCompare(left.runId) + )[0]; +} + +function metricRecordFromTrainingMetricEvent(event: NativeTrainingMetricEvent): MetricRecord { + return { + t: Number.isFinite(event.t) ? event.t : 0, + ...(typeof event.episode === "number" && Number.isFinite(event.episode) ? { episode: event.episode } : {}), + ...(typeof event.step === "number" && Number.isFinite(event.step) ? { step: event.step } : {}), + values: finiteMetricValues(event.values), + createdAt: event.createdAt, + }; +} + +function dedupeMetricPoints( + points: readonly MetricChartSeriesPoint[], +): MetricChartSeriesPoint[] { + const byKey = new Map(); + + for (const point of points) { + byKey.set(metricPointKey(point), point); + } + + return [...byKey.values()]; +} + +function metricPointKey(point: MetricChartSeriesPoint): string { + return [ + point.createdAt, + point.t, + point.episode ?? "", + point.step ?? "", + point.value, + ].join(":"); +} + +function compareMetricPoints(left: MetricChartSeriesPoint, right: MetricChartSeriesPoint): number { + return left.t - right.t || left.createdAt.localeCompare(right.createdAt); +} + +function finiteMetricValues(values: Readonly>): Readonly> { + const nextValues: Record = {}; + + for (const [name, value] of Object.entries(values)) { + if (Number.isFinite(value)) { + nextValues[name] = value; + } + } + + return nextValues; +} + +function metricValuesFromUnknown(value: unknown): Readonly> | undefined { + if (!isRecord(value)) { + return undefined; + } + + const values: Record = {}; + + for (const [name, metric] of Object.entries(value)) { + if (typeof metric === "number" && Number.isFinite(metric)) { + values[name] = metric; + } + } + + return Object.keys(values).length === 0 ? undefined : values; +} + +function numberFromUnknown(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) ? value : undefined; +} + +function positiveInteger(value: number | undefined, fallback: number): number { + if (value === undefined || !Number.isFinite(value) || value <= 0) { + return Math.max(1, Math.floor(fallback)); + } + + return Math.floor(value); +} + +function clampProgress(value: number): number { + if (!Number.isFinite(value)) { + return 0; + } + + return Math.max(0, Math.min(1, value)); +} + +function isTerminalTrainingStatus(status: NativeTrainingMetricEvent["status"]): boolean { + return status === "completed" || status === "cancelled" || status === "failed"; +} + function runViewsForSelection( selectedRun: StudioRunView | undefined, loadedRunViews: readonly StudioRunView[], diff --git a/apps/studio/src/native.ts b/apps/studio/src/native.ts index f53fffb..bb9567d 100644 --- a/apps/studio/src/native.ts +++ b/apps/studio/src/native.ts @@ -48,6 +48,26 @@ export type NativeLocalTrainingRunResult = { readonly trainingSession: unknown; }; +export type NativeTrainingMetricEvent = { + readonly projectPath?: string; + readonly sessionId?: string; + readonly runId: string; + readonly envId: string; + readonly algorithm?: string; + readonly status: "running" | "completed" | "cancelled" | "failed"; + readonly requestedEpisodes?: number; + readonly completedEpisodes?: number; + readonly progress?: number; + readonly t: number; + readonly episode?: number; + readonly step?: number; + readonly values: Readonly>; + readonly createdAt: string; + readonly latestEpisode?: unknown; + readonly summary?: unknown; + readonly error?: string; +}; + export type NativeStudioApi = { readonly isAvailable: () => boolean; readonly openIgnitionProjectDirectory: () => Promise; @@ -62,6 +82,9 @@ export type NativeStudioApi = { readonly projectPath: string; readonly runId: string; }) => Promise; + readonly listenToTrainingMetricEvents?: ( + listener: (event: NativeTrainingMetricEvent) => void, + ) => Promise<() => void>; }; type TauriWindow = Window & { @@ -121,6 +144,17 @@ export const defaultNativeStudioApi: NativeStudioApi = { runId: options.runId, }); }, + async listenToTrainingMetricEvents(listener) { + if (!isTauriRuntime()) { + return () => {}; + } + + const { listen } = await import("@tauri-apps/api/event"); + + return listen("ignitionrl://training-metric", (event) => { + listener(event.payload); + }); + }, async openStudioArtifactFile() { if (!isTauriRuntime()) { return undefined;