diff --git a/crates/ignitionrl-engine/README.md b/crates/ignitionrl-engine/README.md index 60987c5..4df2e73 100644 --- a/crates/ignitionrl-engine/README.md +++ b/crates/ignitionrl-engine/README.md @@ -14,4 +14,9 @@ Supported prototype paths: - `run.start` - `run.stop` - `run.action` +- `checkpoint.load` +- `checkpoint.save` +The prototype emits ordered `run.status`, `transition`, `metrics` and +`checkpoint` events so Studio can exercise its local-training data path before a +real learner backend is wired in. diff --git a/crates/ignitionrl-engine/src/lib.rs b/crates/ignitionrl-engine/src/lib.rs index f1a739b..7e4cc6e 100644 --- a/crates/ignitionrl-engine/src/lib.rs +++ b/crates/ignitionrl-engine/src/lib.rs @@ -1,6 +1,6 @@ use ignitionrl_core::{ - EncodedAction, JsonObject, RolloutSummary, RunStatus, ENGINE_IPC_PROTOCOL, - ENGINE_IPC_PROTOCOL_VERSION, + CheckpointRecord, EncodedAction, JsonObject, MetricBatch, MetricPoint, RolloutSummary, + RunStatus, ENGINE_IPC_PROTOCOL, ENGINE_IPC_PROTOCOL_VERSION, }; use serde::Deserialize; use serde_json::{json, Map, Value}; @@ -19,7 +19,11 @@ const ENGINE_CAPABILITIES: &[&str] = &[ "run.start", "run.stop", "run.action", + "checkpoint.load", + "checkpoint.save", "events.transition", + "events.metrics", + "events.checkpoint", "events.status", ]; @@ -76,6 +80,8 @@ impl JsonlEngine { "run.start" => self.handle_start(request), "run.stop" => self.handle_stop(request), "run.action" => self.handle_action(request), + "checkpoint.load" => self.handle_checkpoint_load(request), + "checkpoint.save" => self.handle_checkpoint_save(request), _ => EngineLineResult::message(self.error_response( RequestInfo::from_request(&request), "unsupported_capability", @@ -186,17 +192,18 @@ impl JsonlEngine { } fn handle_stop(&mut self, request: RawRequest) -> EngineLineResult { - let run_id = match request.payload.get("runId").and_then(Value::as_str) { - Some(run_id) => run_id.to_string(), - None => { + let payload = match RunControlPayload::from_value(&request.payload, "run.stop") { + Ok(payload) => payload, + Err(message) => { return EngineLineResult::message(self.error_response( RequestInfo::from_request(&request), "invalid_request", - "run.stop payload must include runId.", + message, None, )); } }; + let run_id = payload.run_id; let Some(run) = self.runs.remove(&run_id) else { return EngineLineResult::message(self.error_response( RequestInfo::from_request(&request).with_run_id(run_id.clone()), @@ -205,7 +212,8 @@ impl JsonlEngine { None, )); }; - let summary = run.summary("run_stopped"); + let metrics = run.metric_batch(&request.timestamp); + let summary = run.cancelled_summary(payload.reason.as_deref().unwrap_or("run_stopped")); let status = status_payload( &run_id, Some(&run.env_id), @@ -219,6 +227,7 @@ impl JsonlEngine { EngineLineResult { messages: vec![ self.success_response(&request, Some(&run_id), status.clone()), + self.event_message("metrics", Some(&run_id), metrics, &request.timestamp), self.event_message("run.status", Some(&run_id), status, &request.timestamp), ], shutdown: false, @@ -256,13 +265,130 @@ impl JsonlEngine { )); } - let transition = run.transition_payload(payload); + let (run_id, transition, metrics, completed_status, should_complete) = { + let transition = run.transition_payload(payload); + let metrics = run.metric_batch(&request.timestamp); + let should_complete = transition + .get("done") + .and_then(Value::as_bool) + .unwrap_or(false); + let completed_status = should_complete.then(|| { + status_payload( + &run.run_id, + Some(&run.env_id), + RunStatus::Completed, + Some(&run.started_at), + &request.timestamp, + Some(run.completed_summary("max_steps")), + None, + ) + }); + + ( + run.run_id.clone(), + transition, + metrics, + completed_status, + should_complete, + ) + }; + + if should_complete { + self.runs.remove(&run_id); + } + + let mut messages = vec![ + self.success_response(&request, Some(&run_id), transition.clone()), + self.event_message("transition", Some(&run_id), transition, &request.timestamp), + self.event_message("metrics", Some(&run_id), metrics, &request.timestamp), + ]; + + if let Some(status) = completed_status { + messages.push(self.event_message( + "run.status", + Some(&run_id), + status, + &request.timestamp, + )); + } + + EngineLineResult { + messages, + shutdown: false, + } + } + + fn handle_checkpoint_save(&mut self, request: RawRequest) -> EngineLineResult { + let payload = match CheckpointSavePayload::from_value(&request.payload) { + Ok(payload) => payload, + Err(message) => { + return EngineLineResult::message(self.error_response( + RequestInfo::from_request(&request), + "invalid_request", + message, + None, + )); + } + }; + + let Some(run) = self.runs.get_mut(&payload.run_id) else { + return EngineLineResult::message(self.error_response( + RequestInfo::from_request(&request).with_run_id(payload.run_id), + "run_not_found", + "Run does not exist.", + None, + )); + }; + + let checkpoint = run.save_checkpoint_payload(payload, &request.timestamp); let run_id = run.run_id.clone(); EngineLineResult { messages: vec![ - self.success_response(&request, Some(&run_id), transition.clone()), - self.event_message("transition", Some(&run_id), transition, &request.timestamp), + self.success_response(&request, Some(&run_id), checkpoint.clone()), + self.event_message("checkpoint", Some(&run_id), checkpoint, &request.timestamp), + ], + shutdown: false, + } + } + + fn handle_checkpoint_load(&mut self, request: RawRequest) -> EngineLineResult { + let payload = match CheckpointLoadPayload::from_value(&request.payload) { + Ok(payload) => payload, + Err(message) => { + return EngineLineResult::message(self.error_response( + RequestInfo::from_request(&request), + "invalid_request", + message, + None, + )); + } + }; + + let Some(run_id) = payload.run_id.clone() else { + return EngineLineResult::message(self.error_response( + RequestInfo::from_request(&request), + "invalid_request", + "checkpoint.load requires runId for JSONL event streaming.", + None, + )); + }; + let Some(run) = self.runs.get(&run_id) else { + return EngineLineResult::message(self.error_response( + RequestInfo::from_request(&request).with_run_id(run_id), + "run_not_found", + "Run does not exist.", + None, + )); + }; + + let checkpoint = run.load_checkpoint_payload(payload, &request.timestamp); + let run_id = run.run_id.clone(); + + EngineLineResult { + messages: vec![ + self.success_response(&request, Some(&run_id), checkpoint.clone()), + self.event_message("checkpoint", Some(&run_id), checkpoint, &request.timestamp), ], shutdown: false, } @@ -555,10 +681,30 @@ impl RunStartPayload { started_at, step_count: 0, total_reward: 0.0, + checkpoint_count: 0, } } } +#[derive(Debug)] +struct RunControlPayload { + run_id: String, + reason: Option, +} + +impl RunControlPayload { + fn from_value(value: &Value, request_type: &str) -> Result { + Ok(Self { + run_id: required_string(value, "runId") + .map_err(|_| format!("{request_type} payload must include runId."))?, + reason: value + .get("reason") + .and_then(Value::as_str) + .map(ToOwned::to_owned), + }) + } +} + #[derive(Debug)] struct ActionPayload { run_id: String, @@ -597,6 +743,51 @@ impl ActionPayload { } } +#[derive(Debug)] +struct CheckpointSavePayload { + run_id: String, + checkpoint_id: Option, + path: Option, + metadata: Option, +} + +impl CheckpointSavePayload { + fn from_value(value: &Value) -> Result { + Ok(Self { + run_id: required_string(value, "runId")?, + checkpoint_id: value + .get("checkpointId") + .and_then(Value::as_str) + .map(ToOwned::to_owned), + path: value + .get("path") + .and_then(Value::as_str) + .map(ToOwned::to_owned), + metadata: optional_json_object(value, "metadata")?, + }) + } +} + +#[derive(Debug)] +struct CheckpointLoadPayload { + run_id: Option, + checkpoint_id: String, + path: String, +} + +impl CheckpointLoadPayload { + fn from_value(value: &Value) -> Result { + Ok(Self { + run_id: value + .get("runId") + .and_then(Value::as_str) + .map(ToOwned::to_owned), + checkpoint_id: required_string(value, "checkpointId")?, + path: required_string(value, "path")?, + }) + } +} + #[derive(Debug)] struct RunState { project_id: String, @@ -609,6 +800,7 @@ struct RunState { started_at: String, step_count: u64, total_reward: f64, + checkpoint_count: u64, } impl RunState { @@ -623,6 +815,7 @@ impl RunState { let next_observation = synthetic_observation(self.observation_len, next_step); self.step_count = self.step_count.max(next_step); + self.total_reward += 0.0; let mut transition = json!({ "runId": self.run_id, @@ -636,8 +829,8 @@ impl RunState { "rewardTerms": {}, "nextObservation": next_observation, "done": done, - "terminated": done, - "truncated": false, + "terminated": false, + "truncated": done, "info": { "engine": "ignitionrl-engine", "algorithm": self.algorithm, @@ -657,7 +850,77 @@ impl RunState { transition } - fn summary(&self, reason: &str) -> RolloutSummary { + fn metric_batch(&self, timestamp: &str) -> Value { + serde_json::to_value(MetricBatch { + run_id: self.run_id.clone(), + env_id: self.env_id.clone(), + created_at: timestamp.to_string(), + metrics: vec![ + MetricPoint { + name: "episodeReward".to_string(), + value: self.total_reward, + t: Some(self.step_count), + episode: Some(0), + step: Some(self.step_count), + }, + MetricPoint { + name: "episodeLength".to_string(), + value: self.step_count as f64, + t: Some(self.step_count), + episode: Some(0), + step: Some(self.step_count), + }, + ], + }) + .expect("MetricBatch serializes to JSON") + } + + fn save_checkpoint_payload( + &mut self, + payload: CheckpointSavePayload, + timestamp: &str, + ) -> Value { + self.checkpoint_count += 1; + let checkpoint_id = payload + .checkpoint_id + .unwrap_or_else(|| format!("checkpoint-{}", self.checkpoint_count)); + + self.checkpoint_payload(checkpoint_id, payload.path, payload.metadata, timestamp) + } + + fn load_checkpoint_payload(&self, payload: CheckpointLoadPayload, timestamp: &str) -> Value { + let mut metadata = JsonObject::new(); + metadata.insert("loaded".to_string(), Value::Bool(true)); + + self.checkpoint_payload( + payload.checkpoint_id, + Some(payload.path), + Some(metadata), + timestamp, + ) + } + + fn checkpoint_payload( + &self, + checkpoint_id: String, + path: Option, + metadata: Option, + timestamp: &str, + ) -> Value { + serde_json::to_value(CheckpointRecord { + run_id: self.run_id.clone(), + env_id: self.env_id.clone(), + checkpoint_id, + algorithm: self.algorithm.clone(), + path, + created_at: timestamp.to_string(), + metadata, + summary: Some(self.partial_summary("checkpoint")), + }) + .expect("CheckpointRecord serializes to JSON") + } + + fn partial_summary(&self, reason: &str) -> RolloutSummary { RolloutSummary { total_reward: self.total_reward, length: self.step_count as usize, @@ -667,6 +930,21 @@ impl RunState { reason: Some(reason.to_string()), } } + + fn cancelled_summary(&self, reason: &str) -> RolloutSummary { + self.partial_summary(reason) + } + + fn completed_summary(&self, reason: &str) -> RolloutSummary { + RolloutSummary { + total_reward: self.total_reward, + length: self.step_count as usize, + success: false, + terminated: false, + truncated: true, + reason: Some(reason.to_string()), + } + } } fn required_string(value: &Value, field: &str) -> Result { @@ -677,6 +955,14 @@ fn required_string(value: &Value, field: &str) -> Result { .ok_or_else(|| format!("Payload must include string {field}.")) } +fn optional_json_object(value: &Value, field: &str) -> Result, String> { + match value.get(field) { + Some(Value::Object(object)) => Ok(Some(object.clone())), + Some(_) => Err(format!("{field} must be a JSON object when provided.")), + None => Ok(None), + } +} + fn observation_len(value: Option<&Value>) -> Option { let shape = value?.get("shape")?.as_array()?; @@ -827,7 +1113,7 @@ mod tests { .to_string(), ); - assert_eq!(action.messages.len(), 2); + assert_eq!(action.messages.len(), 3); assert_eq!(action.messages[0]["payload"]["action"], "right"); assert_eq!( action.messages[0]["payload"]["observation"], @@ -840,6 +1126,9 @@ mod tests { assert_eq!(action.messages[1]["kind"], "event"); assert_eq!(action.messages[1]["type"], "transition"); assert_eq!(action.messages[1]["sequence"], 2); + assert_eq!(action.messages[2]["kind"], "event"); + assert_eq!(action.messages[2]["type"], "metrics"); + assert_eq!(action.messages[2]["sequence"], 3); let shutdown = engine.process_line( &json!({ @@ -859,6 +1148,227 @@ mod tests { assert_eq!(shutdown.messages[0]["payload"]["accepted"], true); } + #[test] + fn streams_metrics_checkpoints_and_status_events_in_order() { + let mut engine = JsonlEngine::new(); + let _ = engine.process_line( + &json!({ + "protocol": ENGINE_IPC_PROTOCOL, + "version": ENGINE_IPC_PROTOCOL_VERSION, + "kind": "request", + "id": "start-1", + "timestamp": "2026-05-28T13:00:00.000Z", + "runId": "run-1", + "type": "run.start", + "payload": { + "projectId": "demo", + "runId": "run-1", + "envId": "GridWorld-v0", + "algorithm": "prototype", + "maxSteps": 8, + "observationSpec": { "type": "vector", "shape": [4], "dtype": "float32" }, + "actionSpec": { "type": "discrete", "values": ["left", "right"] }, + }, + }) + .to_string(), + ); + + let action = engine.process_line( + &json!({ + "protocol": ENGINE_IPC_PROTOCOL, + "version": ENGINE_IPC_PROTOCOL_VERSION, + "kind": "request", + "id": "action-1", + "timestamp": "2026-05-28T13:00:01.000Z", + "runId": "run-1", + "type": "run.action", + "payload": { + "runId": "run-1", + "envId": "GridWorld-v0", + "episodeId": "run-1:episode:0", + "t": 0, + "source": "policy", + "action": "right", + "encodedAction": 1, + }, + }) + .to_string(), + ); + let metrics: MetricBatch = + serde_json::from_value(action.messages[2]["payload"].clone()).unwrap(); + + assert_eq!(metrics.run_id, "run-1"); + assert_eq!(metrics.metrics[0].name, "episodeReward"); + assert_eq!(metrics.metrics[1].name, "episodeLength"); + assert_eq!(action.messages[1]["sequence"], 2); + assert_eq!(action.messages[2]["sequence"], 3); + + let checkpoint = engine.process_line( + &json!({ + "protocol": ENGINE_IPC_PROTOCOL, + "version": ENGINE_IPC_PROTOCOL_VERSION, + "kind": "request", + "id": "checkpoint-1", + "timestamp": "2026-05-28T13:00:02.000Z", + "runId": "run-1", + "type": "checkpoint.save", + "payload": { + "runId": "run-1", + "checkpointId": "manual", + "path": "runs/run-1/checkpoints/manual.json", + "metadata": { "source": "test" }, + }, + }) + .to_string(), + ); + let checkpoint_record: CheckpointRecord = + serde_json::from_value(checkpoint.messages[1]["payload"].clone()).unwrap(); + + assert_eq!(checkpoint.messages[0]["type"], "checkpoint.save"); + assert_eq!(checkpoint.messages[1]["kind"], "event"); + assert_eq!(checkpoint.messages[1]["type"], "checkpoint"); + assert_eq!(checkpoint.messages[1]["sequence"], 4); + assert_eq!(checkpoint_record.checkpoint_id, "manual"); + assert_eq!( + checkpoint_record.metadata.unwrap()["source"], + Value::String("test".to_string()) + ); + + let loaded = engine.process_line( + &json!({ + "protocol": ENGINE_IPC_PROTOCOL, + "version": ENGINE_IPC_PROTOCOL_VERSION, + "kind": "request", + "id": "checkpoint-load-1", + "timestamp": "2026-05-28T13:00:02.500Z", + "runId": "run-1", + "type": "checkpoint.load", + "payload": { + "runId": "run-1", + "checkpointId": "manual", + "path": "runs/run-1/checkpoints/manual.json", + }, + }) + .to_string(), + ); + let loaded_checkpoint: CheckpointRecord = + serde_json::from_value(loaded.messages[1]["payload"].clone()).unwrap(); + + assert_eq!(loaded.messages[0]["type"], "checkpoint.load"); + assert_eq!(loaded.messages[1]["type"], "checkpoint"); + assert_eq!(loaded.messages[1]["sequence"], 5); + assert_eq!( + loaded_checkpoint.path.unwrap(), + "runs/run-1/checkpoints/manual.json" + ); + assert_eq!( + loaded_checkpoint.metadata.unwrap()["loaded"], + Value::Bool(true) + ); + + let stop = engine.process_line( + &json!({ + "protocol": ENGINE_IPC_PROTOCOL, + "version": ENGINE_IPC_PROTOCOL_VERSION, + "kind": "request", + "id": "stop-1", + "timestamp": "2026-05-28T13:00:03.000Z", + "runId": "run-1", + "type": "run.stop", + "payload": { + "runId": "run-1", + "reason": "user_cancelled", + }, + }) + .to_string(), + ); + + assert_eq!(stop.messages.len(), 3); + assert_eq!(stop.messages[0]["payload"]["status"], "cancelled"); + assert_eq!(stop.messages[0]["payload"]["summary"]["length"], 1); + assert_eq!(stop.messages[1]["type"], "metrics"); + assert_eq!(stop.messages[1]["sequence"], 6); + assert_eq!(stop.messages[2]["type"], "run.status"); + assert_eq!(stop.messages[2]["sequence"], 7); + assert_eq!( + stop.messages[2]["payload"]["summary"]["reason"], + "user_cancelled" + ); + } + + #[test] + fn completed_runs_emit_final_status_after_metrics() { + let mut engine = JsonlEngine::new(); + let _ = engine.process_line( + &json!({ + "protocol": ENGINE_IPC_PROTOCOL, + "version": ENGINE_IPC_PROTOCOL_VERSION, + "kind": "request", + "id": "start-1", + "timestamp": "2026-05-28T13:00:00.000Z", + "runId": "run-1", + "type": "run.start", + "payload": { + "projectId": "demo", + "runId": "run-1", + "envId": "GridWorld-v0", + "algorithm": "prototype", + "maxSteps": 1, + "observationSpec": { "type": "vector", "shape": [2], "dtype": "float32" }, + "actionSpec": { "type": "discrete", "values": ["right"] }, + }, + }) + .to_string(), + ); + + let action = engine.process_line( + &json!({ + "protocol": ENGINE_IPC_PROTOCOL, + "version": ENGINE_IPC_PROTOCOL_VERSION, + "kind": "request", + "id": "action-1", + "timestamp": "2026-05-28T13:00:01.000Z", + "runId": "run-1", + "type": "run.action", + "payload": { + "runId": "run-1", + "envId": "GridWorld-v0", + "episodeId": "run-1:episode:0", + "t": 0, + "source": "policy", + "action": "right", + "encodedAction": 0, + }, + }) + .to_string(), + ); + + assert_eq!(action.messages.len(), 4); + assert_eq!(action.messages[0]["payload"]["done"], true); + assert_eq!(action.messages[0]["payload"]["truncated"], true); + assert_eq!(action.messages[2]["type"], "metrics"); + assert_eq!(action.messages[3]["type"], "run.status"); + assert_eq!(action.messages[3]["payload"]["status"], "completed"); + + let missing_after_complete = engine.process_line( + &json!({ + "protocol": ENGINE_IPC_PROTOCOL, + "version": ENGINE_IPC_PROTOCOL_VERSION, + "kind": "request", + "id": "stop-1", + "timestamp": "2026-05-28T13:00:02.000Z", + "type": "run.stop", + "payload": { "runId": "run-1" }, + }) + .to_string(), + ); + + assert_eq!( + missing_after_complete.messages[0]["error"]["code"], + "run_not_found" + ); + } + #[test] fn malformed_requests_emit_protocol_errors() { let mut engine = JsonlEngine::new(); diff --git a/crates/ignitionrl-engine/tests/jsonl_process.rs b/crates/ignitionrl-engine/tests/jsonl_process.rs index 6cbc30a..fbe93f1 100644 --- a/crates/ignitionrl-engine/tests/jsonl_process.rs +++ b/crates/ignitionrl-engine/tests/jsonl_process.rs @@ -68,6 +68,22 @@ fn jsonl_engine_process_smoke() { }, }) .to_string(), + json!({ + "protocol": ENGINE_IPC_PROTOCOL, + "version": ENGINE_IPC_PROTOCOL_VERSION, + "kind": "request", + "id": "checkpoint-1", + "timestamp": "2026-05-28T13:00:02.500Z", + "runId": "run-1", + "type": "checkpoint.save", + "payload": { + "runId": "run-1", + "checkpointId": "manual", + "path": "runs/run-1/checkpoints/manual.json", + "metadata": { "source": "smoke" }, + }, + }) + .to_string(), "{ malformed".to_string(), json!({ "protocol": ENGINE_IPC_PROTOCOL, @@ -111,6 +127,12 @@ fn jsonl_engine_process_smoke() { assert!(messages.iter().any(|message| message["kind"] == "event" && message["type"] == "transition" && message["payload"]["nextObservation"] == json!([1.0, 0.0, 0.0, 0.0]))); + assert!(messages.iter().any(|message| message["kind"] == "event" + && message["type"] == "metrics" + && message["payload"]["metrics"][1]["name"] == "episodeLength")); + assert!(messages.iter().any(|message| message["kind"] == "event" + && message["type"] == "checkpoint" + && message["payload"]["checkpointId"] == "manual")); assert!(messages.iter().any(|message| message["kind"] == "response" && message["ok"] == false && message["error"]["code"] == "invalid_request")); diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 92b66a0..ed3e6c9 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -194,7 +194,8 @@ Responsibilities: - read request envelopes from stdin as newline-delimited JSON; - write response and event envelopes to stdout as newline-delimited JSON; -- validate handshake, run lifecycle and action paths without binding to Burn or Candle; +- validate handshake, run lifecycle, action and checkpoint paths without binding to Burn or Candle; +- stream ordered status, transition, metric and checkpoint events for Studio panels; - return protocol error responses for malformed requests. ### `crates/ignitionrl-backend-burn` diff --git a/docs/ENGINE_IPC_PROTOCOL.md b/docs/ENGINE_IPC_PROTOCOL.md index b8a11f8..7dc4f80 100644 --- a/docs/ENGINE_IPC_PROTOCOL.md +++ b/docs/ENGINE_IPC_PROTOCOL.md @@ -56,7 +56,16 @@ Studio starts with `engine.hello`: "payload": { "supportedProtocolVersions": [1], "transport": "jsonl-process", - "capabilities": ["engine.hello", "engine.shutdown", "run.start", "events.transition"] + "capabilities": [ + "engine.hello", + "engine.shutdown", + "run.start", + "checkpoint.save", + "events.transition", + "events.metrics", + "events.checkpoint", + "events.status" + ] } } ```