Skip to content

Commit b57cab7

Browse files
authored
spike(engine): benchmark native overhead
Closes #43
1 parent 04512f9 commit b57cab7

10 files changed

Lines changed: 679 additions & 1 deletion

File tree

‎Cargo.lock‎

Lines changed: 11 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ resolver = "2"
33
members = [
44
"crates/ignitionrl-core",
55
"crates/ignitionrl-engine",
6+
"crates/ignitionrl-bench",
67
"crates/ignitionrl-backend-burn",
78
"crates/ignitionrl-backend-candle"
89
]

‎README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ IgnitionRL
8484
├─ packages/cli Headless demo/project commands
8585
├─ crates/ignitionrl-core Rust rollout/runtime/data structures
8686
├─ crates/ignitionrl-engine JSONL local engine process prototype
87+
├─ crates/ignitionrl-bench Reproducible native engine benchmark harness
8788
├─ crates/ignitionrl-backend-* Burn/Candle backend experiments
8889
└─ docs Product, roadmap, architecture, ADRs
8990
```
@@ -119,6 +120,7 @@ Trained checkpoints can be run in inference mode through `bun run --cwd packages
119120
Runs can be checked in CI through `bun run --cwd packages/cli start eval <projectDir> <runId> --score-by summary.totalReward --min-score 0 --json`.
120121
Project artifacts can be checked in CI through `bun run --cwd packages/cli start doctor <projectDir> --require-completed --require-summaries --require-metrics --require-traces --require-run-reports --require-studio-run-views --json`.
121122
The repository CI entrypoint is `bun run ci`; it typechecks, tests, builds, checks core coverage, then runs GridWorld and DroneTarget smoke projects through `demo`, `doctor`, `eval`, `environments`, `environment`, `history`, `studio`, `run`, `artifacts`, `episodes`, `metrics`, `rewards`, `checkpoints` and `checkpoint`.
123+
The native engine benchmark can be reproduced with `bun run benchmark:engine -- --json`; it builds release Rust binaries, compares a Bun/TypeScript loop baseline against Rust replay/JSONL/checkpoint paths, and measures the Burn policy/value spike.
122124
The exported Studio run views include artifact inventories, reward-term timelines plus observation/action inspector payloads, so the future desktop UI can explain what the agent saw, chose, received and exported without rerunning the episode.
123125
Policy and learner experiments accept `AbortSignal`, and SDK training sessions expose start/cancel/progress snapshots plus Studio-ready training panel payloads so future Studio controls can mark runs `cancelled` while preserving completed traces, metrics and partial summaries.
124126
The current Studio shell can be launched in browser mode with `bun run dev:studio` and built with `bun run build:studio`. The desktop wrapper can be launched with `bun run dev:studio:desktop` and bundled with `bun run build:studio:desktop`. It loads SDK `StudioWorkspaceView` JSON, renders the first workspace, run history, metric, reward, replay, checkpoint and environment panels, includes an initial DroneTarget R3F viewport with play/pause/stop/step/reset controls, provides named reward-term timeline plus per-step attribution debugging, and uses Tauri native file dialogs to open exported Studio JSON artifacts from disk.

‎ROADMAP.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,7 @@ Do not start the Rust backend until this workflow is usable end-to-end.
176176
- [x] Backend interface trait
177177
- [x] Burn policy/value update spike
178178
- [x] Candle inference/checkpoint spike
179-
- [ ] Benchmark report
179+
- [x] Benchmark report
180180

181181
### Success criteria
182182

‎crates/ignitionrl-bench/Cargo.toml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
[package]
2+
name = "ignitionrl-bench"
3+
version.workspace = true
4+
edition.workspace = true
5+
license.workspace = true
6+
authors.workspace = true
7+
rust-version.workspace = true
8+
description = "Reproducible native engine benchmark harness for IgnitionRL."
9+
10+
[dependencies]
11+
ignitionrl-backend-burn = { path = "../ignitionrl-backend-burn" }
12+
ignitionrl-core = { path = "../ignitionrl-core" }
13+
ignitionrl-engine = { path = "../ignitionrl-engine" }
14+
serde.workspace = true
15+
serde_json.workspace = true

‎crates/ignitionrl-bench/README.md‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# ignitionrl-bench
2+
3+
Reproducible benchmark harness for the native engine spike.
4+
5+
Run from the repository root:
6+
7+
```sh
8+
bun run benchmark:engine -- --json
9+
```
10+
11+
The Bun wrapper builds release binaries, measures a TypeScript loop baseline,
12+
measures JSONL engine cold start, then runs this Rust binary for replay-buffer,
13+
transition serialization, JSONL action, checkpoint and Burn policy/value update
14+
costs.
Lines changed: 303 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,303 @@
1+
use ignitionrl_backend_burn::{
2+
run_burn_policy_value_spike, toy_policy_value_batch, BurnPolicyValueConfig,
3+
};
4+
use ignitionrl_core::{
5+
CheckpointRecord, EncodedAction, ReplayBuffer, RewardTerms, RolloutSummary, Transition,
6+
ENGINE_IPC_PROTOCOL, ENGINE_IPC_PROTOCOL_VERSION,
7+
};
8+
use ignitionrl_engine::{JsonlEngine, ENGINE_TRANSPORT};
9+
use serde::Serialize;
10+
use serde_json::{json, Value};
11+
use std::{
12+
error::Error,
13+
fs,
14+
hint::black_box,
15+
path::PathBuf,
16+
time::{Duration, Instant},
17+
};
18+
19+
const REPLAY_ITERATIONS: usize = 20_000;
20+
const SERDE_ITERATIONS: usize = 20_000;
21+
const JSONL_ITERATIONS: usize = 2_000;
22+
const CHECKPOINT_SERDE_ITERATIONS: usize = 5_000;
23+
const CHECKPOINT_IO_ITERATIONS: usize = 200;
24+
const BURN_RUNS: usize = 3;
25+
const BURN_UPDATE_STEPS: usize = 16;
26+
27+
fn main() -> Result<(), Box<dyn Error>> {
28+
let report = BenchReport {
29+
schema_version: 1,
30+
profile: if cfg!(debug_assertions) {
31+
"debug"
32+
} else {
33+
"release"
34+
},
35+
iterations: BenchIterations {
36+
replay: REPLAY_ITERATIONS,
37+
transition_serde: SERDE_ITERATIONS,
38+
jsonl_actions: JSONL_ITERATIONS,
39+
checkpoint_serde: CHECKPOINT_SERDE_ITERATIONS,
40+
checkpoint_io: CHECKPOINT_IO_ITERATIONS,
41+
burn_runs: BURN_RUNS,
42+
burn_update_steps: BURN_UPDATE_STEPS,
43+
},
44+
results: BenchResults {
45+
rust_replay_buffer_push_us: bench_replay_buffer_push()?,
46+
rust_transition_serde_round_trip_us: bench_transition_serde()?,
47+
rust_jsonl_action_round_trip_us: bench_jsonl_actions()?,
48+
rust_checkpoint_serde_round_trip_us: bench_checkpoint_serde()?,
49+
rust_checkpoint_file_write_read_us: bench_checkpoint_io()?,
50+
burn_policy_value_batch_update_ms: bench_burn_policy_value_spike()?,
51+
},
52+
};
53+
54+
println!("{}", serde_json::to_string_pretty(&report)?);
55+
56+
Ok(())
57+
}
58+
59+
#[derive(Debug, Serialize)]
60+
#[serde(rename_all = "camelCase")]
61+
struct BenchReport {
62+
schema_version: u16,
63+
profile: &'static str,
64+
iterations: BenchIterations,
65+
results: BenchResults,
66+
}
67+
68+
#[derive(Debug, Serialize)]
69+
#[serde(rename_all = "camelCase")]
70+
struct BenchIterations {
71+
replay: usize,
72+
transition_serde: usize,
73+
jsonl_actions: usize,
74+
checkpoint_serde: usize,
75+
checkpoint_io: usize,
76+
burn_runs: usize,
77+
burn_update_steps: usize,
78+
}
79+
80+
#[derive(Debug, Serialize)]
81+
#[serde(rename_all = "camelCase")]
82+
struct BenchResults {
83+
rust_replay_buffer_push_us: f64,
84+
rust_transition_serde_round_trip_us: f64,
85+
rust_jsonl_action_round_trip_us: f64,
86+
rust_checkpoint_serde_round_trip_us: f64,
87+
rust_checkpoint_file_write_read_us: f64,
88+
burn_policy_value_batch_update_ms: BurnBenchResult,
89+
}
90+
91+
#[derive(Debug, Serialize)]
92+
#[serde(rename_all = "camelCase")]
93+
struct BurnBenchResult {
94+
mean_total_ms: f64,
95+
mean_per_update_ms: f64,
96+
sample_count: usize,
97+
update_steps: usize,
98+
loss_delta: f32,
99+
}
100+
101+
fn bench_replay_buffer_push() -> Result<f64, Box<dyn Error>> {
102+
let mut buffer = ReplayBuffer::new(REPLAY_ITERATIONS)
103+
.map_err(|error| format!("unable to create replay buffer: {error:?}"))?;
104+
let started = Instant::now();
105+
106+
for index in 0..REPLAY_ITERATIONS {
107+
black_box(buffer.push(sample_transition(index as u64)));
108+
}
109+
110+
Ok(per_op_us(started.elapsed(), REPLAY_ITERATIONS))
111+
}
112+
113+
fn bench_transition_serde() -> Result<f64, Box<dyn Error>> {
114+
let transition = sample_transition(0);
115+
let started = Instant::now();
116+
117+
for _ in 0..SERDE_ITERATIONS {
118+
let bytes = serde_json::to_vec(black_box(&transition))?;
119+
let parsed: Transition = serde_json::from_slice(black_box(&bytes))?;
120+
black_box(parsed);
121+
}
122+
123+
Ok(per_op_us(started.elapsed(), SERDE_ITERATIONS))
124+
}
125+
126+
fn bench_jsonl_actions() -> Result<f64, Box<dyn Error>> {
127+
let mut engine = JsonlEngine::new();
128+
let start_request = json!({
129+
"protocol": ENGINE_IPC_PROTOCOL,
130+
"version": ENGINE_IPC_PROTOCOL_VERSION,
131+
"kind": "request",
132+
"id": "start-1",
133+
"timestamp": "2026-05-28T13:00:00.000Z",
134+
"runId": "bench-run",
135+
"type": "run.start",
136+
"payload": {
137+
"projectId": "bench",
138+
"runId": "bench-run",
139+
"envId": "BenchEnv-v0",
140+
"algorithm": "benchmark",
141+
"maxSteps": JSONL_ITERATIONS + 1,
142+
"observationSpec": { "type": "vector", "shape": [4], "dtype": "float32" },
143+
"actionSpec": { "type": "discrete", "values": ["left", "right"] },
144+
},
145+
})
146+
.to_string();
147+
148+
black_box(engine.process_line(&start_request));
149+
150+
let started = Instant::now();
151+
152+
for index in 0..JSONL_ITERATIONS {
153+
let request = json!({
154+
"protocol": ENGINE_IPC_PROTOCOL,
155+
"version": ENGINE_IPC_PROTOCOL_VERSION,
156+
"kind": "request",
157+
"id": format!("action-{index}"),
158+
"timestamp": "2026-05-28T13:00:01.000Z",
159+
"runId": "bench-run",
160+
"type": "run.action",
161+
"payload": {
162+
"runId": "bench-run",
163+
"envId": "BenchEnv-v0",
164+
"episodeId": "bench-run:episode:0",
165+
"t": index,
166+
"source": "policy",
167+
"action": "right",
168+
"encodedAction": 1,
169+
},
170+
})
171+
.to_string();
172+
173+
let result = engine.process_line(black_box(&request));
174+
black_box(result);
175+
}
176+
177+
Ok(per_op_us(started.elapsed(), JSONL_ITERATIONS))
178+
}
179+
180+
fn bench_checkpoint_serde() -> Result<f64, Box<dyn Error>> {
181+
let checkpoint = sample_checkpoint();
182+
let started = Instant::now();
183+
184+
for _ in 0..CHECKPOINT_SERDE_ITERATIONS {
185+
let bytes = serde_json::to_vec(black_box(&checkpoint))?;
186+
let parsed: CheckpointRecord = serde_json::from_slice(black_box(&bytes))?;
187+
black_box(parsed);
188+
}
189+
190+
Ok(per_op_us(started.elapsed(), CHECKPOINT_SERDE_ITERATIONS))
191+
}
192+
193+
fn bench_checkpoint_io() -> Result<f64, Box<dyn Error>> {
194+
let checkpoint = sample_checkpoint();
195+
let bytes = serde_json::to_vec(&checkpoint)?;
196+
let path = temp_checkpoint_path();
197+
let started = Instant::now();
198+
199+
for _ in 0..CHECKPOINT_IO_ITERATIONS {
200+
fs::write(&path, black_box(&bytes))?;
201+
let loaded = fs::read(&path)?;
202+
let parsed: CheckpointRecord = serde_json::from_slice(black_box(&loaded))?;
203+
black_box(parsed);
204+
}
205+
206+
let _ = fs::remove_file(path);
207+
208+
Ok(per_op_us(started.elapsed(), CHECKPOINT_IO_ITERATIONS))
209+
}
210+
211+
fn bench_burn_policy_value_spike() -> Result<BurnBenchResult, Box<dyn Error>> {
212+
let mut config = BurnPolicyValueConfig::new(4, 3);
213+
config.update_steps = BURN_UPDATE_STEPS;
214+
215+
let base_batch = toy_policy_value_batch();
216+
let mut samples = Vec::with_capacity(base_batch.len() * 8);
217+
for _ in 0..8 {
218+
samples.extend(base_batch.clone());
219+
}
220+
221+
let mut total = Duration::ZERO;
222+
let mut last_loss_delta = 0.0;
223+
224+
for _ in 0..BURN_RUNS {
225+
let started = Instant::now();
226+
let report = run_burn_policy_value_spike(config.clone(), &samples)?;
227+
total += started.elapsed();
228+
last_loss_delta = report.loss_delta();
229+
black_box(report);
230+
}
231+
232+
let mean_total_ms = total.as_secs_f64() * 1000.0 / BURN_RUNS as f64;
233+
234+
Ok(BurnBenchResult {
235+
mean_total_ms,
236+
mean_per_update_ms: mean_total_ms / BURN_UPDATE_STEPS as f64,
237+
sample_count: samples.len(),
238+
update_steps: BURN_UPDATE_STEPS,
239+
loss_delta: last_loss_delta,
240+
})
241+
}
242+
243+
fn sample_transition(t: u64) -> Transition {
244+
let mut reward_terms = RewardTerms::new();
245+
reward_terms.insert("step_penalty".to_string(), -0.01);
246+
247+
Transition {
248+
run_id: "bench-run".to_string(),
249+
env_id: "BenchEnv-v0".to_string(),
250+
episode_id: "bench-run:episode:0".to_string(),
251+
t,
252+
observation: vec![0.0, 0.0, 4.0, 4.0],
253+
action: Value::String("right".to_string()),
254+
encoded_action: EncodedAction::Scalar(1.0),
255+
reward: -0.01,
256+
reward_terms,
257+
next_observation: vec![1.0, 0.0, 4.0, 4.0],
258+
done: false,
259+
terminated: false,
260+
truncated: false,
261+
reason: None,
262+
success: None,
263+
info: None,
264+
}
265+
}
266+
267+
fn sample_checkpoint() -> CheckpointRecord {
268+
let mut metadata = serde_json::Map::new();
269+
metadata.insert(
270+
"transport".to_string(),
271+
Value::String(ENGINE_TRANSPORT.to_string()),
272+
);
273+
metadata.insert("benchmark".to_string(), Value::Bool(true));
274+
275+
CheckpointRecord {
276+
run_id: "bench-run".to_string(),
277+
env_id: "BenchEnv-v0".to_string(),
278+
checkpoint_id: "checkpoint-final".to_string(),
279+
algorithm: "benchmark".to_string(),
280+
path: Some("runs/bench-run/checkpoints/checkpoint-final.json".to_string()),
281+
created_at: "2026-05-28T13:00:00.000Z".to_string(),
282+
metadata: Some(metadata),
283+
summary: Some(RolloutSummary {
284+
total_reward: 1.0,
285+
length: 128,
286+
success: true,
287+
terminated: true,
288+
truncated: false,
289+
reason: Some("benchmark".to_string()),
290+
}),
291+
}
292+
}
293+
294+
fn temp_checkpoint_path() -> PathBuf {
295+
std::env::temp_dir().join(format!(
296+
"ignitionrl-bench-checkpoint-{}.json",
297+
std::process::id()
298+
))
299+
}
300+
301+
fn per_op_us(duration: Duration, iterations: usize) -> f64 {
302+
duration.as_secs_f64() * 1_000_000.0 / iterations as f64
303+
}

0 commit comments

Comments
 (0)