|
| 1 | +# Build Your First Environment |
| 2 | + |
| 3 | +This guide starts from an empty TypeScript project and ends with a working |
| 4 | +IgnitionRL environment that can reset, step, emit observations, validate |
| 5 | +actions, explain rewards and produce a replay trace. |
| 6 | + |
| 7 | +The example is a small `Target2D-v0` task: an agent starts at the origin and |
| 8 | +must move toward a seeded target. |
| 9 | + |
| 10 | +## 1. Create a Blank Project |
| 11 | + |
| 12 | +```sh |
| 13 | +mkdir target-2d-env |
| 14 | +cd target-2d-env |
| 15 | +bun init -y |
| 16 | +bun add @ignitionrl/core |
| 17 | +mkdir src |
| 18 | +``` |
| 19 | + |
| 20 | +Inside this repository, the same APIs are available from |
| 21 | +`packages/core/src/index.ts`; in a user project, import from |
| 22 | +`@ignitionrl/core`. |
| 23 | + |
| 24 | +## 2. Define State and Helpers |
| 25 | + |
| 26 | +Create `src/target-2d.ts`. |
| 27 | + |
| 28 | +```ts |
| 29 | +import { defineEnvironment, reward } from "@ignitionrl/core"; |
| 30 | + |
| 31 | +type Vec2 = { |
| 32 | + x: number; |
| 33 | + y: number; |
| 34 | +}; |
| 35 | + |
| 36 | +export type Target2DState = { |
| 37 | + agent: Vec2; |
| 38 | + target: Vec2; |
| 39 | + previousDistance: number; |
| 40 | + steps: number; |
| 41 | +}; |
| 42 | + |
| 43 | +const ACTIONS = ["up", "down", "left", "right"] as const; |
| 44 | + |
| 45 | +export type Target2DAction = typeof ACTIONS[number]; |
| 46 | + |
| 47 | +const STEP_SIZE = 0.25; |
| 48 | +const TARGET_RADIUS = 0.5; |
| 49 | +const MAX_STEPS = 120; |
| 50 | + |
| 51 | +function distance(a: Vec2, b: Vec2): number { |
| 52 | + return Math.hypot(a.x - b.x, a.y - b.y); |
| 53 | +} |
| 54 | +``` |
| 55 | + |
| 56 | +The state can be any serializable shape that your environment owns. Keep the |
| 57 | +state explicit: observations should be derived from it, rewards should be |
| 58 | +computed from it and `step()` should return the next state without mutating the |
| 59 | +previous one. |
| 60 | + |
| 61 | +## 3. Add Reset and Seeding |
| 62 | + |
| 63 | +In IgnitionRL, reset logic lives in `createInitialState()`. The runner calls it |
| 64 | +when it is created and every time you call `runner.reset()`. |
| 65 | + |
| 66 | +```ts |
| 67 | +export const Target2D = defineEnvironment({ |
| 68 | + id: "Target2D-v0", |
| 69 | + metadata: { |
| 70 | + description: "A compact 2D target-reaching tutorial environment.", |
| 71 | + maxSteps: MAX_STEPS, |
| 72 | + tags: ["tutorial", "2d", "discrete"], |
| 73 | + observationLabels: [ |
| 74 | + "agent.x", |
| 75 | + "agent.y", |
| 76 | + "target.dx", |
| 77 | + "target.dy", |
| 78 | + "previousDistance", |
| 79 | + ], |
| 80 | + actionLabels: ACTIONS, |
| 81 | + }, |
| 82 | + |
| 83 | + createInitialState: ({ rng }): Target2DState => { |
| 84 | + const agent = { x: 0, y: 0 }; |
| 85 | + const target = { |
| 86 | + x: rng.float(-5, 5), |
| 87 | + y: rng.float(-5, 5), |
| 88 | + }; |
| 89 | + |
| 90 | + return { |
| 91 | + agent, |
| 92 | + target, |
| 93 | + previousDistance: distance(agent, target), |
| 94 | + steps: 0, |
| 95 | + }; |
| 96 | + }, |
| 97 | +``` |
| 98 | +
|
| 99 | +Use the provided `rng` instead of `Math.random()`. A runner created with the |
| 100 | +same seed and stepped with the same action sequence should produce the same |
| 101 | +observations, rewards and done results. |
| 102 | +
|
| 103 | +## 4. Return Observations |
| 104 | +
|
| 105 | +Observations are numeric vectors. Arrays and numeric typed arrays are accepted. |
| 106 | +
|
| 107 | +```ts |
| 108 | + observations: ({ state }) => [ |
| 109 | + state.agent.x, |
| 110 | + state.agent.y, |
| 111 | + state.target.x - state.agent.x, |
| 112 | + state.target.y - state.agent.y, |
| 113 | + state.previousDistance, |
| 114 | + ], |
| 115 | +``` |
| 116 | +
|
| 117 | +Keep the observation order stable. Learners treat dimension `0`, dimension `1` |
| 118 | +and so on as separate signals, so reordering values between runs changes the |
| 119 | +meaning of a checkpoint. |
| 120 | +
|
| 121 | +## 5. Declare Actions |
| 122 | +
|
| 123 | +This guide uses a discrete action space with literal values. |
| 124 | +
|
| 125 | +```ts |
| 126 | + actions: { |
| 127 | + type: "discrete", |
| 128 | + values: ACTIONS, |
| 129 | + labels: ACTIONS, |
| 130 | + }, |
| 131 | +``` |
| 132 | +
|
| 133 | +Other supported action spaces are: |
| 134 | +
|
| 135 | +```ts |
| 136 | +// Discrete index action: action is 0, 1, 2, ... |
| 137 | +{ type: "discrete", n: 4, labels: ["up", "down", "left", "right"] } |
| 138 | + |
| 139 | +// Continuous action: action is a number array with shape size 4. |
| 140 | +{ type: "continuous", shape: [4], low: -1, high: 1, labels: ["throttle", "yaw", "pitch", "roll"] } |
| 141 | + |
| 142 | +// Multi-discrete action: action is a number array with one integer per branch. |
| 143 | +{ type: "multi-discrete", nvec: [3, 2], labels: ["move", "fire"] } |
| 144 | +``` |
| 145 | +
|
| 146 | +## 6. Step the Environment |
| 147 | +
|
| 148 | +`step()` receives the current state and a validated action. Return the next |
| 149 | +state. |
| 150 | +
|
| 151 | +```ts |
| 152 | + step: ({ state, action }) => { |
| 153 | + const next: Target2DState = structuredClone(state); |
| 154 | + |
| 155 | + next.previousDistance = distance(state.agent, state.target); |
| 156 | + |
| 157 | + if (action === "up") next.agent.y += STEP_SIZE; |
| 158 | + if (action === "down") next.agent.y -= STEP_SIZE; |
| 159 | + if (action === "left") next.agent.x -= STEP_SIZE; |
| 160 | + if (action === "right") next.agent.x += STEP_SIZE; |
| 161 | + |
| 162 | + next.steps += 1; |
| 163 | + |
| 164 | + return next; |
| 165 | + }, |
| 166 | +``` |
| 167 | +
|
| 168 | +Avoid mutating `state` in place. Traces, reward debugging and comparisons are |
| 169 | +easier to reason about when `state` and `nextState` are distinct snapshots. |
| 170 | +
|
| 171 | +## 7. Name Reward Terms |
| 172 | +
|
| 173 | +Rewards can be a scalar internally, but users debug them as named terms. Use |
| 174 | +one term for each reason the agent gains or loses reward. |
| 175 | +
|
| 176 | +```ts |
| 177 | + reward: ({ state, nextState }) => { |
| 178 | + const previousDistance = distance(state.agent, state.target); |
| 179 | + const nextDistance = distance(nextState.agent, nextState.target); |
| 180 | + |
| 181 | + return reward() |
| 182 | + .add("progress", previousDistance - nextDistance) |
| 183 | + .add("target_reached", nextDistance < TARGET_RADIUS, 10) |
| 184 | + .stepPenalty(-0.01); |
| 185 | + }, |
| 186 | +``` |
| 187 | +
|
| 188 | +The reward builder also supports penalties: |
| 189 | +
|
| 190 | +```ts |
| 191 | +reward() |
| 192 | + .add("progress", 0.1) |
| 193 | + .penalty("collision", didCollide, 3) |
| 194 | + .stepPenalty(-0.01) |
| 195 | +``` |
| 196 | +
|
| 197 | +## 8. End Episodes |
| 198 | +
|
| 199 | +`done()` can return a boolean or a structured result. Prefer structured results |
| 200 | +when you know whether the episode succeeded or was truncated. |
| 201 | +
|
| 202 | +```ts |
| 203 | + done: ({ nextState }) => { |
| 204 | + const nextDistance = distance(nextState.agent, nextState.target); |
| 205 | + |
| 206 | + if (nextDistance < TARGET_RADIUS) { |
| 207 | + return { done: true, reason: "target_reached", success: true }; |
| 208 | + } |
| 209 | + |
| 210 | + if (nextState.steps >= MAX_STEPS) { |
| 211 | + return { done: true, reason: "max_steps", truncated: true }; |
| 212 | + } |
| 213 | + |
| 214 | + return false; |
| 215 | + }, |
| 216 | +}); |
| 217 | +``` |
| 218 | + |
| 219 | +The full `src/target-2d.ts` file now exports `Target2D`, a typed environment |
| 220 | +with reset, observation, action, step, reward and done behavior. |
| 221 | + |
| 222 | +## 9. Run a Smoke Script |
| 223 | + |
| 224 | +Create `src/smoke.ts`. |
| 225 | + |
| 226 | +```ts |
| 227 | +import { randomPolicy } from "@ignitionrl/core"; |
| 228 | +import { Target2D } from "./target-2d.ts"; |
| 229 | + |
| 230 | +const spec = Target2D.getSpec({ seed: "tutorial-spec" }); |
| 231 | + |
| 232 | +console.log("Environment:", spec.id); |
| 233 | +console.log("Observation shape:", spec.observation.shape); |
| 234 | +console.log("Action spec:", spec.actions); |
| 235 | + |
| 236 | +const runner = Target2D.createRunner({ |
| 237 | + seed: "tutorial-run", |
| 238 | + runId: "tutorial-target-2d", |
| 239 | + maxSteps: 120, |
| 240 | + collectTrace: true, |
| 241 | +}); |
| 242 | + |
| 243 | +const reset = runner.reset({ seed: "tutorial-reset" }); |
| 244 | + |
| 245 | +console.log("Initial observation:", reset.observation); |
| 246 | + |
| 247 | +for (const action of ["right", "up", "right", "up"] as const) { |
| 248 | + const step = runner.step(action); |
| 249 | + |
| 250 | + console.log({ |
| 251 | + t: step.t, |
| 252 | + action: step.action, |
| 253 | + reward: step.reward.total, |
| 254 | + terms: step.reward.terms, |
| 255 | + done: step.done, |
| 256 | + reason: step.reason, |
| 257 | + }); |
| 258 | + |
| 259 | + if (step.done) break; |
| 260 | +} |
| 261 | + |
| 262 | +const trace = await Target2D |
| 263 | + .createRunner({ seed: "tutorial-random", maxSteps: 120, collectTrace: true }) |
| 264 | + .runEpisode(randomPolicy(Target2D.actions, { seed: "tutorial-policy" }), { |
| 265 | + reset: false, |
| 266 | + maxSteps: 120, |
| 267 | + }); |
| 268 | + |
| 269 | +console.log("Random baseline:", trace.summary); |
| 270 | +console.log("Trace steps:", trace.steps.length); |
| 271 | +``` |
| 272 | + |
| 273 | +Run it: |
| 274 | + |
| 275 | +```sh |
| 276 | +bun src/smoke.ts |
| 277 | +``` |
| 278 | + |
| 279 | +You should see: |
| 280 | + |
| 281 | +- an environment spec with a vector observation shape; |
| 282 | +- an action spec with four discrete values; |
| 283 | +- per-step rewards with named terms; |
| 284 | +- a replay trace summary from the random baseline episode. |
| 285 | + |
| 286 | +## 10. What IgnitionRL Validates |
| 287 | + |
| 288 | +IgnitionRL validates the contract at definition time and step time: |
| 289 | + |
| 290 | +- `defineEnvironment()` checks that `id`, action spec and required functions are valid. |
| 291 | +- `getSpec()` calls `createInitialState()` and `observations()` to infer observation shape. |
| 292 | +- `runner.step(action)` validates the action before calling your `step()`. |
| 293 | +- every observation is normalized into a finite number array. |
| 294 | +- every reward is normalized into `{ total, terms }`. |
| 295 | +- every done result is normalized into `{ done, reason?, success?, truncated? }`. |
| 296 | + |
| 297 | +## Troubleshooting |
| 298 | + |
| 299 | +| Symptom | Likely cause | Fix | |
| 300 | +| --- | --- | --- | |
| 301 | +| `[IgnitionRL] observations() must return a numeric array.` | `observations()` returned an object, scalar, `undefined` or mixed structure. | Return `number[]`, `Float32Array` or another numeric typed array. | |
| 302 | +| `[IgnitionRL] observation[2] must be finite, got NaN.` | An observation dimension is `NaN` or `Infinity`. | Guard division by zero, missing state fields and invalid physics values. | |
| 303 | +| `[IgnitionRL] Invalid discrete action: jump.` | The policy returned a value not listed in `actions.values`. | Return one of the exact literal values, or use `env.sampleAction()` while debugging. | |
| 304 | +| `[IgnitionRL] Continuous action length must be 4, got 3.` | A continuous policy returned the wrong vector size. | Match the flattened size of `actions.shape`. | |
| 305 | +| `[IgnitionRL] continuous action[0] must be in [-1, 1]` | A continuous policy exceeded `low` or `high`. | Clamp action outputs or widen the action spec intentionally. | |
| 306 | +| `[IgnitionRL] reward() must return a RewardBuilder or RewardResult.` | `reward()` returned a raw number or a malformed object. | Return `reward().add(...)` or `{ total, terms }`. | |
| 307 | +| `[IgnitionRL] Reward term names must be non-empty.` | A reward term name is `""` or whitespace. | Name every reward cause, such as `progress`, `collision` or `step_penalty`. | |
| 308 | +| `[IgnitionRL] done() must return a boolean or DoneResult.` | `done()` returned `undefined`, a number or another shape. | Return `false`, `true` or `{ done: boolean, reason?: string }`. | |
| 309 | +| `[IgnitionRL] Episode is done. Call reset() before stepping again.` | Code called `runner.step()` after termination. | Call `runner.reset()` before continuing. | |
| 310 | +| Runs are not reproducible. | Randomness is coming from `Math.random()` or time-based state. | Use the provided `rng` in `createInitialState()` and policy code. | |
| 311 | + |
| 312 | +## Next Step |
| 313 | + |
| 314 | +Once the environment works under `@ignitionrl/core`, wire it into a local |
| 315 | +project through the SDK or one of the CLI templates. The current first-class CLI |
| 316 | +training command supports the built-in `GridWorld-v0`, `Target2D-v0` and |
| 317 | +`DroneTarget-v0` environments while custom project registration matures. |
0 commit comments