Skip to content

Commit 719e0cc

Browse files
salim4ncursoragent
andauthored
docs(skills): add ignition-train skill for DQN, PPO, and Q-Table (#116)
Teach algorithm choice, train/stop/setSpeed, and measured Q-Table vs random on GridWorld with declared seeds and budget. Examples live under examples/skill-train*. Co-authored-by: Cursor Agent <cursoragent@cursor.com> Co-authored-by: Salim <salim4n@users.noreply.github.com>
1 parent 0ee79cb commit 719e0cc

6 files changed

Lines changed: 490 additions & 0 deletions

File tree

‎examples/skill-train.test.ts‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
import { readFileSync } from 'node:fs';
2+
import { dirname, join } from 'node:path';
3+
import { fileURLToPath } from 'node:url';
4+
import { describe, expect, it } from 'vitest';
5+
import {
6+
DQN_DEFAULTS,
7+
PPO_DEFAULTS,
8+
QTABLE_DEFAULTS,
9+
} from '@ignitionai/backend-tfjs';
10+
import {
11+
TRAIN_EPISODES,
12+
TRAIN_MAX_STEPS,
13+
TRAIN_SEEDS,
14+
assertNoStepsAfterStop,
15+
measureQTableVsRandom,
16+
} from './skill-train';
17+
18+
describe('examples/skill-train', () => {
19+
it('Q-table mean return exceeds random on declared GridWorld seeds', async () => {
20+
// Budget and seeds are module constants, declared before this call.
21+
expect(TRAIN_SEEDS.length).toBeGreaterThanOrEqual(3);
22+
expect(TRAIN_EPISODES).toBeGreaterThan(0);
23+
expect(TRAIN_MAX_STEPS).toBeGreaterThan(0);
24+
25+
const result = await measureQTableVsRandom();
26+
// eslint-disable-next-line no-console
27+
console.log('[skill-train] declared budget', result.budget, 'seeds', result.seeds);
28+
// eslint-disable-next-line no-console
29+
console.log('[skill-train] per-seed returns', JSON.stringify(result.perSeed, null, 2));
30+
// eslint-disable-next-line no-console
31+
console.log('[skill-train] trainedMean', result.trainedMean, 'randomMean', result.randomMean);
32+
33+
expect(result.trainedMean).toBeGreaterThan(result.randomMean);
34+
});
35+
36+
it('records no automatic steps after stop() settles', async () => {
37+
const counts = await assertNoStepsAfterStop();
38+
// eslint-disable-next-line no-console
39+
console.log('[skill-train] stop() step counts', counts);
40+
expect(counts.stepCountAfterSettle - counts.stepCountAtStop).toBeLessThanOrEqual(1);
41+
expect(counts.stepCountAfterWait).toBe(counts.stepCountAfterSettle);
42+
});
43+
44+
it('env.train defaults match defaults.ts', () => {
45+
expect(DQN_DEFAULTS).toEqual({
46+
hiddenLayers: [64, 64],
47+
gamma: 0.99,
48+
epsilon: 1.0,
49+
epsilonDecay: 0.995,
50+
minEpsilon: 0.01,
51+
lr: 0.001,
52+
batchSize: 32,
53+
memorySize: 10000,
54+
targetUpdateFrequency: 100,
55+
});
56+
expect(PPO_DEFAULTS).toEqual({
57+
hiddenLayers: [64, 64],
58+
gamma: 0.99,
59+
gaeLambda: 0.95,
60+
clipRatio: 0.2,
61+
lr: 3e-4,
62+
epochs: 4,
63+
batchSize: 64,
64+
entropyCoef: 0.01,
65+
valueLossCoef: 0.5,
66+
});
67+
expect(QTABLE_DEFAULTS).toEqual({
68+
stateBins: 10,
69+
gamma: 0.99,
70+
epsilon: 1.0,
71+
epsilonDecay: 0.995,
72+
minEpsilon: 0.01,
73+
lr: 0.1,
74+
});
75+
});
76+
});
77+
78+
describe('skills/ignition-train frontmatter', () => {
79+
it('name matches folder, description ≤1024, body ≤500 lines', () => {
80+
const skillPath = join(
81+
dirname(fileURLToPath(import.meta.url)),
82+
'../skills/ignition-train/SKILL.md',
83+
);
84+
const text = readFileSync(skillPath, 'utf8');
85+
const match = text.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/);
86+
expect(match).not.toBeNull();
87+
const front = match![1];
88+
const body = match![2];
89+
const name = front.match(/^name:\s*(.+)$/m)?.[1]?.trim();
90+
const description = front.match(/^description:\s*(.+)$/m)?.[1]?.trim() ?? '';
91+
expect(name).toBe('ignition-train');
92+
expect(description.length).toBeGreaterThan(0);
93+
expect(description.length).toBeLessThanOrEqual(1024);
94+
expect(body.split('\n').length).toBeLessThanOrEqual(500);
95+
});
96+
});

‎examples/skill-train.ts‎

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
import { IgnitionEnvTFJS } from '@ignitionai/backend-tfjs';
2+
import { GridWorldEnv } from '@ignitionai/environments';
3+
4+
/**
5+
* Declared before any training or evaluation in this module's runners.
6+
* Do not change these after a measured run has started.
7+
*/
8+
export const TRAIN_SEEDS = [11, 22, 33] as const;
9+
export const TRAIN_EPISODES = 180;
10+
export const TRAIN_MAX_STEPS = 80;
11+
export const EVAL_EPISODES = 25;
12+
export const EVAL_MAX_STEPS = 80;
13+
export const GRID_SIZE = 5;
14+
export const QTABLE_OVERRIDES = {
15+
stateBins: 5,
16+
lr: 0.3,
17+
gamma: 0.99,
18+
epsilon: 1,
19+
epsilonDecay: 0.99,
20+
minEpsilon: 0.05,
21+
} as const;
22+
23+
export interface SeedReturns {
24+
seed: number;
25+
trainedMean: number;
26+
randomMean: number;
27+
trainedEpisodes: number[];
28+
randomEpisodes: number[];
29+
}
30+
31+
export async function withSeed<T>(seed: number, fn: () => Promise<T> | T): Promise<T> {
32+
const previous = Math.random;
33+
let state = seed >>> 0;
34+
Math.random = () => {
35+
state = (Math.imul(state, 1664525) + 1013904223) >>> 0;
36+
return state / 4294967296;
37+
};
38+
try {
39+
return await fn();
40+
} finally {
41+
Math.random = previous;
42+
}
43+
}
44+
45+
function sleep(ms: number): Promise<void> {
46+
return new Promise((resolve) => setTimeout(resolve, ms));
47+
}
48+
49+
async function runEpisode(
50+
env: IgnitionEnvTFJS,
51+
greedy: boolean,
52+
maxSteps: number,
53+
): Promise<number> {
54+
env.reset();
55+
let total = 0;
56+
for (let step = 0; step < maxSteps; step++) {
57+
const result = greedy ? await env.inferStep() : await env.step();
58+
total += result.reward;
59+
if (result.terminated || result.truncated) break;
60+
}
61+
return total;
62+
}
63+
64+
async function randomPolicyReturns(episodes: number, maxSteps: number): Promise<number[]> {
65+
const world = new GridWorldEnv(GRID_SIZE);
66+
const returns: number[] = [];
67+
for (let ep = 0; ep < episodes; ep++) {
68+
world.reset();
69+
let total = 0;
70+
for (let step = 0; step < maxSteps; step++) {
71+
world.step(Math.floor(Math.random() * world.actions.length));
72+
total += world.reward();
73+
if (world.done()) break;
74+
}
75+
returns.push(total);
76+
}
77+
return returns;
78+
}
79+
80+
async function trainThenEvaluate(seed: number): Promise<SeedReturns> {
81+
const world = new GridWorldEnv(GRID_SIZE);
82+
const env = new IgnitionEnvTFJS(world);
83+
env.train('qtable', { ...QTABLE_OVERRIDES });
84+
env.stop();
85+
86+
const trainedEpisodes: number[] = [];
87+
for (let ep = 0; ep < TRAIN_EPISODES; ep++) {
88+
trainedEpisodes.push(await runEpisode(env, false, TRAIN_MAX_STEPS));
89+
}
90+
91+
const evalTrained: number[] = [];
92+
for (let ep = 0; ep < EVAL_EPISODES; ep++) {
93+
evalTrained.push(await runEpisode(env, true, EVAL_MAX_STEPS));
94+
}
95+
96+
const randomEpisodes = await randomPolicyReturns(EVAL_EPISODES, EVAL_MAX_STEPS);
97+
env.stop();
98+
env.agent?.dispose?.();
99+
100+
const mean = (xs: number[]) => xs.reduce((a, b) => a + b, 0) / xs.length;
101+
return {
102+
seed,
103+
trainedMean: mean(evalTrained),
104+
randomMean: mean(randomEpisodes),
105+
trainedEpisodes: evalTrained,
106+
randomEpisodes,
107+
};
108+
}
109+
110+
export async function measureQTableVsRandom(): Promise<{
111+
seeds: typeof TRAIN_SEEDS;
112+
budget: {
113+
trainEpisodes: number;
114+
trainMaxSteps: number;
115+
evalEpisodes: number;
116+
evalMaxSteps: number;
117+
gridSize: number;
118+
};
119+
perSeed: SeedReturns[];
120+
trainedMean: number;
121+
randomMean: number;
122+
}> {
123+
const perSeed: SeedReturns[] = [];
124+
for (const seed of TRAIN_SEEDS) {
125+
perSeed.push(await withSeed(seed, () => trainThenEvaluate(seed)));
126+
}
127+
const trainedMean = perSeed.reduce((a, r) => a + r.trainedMean, 0) / perSeed.length;
128+
const randomMean = perSeed.reduce((a, r) => a + r.randomMean, 0) / perSeed.length;
129+
return {
130+
seeds: TRAIN_SEEDS,
131+
budget: {
132+
trainEpisodes: TRAIN_EPISODES,
133+
trainMaxSteps: TRAIN_MAX_STEPS,
134+
evalEpisodes: EVAL_EPISODES,
135+
evalMaxSteps: EVAL_MAX_STEPS,
136+
gridSize: GRID_SIZE,
137+
},
138+
perSeed,
139+
trainedMean,
140+
randomMean,
141+
};
142+
}
143+
144+
export async function assertNoStepsAfterStop(): Promise<{
145+
stepCountAtStop: number;
146+
stepCountAfterSettle: number;
147+
stepCountAfterWait: number;
148+
}> {
149+
const env = new IgnitionEnvTFJS(new GridWorldEnv(GRID_SIZE));
150+
env.setSpeed(1);
151+
env.train('qtable', { ...QTABLE_OVERRIDES });
152+
153+
const deadline = Date.now() + 2000;
154+
while (env.stepCount < 2 && Date.now() < deadline) {
155+
await sleep(10);
156+
}
157+
if (env.stepCount < 2) {
158+
throw new Error(`automatic loop did not step before stop(); lastError=${String(env.lastError)}`);
159+
}
160+
161+
env.stop();
162+
const stepCountAtStop = env.stepCount;
163+
await sleep(20);
164+
const stepCountAfterSettle = env.stepCount;
165+
await sleep(200);
166+
const stepCountAfterWait = env.stepCount;
167+
env.agent?.dispose?.();
168+
return { stepCountAtStop, stepCountAfterSettle, stepCountAfterWait };
169+
}

‎skills/ignition-train/SKILL.md‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
---
2+
name: ignition-train
3+
description: Choose DQN, PPO, or Q-Table, configure hyperparameters, then start and stop IgnitionEnv training. Use when picking an algorithm from action and observation size, calling train/stop/setSpeed, overriding defaults, or running a measured training budget.
4+
---
5+
6+
# Train an IgnitionRL agent
7+
8+
Use this skill to pick an algorithm that **this `dev` SHA actually runs**, configure it, start the loop, and stop it. Do not claim the agent converged unless you measured returns against a declared budget and seeds.
9+
10+
## Choose an algorithm
11+
12+
`IgnitionEnv.train(algorithm, overrides)` accepts `AlgorithmType`: `'dqn' | 'double-dqn' | 'ppo' | 'qtable'`. Observation size is `observe().length`. Action size is `actions.length` or the numeric `actions` count. PPO on this SHA samples a **discrete** categorical action (`PPOAgent.getAction` returns a single index). Continuous PPO is not on this SHA.
13+
14+
| Observation | Actions | Use | Why |
15+
|---|---|---|---|
16+
| Small enumerable state (grid cells, few bins per dim, table size `stateBins ** obsDim` stays tractable) | Discrete | `qtable` | Tabular update, no network |
17+
| Vector obs, discrete actions (default) | Discrete | `dqn` | Replay + target net; also `'double-dqn'` |
18+
| Vector obs, discrete actions, on-policy / stability | Discrete | `ppo` | Clipped categorical policy; needs a full rollout before `shouldTrain()` |
19+
| Box / continuous actions | Continuous | **unsupported here** | Not in this SHA |
20+
21+
Q-Table bins every observation dim. Default `stateBins` is 10, so 4D obs → 10⁴ cells. Pass `stateLow` / `stateHigh` when obs are not in `[0, 1]`.
22+
23+
## Start, speed, stop
24+
25+
```ts
26+
import { IgnitionEnvTFJS } from '@ignitionai/backend-tfjs';
27+
import { GridWorldEnv } from '@ignitionai/environments';
28+
29+
const env = new IgnitionEnvTFJS(new GridWorldEnv(5));
30+
env.setSpeed(10); // 1x default, 10x fast, >10 turbo
31+
env.train('qtable', { stateBins: 5, lr: 0.3 });
32+
// ... later
33+
env.stop();
34+
```
35+
36+
- `train()` with no algorithm uses `'dqn'`. It creates the agent, then `start()` (timer loop).
37+
- Same algorithm again **without** overrides resumes. Same algorithm **with** overrides throws; create a new runner.
38+
- Switching algorithm replaces the agent (previous `dispose` waits on an in-flight transition).
39+
- `stop()` clears the timer and bumps a generation id so later ticks return. A transition already inside `performStep` may still finish. After that, no further automatic steps.
40+
- `setSpeed(multiplier)` changes `stepIntervalMs` / `stepsPerTick` only (not learning rate).
41+
- Manual `step()` / `inferStep()` work after `stop()`. They share the same serialized transition queue.
42+
43+
`packages/backend-tfjs/src/tools/trainer.ts` `trainAgent` is a separate DQN checkpoint loop. Prefer `IgnitionEnv` for env-driven training.
44+
45+
## Defaults (`env.train`) vs constructing an agent
46+
47+
`train()` merges `ALGORITHM_DEFAULTS` then `{ ...overrides, inputSize, actionSize }`. Cite **`defaults.ts`**, not constructor fallbacks:
48+
49+
| Key | DQN / double-dqn | PPO | Q-Table |
50+
|---|---|---|---|
51+
| `hiddenLayers` | `[64, 64]` | `[64, 64]` | — |
52+
| `lr` | `0.001` | `3e-4` | `0.1` |
53+
| `gamma` | `0.99` | `0.99` | `0.99` |
54+
| `epsilon` / decay / min | `1.0` / `0.995` / `0.01` | — | `1.0` / `0.995` / `0.01` |
55+
| `batchSize` | `32` | `64` | — |
56+
| `memorySize` | `10000` | — | — |
57+
| `targetUpdateFrequency` | `100` | — | — |
58+
| `gaeLambda` / `clipRatio` / `epochs` | — | `0.95` / `0.2` / `4` | — |
59+
| `entropyCoef` / `valueLossCoef` | — | `0.01` / `0.5` | — |
60+
| `stateBins` | — | — | `10` |
61+
| extra | `double-dqn` sets `doubleQ: true` | `rolloutSize` default **128** in `shouldTrain` (not in `defaults.ts`) | `stateLow`/`stateHigh` default `[0,1]` per dim |
62+
63+
Direct `new DQNAgent(...)` uses constructor fallbacks `hiddenLayers: [24, 24]` and `targetUpdateFrequency: 1000` if you omit those keys — different from `env.train('dqn')`. `buildQNetwork` (DQN MLP, ReLU, linear Q-head) also defaults to `[24, 24]`. PPO builds a separate tanh MLP in `ppo.ts`, not `BuildMLP.ts`.
64+
65+
Schemas: `DQNConfigSchema`, `PPOConfigSchema`, `QTableConfigSchema`. DQN-only optional `seed` (1…1e6). Q-Table has **no** `seed` field — seed `Math.random` around a run if you need repeatable exploration.
66+
67+
## Measured GridWorld + Q-Table
68+
69+
Do this **before** any `step()`:
70+
71+
1. Write seeds (≥ 3) and budget (episodes and/or transitions, eval episodes, max steps).
72+
2. Train each seed with `train('qtable', …)` then `stop()`, then a **fixed** number of `step()` calls (or episode loop).
73+
3. Evaluate greedy (`inferStep`) vs a random policy on the **same seeds**.
74+
4. Compare **measured** mean returns. Paste raw per-seed numbers. Do not say “it converged” if you only watched the loop.
75+
76+
Runnable code: `examples/skill-train.ts`. Tests: `examples/skill-train.test.ts` and `skills/ignition-train/skill-train.test.ts`.
77+
78+
## More detail
79+
80+
- Algorithm choice and PPO discrete-only note: [references/algorithm-choice.md](references/algorithm-choice.md)
81+
- Hyperparameters and file pointers: [references/hyperparameters.md](references/hyperparameters.md)
Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# Algorithm choice (this SHA only)
2+
3+
Action size is `typeof env.actions === 'number' ? env.actions : env.actions.length`. Input size is the first `observe()` vector length (`IgnitionEnv` constructor / `train()`).
4+
5+
## What each factory builds
6+
7+
`IgnitionEnvTFJS` registers:
8+
9+
- `dqn` → `DQNAgent`
10+
- `double-dqn` → `DQNAgent` with `doubleQ: true`
11+
- `ppo` → `PPOAgent` (softmax over `actionSize`, `sampleCategorical`)
12+
- `qtable` → `QTableAgent` (epsilon-greedy table)
13+
14+
PPO `getAction` always returns one integer in `0 … actionSize-1`. There is no Gaussian / box action head on this SHA.
15+
16+
## Table size check for Q-Table
17+
18+
Approximate cells: `(stateBins ?? 10) ** inputSize`. Schema rejects encodings that exceed `Number.MAX_SAFE_INTEGER`. If `stateBins ** inputSize` is huge, use DQN instead.
19+
20+
GridWorld (`GridWorldEnv`): 4 named discrete actions, observation `[agentRow, agentCol, targetRow, targetCol]` each in `[0, 1]`. Target is fixed, so Q-Table still works if `stateBins` is at least the grid size.
21+
22+
## Docs vs code
23+
24+
The algorithm pages describe defaults and intuition. Prefer `defaults.ts` + agent sources when they disagree (for example Q-Table `epsilon` decays on every `train()` update, not once per episode; PPO docs mention continuous control that this SHA does not implement).

0 commit comments

Comments
 (0)