Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 96 additions & 0 deletions examples/skill-train.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
import { readFileSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { describe, expect, it } from 'vitest';
import {
DQN_DEFAULTS,
PPO_DEFAULTS,
QTABLE_DEFAULTS,
} from '@ignitionai/backend-tfjs';
import {
TRAIN_EPISODES,
TRAIN_MAX_STEPS,
TRAIN_SEEDS,
assertNoStepsAfterStop,
measureQTableVsRandom,
} from './skill-train';

describe('examples/skill-train', () => {
it('Q-table mean return exceeds random on declared GridWorld seeds', async () => {
// Budget and seeds are module constants, declared before this call.
expect(TRAIN_SEEDS.length).toBeGreaterThanOrEqual(3);
expect(TRAIN_EPISODES).toBeGreaterThan(0);
expect(TRAIN_MAX_STEPS).toBeGreaterThan(0);

const result = await measureQTableVsRandom();
// eslint-disable-next-line no-console
console.log('[skill-train] declared budget', result.budget, 'seeds', result.seeds);
// eslint-disable-next-line no-console
console.log('[skill-train] per-seed returns', JSON.stringify(result.perSeed, null, 2));
// eslint-disable-next-line no-console
console.log('[skill-train] trainedMean', result.trainedMean, 'randomMean', result.randomMean);

expect(result.trainedMean).toBeGreaterThan(result.randomMean);
});

it('records no automatic steps after stop() settles', async () => {
const counts = await assertNoStepsAfterStop();
// eslint-disable-next-line no-console
console.log('[skill-train] stop() step counts', counts);
expect(counts.stepCountAfterSettle - counts.stepCountAtStop).toBeLessThanOrEqual(1);
expect(counts.stepCountAfterWait).toBe(counts.stepCountAfterSettle);
});

it('env.train defaults match defaults.ts', () => {
expect(DQN_DEFAULTS).toEqual({
hiddenLayers: [64, 64],
gamma: 0.99,
epsilon: 1.0,
epsilonDecay: 0.995,
minEpsilon: 0.01,
lr: 0.001,
batchSize: 32,
memorySize: 10000,
targetUpdateFrequency: 100,
});
expect(PPO_DEFAULTS).toEqual({
hiddenLayers: [64, 64],
gamma: 0.99,
gaeLambda: 0.95,
clipRatio: 0.2,
lr: 3e-4,
epochs: 4,
batchSize: 64,
entropyCoef: 0.01,
valueLossCoef: 0.5,
});
expect(QTABLE_DEFAULTS).toEqual({
stateBins: 10,
gamma: 0.99,
epsilon: 1.0,
epsilonDecay: 0.995,
minEpsilon: 0.01,
lr: 0.1,
});
});
});

describe('skills/ignition-train frontmatter', () => {
it('name matches folder, description ≤1024, body ≤500 lines', () => {
const skillPath = join(
dirname(fileURLToPath(import.meta.url)),
'../skills/ignition-train/SKILL.md',
);
const text = readFileSync(skillPath, 'utf8');
const match = text.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/);
expect(match).not.toBeNull();
const front = match![1];
const body = match![2];
const name = front.match(/^name:\s*(.+)$/m)?.[1]?.trim();
const description = front.match(/^description:\s*(.+)$/m)?.[1]?.trim() ?? '';
expect(name).toBe('ignition-train');
expect(description.length).toBeGreaterThan(0);
expect(description.length).toBeLessThanOrEqual(1024);
expect(body.split('\n').length).toBeLessThanOrEqual(500);
});
});
169 changes: 169 additions & 0 deletions examples/skill-train.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
import { IgnitionEnvTFJS } from '@ignitionai/backend-tfjs';
import { GridWorldEnv } from '@ignitionai/environments';

/**
* Declared before any training or evaluation in this module's runners.
* Do not change these after a measured run has started.
*/
export const TRAIN_SEEDS = [11, 22, 33] as const;
export const TRAIN_EPISODES = 180;
export const TRAIN_MAX_STEPS = 80;
export const EVAL_EPISODES = 25;
export const EVAL_MAX_STEPS = 80;
export const GRID_SIZE = 5;
export const QTABLE_OVERRIDES = {
stateBins: 5,
lr: 0.3,
gamma: 0.99,
epsilon: 1,
epsilonDecay: 0.99,
minEpsilon: 0.05,
} as const;

export interface SeedReturns {
seed: number;
trainedMean: number;
randomMean: number;
trainedEpisodes: number[];
randomEpisodes: number[];
}

export async function withSeed<T>(seed: number, fn: () => Promise<T> | T): Promise<T> {
const previous = Math.random;
let state = seed >>> 0;
Math.random = () => {
state = (Math.imul(state, 1664525) + 1013904223) >>> 0;
return state / 4294967296;
};
try {
return await fn();
} finally {
Math.random = previous;
}
}

function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}

async function runEpisode(
env: IgnitionEnvTFJS,
greedy: boolean,
maxSteps: number,
): Promise<number> {
env.reset();
let total = 0;
for (let step = 0; step < maxSteps; step++) {
const result = greedy ? await env.inferStep() : await env.step();
total += result.reward;
if (result.terminated || result.truncated) break;
}
return total;
}

async function randomPolicyReturns(episodes: number, maxSteps: number): Promise<number[]> {
const world = new GridWorldEnv(GRID_SIZE);
const returns: number[] = [];
for (let ep = 0; ep < episodes; ep++) {
world.reset();
let total = 0;
for (let step = 0; step < maxSteps; step++) {
world.step(Math.floor(Math.random() * world.actions.length));
total += world.reward();
if (world.done()) break;
}
returns.push(total);
}
return returns;
}

async function trainThenEvaluate(seed: number): Promise<SeedReturns> {
const world = new GridWorldEnv(GRID_SIZE);
const env = new IgnitionEnvTFJS(world);
env.train('qtable', { ...QTABLE_OVERRIDES });
env.stop();

const trainedEpisodes: number[] = [];
for (let ep = 0; ep < TRAIN_EPISODES; ep++) {
trainedEpisodes.push(await runEpisode(env, false, TRAIN_MAX_STEPS));
}

const evalTrained: number[] = [];
for (let ep = 0; ep < EVAL_EPISODES; ep++) {
evalTrained.push(await runEpisode(env, true, EVAL_MAX_STEPS));
}

const randomEpisodes = await randomPolicyReturns(EVAL_EPISODES, EVAL_MAX_STEPS);
env.stop();
env.agent?.dispose?.();

const mean = (xs: number[]) => xs.reduce((a, b) => a + b, 0) / xs.length;
return {
seed,
trainedMean: mean(evalTrained),
randomMean: mean(randomEpisodes),
trainedEpisodes: evalTrained,
randomEpisodes,
};
}

export async function measureQTableVsRandom(): Promise<{
seeds: typeof TRAIN_SEEDS;
budget: {
trainEpisodes: number;
trainMaxSteps: number;
evalEpisodes: number;
evalMaxSteps: number;
gridSize: number;
};
perSeed: SeedReturns[];
trainedMean: number;
randomMean: number;
}> {
const perSeed: SeedReturns[] = [];
for (const seed of TRAIN_SEEDS) {
perSeed.push(await withSeed(seed, () => trainThenEvaluate(seed)));
}
const trainedMean = perSeed.reduce((a, r) => a + r.trainedMean, 0) / perSeed.length;
const randomMean = perSeed.reduce((a, r) => a + r.randomMean, 0) / perSeed.length;
return {
seeds: TRAIN_SEEDS,
budget: {
trainEpisodes: TRAIN_EPISODES,
trainMaxSteps: TRAIN_MAX_STEPS,
evalEpisodes: EVAL_EPISODES,
evalMaxSteps: EVAL_MAX_STEPS,
gridSize: GRID_SIZE,
},
perSeed,
trainedMean,
randomMean,
};
}

export async function assertNoStepsAfterStop(): Promise<{
stepCountAtStop: number;
stepCountAfterSettle: number;
stepCountAfterWait: number;
}> {
const env = new IgnitionEnvTFJS(new GridWorldEnv(GRID_SIZE));
env.setSpeed(1);
env.train('qtable', { ...QTABLE_OVERRIDES });

const deadline = Date.now() + 2000;
while (env.stepCount < 2 && Date.now() < deadline) {
await sleep(10);
}
if (env.stepCount < 2) {
throw new Error(`automatic loop did not step before stop(); lastError=${String(env.lastError)}`);
}

env.stop();
const stepCountAtStop = env.stepCount;
await sleep(20);
const stepCountAfterSettle = env.stepCount;
await sleep(200);
const stepCountAfterWait = env.stepCount;
env.agent?.dispose?.();
return { stepCountAtStop, stepCountAfterSettle, stepCountAfterWait };
}
81 changes: 81 additions & 0 deletions skills/ignition-train/SKILL.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
---
name: ignition-train
description: Choose DQN, PPO, or Q-Table, configure hyperparameters, then start and stop IgnitionEnv training. Use when picking an algorithm from action and observation size, calling train/stop/setSpeed, overriding defaults, or running a measured training budget.
---

# Train an IgnitionRL agent

Use this skill to pick an algorithm that **this `dev` SHA actually runs**, configure it, start the loop, and stop it. Do not claim the agent converged unless you measured returns against a declared budget and seeds.

## Choose an algorithm

`IgnitionEnv.train(algorithm, overrides)` accepts `AlgorithmType`: `'dqn' | 'double-dqn' | 'ppo' | 'qtable'`. Observation size is `observe().length`. Action size is `actions.length` or the numeric `actions` count. PPO on this SHA samples a **discrete** categorical action (`PPOAgent.getAction` returns a single index). Continuous PPO is not on this SHA.

| Observation | Actions | Use | Why |
|---|---|---|---|
| Small enumerable state (grid cells, few bins per dim, table size `stateBins ** obsDim` stays tractable) | Discrete | `qtable` | Tabular update, no network |
| Vector obs, discrete actions (default) | Discrete | `dqn` | Replay + target net; also `'double-dqn'` |
| Vector obs, discrete actions, on-policy / stability | Discrete | `ppo` | Clipped categorical policy; needs a full rollout before `shouldTrain()` |
| Box / continuous actions | Continuous | **unsupported here** | Not in this SHA |

Q-Table bins every observation dim. Default `stateBins` is 10, so 4D obs → 10⁴ cells. Pass `stateLow` / `stateHigh` when obs are not in `[0, 1]`.

## Start, speed, stop

```ts
import { IgnitionEnvTFJS } from '@ignitionai/backend-tfjs';
import { GridWorldEnv } from '@ignitionai/environments';

const env = new IgnitionEnvTFJS(new GridWorldEnv(5));
env.setSpeed(10); // 1x default, 10x fast, >10 turbo
env.train('qtable', { stateBins: 5, lr: 0.3 });
// ... later
env.stop();
```

- `train()` with no algorithm uses `'dqn'`. It creates the agent, then `start()` (timer loop).
- Same algorithm again **without** overrides resumes. Same algorithm **with** overrides throws; create a new runner.
- Switching algorithm replaces the agent (previous `dispose` waits on an in-flight transition).
- `stop()` clears the timer and bumps a generation id so later ticks return. A transition already inside `performStep` may still finish. After that, no further automatic steps.
- `setSpeed(multiplier)` changes `stepIntervalMs` / `stepsPerTick` only (not learning rate).
- Manual `step()` / `inferStep()` work after `stop()`. They share the same serialized transition queue.

`packages/backend-tfjs/src/tools/trainer.ts` `trainAgent` is a separate DQN checkpoint loop. Prefer `IgnitionEnv` for env-driven training.

## Defaults (`env.train`) vs constructing an agent

`train()` merges `ALGORITHM_DEFAULTS` then `{ ...overrides, inputSize, actionSize }`. Cite **`defaults.ts`**, not constructor fallbacks:

| Key | DQN / double-dqn | PPO | Q-Table |
|---|---|---|---|
| `hiddenLayers` | `[64, 64]` | `[64, 64]` | — |
| `lr` | `0.001` | `3e-4` | `0.1` |
| `gamma` | `0.99` | `0.99` | `0.99` |
| `epsilon` / decay / min | `1.0` / `0.995` / `0.01` | — | `1.0` / `0.995` / `0.01` |
| `batchSize` | `32` | `64` | — |
| `memorySize` | `10000` | — | — |
| `targetUpdateFrequency` | `100` | — | — |
| `gaeLambda` / `clipRatio` / `epochs` | — | `0.95` / `0.2` / `4` | — |
| `entropyCoef` / `valueLossCoef` | — | `0.01` / `0.5` | — |
| `stateBins` | — | — | `10` |
| extra | `double-dqn` sets `doubleQ: true` | `rolloutSize` default **128** in `shouldTrain` (not in `defaults.ts`) | `stateLow`/`stateHigh` default `[0,1]` per dim |

Direct `new DQNAgent(...)` uses constructor fallbacks `hiddenLayers: [24, 24]` and `targetUpdateFrequency: 1000` if you omit those keys — different from `env.train('dqn')`. `buildQNetwork` (DQN MLP, ReLU, linear Q-head) also defaults to `[24, 24]`. PPO builds a separate tanh MLP in `ppo.ts`, not `BuildMLP.ts`.

Schemas: `DQNConfigSchema`, `PPOConfigSchema`, `QTableConfigSchema`. DQN-only optional `seed` (1…1e6). Q-Table has **no** `seed` field — seed `Math.random` around a run if you need repeatable exploration.

## Measured GridWorld + Q-Table

Do this **before** any `step()`:

1. Write seeds (≥ 3) and budget (episodes and/or transitions, eval episodes, max steps).
2. Train each seed with `train('qtable', …)` then `stop()`, then a **fixed** number of `step()` calls (or episode loop).
3. Evaluate greedy (`inferStep`) vs a random policy on the **same seeds**.
4. Compare **measured** mean returns. Paste raw per-seed numbers. Do not say “it converged” if you only watched the loop.

Runnable code: `examples/skill-train.ts`. Tests: `examples/skill-train.test.ts` and `skills/ignition-train/skill-train.test.ts`.

## More detail

- Algorithm choice and PPO discrete-only note: [references/algorithm-choice.md](references/algorithm-choice.md)
- Hyperparameters and file pointers: [references/hyperparameters.md](references/hyperparameters.md)
24 changes: 24 additions & 0 deletions skills/ignition-train/references/algorithm-choice.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
# Algorithm choice (this SHA only)

Action size is `typeof env.actions === 'number' ? env.actions : env.actions.length`. Input size is the first `observe()` vector length (`IgnitionEnv` constructor / `train()`).

## What each factory builds

`IgnitionEnvTFJS` registers:

- `dqn` → `DQNAgent`
- `double-dqn` → `DQNAgent` with `doubleQ: true`
- `ppo` → `PPOAgent` (softmax over `actionSize`, `sampleCategorical`)
- `qtable` → `QTableAgent` (epsilon-greedy table)

PPO `getAction` always returns one integer in `0 … actionSize-1`. There is no Gaussian / box action head on this SHA.

## Table size check for Q-Table

Approximate cells: `(stateBins ?? 10) ** inputSize`. Schema rejects encodings that exceed `Number.MAX_SAFE_INTEGER`. If `stateBins ** inputSize` is huge, use DQN instead.

GridWorld (`GridWorldEnv`): 4 named discrete actions, observation `[agentRow, agentCol, targetRow, targetCol]` each in `[0, 1]`. Target is fixed, so Q-Table still works if `stateBins` is at least the grid size.

## Docs vs code

The algorithm pages describe defaults and intuition. Prefer `defaults.ts` + agent sources when they disagree (for example Q-Table `epsilon` decays on every `train()` update, not once per episode; PPO docs mention continuous control that this SHA does not implement).
Loading
Loading