diff --git a/examples/skill-train.test.ts b/examples/skill-train.test.ts new file mode 100644 index 0000000..b226c83 --- /dev/null +++ b/examples/skill-train.test.ts @@ -0,0 +1,96 @@ +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { + DQN_DEFAULTS, + PPO_DEFAULTS, + QTABLE_DEFAULTS, +} from '@ignitionai/backend-tfjs'; +import { + TRAIN_EPISODES, + TRAIN_MAX_STEPS, + TRAIN_SEEDS, + assertNoStepsAfterStop, + measureQTableVsRandom, +} from './skill-train'; + +describe('examples/skill-train', () => { + it('Q-table mean return exceeds random on declared GridWorld seeds', async () => { + // Budget and seeds are module constants, declared before this call. + expect(TRAIN_SEEDS.length).toBeGreaterThanOrEqual(3); + expect(TRAIN_EPISODES).toBeGreaterThan(0); + expect(TRAIN_MAX_STEPS).toBeGreaterThan(0); + + const result = await measureQTableVsRandom(); + // eslint-disable-next-line no-console + console.log('[skill-train] declared budget', result.budget, 'seeds', result.seeds); + // eslint-disable-next-line no-console + console.log('[skill-train] per-seed returns', JSON.stringify(result.perSeed, null, 2)); + // eslint-disable-next-line no-console + console.log('[skill-train] trainedMean', result.trainedMean, 'randomMean', result.randomMean); + + expect(result.trainedMean).toBeGreaterThan(result.randomMean); + }); + + it('records no automatic steps after stop() settles', async () => { + const counts = await assertNoStepsAfterStop(); + // eslint-disable-next-line no-console + console.log('[skill-train] stop() step counts', counts); + expect(counts.stepCountAfterSettle - counts.stepCountAtStop).toBeLessThanOrEqual(1); + expect(counts.stepCountAfterWait).toBe(counts.stepCountAfterSettle); + }); + + it('env.train defaults match defaults.ts', () => { + expect(DQN_DEFAULTS).toEqual({ + hiddenLayers: [64, 64], + gamma: 0.99, + epsilon: 1.0, + epsilonDecay: 0.995, + minEpsilon: 0.01, + lr: 0.001, + batchSize: 32, + memorySize: 10000, + targetUpdateFrequency: 100, + }); + expect(PPO_DEFAULTS).toEqual({ + hiddenLayers: [64, 64], + gamma: 0.99, + gaeLambda: 0.95, + clipRatio: 0.2, + lr: 3e-4, + epochs: 4, + batchSize: 64, + entropyCoef: 0.01, + valueLossCoef: 0.5, + }); + expect(QTABLE_DEFAULTS).toEqual({ + stateBins: 10, + gamma: 0.99, + epsilon: 1.0, + epsilonDecay: 0.995, + minEpsilon: 0.01, + lr: 0.1, + }); + }); +}); + +describe('skills/ignition-train frontmatter', () => { + it('name matches folder, description ≤1024, body ≤500 lines', () => { + const skillPath = join( + dirname(fileURLToPath(import.meta.url)), + '../skills/ignition-train/SKILL.md', + ); + const text = readFileSync(skillPath, 'utf8'); + const match = text.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/); + expect(match).not.toBeNull(); + const front = match![1]; + const body = match![2]; + const name = front.match(/^name:\s*(.+)$/m)?.[1]?.trim(); + const description = front.match(/^description:\s*(.+)$/m)?.[1]?.trim() ?? ''; + expect(name).toBe('ignition-train'); + expect(description.length).toBeGreaterThan(0); + expect(description.length).toBeLessThanOrEqual(1024); + expect(body.split('\n').length).toBeLessThanOrEqual(500); + }); +}); diff --git a/examples/skill-train.ts b/examples/skill-train.ts new file mode 100644 index 0000000..bf519c5 --- /dev/null +++ b/examples/skill-train.ts @@ -0,0 +1,169 @@ +import { IgnitionEnvTFJS } from '@ignitionai/backend-tfjs'; +import { GridWorldEnv } from '@ignitionai/environments'; + +/** + * Declared before any training or evaluation in this module's runners. + * Do not change these after a measured run has started. + */ +export const TRAIN_SEEDS = [11, 22, 33] as const; +export const TRAIN_EPISODES = 180; +export const TRAIN_MAX_STEPS = 80; +export const EVAL_EPISODES = 25; +export const EVAL_MAX_STEPS = 80; +export const GRID_SIZE = 5; +export const QTABLE_OVERRIDES = { + stateBins: 5, + lr: 0.3, + gamma: 0.99, + epsilon: 1, + epsilonDecay: 0.99, + minEpsilon: 0.05, +} as const; + +export interface SeedReturns { + seed: number; + trainedMean: number; + randomMean: number; + trainedEpisodes: number[]; + randomEpisodes: number[]; +} + +export async function withSeed(seed: number, fn: () => Promise | T): Promise { + const previous = Math.random; + let state = seed >>> 0; + Math.random = () => { + state = (Math.imul(state, 1664525) + 1013904223) >>> 0; + return state / 4294967296; + }; + try { + return await fn(); + } finally { + Math.random = previous; + } +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function runEpisode( + env: IgnitionEnvTFJS, + greedy: boolean, + maxSteps: number, +): Promise { + env.reset(); + let total = 0; + for (let step = 0; step < maxSteps; step++) { + const result = greedy ? await env.inferStep() : await env.step(); + total += result.reward; + if (result.terminated || result.truncated) break; + } + return total; +} + +async function randomPolicyReturns(episodes: number, maxSteps: number): Promise { + const world = new GridWorldEnv(GRID_SIZE); + const returns: number[] = []; + for (let ep = 0; ep < episodes; ep++) { + world.reset(); + let total = 0; + for (let step = 0; step < maxSteps; step++) { + world.step(Math.floor(Math.random() * world.actions.length)); + total += world.reward(); + if (world.done()) break; + } + returns.push(total); + } + return returns; +} + +async function trainThenEvaluate(seed: number): Promise { + const world = new GridWorldEnv(GRID_SIZE); + const env = new IgnitionEnvTFJS(world); + env.train('qtable', { ...QTABLE_OVERRIDES }); + env.stop(); + + const trainedEpisodes: number[] = []; + for (let ep = 0; ep < TRAIN_EPISODES; ep++) { + trainedEpisodes.push(await runEpisode(env, false, TRAIN_MAX_STEPS)); + } + + const evalTrained: number[] = []; + for (let ep = 0; ep < EVAL_EPISODES; ep++) { + evalTrained.push(await runEpisode(env, true, EVAL_MAX_STEPS)); + } + + const randomEpisodes = await randomPolicyReturns(EVAL_EPISODES, EVAL_MAX_STEPS); + env.stop(); + env.agent?.dispose?.(); + + const mean = (xs: number[]) => xs.reduce((a, b) => a + b, 0) / xs.length; + return { + seed, + trainedMean: mean(evalTrained), + randomMean: mean(randomEpisodes), + trainedEpisodes: evalTrained, + randomEpisodes, + }; +} + +export async function measureQTableVsRandom(): Promise<{ + seeds: typeof TRAIN_SEEDS; + budget: { + trainEpisodes: number; + trainMaxSteps: number; + evalEpisodes: number; + evalMaxSteps: number; + gridSize: number; + }; + perSeed: SeedReturns[]; + trainedMean: number; + randomMean: number; +}> { + const perSeed: SeedReturns[] = []; + for (const seed of TRAIN_SEEDS) { + perSeed.push(await withSeed(seed, () => trainThenEvaluate(seed))); + } + const trainedMean = perSeed.reduce((a, r) => a + r.trainedMean, 0) / perSeed.length; + const randomMean = perSeed.reduce((a, r) => a + r.randomMean, 0) / perSeed.length; + return { + seeds: TRAIN_SEEDS, + budget: { + trainEpisodes: TRAIN_EPISODES, + trainMaxSteps: TRAIN_MAX_STEPS, + evalEpisodes: EVAL_EPISODES, + evalMaxSteps: EVAL_MAX_STEPS, + gridSize: GRID_SIZE, + }, + perSeed, + trainedMean, + randomMean, + }; +} + +export async function assertNoStepsAfterStop(): Promise<{ + stepCountAtStop: number; + stepCountAfterSettle: number; + stepCountAfterWait: number; +}> { + const env = new IgnitionEnvTFJS(new GridWorldEnv(GRID_SIZE)); + env.setSpeed(1); + env.train('qtable', { ...QTABLE_OVERRIDES }); + + const deadline = Date.now() + 2000; + while (env.stepCount < 2 && Date.now() < deadline) { + await sleep(10); + } + if (env.stepCount < 2) { + throw new Error(`automatic loop did not step before stop(); lastError=${String(env.lastError)}`); + } + + env.stop(); + const stepCountAtStop = env.stepCount; + await sleep(20); + const stepCountAfterSettle = env.stepCount; + await sleep(200); + const stepCountAfterWait = env.stepCount; + env.agent?.dispose?.(); + return { stepCountAtStop, stepCountAfterSettle, stepCountAfterWait }; +} diff --git a/skills/ignition-train/SKILL.md b/skills/ignition-train/SKILL.md new file mode 100644 index 0000000..1c14748 --- /dev/null +++ b/skills/ignition-train/SKILL.md @@ -0,0 +1,81 @@ +--- +name: ignition-train +description: Choose DQN, PPO, or Q-Table, configure hyperparameters, then start and stop IgnitionEnv training. Use when picking an algorithm from action and observation size, calling train/stop/setSpeed, overriding defaults, or running a measured training budget. +--- + +# Train an IgnitionRL agent + +Use this skill to pick an algorithm that **this `dev` SHA actually runs**, configure it, start the loop, and stop it. Do not claim the agent converged unless you measured returns against a declared budget and seeds. + +## Choose an algorithm + +`IgnitionEnv.train(algorithm, overrides)` accepts `AlgorithmType`: `'dqn' | 'double-dqn' | 'ppo' | 'qtable'`. Observation size is `observe().length`. Action size is `actions.length` or the numeric `actions` count. PPO on this SHA samples a **discrete** categorical action (`PPOAgent.getAction` returns a single index). Continuous PPO is not on this SHA. + +| Observation | Actions | Use | Why | +|---|---|---|---| +| Small enumerable state (grid cells, few bins per dim, table size `stateBins ** obsDim` stays tractable) | Discrete | `qtable` | Tabular update, no network | +| Vector obs, discrete actions (default) | Discrete | `dqn` | Replay + target net; also `'double-dqn'` | +| Vector obs, discrete actions, on-policy / stability | Discrete | `ppo` | Clipped categorical policy; needs a full rollout before `shouldTrain()` | +| Box / continuous actions | Continuous | **unsupported here** | Not in this SHA | + +Q-Table bins every observation dim. Default `stateBins` is 10, so 4D obs → 10⁴ cells. Pass `stateLow` / `stateHigh` when obs are not in `[0, 1]`. + +## Start, speed, stop + +```ts +import { IgnitionEnvTFJS } from '@ignitionai/backend-tfjs'; +import { GridWorldEnv } from '@ignitionai/environments'; + +const env = new IgnitionEnvTFJS(new GridWorldEnv(5)); +env.setSpeed(10); // 1x default, 10x fast, >10 turbo +env.train('qtable', { stateBins: 5, lr: 0.3 }); +// ... later +env.stop(); +``` + +- `train()` with no algorithm uses `'dqn'`. It creates the agent, then `start()` (timer loop). +- Same algorithm again **without** overrides resumes. Same algorithm **with** overrides throws; create a new runner. +- Switching algorithm replaces the agent (previous `dispose` waits on an in-flight transition). +- `stop()` clears the timer and bumps a generation id so later ticks return. A transition already inside `performStep` may still finish. After that, no further automatic steps. +- `setSpeed(multiplier)` changes `stepIntervalMs` / `stepsPerTick` only (not learning rate). +- Manual `step()` / `inferStep()` work after `stop()`. They share the same serialized transition queue. + +`packages/backend-tfjs/src/tools/trainer.ts` `trainAgent` is a separate DQN checkpoint loop. Prefer `IgnitionEnv` for env-driven training. + +## Defaults (`env.train`) vs constructing an agent + +`train()` merges `ALGORITHM_DEFAULTS` then `{ ...overrides, inputSize, actionSize }`. Cite **`defaults.ts`**, not constructor fallbacks: + +| Key | DQN / double-dqn | PPO | Q-Table | +|---|---|---|---| +| `hiddenLayers` | `[64, 64]` | `[64, 64]` | — | +| `lr` | `0.001` | `3e-4` | `0.1` | +| `gamma` | `0.99` | `0.99` | `0.99` | +| `epsilon` / decay / min | `1.0` / `0.995` / `0.01` | — | `1.0` / `0.995` / `0.01` | +| `batchSize` | `32` | `64` | — | +| `memorySize` | `10000` | — | — | +| `targetUpdateFrequency` | `100` | — | — | +| `gaeLambda` / `clipRatio` / `epochs` | — | `0.95` / `0.2` / `4` | — | +| `entropyCoef` / `valueLossCoef` | — | `0.01` / `0.5` | — | +| `stateBins` | — | — | `10` | +| extra | `double-dqn` sets `doubleQ: true` | `rolloutSize` default **128** in `shouldTrain` (not in `defaults.ts`) | `stateLow`/`stateHigh` default `[0,1]` per dim | + +Direct `new DQNAgent(...)` uses constructor fallbacks `hiddenLayers: [24, 24]` and `targetUpdateFrequency: 1000` if you omit those keys — different from `env.train('dqn')`. `buildQNetwork` (DQN MLP, ReLU, linear Q-head) also defaults to `[24, 24]`. PPO builds a separate tanh MLP in `ppo.ts`, not `BuildMLP.ts`. + +Schemas: `DQNConfigSchema`, `PPOConfigSchema`, `QTableConfigSchema`. DQN-only optional `seed` (1…1e6). Q-Table has **no** `seed` field — seed `Math.random` around a run if you need repeatable exploration. + +## Measured GridWorld + Q-Table + +Do this **before** any `step()`: + +1. Write seeds (≥ 3) and budget (episodes and/or transitions, eval episodes, max steps). +2. Train each seed with `train('qtable', …)` then `stop()`, then a **fixed** number of `step()` calls (or episode loop). +3. Evaluate greedy (`inferStep`) vs a random policy on the **same seeds**. +4. Compare **measured** mean returns. Paste raw per-seed numbers. Do not say “it converged” if you only watched the loop. + +Runnable code: `examples/skill-train.ts`. Tests: `examples/skill-train.test.ts` and `skills/ignition-train/skill-train.test.ts`. + +## More detail + +- Algorithm choice and PPO discrete-only note: [references/algorithm-choice.md](references/algorithm-choice.md) +- Hyperparameters and file pointers: [references/hyperparameters.md](references/hyperparameters.md) diff --git a/skills/ignition-train/references/algorithm-choice.md b/skills/ignition-train/references/algorithm-choice.md new file mode 100644 index 0000000..d46dcae --- /dev/null +++ b/skills/ignition-train/references/algorithm-choice.md @@ -0,0 +1,24 @@ +# Algorithm choice (this SHA only) + +Action size is `typeof env.actions === 'number' ? env.actions : env.actions.length`. Input size is the first `observe()` vector length (`IgnitionEnv` constructor / `train()`). + +## What each factory builds + +`IgnitionEnvTFJS` registers: + +- `dqn` → `DQNAgent` +- `double-dqn` → `DQNAgent` with `doubleQ: true` +- `ppo` → `PPOAgent` (softmax over `actionSize`, `sampleCategorical`) +- `qtable` → `QTableAgent` (epsilon-greedy table) + +PPO `getAction` always returns one integer in `0 … actionSize-1`. There is no Gaussian / box action head on this SHA. + +## Table size check for Q-Table + +Approximate cells: `(stateBins ?? 10) ** inputSize`. Schema rejects encodings that exceed `Number.MAX_SAFE_INTEGER`. If `stateBins ** inputSize` is huge, use DQN instead. + +GridWorld (`GridWorldEnv`): 4 named discrete actions, observation `[agentRow, agentCol, targetRow, targetCol]` each in `[0, 1]`. Target is fixed, so Q-Table still works if `stateBins` is at least the grid size. + +## Docs vs code + +The algorithm pages describe defaults and intuition. Prefer `defaults.ts` + agent sources when they disagree (for example Q-Table `epsilon` decays on every `train()` update, not once per episode; PPO docs mention continuous control that this SHA does not implement). diff --git a/skills/ignition-train/references/hyperparameters.md b/skills/ignition-train/references/hyperparameters.md new file mode 100644 index 0000000..e001c2c --- /dev/null +++ b/skills/ignition-train/references/hyperparameters.md @@ -0,0 +1,31 @@ +# Hyperparameters (sourced) + +## `env.train` defaults + +File: `packages/backend-tfjs/src/defaults.ts` + +- `DQN_DEFAULTS`: `hiddenLayers [64, 64]`, `gamma 0.99`, `epsilon 1.0`, `epsilonDecay 0.995`, `minEpsilon 0.01`, `lr 0.001`, `batchSize 32`, `memorySize 10000`, `targetUpdateFrequency 100` +- `PPO_DEFAULTS`: `hiddenLayers [64, 64]`, `gamma 0.99`, `gaeLambda 0.95`, `clipRatio 0.2`, `lr 3e-4`, `epochs 4`, `batchSize 64`, `entropyCoef 0.01`, `valueLossCoef 0.5` +- `QTABLE_DEFAULTS`: `stateBins 10`, `gamma 0.99`, `epsilon 1.0`, `epsilonDecay 0.995`, `minEpsilon 0.01`, `lr 0.1` +- `'double-dqn'` copies DQN defaults and sets `doubleQ: true` + +Merge: `packages/core/src/defaults.ts` `mergeDefaults` (shallow; arrays replaced). + +## Runner + +File: `packages/core/src/ignition-env.ts` + +- `train(algorithm?, overrides?)` — default algorithm `'dqn'` +- `stop()` — cancel future ticks +- `setSpeed(multiplier)` — `≤1` → 50 ms / 1 step; `≤10` → 10 ms / `round(multiplier)` steps; else 1 ms / `round(multiplier / 2)` steps +- `step()` / `inferStep()` — one serialized transition (`greedy` true on infer) + +## Networks and trainer helper + +- `packages/backend-tfjs/src/model/BuildMLP.ts` — `buildQNetwork(inputSize, outputSize, hiddenLayers = [24, 24], lr = 0.001, seed?)` for DQN +- `packages/backend-tfjs/src/agents/ppo.ts` — local tanh `buildMLP` for actor and critic +- `packages/backend-tfjs/src/tools/trainer.ts` — `trainAgent({ config, maxSteps, getEnvStep, ... })` DQN-only helper + +## Schemas + +`packages/backend-tfjs/src/schemas.ts`: `DQNConfigSchema`, `PPOConfigSchema`, `QTableConfigSchema`. diff --git a/skills/ignition-train/skill-train.test.ts b/skills/ignition-train/skill-train.test.ts new file mode 100644 index 0000000..001d976 --- /dev/null +++ b/skills/ignition-train/skill-train.test.ts @@ -0,0 +1,89 @@ +import { readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { + DQN_DEFAULTS, + PPO_DEFAULTS, + QTABLE_DEFAULTS, +} from '@ignitionai/backend-tfjs'; +import { + TRAIN_EPISODES, + TRAIN_MAX_STEPS, + TRAIN_SEEDS, + assertNoStepsAfterStop, + measureQTableVsRandom, +} from '../../examples/skill-train'; + +describe('ignition-train skill', () => { + it('Q-table mean return exceeds random on declared GridWorld seeds', async () => { + expect(TRAIN_SEEDS.length).toBeGreaterThanOrEqual(3); + expect(TRAIN_EPISODES).toBeGreaterThan(0); + expect(TRAIN_MAX_STEPS).toBeGreaterThan(0); + + const result = await measureQTableVsRandom(); + // eslint-disable-next-line no-console + console.log('[ignition-train] declared budget', result.budget, 'seeds', result.seeds); + // eslint-disable-next-line no-console + console.log('[ignition-train] per-seed returns', JSON.stringify(result.perSeed, null, 2)); + // eslint-disable-next-line no-console + console.log('[ignition-train] trainedMean', result.trainedMean, 'randomMean', result.randomMean); + + expect(result.trainedMean).toBeGreaterThan(result.randomMean); + }); + + it('records no automatic steps after stop() settles', async () => { + const counts = await assertNoStepsAfterStop(); + // eslint-disable-next-line no-console + console.log('[ignition-train] stop() step counts', counts); + expect(counts.stepCountAfterSettle - counts.stepCountAtStop).toBeLessThanOrEqual(1); + expect(counts.stepCountAfterWait).toBe(counts.stepCountAfterSettle); + }); + + it('env.train defaults match defaults.ts', () => { + expect(DQN_DEFAULTS).toMatchObject({ + hiddenLayers: [64, 64], + gamma: 0.99, + epsilon: 1.0, + epsilonDecay: 0.995, + minEpsilon: 0.01, + lr: 0.001, + batchSize: 32, + memorySize: 10000, + targetUpdateFrequency: 100, + }); + expect(PPO_DEFAULTS).toMatchObject({ + hiddenLayers: [64, 64], + gamma: 0.99, + gaeLambda: 0.95, + clipRatio: 0.2, + lr: 3e-4, + epochs: 4, + batchSize: 64, + entropyCoef: 0.01, + valueLossCoef: 0.5, + }); + expect(QTABLE_DEFAULTS).toMatchObject({ + stateBins: 10, + gamma: 0.99, + epsilon: 1.0, + epsilonDecay: 0.995, + minEpsilon: 0.01, + lr: 0.1, + }); + }); + + it('frontmatter is valid', () => { + const skillPath = join(dirname(fileURLToPath(import.meta.url)), 'SKILL.md'); + const text = readFileSync(skillPath, 'utf8'); + const match = text.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/); + expect(match).not.toBeNull(); + const front = match![1]; + const body = match![2]; + const name = front.match(/^name:\s*(.+)$/m)?.[1]?.trim(); + const description = front.match(/^description:\s*(.+)$/m)?.[1]?.trim() ?? ''; + expect(name).toBe('ignition-train'); + expect(description.length).toBeLessThanOrEqual(1024); + expect(body.split('\n').length).toBeLessThanOrEqual(500); + }); +});