diff --git a/examples/skill-debug-training.test.ts b/examples/skill-debug-training.test.ts new file mode 100644 index 0000000..bbec748 --- /dev/null +++ b/examples/skill-debug-training.test.ts @@ -0,0 +1,72 @@ +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { + REJECTED_LR, + SEEDS, + TRAIN_LR, + TRAIN_STEPS, + measureTensorStability, + runDeclaredProtocol, + tryRejectedLearningRate, +} from './skill-debug-training'; + +describe('skill ignition-debug-training', () => { + it('rejects Protocol A lr >= 1 before any training', () => { + const message = tryRejectedLearningRate(); + expect(REJECTED_LR).toBe(1); + expect(message).toMatch(/lr must be < 1/); + }); + + it('keeps TensorFlow.js tensor count flat after DQN warmup', async () => { + const { before, after, delta } = await measureTensorStability(SEEDS[0]); + expect(before).toBeGreaterThan(0); + expect(after).toBe(before); + expect(delta).toBe(0); + }); + + it('reproduces flat CartPole reward on 3 seeds and restores a non-zero return after the fix', async () => { + const { broken, fixed } = await runDeclaredProtocol(); + console.log(JSON.stringify({ broken, fixed }, null, 2)); + + expect(broken).toHaveLength(SEEDS.length); + expect(fixed).toHaveLength(SEEDS.length); + expect(TRAIN_STEPS).toBe(64); + expect(TRAIN_LR).toBe(0.001); + + for (const row of broken) { + expect(row.rewardMode).toBe('zero'); + expect(row.meanReturn).toBe(0); + expect(row.finiteRewards).toBe(true); + } + + for (const row of fixed) { + expect(row.rewardMode).toBe('cartpole'); + expect(row.meanReturn).toBeGreaterThan(0); + expect(row.qFinite, `seed ${row.seed} Q ${row.qSample}`).toBe(true); + expect(row.finiteRewards).toBe(true); + expect(row.finiteObservations).toBe(true); + } + }); +}); + +describe('skills/ignition-debug-training frontmatter', () => { + it('has name = folder, description <= 1024, body <= 500 lines', () => { + const raw = readFileSync( + resolve(__dirname, '../skills/ignition-debug-training/SKILL.md'), + 'utf8', + ); + expect(raw.startsWith('---\n')).toBe(true); + const end = raw.indexOf('\n---\n', 4); + expect(end).toBeGreaterThan(0); + const front = raw.slice(4, end); + const body = raw.slice(end + 5); + const name = /(?:^|\n)name:\s*(.+)/.exec(front)?.[1]?.trim(); + const description = /(?:^|\n)description:\s*(.+)/.exec(front)?.[1]?.trim(); + expect(name).toBe('ignition-debug-training'); + expect(description).toBeDefined(); + expect(description!.length).toBeGreaterThan(0); + expect(description!.length).toBeLessThanOrEqual(1024); + expect(body.split(/\r?\n/).length).toBeLessThanOrEqual(500); + }); +}); diff --git a/examples/skill-debug-training.ts b/examples/skill-debug-training.ts new file mode 100644 index 0000000..396d8bc --- /dev/null +++ b/examples/skill-debug-training.ts @@ -0,0 +1,302 @@ +/** + * CartPole debug protocol for skill ignition-debug-training. + * + * PROTOCOL A (declared before first run): + * SEEDS, TRAIN_STEPS, network as below, BROKEN_LR = 1, FIXED_LR = 0.001 + * Result: DQNConfigSchema rejects lr >= 1 ("lr must be < 1"). No training. + * + * PROTOCOL B (declared after A, before any Protocol B run): + * SEEDS = [7, 11, 19] + * TRAIN_STEPS = 64 + * hiddenLayers = [8] + * batchSize = 8 + * memorySize = 64 + * targetUpdateFrequency = 100 + * backend = 'cpu' + * lr = 0.001 + * BROKEN_OBS_SCALE = 1000 + * FIXED_OBS_SCALE = 1 + * + * Hypothesis B: multiplying CartPole observations by 1000 yields non-finite + * Q-values (or non-finite rewards/obs) on every seed; scale 1 keeps Q finite. + * Result B: Q stayed finite (seed 7 qSample ≈ [79.4, 104.6], meanReturn 12). + * + * PROTOCOL C (declared after B, before any Protocol C run): + * Same seeds, TRAIN_STEPS, network, backend, FIXED_OBS_SCALE = 1 + * BROKEN_LR = 0.99 (legal: schema requires lr < 1) + * FIXED_LR = 0.001 + * Hypothesis C: lr 0.99 makes Q non-finite on every seed; lr 0.001 keeps Q finite. + * Result C: Q stayed finite on seeds 7/11/19 (see test log). Not a NaN failure. + * + * PROTOCOL D (declared after C, before any Protocol D run): + * Same seeds, TRAIN_STEPS = 64, network, backend, lr = 0.001, obs scale = 1 + * Broken: TrainingEnv.reward() always returns 0 (flat reward on CartPole physics) + * Fixed: CartPoleEnv.reward() (1 while up, 0 on done) + * Hypothesis D: broken meanReturn is 0 on every seed; fixed meanReturn is > 0. + * No claim that the pole is solved. + */ +import { CartPoleEnv } from '@ignitionai/environments'; +import { DQNAgent, IgnitionEnvTFJS } from '@ignitionai/backend-tfjs'; +import type { TrainingEnv } from '@ignitionai/core'; +// Root vitest aliases @ignitionai/* only. Resolve TF.js from the backend workspace package. +import * as tf from '../packages/backend-tfjs/node_modules/@tensorflow/tfjs'; + +export const SEEDS = [7, 11, 19] as const; +export const TRAIN_STEPS = 64; +export const TENSOR_STABLE_STEPS = 24; +export const TRAIN_LR = 0.001; +export const BROKEN_LR = 0.99; +export const BROKEN_OBS_SCALE = 1; +export const FIXED_OBS_SCALE = 1; +export const REJECTED_LR = 1; +export type RewardMode = 'cartpole' | 'zero'; + +export const SHARED_DQN_OVERRIDES = { + backend: 'cpu' as const, + hiddenLayers: [8], + batchSize: 8, + memorySize: 64, + targetUpdateFrequency: 100, + epsilon: 1, + epsilonDecay: 0.995, + minEpsilon: 0.01, + gamma: 0.99, +}; + +function mulberry32(seed: number): () => number { + let state = seed >>> 0; + return () => { + state = (Math.imul(state, 1664525) + 1013904223) >>> 0; + return state / 4294967296; + }; +} + +/** CartPole with seeded resets. Agent `seed` does not control CartPoleEnv.reset(). */ +export class SeededCartPole implements TrainingEnv { + readonly inner = new CartPoleEnv(); + readonly actions = this.inner.actions; + private readonly random: () => number; + private readonly observationScale: number; + private readonly rewardMode: RewardMode; + + constructor( + seed: number, + observationScale = 1, + rewardMode: RewardMode = 'cartpole', + ) { + this.random = mulberry32(seed ^ 0x9e3779b9); + this.observationScale = observationScale; + this.rewardMode = rewardMode; + this.reset(); + } + + observe(): number[] { + return this.inner.observe().map((x) => x * this.observationScale); + } + + step(action: number | number[]): void { + this.inner.step(action); + } + + reward(): number { + switch (this.rewardMode) { + case 'zero': + return 0; + case 'cartpole': + return this.inner.reward(); + default: { + const exhaustive: never = this.rewardMode; + throw new Error(`unhandled rewardMode: ${String(exhaustive)}`); + } + } + } + + done(): boolean { + return this.inner.done(); + } + + reset(): void { + this.inner.reset(); + this.inner.state = { + x: (this.random() - 0.5) * 0.1, + xDot: (this.random() - 0.5) * 0.1, + theta: (this.random() - 0.5) * 0.1, + thetaDot: (this.random() - 0.5) * 0.1, + stepCount: 0, + }; + } +} + +export interface TrialResult { + seed: number; + lr: number; + observationScale: number; + rewardMode: RewardMode; + steps: number; + episodes: number; + meanReturn: number; + maxReturn: number; + minReturn: number; + finiteRewards: boolean; + finiteObservations: boolean; + qFinite: boolean; + qSample: number[]; + epsilon: number | null; + trainStepCounter: number | null; + tensorsAfterWarmup: number | null; + tensorsAfterStableWindow: number | null; + tensorDelta: number | null; +} + +export function probeQValues(agent: DQNAgent, observation: number[]): number[] { + return tf.tidy(() => { + const input = tf.tensor2d([observation]); + const out = agent.getModel().predict(input) as tf.Tensor; + return Array.from(out.dataSync()); + }); +} + +export function valuesAreFinite(xs: number[]): boolean { + return xs.every((x) => Number.isFinite(x)); +} + +export async function runTrial( + seed: number, + observationScale: number, + lr = TRAIN_LR, + rewardMode: RewardMode = 'cartpole', +): Promise { + await tf.setBackend('cpu'); + await tf.ready(); + + const pole = new SeededCartPole(seed, observationScale, rewardMode); + const env = new IgnitionEnvTFJS(pole); + const episodeReturns: number[] = []; + let current = 0; + let finiteRewards = true; + let finiteObservations = true; + let tensorsAfterWarmup: number | null = null; + let tensorsAfterStableWindow: number | null = null; + + env.train('dqn', { + ...SHARED_DQN_OVERRIDES, + lr, + seed, + }); + env.stop(); + + try { + for (let i = 0; i < TRAIN_STEPS; i++) { + const result = await env.step(); + if (!Number.isFinite(result.reward)) finiteRewards = false; + if (!valuesAreFinite(result.observation)) finiteObservations = false; + current += result.reward; + if (result.terminated || result.truncated) { + episodeReturns.push(current); + current = 0; + } + if (i + 1 === SHARED_DQN_OVERRIDES.batchSize) { + tensorsAfterWarmup = tf.memory().numTensors; + } + if ( + tensorsAfterWarmup !== null && + i + 1 === SHARED_DQN_OVERRIDES.batchSize + TENSOR_STABLE_STEPS + ) { + tensorsAfterStableWindow = tf.memory().numTensors; + } + } + if (current > 0 || episodeReturns.length === 0) { + episodeReturns.push(current); + } + + const agent = env.agent as DQNAgent; + const qSample = probeQValues(agent, pole.observe()); + const state = agent.getState(); + const epsilon = typeof state.epsilon === 'number' ? state.epsilon : null; + const trainStepCounter = + typeof state.trainStepCounter === 'number' ? state.trainStepCounter : null; + const meanReturn = + episodeReturns.reduce((a, b) => a + b, 0) / episodeReturns.length; + + return { + seed, + lr, + observationScale, + rewardMode, + steps: TRAIN_STEPS, + episodes: episodeReturns.length, + meanReturn, + maxReturn: Math.max(...episodeReturns), + minReturn: Math.min(...episodeReturns), + finiteRewards, + finiteObservations, + qFinite: valuesAreFinite(qSample), + qSample, + epsilon, + trainStepCounter, + tensorsAfterWarmup, + tensorsAfterStableWindow, + tensorDelta: + tensorsAfterWarmup === null || tensorsAfterStableWindow === null + ? null + : tensorsAfterStableWindow - tensorsAfterWarmup, + }; + } finally { + env.stop(); + env.agent?.dispose?.(); + } +} + +export async function runDeclaredProtocol(): Promise<{ + broken: TrialResult[]; + fixed: TrialResult[]; +}> { + const broken: TrialResult[] = []; + const fixed: TrialResult[] = []; + for (const seed of SEEDS) { + broken.push(await runTrial(seed, FIXED_OBS_SCALE, TRAIN_LR, 'zero')); + } + for (const seed of SEEDS) { + fixed.push(await runTrial(seed, FIXED_OBS_SCALE, TRAIN_LR, 'cartpole')); + } + return { broken, fixed }; +} + +export function tryRejectedLearningRate(): string { + const env = new IgnitionEnvTFJS(new SeededCartPole(SEEDS[0], FIXED_OBS_SCALE)); + try { + env.train('dqn', { ...SHARED_DQN_OVERRIDES, lr: REJECTED_LR, seed: SEEDS[0] }); + env.stop(); + env.agent?.dispose?.(); + return 'NO_THROW'; + } catch (error) { + env.stop(); + return error instanceof Error ? error.message : String(error); + } +} + +export async function measureTensorStability(seed: number): Promise<{ + before: number; + after: number; + delta: number; +}> { + await tf.setBackend('cpu'); + await tf.ready(); + const env = new IgnitionEnvTFJS(new SeededCartPole(seed, FIXED_OBS_SCALE)); + env.train('dqn', { ...SHARED_DQN_OVERRIDES, lr: TRAIN_LR, seed }); + env.stop(); + try { + for (let i = 0; i < SHARED_DQN_OVERRIDES.batchSize; i++) { + await env.step(); + } + const before = tf.memory().numTensors; + for (let i = 0; i < TENSOR_STABLE_STEPS; i++) { + await env.step(); + } + const after = tf.memory().numTensors; + return { before, after, delta: after - before }; + } finally { + env.stop(); + env.agent?.dispose?.(); + } +} diff --git a/skills/ignition-debug-training/SKILL.md b/skills/ignition-debug-training/SKILL.md new file mode 100644 index 0000000..82cc4a7 --- /dev/null +++ b/skills/ignition-debug-training/SKILL.md @@ -0,0 +1,97 @@ +--- +name: ignition-debug-training +description: Diagnose an IgnitionRL training run that is not learning. Use when reward is flat, values are NaN, training is unstable, the policy forgets or collapses, DQN epsilon is wrong, observations are unscaled, the learning rate is too high, TensorFlow.js tensors leak, or the TF.js backend is mis-selected. Start from a symptom, pick a cause, run a check that can falsify it, then change one thing and re-measure. +--- + +# Debug a training run that does not learn + +Work from a **symptom**, pick a **cause**, run a **check that can fail**, then apply **one** change and re-measure. Do not claim the agent learned unless you recorded returns. Declare **budget** (step count, network size) and **seeds** (at least three) before any run. + +Executable protocol: `examples/skill-debug-training.ts` (test: `examples/skill-debug-training.test.ts`). + +Full symptom table: [references/symptom-table.md](references/symptom-table.md). + +## Metrics this API actually exposes + +Record only these. If a number is not in this list, compute it yourself from `StepResult` or say it is missing. + +| What you can read | Where | +|---|---| +| Per-step `reward`, `observation`, `terminated`, `truncated` | `IgnitionEnv.step()` → `StepResult` | +| Steps taken since last `reset()` | `IgnitionEnv.stepCount` | +| Automatic-loop failure | `IgnitionEnv.lastError` | +| DQN `epsilon`, `trainStepCounter` | `DQNAgent.getState()` | +| PPO `trainStepCounter` | `PPOAgent.getState()` | +| Q-table `epsilon` | `QTableAgent.currentEpsilon` | +| Q-values (not a training metric; you must probe) | `DQNAgent.getModel().predict(...)` | +| Live tensor count | `tf.memory().numTensors` | +| Active TF.js backend | `tf.getBackend()` after `setBackend` | + +**Not exposed** (do not invent them): training loss, policy entropy, explained variance, gradient norms, mean Q as a logger, episode return as a framework field. `DQNAgent.getState().bestReward` exists but is updated only by `maybeSaveBestCheckpoint`, not by `train()`. PPO computes entropy inside `train()` and never returns it. + +Episode return is the sum of `StepResult.reward` from reset until `terminated || truncated`. Mean return over a budget is the number you compare across seeds. + +## Procedure + +1. **Stop the timer loop.** `train()` starts an automatic loop (`IgnitionEnv.train` → `start`). Call `stop()` and drive `await env.step()` yourself so counts are exact. +2. **Write the protocol first.** Example: seeds `[7, 11, 19]`, `TRAIN_STEPS = 64`, `hiddenLayers: [8]`, `batchSize: 8`, `backend: 'cpu'`. Then run. Paste raw JSON, including failures. +3. **One hypothesis per run.** Change learning rate *or* epsilon *or* observation scaling, not all three. +4. **Falsify.** If Q-values stay finite with `lr: 1`, the “LR too high → NaN” story is false for that budget; pick another row in the table. +5. **Fix and re-measure** on the same seeds and budget. Report both sides. + +CartPole physics: `CartPoleEnv` (`packages/environments/src/cartpole.ts`). Reward is `1` while up, `0` on `done()`. `observe()` is **not** scaled to `[-1, 1]`. `reset()` uses `Math.random()` (not the agent `seed`). For a fair comparison, wrap the env and write `state` from your own RNG after `reset()` (the example does this). `done()` treats the 500-step cap as a terminal failure; there is no `truncated()`. + +## Symptom → cause → check → fix + +Use this abbreviated map. Details and citations: [references/symptom-table.md](references/symptom-table.md). + +| Symptom | Likely cause | Check that decides | Fix | +|---|---|---|---| +| Reward always 0 or constant | `reward()` ignores the task; or always `done()` | Print `reward` and `terminated` for 20 random actions | Return a shaped signal; fix `done()` | +| `reward` or Q-values are `NaN`/`Inf` | `lr` too large; exploding obs; leaked tensors corrupting math | `Number.isFinite` on rewards and a `tf.tidy` Q probe | Lower `lr` (DQN default `0.001`); scale `observe()`; wrap graphs in `tf.tidy` | +| Unstable returns (wild swings) | `lr` high; target net too slow/fast; no replay warmup | Compare `lr: 1` vs `lr: 0.001` on 3 seeds; read `trainStepCounter` | Use `DQN_DEFAULTS.lr`; `targetUpdateFrequency: 100` | +| Late collapse / “forgetting” | Off-policy overwrite; epsilon already at `minEpsilon`; CartPole cap marked `terminated` | Plot mean return by slice of the budget; read `epsilon` from `getState()` | Slow `epsilonDecay`; treat time-limit as `truncated` in **your** env (CartPole does not) | +| Policy collapse (one action) | PPO entropy too small; DQN greedy too early | Histogram of actions during `step()` (not entropy: not logged) | Raise `PPOConfig.entropyCoef` (default `0.01`); keep `epsilon` from decaying to 0 instantly | +| DQN never exploits / never explores | `epsilon = 1` and `epsilonDecay = 1`; or `epsilon = 0` | `getState().epsilon` after N trained steps | Defaults: `epsilon: 1`, `epsilonDecay: 0.995`, `minEpsilon: 0.01` | +| Learns in notebook, fails in game | Observations not normalized; scale mismatch vs training | Print min/max of each `observe()` dim over 50 steps | Scale each dim toward `[-1, 1]` as the root README recommends | +| `lr` “too strong” | Default is already `0.001`; `lr >= 1` is **rejected** by `DQNConfigSchema`; `0.1`–`0.99` is legal but aggressive | Catch the schema error, or compare legal `lr` values on 3 seeds | Remove the override or set `lr: 0.001` | +| Memory grows / browser tab dies | Missing `tf.tidy` / `tf.dispose`; `getAction` tensors not disposed | `tf.memory().numTensors` before vs after N `step()`s | Agents already tidy `train()` and dispose `getAction` tensors; **your** `predict` probes must be in `tf.tidy`. Call `dispose()` when replacing an agent | +| Silent CPU fallback / slow train | `setBackend('webgpu')` unavailable; `backend: 'auto'` does nothing | `getAvailableBackends()` then `tf.getBackend()` | Pass `backend: 'cpu'` in Node tests; `'webgl'`/`'webgpu'` only after they appear in the registry | + +## TensorFlow.js hygiene + +- `DQNAgent.train` builds the Bellman batch inside `tf.tidy` and `tf.dispose`s the fit tensors in `finally`. +- `DQNAgent.getAction` disposes `stateTensor`, `qValues`, and `argMaxTensor`. +- `PPOAgent.train` wraps gradient work in `tf.tidy`. +- After warmup (replay at `batchSize`), `tf.memory().numTensors` must stay **flat** across N further `env.step()` calls. If it climbs, the leak is in **your** probe code or a custom env, until proven otherwise. +- Replacing an agent: `IgnitionEnv` disposes the previous agent asynchronously after `train()` with a new algorithm. For tests, call `env.stop(); env.agent?.dispose?.()`. + +## Backend choice + +`setBackend` in `packages/backend-tfjs/src/utils/backend-selector.ts`: + +- `'auto'` **returns immediately** and does not pick a backend. +- `'node'` `require`s `@tensorflow/tfjs-node` and falls back to CPU if missing. +- `'wasm'` same pattern with `@tensorflow/tfjs-backend-wasm`. +- Other names: `tf.setBackend(name)`, CPU on failure. +- `getAvailableBackends()` lists `engine().registryFactory` keys. + +Call `await tf.setBackend('cpu'); await tf.ready();` in Node before `train()`, matching package tests. `DQNAgent` fires `setBackend(config.backend)` without awaiting it. + +## CartPole failure used by the example + +Shared budget (all protocols): seeds `7`, `11`, `19`; `64` steps; `hiddenLayers: [8]`; `batchSize: 8`; `memorySize: 64`; `targetUpdateFrequency: 100`; `backend: 'cpu'`. + +| Protocol | Declared before the run | Measured result | +|---|---|---| +| A | `lr: 1` vs `0.001` | Schema rejects `lr >= 1`. No training. | +| B | observe × `1000` vs × `1`, `lr: 0.001` | Q stayed finite (seed 7 ≈ `[79.4, 104.6]`). Hypothesis false at this budget. | +| C | `lr: 0.99` vs `0.001` | Q stayed finite on 3 seeds. Hypothesis false at this budget. | +| D | `reward() === 0` vs `CartPoleEnv.reward()` | Broken mean return is 0; restored reward has mean return > 0. Not a CartPole solve. | + +## Do not + +- Do not use APIs from unmerged PRs. +- Do not mock `IgnitionEnvTFJS`, `DQNAgent`, or `CartPoleEnv`. +- Do not skip a failing seed. +- Do not “fix” `@ignitionai/*` in this skill’s PR. Report gaps instead. diff --git a/skills/ignition-debug-training/references/symptom-table.md b/skills/ignition-debug-training/references/symptom-table.md new file mode 100644 index 0000000..535cec8 --- /dev/null +++ b/skills/ignition-debug-training/references/symptom-table.md @@ -0,0 +1,72 @@ +# Symptom table (IgnitionRL on `dev` SHA `343a76952f9dcd6594674c3526ee03ee45b4d93e`) + +Signatures are cited as `file:line` in the pull request. This file only names the symbols. + +## Flat reward + +- **Cause:** `TrainingEnv.reward()` ignores the task, or `done()` is always true so CartPole returns `0`. +- **Check:** Drive 20 `env.step()` calls. If every `StepResult.reward` is identical, the environment has no learning signal. CartPole itself returns `1` while the pole is up and `0` when `done()` is true. +- **Fix:** Change **your** `reward()` / `done()`. Do not expect a framework logger. + +## NaN / Inf + +- **Cause:** `DQNConfig.lr` far above `0.001`; observations with huge magnitude; a tensor leak feeding garbage into `predict`. +- **Check:** After the budget, probe Q-values with `DQNAgent.getModel()` inside `tf.tidy`. Also test `Number.isFinite` on every `StepResult.reward` and observation component. +- **Fix:** `lr: 0.001` (`DQN_DEFAULTS`). Scale observations. Keep probes in `tf.tidy`. + +## Instability + +- **Cause:** Large `lr`; tiny `targetUpdateFrequency` (or huge, so the target never moves); training before the replay buffer has `batchSize` samples (`DQNAgent.train` returns early). +- **Check:** Read `getState().trainStepCounter` — it only increases when `memory.size() >= batchSize`. Compare mean returns across three seeds; high variance plus non-finite Q is instability, not “almost learned”. +- **Fix:** Defaults in `DQN_DEFAULTS`: `lr: 0.001`, `targetUpdateFrequency: 100`, `batchSize: 32` (example uses `8` only to stay under 60 s CPU). + +## Forgetting + +- **Cause:** Replay overwriting early successes; epsilon already at `minEpsilon` so the policy is greedy on a weak Q; CartPole’s 500-step cap reported as `terminated` via `done()`, so DQN treats timeout as a true terminal (`q = reward` with no bootstrap). +- **Check:** Split the budget into early/late mean returns. Read `epsilon`. Confirm `truncated` is always `false` on stock CartPole (`IgnitionEnv` sets `truncated = env.truncated?.() ?? false`). +- **Fix:** In a **custom** env, implement `truncated()` for time limits. Slow `epsilonDecay` (default `0.995`). This skill cannot change `CartPoleEnv`. + +## Policy collapse + +- **Cause:** PPO `entropyCoef` too small (default `0.01`); DQN `epsilon` decayed to `minEpsilon` before the Q-network is useful. +- **Check:** Count chosen actions over the budget. Do **not** print “entropy” — PPO computes it inside `train()` and does not return it. DQN has no entropy term. +- **Fix:** Raise `entropyCoef` for PPO. For DQN, start at `epsilon: 1` and decay slowly. + +## DQN epsilon mis-set + +- **Cause:** `epsilon: 0` (no exploration) or `epsilonDecay: 1` with `epsilon: 1` (pure random forever). +- **Check:** `DQNAgent.getState().epsilon` after N **trained** steps (warmup does not decay epsilon because `train()` returns before the decay). +- **Fix:** `epsilon: 1.0`, `epsilonDecay: 0.995`, `minEpsilon: 0.01`. + +## Unnormalized observations + +- **Cause:** `CartPoleEnv.observe()` returns raw physics: cart `x` in about `[-2.4, 2.4]`, `theta` in radians (~`±0.209` at failure), velocities unbounded. The root README asks for `[-1, 1]`. +- **Check:** Min/max per dimension over 50 steps. If you wrap `observe()` with `× 1e3`, Q-values often become non-finite under the same `lr`. +- **Fix:** Divide each dim by a known scale (e.g. `x / 2.4`, `theta / 0.209`) in **your** wrapper. + +## Learning rate too high + +- **Cause:** Override `lr: 0.1` or `1` on `env.train('dqn', { lr })`. Default is `0.001`. +- **Check:** Same seeds, only `lr` changes. Non-finite Q or mean return stuck at random is a failed run, not a maybe. +- **Fix:** Drop the override or set `lr: 0.001`. + +## Tensor leaks + +- **Cause:** Calling `model.predict` without `tf.tidy` / `tf.dispose`; never calling `DQNAgent.dispose()`. +- **Check:** After warmup, `const n = tf.memory().numTensors`; run N `env.step()`; `tf.memory().numTensors === n`. +- **Fix:** Copy the agent pattern: tidy the graph, dispose fit inputs, dispose the agent when finished. + +## Backend selection + +- **Cause:** `backend: 'auto'` in `setBackend` is a no-op. Requesting `webgpu` in Node falls back to CPU with a console warning. +- **Check:** `getAvailableBackends()` then `tf.getBackend()` after `await tf.ready()`. +- **Fix:** Node tests: `backend: 'cpu'`. Browser: try `webgpu`, then `webgl`, then `wasm`, then `cpu`. + +## Gaps in the package (report, do not patch here) + +1. No training-loss or entropy metric on `getState()`. +2. `bestReward` on DQN/PPO is unused by the normal `train()` path. +3. `CartPoleEnv.reset()` is unseeded; agent `seed` does not control physics. +4. CartPole time limit is `done() === true` without `truncated()`. +5. `setBackend('auto')` does not select a backend. +6. `DQNAgent` does not `await` `setBackend`. diff --git a/skills/ignition-debug-training/skill.test.ts b/skills/ignition-debug-training/skill.test.ts new file mode 100644 index 0000000..8bb8a5e --- /dev/null +++ b/skills/ignition-debug-training/skill.test.ts @@ -0,0 +1,18 @@ +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { expect, it } from 'vitest'; + +it('SKILL.md frontmatter matches Agent Skills conventions', () => { + const raw = readFileSync( + resolve(process.cwd(), 'skills/ignition-debug-training/SKILL.md'), + 'utf8', + ); + const end = raw.indexOf('\n---\n', 4); + const front = raw.slice(4, end); + const body = raw.slice(end + 5); + const name = /(?:^|\n)name:\s*(.+)/.exec(front)?.[1]?.trim(); + const description = /(?:^|\n)description:\s*(.+)/.exec(front)?.[1]?.trim(); + expect(name).toBe('ignition-debug-training'); + expect(description!.length).toBeLessThanOrEqual(1024); + expect(body.split(/\r?\n/).length).toBeLessThanOrEqual(500); +});