|
| 1 | +import { |
| 2 | + IgnitionEnv, |
| 3 | + type ActionSpace, |
| 4 | + type BoxSpace, |
| 5 | + type DiscreteSpace, |
| 6 | + type MultiDiscreteSpace, |
| 7 | + type ObservationSpace, |
| 8 | + type StepResult, |
| 9 | + type TrainingEnv, |
| 10 | +} from '@ignitionai/core'; |
| 11 | +import { IgnitionEnvTFJS } from '@ignitionai/backend-tfjs'; |
| 12 | + |
| 13 | +/** |
| 14 | + * Declared before any training run. |
| 15 | + * |
| 16 | + * Budget: 200 environment steps per seed (manual `step()`, not the async loop). |
| 17 | + * Seeds: 7, 11, 19. Algorithm: qtable. No convergence claim without measurement. |
| 18 | + */ |
| 19 | +export const TRAINING_BUDGET_STEPS = 200; |
| 20 | +export const TRAINING_SEEDS = [7, 11, 19] as const; |
| 21 | +export const TRAINING_OVERRIDES = { |
| 22 | + lr: 0.3, |
| 23 | + epsilon: 1, |
| 24 | + epsilonDecay: 0.99, |
| 25 | + minEpsilon: 0.05, |
| 26 | + stateBins: 8, |
| 27 | + stateLow: [0], |
| 28 | + stateHigh: [1], |
| 29 | +} as const; |
| 30 | + |
| 31 | +/** 1D line: start at 0, goal at `length - 1`, discrete left/right actions. */ |
| 32 | +export class LineWorldEnv implements TrainingEnv { |
| 33 | + actions = ['left', 'right']; |
| 34 | + position = 0; |
| 35 | + stepCount = 0; |
| 36 | + readonly length: number; |
| 37 | + readonly maxSteps: number; |
| 38 | + |
| 39 | + constructor(length = 5, maxSteps = 12) { |
| 40 | + this.length = length; |
| 41 | + this.maxSteps = maxSteps; |
| 42 | + } |
| 43 | + |
| 44 | + observe(): number[] { |
| 45 | + return [this.position / (this.length - 1)]; |
| 46 | + } |
| 47 | + |
| 48 | + step(action: number | number[]): void { |
| 49 | + const a = typeof action === 'number' ? action : action[0]; |
| 50 | + if (a === 1) { |
| 51 | + this.position = Math.min(this.length - 1, this.position + 1); |
| 52 | + } else { |
| 53 | + this.position = Math.max(0, this.position - 1); |
| 54 | + } |
| 55 | + this.stepCount += 1; |
| 56 | + } |
| 57 | + |
| 58 | + reward(): number { |
| 59 | + if (this.terminated()) return 1; |
| 60 | + return -0.01; |
| 61 | + } |
| 62 | + |
| 63 | + done(): boolean { |
| 64 | + return this.terminated() || this.truncated(); |
| 65 | + } |
| 66 | + |
| 67 | + terminated(): boolean { |
| 68 | + return this.position === this.length - 1; |
| 69 | + } |
| 70 | + |
| 71 | + truncated(): boolean { |
| 72 | + return !this.terminated() && this.stepCount >= this.maxSteps; |
| 73 | + } |
| 74 | + |
| 75 | + reset(): void { |
| 76 | + this.position = 0; |
| 77 | + this.stepCount = 0; |
| 78 | + } |
| 79 | + |
| 80 | + actionSpace(): DiscreteSpace { |
| 81 | + return { type: 'discrete', n: this.actions.length }; |
| 82 | + } |
| 83 | + |
| 84 | + observationSpace(): BoxSpace { |
| 85 | + return { type: 'box', low: [0], high: [1], shape: [1] }; |
| 86 | + } |
| 87 | +} |
| 88 | + |
| 89 | +/** Missing `done()` — rejected by `validateTrainingEnv`. */ |
| 90 | +export function createInvalidTrainingEnv(): unknown { |
| 91 | + return { |
| 92 | + actions: ['left', 'right'], |
| 93 | + observe(): number[] { |
| 94 | + return [0]; |
| 95 | + }, |
| 96 | + step(): void {}, |
| 97 | + reward(): number { |
| 98 | + return 0; |
| 99 | + }, |
| 100 | + reset(): void {}, |
| 101 | + }; |
| 102 | +} |
| 103 | + |
| 104 | +export function spaceKind(space: ActionSpace | ObservationSpace): string { |
| 105 | + switch (space.type) { |
| 106 | + case 'discrete': |
| 107 | + return `discrete n=${(space as DiscreteSpace).n}`; |
| 108 | + case 'box': |
| 109 | + return `box shape=${(space as BoxSpace).shape.join('x')}`; |
| 110 | + case 'multidiscrete': |
| 111 | + return `multidiscrete nvec=${(space as MultiDiscreteSpace).nvec.join(',')}`; |
| 112 | + default: { |
| 113 | + const unexpected: never = space; |
| 114 | + throw new Error(`unhandled space: ${JSON.stringify(unexpected)}`); |
| 115 | + } |
| 116 | + } |
| 117 | +} |
| 118 | + |
| 119 | +function mulberry32(seed: number): () => number { |
| 120 | + let state = seed >>> 0; |
| 121 | + return () => { |
| 122 | + state += 0x6d2b79f5; |
| 123 | + let t = state; |
| 124 | + t = Math.imul(t ^ (t >>> 15), t | 1); |
| 125 | + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); |
| 126 | + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; |
| 127 | + }; |
| 128 | +} |
| 129 | + |
| 130 | +export interface TrainingRunMeasurement { |
| 131 | + seed: number; |
| 132 | + budgetSteps: number; |
| 133 | + completedSteps: number; |
| 134 | + terminatedEpisodes: number; |
| 135 | + truncatedEpisodes: number; |
| 136 | + episodeReturns: number[]; |
| 137 | +} |
| 138 | + |
| 139 | +export async function runLineWorldTraining( |
| 140 | + seed: number, |
| 141 | + budgetSteps: number = TRAINING_BUDGET_STEPS, |
| 142 | +): Promise<TrainingRunMeasurement> { |
| 143 | + const rng = mulberry32(seed); |
| 144 | + const originalRandom = Math.random; |
| 145 | + Math.random = rng; |
| 146 | + const world = new LineWorldEnv(); |
| 147 | + const runner = new IgnitionEnvTFJS(world); |
| 148 | + try { |
| 149 | + runner.train('qtable', { ...TRAINING_OVERRIDES }); |
| 150 | + runner.stop(); |
| 151 | + const episodeReturns: number[] = []; |
| 152 | + let episodeReturn = 0; |
| 153 | + let terminatedEpisodes = 0; |
| 154 | + let truncatedEpisodes = 0; |
| 155 | + let completedSteps = 0; |
| 156 | + for (let i = 0; i < budgetSteps; i++) { |
| 157 | + const result: StepResult = await runner.step(); |
| 158 | + completedSteps += 1; |
| 159 | + episodeReturn += result.reward; |
| 160 | + if (result.terminated || result.truncated) { |
| 161 | + if (result.terminated) terminatedEpisodes += 1; |
| 162 | + if (result.truncated) truncatedEpisodes += 1; |
| 163 | + episodeReturns.push(episodeReturn); |
| 164 | + episodeReturn = 0; |
| 165 | + } |
| 166 | + } |
| 167 | + return { |
| 168 | + seed, |
| 169 | + budgetSteps, |
| 170 | + completedSteps, |
| 171 | + terminatedEpisodes, |
| 172 | + truncatedEpisodes, |
| 173 | + episodeReturns, |
| 174 | + }; |
| 175 | + } finally { |
| 176 | + runner.stop(); |
| 177 | + runner.agent?.dispose?.(); |
| 178 | + Math.random = originalRandom; |
| 179 | + } |
| 180 | +} |
| 181 | + |
| 182 | +/** Scripted always-right agent for episode-end semantics (real `IgnitionEnv.step`). */ |
| 183 | +export async function scriptedStep(world: TrainingEnv): Promise<StepResult> { |
| 184 | + const runner = new IgnitionEnv(world); |
| 185 | + runner.agent = { |
| 186 | + getAction: async () => 1, |
| 187 | + remember() {}, |
| 188 | + train: async () => {}, |
| 189 | + }; |
| 190 | + return runner.step(); |
| 191 | +} |
0 commit comments