|
| 1 | +import { readFileSync } from 'node:fs'; |
| 2 | +import { resolve } from 'node:path'; |
| 3 | +import { describe, expect, it } from 'vitest'; |
| 4 | +import type { TrainingEnv } from '@ignitionai/core'; |
| 5 | +import { validateTrainingEnv } from '@ignitionai/core'; |
| 6 | +import { MountainCarEnv } from '@ignitionai/environments'; |
| 7 | +import { |
| 8 | + MOUNTAIN_CAR_COMPARISON, |
| 9 | + MOUNTAIN_CAR_GOAL_POS, |
| 10 | + MountainCarTerminalWrapper, |
| 11 | + RewardShapingWrapper, |
| 12 | + compareMountainCarRewards, |
| 13 | + mountainCarReachedFlag, |
| 14 | +} from './skill-reward-shaping'; |
| 15 | + |
| 16 | +class ScriptedEnv implements TrainingEnv { |
| 17 | + actions = ['stay', 'go']; |
| 18 | + position: number; |
| 19 | + lastReward = 0; |
| 20 | + private readonly terminalAt: number; |
| 21 | + |
| 22 | + constructor(start: number, terminalAt = 10) { |
| 23 | + this.position = start; |
| 24 | + this.terminalAt = terminalAt; |
| 25 | + } |
| 26 | + |
| 27 | + observe(): number[] { |
| 28 | + return [this.position]; |
| 29 | + } |
| 30 | + |
| 31 | + step(action: number | number[]): void { |
| 32 | + const a = typeof action === 'number' ? action : action[0]; |
| 33 | + this.position += a === 1 ? 1 : 0; |
| 34 | + this.lastReward = -1; |
| 35 | + } |
| 36 | + |
| 37 | + reward(): number { |
| 38 | + return this.lastReward; |
| 39 | + } |
| 40 | + |
| 41 | + done(): boolean { |
| 42 | + return this.position >= this.terminalAt; |
| 43 | + } |
| 44 | + |
| 45 | + reset(): void { |
| 46 | + this.position = 0; |
| 47 | + this.lastReward = 0; |
| 48 | + } |
| 49 | +} |
| 50 | + |
| 51 | +describe('ignition-reward-shaping frontmatter', () => { |
| 52 | + it('name matches folder, description ≤ 1024, body ≤ 500 lines', () => { |
| 53 | + const skillPath = resolve(process.cwd(), 'skills/ignition-reward-shaping/SKILL.md'); |
| 54 | + const raw = readFileSync(skillPath, 'utf8'); |
| 55 | + expect(raw.startsWith('---')).toBe(true); |
| 56 | + const end = raw.indexOf('\n---', 3); |
| 57 | + expect(end).toBeGreaterThan(0); |
| 58 | + const frontmatter = raw.slice(4, end); |
| 59 | + const body = raw.slice(end + 4).replace(/^\n/, ''); |
| 60 | + const name = /(?:^|\n)name:\s*(.+)/.exec(frontmatter)?.[1]?.trim(); |
| 61 | + const description = /(?:^|\n)description:\s*(.+)/.exec(frontmatter)?.[1]?.trim(); |
| 62 | + expect(name).toBe('ignition-reward-shaping'); |
| 63 | + expect(description).toBeDefined(); |
| 64 | + expect(description!.length).toBeGreaterThan(0); |
| 65 | + expect(description!.length).toBeLessThanOrEqual(1024); |
| 66 | + expect(body.split('\n').length).toBeLessThanOrEqual(500); |
| 67 | + }); |
| 68 | +}); |
| 69 | + |
| 70 | +describe('RewardShapingWrapper hand-calculated transitions', () => { |
| 71 | + it('adds F = γ Φ(s′) − Φ(s) to a scripted inner reward', () => { |
| 72 | + const inner = new ScriptedEnv(2); |
| 73 | + const wrapped = new RewardShapingWrapper(inner, { |
| 74 | + gamma: 0.99, |
| 75 | + potential: (obs) => obs[0], |
| 76 | + zeroPotentialOnTerminate: false, |
| 77 | + }); |
| 78 | + validateTrainingEnv(wrapped); |
| 79 | + expect(inner.reward()).toBe(0); |
| 80 | + wrapped.step(1); |
| 81 | + // s=2 → s′=3, inner r=-1, F=0.99*3-2=0.97 → r′=-0.03 |
| 82 | + expect(inner.position).toBe(3); |
| 83 | + expect(inner.reward()).toBe(-1); |
| 84 | + expect(wrapped.reward()).toBeCloseTo(-1 + 0.99 * 3 - 2, 12); |
| 85 | + }); |
| 86 | + |
| 87 | + it('applies scale and time penalty on the same transition', () => { |
| 88 | + const inner = new ScriptedEnv(2); |
| 89 | + const wrapped = new RewardShapingWrapper(inner, { |
| 90 | + gamma: 0.99, |
| 91 | + potential: (obs) => obs[0], |
| 92 | + scale: 2, |
| 93 | + timePenalty: -0.1, |
| 94 | + zeroPotentialOnTerminate: false, |
| 95 | + }); |
| 96 | + wrapped.step(1); |
| 97 | + // 2*(-1) + (0.99*3-2) + (-0.1) = -2 + 0.97 - 0.1 = -1.13 |
| 98 | + expect(wrapped.reward()).toBeCloseTo(-1.13, 12); |
| 99 | + }); |
| 100 | + |
| 101 | + it('sign flip (scale=-1) inverts the inner term only', () => { |
| 102 | + const inner = new ScriptedEnv(2); |
| 103 | + const wrapped = new RewardShapingWrapper(inner, { |
| 104 | + gamma: 0.99, |
| 105 | + potential: (obs) => obs[0], |
| 106 | + scale: -1, |
| 107 | + zeroPotentialOnTerminate: false, |
| 108 | + }); |
| 109 | + wrapped.step(1); |
| 110 | + // -1*(-1) + 0.97 = 1.97 |
| 111 | + expect(wrapped.reward()).toBeCloseTo(1.97, 12); |
| 112 | + }); |
| 113 | + |
| 114 | + it('zeros Φ(s′) on a true terminal', () => { |
| 115 | + const inner = new ScriptedEnv(9, 10); |
| 116 | + const wrapped = new RewardShapingWrapper(inner, { |
| 117 | + gamma: 0.99, |
| 118 | + potential: (obs) => obs[0], |
| 119 | + zeroPotentialOnTerminate: true, |
| 120 | + terminated: (env) => (unwrapScripted(env).position >= 10), |
| 121 | + }); |
| 122 | + wrapped.step(1); |
| 123 | + expect(inner.position).toBe(10); |
| 124 | + expect(wrapped.terminated()).toBe(true); |
| 125 | + // F = 0.99*0 - 9 = -9, r′ = -1 - 9 = -10 |
| 126 | + expect(wrapped.reward()).toBeCloseTo(-10, 12); |
| 127 | + }); |
| 128 | + |
| 129 | + it('matches MountainCar physics from (position,velocity)=(0,0) with action 2', () => { |
| 130 | + const inner = new MountainCarEnv(); |
| 131 | + inner.position = 0; |
| 132 | + inner.velocity = 0; |
| 133 | + inner.stepCount = 0; |
| 134 | + const wrapped = new RewardShapingWrapper(inner, { |
| 135 | + gamma: 0.99, |
| 136 | + potential: (obs) => obs[0], |
| 137 | + zeroPotentialOnTerminate: false, |
| 138 | + }); |
| 139 | + wrapped.step(2); |
| 140 | + // force=(2-1)*0.001=0.001; vel=0.001-0.0025*cos(0)=-0.0015; pos=-0.0015 |
| 141 | + expect(inner.position).toBeCloseTo(-0.0015, 12); |
| 142 | + expect(inner.velocity).toBeCloseTo(-0.0015, 12); |
| 143 | + expect(inner.reward()).toBe(-1); |
| 144 | + // F=0.99*(-0.0015)-0=-0.001485 → r′=-1.001485 |
| 145 | + expect(wrapped.reward()).toBeCloseTo(-1.001485, 12); |
| 146 | + expect(mountainCarReachedFlag(inner)).toBe(false); |
| 147 | + }); |
| 148 | + |
| 149 | + it('goal transition on MountainCar uses Φ(s′)=0 and does not use timeout as success', () => { |
| 150 | + const inner = new MountainCarEnv(); |
| 151 | + inner.position = 0.499; |
| 152 | + inner.velocity = 0.07; |
| 153 | + inner.stepCount = 0; |
| 154 | + const terminals = new MountainCarTerminalWrapper(inner); |
| 155 | + const wrapped = new RewardShapingWrapper(terminals, { |
| 156 | + gamma: 0.99, |
| 157 | + potential: (obs) => obs[0], |
| 158 | + zeroPotentialOnTerminate: true, |
| 159 | + }); |
| 160 | + wrapped.step(2); |
| 161 | + // vel clamped 0.07; pos=0.499+0.07=0.569 ≥ 0.5 |
| 162 | + expect(inner.position).toBeCloseTo(0.569, 12); |
| 163 | + expect(inner.position).toBeGreaterThanOrEqual(MOUNTAIN_CAR_GOAL_POS); |
| 164 | + expect(wrapped.terminated()).toBe(true); |
| 165 | + expect(wrapped.truncated()).toBe(false); |
| 166 | + expect(mountainCarReachedFlag(inner)).toBe(true); |
| 167 | + // F=0.99*0-0.499=-0.499 → r′=-1.499 |
| 168 | + expect(wrapped.reward()).toBeCloseTo(-1.499, 12); |
| 169 | + }); |
| 170 | +}); |
| 171 | + |
| 172 | +function unwrapScripted(env: TrainingEnv): ScriptedEnv { |
| 173 | + let current: TrainingEnv = env; |
| 174 | + while (current instanceof RewardShapingWrapper) { |
| 175 | + current = current.innerEnv(); |
| 176 | + } |
| 177 | + return current as ScriptedEnv; |
| 178 | +} |
| 179 | + |
| 180 | +describe('MountainCar sparse vs shaped (true flag successes)', () => { |
| 181 | + it('runs the declared 3-seed budget and reports flag counts without claiming a winner', async () => { |
| 182 | + expect(MOUNTAIN_CAR_COMPARISON.seeds).toHaveLength(3); |
| 183 | + const report = await compareMountainCarRewards(); |
| 184 | + expect(report.rows).toHaveLength(6); |
| 185 | + for (const row of report.rows) { |
| 186 | + expect(row.episodes).toBe(MOUNTAIN_CAR_COMPARISON.episodesPerSeed); |
| 187 | + expect(row.flagSuccesses + row.timeouts).toBe(row.episodes); |
| 188 | + expect(row.flagSuccesses).toBeGreaterThanOrEqual(0); |
| 189 | + expect(Number.isFinite(row.sumReportedReward)).toBe(true); |
| 190 | + } |
| 191 | + const sparseFlags = report.rows.filter((r) => r.variant === 'sparse').map((r) => r.flagSuccesses); |
| 192 | + const shapedFlags = report.rows.filter((r) => r.variant === 'shaped').map((r) => r.flagSuccesses); |
| 193 | + console.log('[skill-reward-shaping] flagSuccesses sparse', sparseFlags, 'shaped', shapedFlags); |
| 194 | + }, 55_000); |
| 195 | +}); |
0 commit comments