Skip to content

Commit 147e705

Browse files
salim4ncursoragent
andauthored
docs(skills): add ignition-reward-shaping wrapper and MountainCar comparison (#117)
Teach potential-based reward shaping as a TrainingEnv wrapper, with hand-calculated transition tests and a declared-budget flag-success comparison on stock MountainCar (sparse vs shaped). Co-authored-by: Cursor Agent <cursoragent@cursor.com> Co-authored-by: Salim <salim4n@users.noreply.github.com>
1 parent 56da7d6 commit 147e705

6 files changed

Lines changed: 712 additions & 0 deletions

File tree

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
import { readFileSync } from 'node:fs';
2+
import { resolve } from 'node:path';
3+
import { describe, expect, it } from 'vitest';
4+
import type { TrainingEnv } from '@ignitionai/core';
5+
import { validateTrainingEnv } from '@ignitionai/core';
6+
import { MountainCarEnv } from '@ignitionai/environments';
7+
import {
8+
MOUNTAIN_CAR_COMPARISON,
9+
MOUNTAIN_CAR_GOAL_POS,
10+
MountainCarTerminalWrapper,
11+
RewardShapingWrapper,
12+
compareMountainCarRewards,
13+
mountainCarReachedFlag,
14+
} from './skill-reward-shaping';
15+
16+
class ScriptedEnv implements TrainingEnv {
17+
actions = ['stay', 'go'];
18+
position: number;
19+
lastReward = 0;
20+
private readonly terminalAt: number;
21+
22+
constructor(start: number, terminalAt = 10) {
23+
this.position = start;
24+
this.terminalAt = terminalAt;
25+
}
26+
27+
observe(): number[] {
28+
return [this.position];
29+
}
30+
31+
step(action: number | number[]): void {
32+
const a = typeof action === 'number' ? action : action[0];
33+
this.position += a === 1 ? 1 : 0;
34+
this.lastReward = -1;
35+
}
36+
37+
reward(): number {
38+
return this.lastReward;
39+
}
40+
41+
done(): boolean {
42+
return this.position >= this.terminalAt;
43+
}
44+
45+
reset(): void {
46+
this.position = 0;
47+
this.lastReward = 0;
48+
}
49+
}
50+
51+
describe('ignition-reward-shaping frontmatter', () => {
52+
it('name matches folder, description ≤ 1024, body ≤ 500 lines', () => {
53+
const skillPath = resolve(process.cwd(), 'skills/ignition-reward-shaping/SKILL.md');
54+
const raw = readFileSync(skillPath, 'utf8');
55+
expect(raw.startsWith('---')).toBe(true);
56+
const end = raw.indexOf('\n---', 3);
57+
expect(end).toBeGreaterThan(0);
58+
const frontmatter = raw.slice(4, end);
59+
const body = raw.slice(end + 4).replace(/^\n/, '');
60+
const name = /(?:^|\n)name:\s*(.+)/.exec(frontmatter)?.[1]?.trim();
61+
const description = /(?:^|\n)description:\s*(.+)/.exec(frontmatter)?.[1]?.trim();
62+
expect(name).toBe('ignition-reward-shaping');
63+
expect(description).toBeDefined();
64+
expect(description!.length).toBeGreaterThan(0);
65+
expect(description!.length).toBeLessThanOrEqual(1024);
66+
expect(body.split('\n').length).toBeLessThanOrEqual(500);
67+
});
68+
});
69+
70+
describe('RewardShapingWrapper hand-calculated transitions', () => {
71+
it('adds F = γ Φ(s′) − Φ(s) to a scripted inner reward', () => {
72+
const inner = new ScriptedEnv(2);
73+
const wrapped = new RewardShapingWrapper(inner, {
74+
gamma: 0.99,
75+
potential: (obs) => obs[0],
76+
zeroPotentialOnTerminate: false,
77+
});
78+
validateTrainingEnv(wrapped);
79+
expect(inner.reward()).toBe(0);
80+
wrapped.step(1);
81+
// s=2 → s′=3, inner r=-1, F=0.99*3-2=0.97 → r′=-0.03
82+
expect(inner.position).toBe(3);
83+
expect(inner.reward()).toBe(-1);
84+
expect(wrapped.reward()).toBeCloseTo(-1 + 0.99 * 3 - 2, 12);
85+
});
86+
87+
it('applies scale and time penalty on the same transition', () => {
88+
const inner = new ScriptedEnv(2);
89+
const wrapped = new RewardShapingWrapper(inner, {
90+
gamma: 0.99,
91+
potential: (obs) => obs[0],
92+
scale: 2,
93+
timePenalty: -0.1,
94+
zeroPotentialOnTerminate: false,
95+
});
96+
wrapped.step(1);
97+
// 2*(-1) + (0.99*3-2) + (-0.1) = -2 + 0.97 - 0.1 = -1.13
98+
expect(wrapped.reward()).toBeCloseTo(-1.13, 12);
99+
});
100+
101+
it('sign flip (scale=-1) inverts the inner term only', () => {
102+
const inner = new ScriptedEnv(2);
103+
const wrapped = new RewardShapingWrapper(inner, {
104+
gamma: 0.99,
105+
potential: (obs) => obs[0],
106+
scale: -1,
107+
zeroPotentialOnTerminate: false,
108+
});
109+
wrapped.step(1);
110+
// -1*(-1) + 0.97 = 1.97
111+
expect(wrapped.reward()).toBeCloseTo(1.97, 12);
112+
});
113+
114+
it('zeros Φ(s′) on a true terminal', () => {
115+
const inner = new ScriptedEnv(9, 10);
116+
const wrapped = new RewardShapingWrapper(inner, {
117+
gamma: 0.99,
118+
potential: (obs) => obs[0],
119+
zeroPotentialOnTerminate: true,
120+
terminated: (env) => (unwrapScripted(env).position >= 10),
121+
});
122+
wrapped.step(1);
123+
expect(inner.position).toBe(10);
124+
expect(wrapped.terminated()).toBe(true);
125+
// F = 0.99*0 - 9 = -9, r′ = -1 - 9 = -10
126+
expect(wrapped.reward()).toBeCloseTo(-10, 12);
127+
});
128+
129+
it('matches MountainCar physics from (position,velocity)=(0,0) with action 2', () => {
130+
const inner = new MountainCarEnv();
131+
inner.position = 0;
132+
inner.velocity = 0;
133+
inner.stepCount = 0;
134+
const wrapped = new RewardShapingWrapper(inner, {
135+
gamma: 0.99,
136+
potential: (obs) => obs[0],
137+
zeroPotentialOnTerminate: false,
138+
});
139+
wrapped.step(2);
140+
// force=(2-1)*0.001=0.001; vel=0.001-0.0025*cos(0)=-0.0015; pos=-0.0015
141+
expect(inner.position).toBeCloseTo(-0.0015, 12);
142+
expect(inner.velocity).toBeCloseTo(-0.0015, 12);
143+
expect(inner.reward()).toBe(-1);
144+
// F=0.99*(-0.0015)-0=-0.001485 → r′=-1.001485
145+
expect(wrapped.reward()).toBeCloseTo(-1.001485, 12);
146+
expect(mountainCarReachedFlag(inner)).toBe(false);
147+
});
148+
149+
it('goal transition on MountainCar uses Φ(s′)=0 and does not use timeout as success', () => {
150+
const inner = new MountainCarEnv();
151+
inner.position = 0.499;
152+
inner.velocity = 0.07;
153+
inner.stepCount = 0;
154+
const terminals = new MountainCarTerminalWrapper(inner);
155+
const wrapped = new RewardShapingWrapper(terminals, {
156+
gamma: 0.99,
157+
potential: (obs) => obs[0],
158+
zeroPotentialOnTerminate: true,
159+
});
160+
wrapped.step(2);
161+
// vel clamped 0.07; pos=0.499+0.07=0.569 ≥ 0.5
162+
expect(inner.position).toBeCloseTo(0.569, 12);
163+
expect(inner.position).toBeGreaterThanOrEqual(MOUNTAIN_CAR_GOAL_POS);
164+
expect(wrapped.terminated()).toBe(true);
165+
expect(wrapped.truncated()).toBe(false);
166+
expect(mountainCarReachedFlag(inner)).toBe(true);
167+
// F=0.99*0-0.499=-0.499 → r′=-1.499
168+
expect(wrapped.reward()).toBeCloseTo(-1.499, 12);
169+
});
170+
});
171+
172+
function unwrapScripted(env: TrainingEnv): ScriptedEnv {
173+
let current: TrainingEnv = env;
174+
while (current instanceof RewardShapingWrapper) {
175+
current = current.innerEnv();
176+
}
177+
return current as ScriptedEnv;
178+
}
179+
180+
describe('MountainCar sparse vs shaped (true flag successes)', () => {
181+
it('runs the declared 3-seed budget and reports flag counts without claiming a winner', async () => {
182+
expect(MOUNTAIN_CAR_COMPARISON.seeds).toHaveLength(3);
183+
const report = await compareMountainCarRewards();
184+
expect(report.rows).toHaveLength(6);
185+
for (const row of report.rows) {
186+
expect(row.episodes).toBe(MOUNTAIN_CAR_COMPARISON.episodesPerSeed);
187+
expect(row.flagSuccesses + row.timeouts).toBe(row.episodes);
188+
expect(row.flagSuccesses).toBeGreaterThanOrEqual(0);
189+
expect(Number.isFinite(row.sumReportedReward)).toBe(true);
190+
}
191+
const sparseFlags = report.rows.filter((r) => r.variant === 'sparse').map((r) => r.flagSuccesses);
192+
const shapedFlags = report.rows.filter((r) => r.variant === 'shaped').map((r) => r.flagSuccesses);
193+
console.log('[skill-reward-shaping] flagSuccesses sparse', sparseFlags, 'shaped', shapedFlags);
194+
}, 55_000);
195+
});

0 commit comments

Comments
 (0)