Skip to content

Commit c91fbae

Browse files
authored
Merge branch 'dev' into skill/train
2 parents 5db6073 + 0ee79cb commit c91fbae

11 files changed

Lines changed: 926 additions & 0 deletions

File tree

‎examples/skill-create-env.test.ts‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
import { describe, expect, it } from 'vitest';
2+
import { IgnitionEnv, validateTrainingEnv } from '@ignitionai/core';
3+
import { CartPoleEnv, GridWorldEnv, MountainCarEnv } from '@ignitionai/environments';
4+
import {
5+
TRAINING_BUDGET_STEPS,
6+
TRAINING_SEEDS,
7+
createInvalidTrainingEnv,
8+
LineWorldEnv,
9+
runLineWorldTraining,
10+
scriptedStep,
11+
spaceKind,
12+
} from './skill-create-env';
13+
14+
/**
15+
* Budget and seeds declared before execution:
16+
* - budget = TRAINING_BUDGET_STEPS (200 env steps / seed)
17+
* - seeds = TRAINING_SEEDS (7, 11, 19)
18+
* Measurements are printed; this file does not assert convergence.
19+
*/
20+
describe('skill-create-env LineWorld', () => {
21+
it('passes TrainingEnv validation and wraps in IgnitionEnv', () => {
22+
const world = new LineWorldEnv();
23+
expect(() => validateTrainingEnv(world)).not.toThrow();
24+
expect(() => new IgnitionEnv(world)).not.toThrow();
25+
expect(spaceKind(world.actionSpace())).toBe('discrete n=2');
26+
expect(spaceKind(world.observationSpace())).toBe('box shape=1');
27+
expect(world.observe()).toEqual([0]);
28+
});
29+
30+
it('rejects an invalid env with the real validation error', () => {
31+
const invalid = createInvalidTrainingEnv();
32+
expect(() => validateTrainingEnv(invalid)).toThrowError(
33+
'[IgnitionAI] TrainingEnv must implement done() as a function',
34+
);
35+
expect(
36+
() => new IgnitionEnv(invalid as never),
37+
).toThrowError('[IgnitionAI] TrainingEnv must implement done() as a function');
38+
});
39+
40+
it('reports terminated when the 1D goal is reached', async () => {
41+
const world = new LineWorldEnv(3, 20);
42+
world.position = 1;
43+
const result = await scriptedStep(world);
44+
expect(result).toMatchObject({
45+
observation: [1],
46+
reward: 1,
47+
terminated: true,
48+
truncated: false,
49+
});
50+
expect(world.position).toBe(0);
51+
});
52+
53+
it('reports truncated when the step limit hits before the goal', async () => {
54+
const world = new LineWorldEnv(5, 1);
55+
const result = await scriptedStep(world);
56+
expect(result.terminated).toBe(false);
57+
expect(result.truncated).toBe(true);
58+
expect(result.reward).toBe(-0.01);
59+
expect(world.position).toBe(0);
60+
});
61+
62+
it('runs the declared qtable budget on three seeds and records raw returns', async () => {
63+
expect(TRAINING_SEEDS).toHaveLength(3);
64+
const measurements = [];
65+
for (const seed of TRAINING_SEEDS) {
66+
const measurement = await runLineWorldTraining(seed, TRAINING_BUDGET_STEPS);
67+
measurements.push(measurement);
68+
expect(measurement.completedSteps).toBe(TRAINING_BUDGET_STEPS);
69+
expect(measurement.terminatedEpisodes + measurement.truncatedEpisodes).toBe(
70+
measurement.episodeReturns.length,
71+
);
72+
}
73+
// Raw measurements (failures included). Not a convergence claim.
74+
console.log(JSON.stringify({ budgetSteps: TRAINING_BUDGET_STEPS, measurements }, null, 2));
75+
});
76+
77+
it('reference envs still satisfy TrainingEnv validation', () => {
78+
expect(() => validateTrainingEnv(new GridWorldEnv())).not.toThrow();
79+
expect(() => validateTrainingEnv(new CartPoleEnv())).not.toThrow();
80+
expect(() => validateTrainingEnv(new MountainCarEnv())).not.toThrow();
81+
});
82+
});

‎examples/skill-create-env.ts‎

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
import {
2+
IgnitionEnv,
3+
type ActionSpace,
4+
type BoxSpace,
5+
type DiscreteSpace,
6+
type MultiDiscreteSpace,
7+
type ObservationSpace,
8+
type StepResult,
9+
type TrainingEnv,
10+
} from '@ignitionai/core';
11+
import { IgnitionEnvTFJS } from '@ignitionai/backend-tfjs';
12+
13+
/**
14+
* Declared before any training run.
15+
*
16+
* Budget: 200 environment steps per seed (manual `step()`, not the async loop).
17+
* Seeds: 7, 11, 19. Algorithm: qtable. No convergence claim without measurement.
18+
*/
19+
export const TRAINING_BUDGET_STEPS = 200;
20+
export const TRAINING_SEEDS = [7, 11, 19] as const;
21+
export const TRAINING_OVERRIDES = {
22+
lr: 0.3,
23+
epsilon: 1,
24+
epsilonDecay: 0.99,
25+
minEpsilon: 0.05,
26+
stateBins: 8,
27+
stateLow: [0],
28+
stateHigh: [1],
29+
} as const;
30+
31+
/** 1D line: start at 0, goal at `length - 1`, discrete left/right actions. */
32+
export class LineWorldEnv implements TrainingEnv {
33+
actions = ['left', 'right'];
34+
position = 0;
35+
stepCount = 0;
36+
readonly length: number;
37+
readonly maxSteps: number;
38+
39+
constructor(length = 5, maxSteps = 12) {
40+
this.length = length;
41+
this.maxSteps = maxSteps;
42+
}
43+
44+
observe(): number[] {
45+
return [this.position / (this.length - 1)];
46+
}
47+
48+
step(action: number | number[]): void {
49+
const a = typeof action === 'number' ? action : action[0];
50+
if (a === 1) {
51+
this.position = Math.min(this.length - 1, this.position + 1);
52+
} else {
53+
this.position = Math.max(0, this.position - 1);
54+
}
55+
this.stepCount += 1;
56+
}
57+
58+
reward(): number {
59+
if (this.terminated()) return 1;
60+
return -0.01;
61+
}
62+
63+
done(): boolean {
64+
return this.terminated() || this.truncated();
65+
}
66+
67+
terminated(): boolean {
68+
return this.position === this.length - 1;
69+
}
70+
71+
truncated(): boolean {
72+
return !this.terminated() && this.stepCount >= this.maxSteps;
73+
}
74+
75+
reset(): void {
76+
this.position = 0;
77+
this.stepCount = 0;
78+
}
79+
80+
actionSpace(): DiscreteSpace {
81+
return { type: 'discrete', n: this.actions.length };
82+
}
83+
84+
observationSpace(): BoxSpace {
85+
return { type: 'box', low: [0], high: [1], shape: [1] };
86+
}
87+
}
88+
89+
/** Missing `done()` — rejected by `validateTrainingEnv`. */
90+
export function createInvalidTrainingEnv(): unknown {
91+
return {
92+
actions: ['left', 'right'],
93+
observe(): number[] {
94+
return [0];
95+
},
96+
step(): void {},
97+
reward(): number {
98+
return 0;
99+
},
100+
reset(): void {},
101+
};
102+
}
103+
104+
export function spaceKind(space: ActionSpace | ObservationSpace): string {
105+
switch (space.type) {
106+
case 'discrete':
107+
return `discrete n=${(space as DiscreteSpace).n}`;
108+
case 'box':
109+
return `box shape=${(space as BoxSpace).shape.join('x')}`;
110+
case 'multidiscrete':
111+
return `multidiscrete nvec=${(space as MultiDiscreteSpace).nvec.join(',')}`;
112+
default: {
113+
const unexpected: never = space;
114+
throw new Error(`unhandled space: ${JSON.stringify(unexpected)}`);
115+
}
116+
}
117+
}
118+
119+
function mulberry32(seed: number): () => number {
120+
let state = seed >>> 0;
121+
return () => {
122+
state += 0x6d2b79f5;
123+
let t = state;
124+
t = Math.imul(t ^ (t >>> 15), t | 1);
125+
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
126+
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
127+
};
128+
}
129+
130+
export interface TrainingRunMeasurement {
131+
seed: number;
132+
budgetSteps: number;
133+
completedSteps: number;
134+
terminatedEpisodes: number;
135+
truncatedEpisodes: number;
136+
episodeReturns: number[];
137+
}
138+
139+
export async function runLineWorldTraining(
140+
seed: number,
141+
budgetSteps: number = TRAINING_BUDGET_STEPS,
142+
): Promise<TrainingRunMeasurement> {
143+
const rng = mulberry32(seed);
144+
const originalRandom = Math.random;
145+
Math.random = rng;
146+
const world = new LineWorldEnv();
147+
const runner = new IgnitionEnvTFJS(world);
148+
try {
149+
runner.train('qtable', { ...TRAINING_OVERRIDES });
150+
runner.stop();
151+
const episodeReturns: number[] = [];
152+
let episodeReturn = 0;
153+
let terminatedEpisodes = 0;
154+
let truncatedEpisodes = 0;
155+
let completedSteps = 0;
156+
for (let i = 0; i < budgetSteps; i++) {
157+
const result: StepResult = await runner.step();
158+
completedSteps += 1;
159+
episodeReturn += result.reward;
160+
if (result.terminated || result.truncated) {
161+
if (result.terminated) terminatedEpisodes += 1;
162+
if (result.truncated) truncatedEpisodes += 1;
163+
episodeReturns.push(episodeReturn);
164+
episodeReturn = 0;
165+
}
166+
}
167+
return {
168+
seed,
169+
budgetSteps,
170+
completedSteps,
171+
terminatedEpisodes,
172+
truncatedEpisodes,
173+
episodeReturns,
174+
};
175+
} finally {
176+
runner.stop();
177+
runner.agent?.dispose?.();
178+
Math.random = originalRandom;
179+
}
180+
}
181+
182+
/** Scripted always-right agent for episode-end semantics (real `IgnitionEnv.step`). */
183+
export async function scriptedStep(world: TrainingEnv): Promise<StepResult> {
184+
const runner = new IgnitionEnv(world);
185+
runner.agent = {
186+
getAction: async () => 1,
187+
remember() {},
188+
train: async () => {},
189+
};
190+
return runner.step();
191+
}

‎examples/skill-quickstart.test.ts‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
import { describe, expect, it } from 'vitest';
2+
import {
3+
QUICKSTART_BUDGET,
4+
QUICKSTART_SEEDS,
5+
proveStopHaltsAutomaticLoop,
6+
runQuickstartBudget,
7+
} from './skill-quickstart';
8+
9+
describe('skill-quickstart CartPole loop', () => {
10+
it('declares budget and at least three seeds before any run', () => {
11+
expect(QUICKSTART_SEEDS).toEqual([11, 29, 47]);
12+
expect(QUICKSTART_BUDGET.transitionsPerSeed).toBe(8);
13+
expect(QUICKSTART_BUDGET.backend).toBe('cpu');
14+
});
15+
16+
it.each([...QUICKSTART_SEEDS])(
17+
'runs the declared budget on seed %i without claiming convergence',
18+
async (seed) => {
19+
const snapshot = await runQuickstartBudget(seed);
20+
console.log(`QUICKSTART_RAW ${JSON.stringify(snapshot)}`);
21+
expect(snapshot.seed).toBe(seed);
22+
expect(snapshot.stepCount).toBe(QUICKSTART_BUDGET.transitionsPerSeed);
23+
expect(snapshot.rewards).toHaveLength(QUICKSTART_BUDGET.transitionsPerSeed);
24+
expect(snapshot.rewards.every((reward) => Number.isFinite(reward))).toBe(true);
25+
expect(snapshot.observationSize).toBe(4);
26+
expect(snapshot.lastError).toBeNull();
27+
},
28+
);
29+
30+
it('stop() cancels further automatic ticks', async () => {
31+
const proof = await proveStopHaltsAutomaticLoop(QUICKSTART_SEEDS[0]);
32+
console.log(`QUICKSTART_STOP_RAW ${JSON.stringify(proof)}`);
33+
expect(proof.afterStop).toBeGreaterThanOrEqual(1);
34+
expect(proof.later - proof.afterStop).toBeLessThanOrEqual(1);
35+
expect(proof.lastError).toBeNull();
36+
});
37+
});

0 commit comments

Comments
 (0)