Skip to content

Commit 56da7d6

Browse files
salim4ncursoragent
andauthored
docs(skills): add ignition-export-onnx skill (#118)
* docs(skills): add ignition-export-onnx skill and example Add the Agent Skill for exporting a TF.js DQN policy to ONNX and serving it with OnnxAgent, plus a Vitest example that trains on declared seeds without claiming convergence. Python conversion is skipped unless the pinned toolchain is present. Co-authored-by: Salim <salim4n@users.noreply.github.com> * test(skills): run real TF.js to ONNX conversion when the venv exists Probe .venv-onnx / IGNITION_ONNX_PYTHON for tensorflowjs, tf2onnx, and onnxruntime. Convert the saved DQN for seeds 7, 13, and 29 and compare Q-values and greedy actions. Skip only if that toolchain is missing. Document the uv/pip install commands in the skill. Co-authored-by: Salim <salim4n@users.noreply.github.com> --------- Co-authored-by: Cursor Agent <cursoragent@cursor.com> Co-authored-by: Salim <salim4n@users.noreply.github.com>
1 parent 77a0104 commit 56da7d6

8 files changed

Lines changed: 734 additions & 0 deletions

File tree

‎examples/skill-export-onnx.test.ts‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
import { afterAll, describe, expect, it } from 'vitest';
2+
import { OnnxAgent } from '@ignitionai/backend-onnx';
3+
import {
4+
EXPORT_MEASUREMENT,
5+
collectGreedyTfjsPolicy,
6+
conversionScriptFor,
7+
declareExportBudget,
8+
disposeTfjsVariables,
9+
probePythonConversion,
10+
probeTfjsNodeFileHandler,
11+
trySaveAndConvert,
12+
compareOnnxToTfjs,
13+
} from './skill-export-onnx';
14+
15+
const budgetText = declareExportBudget();
16+
process.stdout.write(`${budgetText}\n`);
17+
18+
const pythonProbe = probePythonConversion();
19+
const tfjsNodeProbe = probeTfjsNodeFileHandler();
20+
const conversionReady = pythonProbe.available && tfjsNodeProbe.available;
21+
process.stdout.write(
22+
`CONVERSION_PROBES ${JSON.stringify({ pythonProbe, tfjsNodeProbe, conversionReady })}\n`,
23+
);
24+
const conversionSkipReason = [
25+
pythonProbe.available ? null : pythonProbe.reason,
26+
tfjsNodeProbe.available ? null : tfjsNodeProbe.reason,
27+
]
28+
.filter((part): part is string => part !== null)
29+
.join(' ');
30+
31+
afterAll(() => {
32+
disposeTfjsVariables();
33+
});
34+
35+
describe('skill-export-onnx', () => {
36+
it('generateConversionScript names tensorflowjs Layers → tf2onnx', () => {
37+
const script = conversionScriptFor('./export');
38+
expect(script.startsWith('#!/bin/bash')).toBe(true);
39+
expect(script).toContain('tensorflowjs.converters.converter');
40+
expect(script).toContain('--input_format=tfjs_layers_model');
41+
expect(script).toContain('--output_format=keras_saved_model');
42+
expect(script).toContain('tf2onnx.convert');
43+
expect(script).toContain('--opset 13');
44+
});
45+
46+
it('OnnxAgent requires load() before greedy getAction', async () => {
47+
const agent = new OnnxAgent({ modelPath: './missing.onnx', actionSize: 2, outputName: 'output' });
48+
await expect(agent.getAction([0, 0, 0, 0])).rejects.toThrow('call load()');
49+
});
50+
51+
it('TF.js greedy actions on declared seeds and fixed observations (no convergence claim)', async () => {
52+
const rows: Array<{ seed: number; steps: number; actions: number[] }> = [];
53+
for (const seed of EXPORT_MEASUREMENT.seeds) {
54+
const result = await collectGreedyTfjsPolicy(seed);
55+
try {
56+
expect(result.steps).toBe(EXPORT_MEASUREMENT.trainingSteps);
57+
expect(result.actions).toHaveLength(3);
58+
for (const action of result.actions) {
59+
expect(action).toBeGreaterThanOrEqual(0);
60+
expect(action).toBeLessThan(2);
61+
}
62+
rows.push({ seed: result.seed, steps: result.steps, actions: result.actions });
63+
} finally {
64+
result.runner.stop();
65+
result.agent.dispose();
66+
}
67+
}
68+
process.stdout.write(`TFJS_GREEDY ${JSON.stringify(rows)}\n`);
69+
expect(rows).toHaveLength(EXPORT_MEASUREMENT.seeds.length);
70+
}, 60_000);
71+
72+
it.skipIf(!conversionReady)(
73+
`ONNX greedy Q-values and actions match TF.js for seeds ${EXPORT_MEASUREMENT.seeds.join(', ')} (${conversionReady ? 'toolchain present' : conversionSkipReason})`,
74+
async () => {
75+
const python = pythonProbe.python;
76+
if (!python) {
77+
throw new Error('skipIf should have excluded this test');
78+
}
79+
const rows: Array<{
80+
seed: number;
81+
maxAbsoluteError: number;
82+
tfjsActions: number[];
83+
onnxActions: number[];
84+
inputName: string;
85+
outputName: string;
86+
}> = [];
87+
for (const seed of EXPORT_MEASUREMENT.seeds) {
88+
const tfjsPolicy = await collectGreedyTfjsPolicy(seed);
89+
try {
90+
const { modelPath, log } = await trySaveAndConvert(tfjsPolicy.agent.getModel(), python);
91+
process.stdout.write(`CONVERT_SH seed=${seed}\n${log}\n`);
92+
const comparison = await compareOnnxToTfjs({
93+
modelPath,
94+
actionSize: 2,
95+
tfjsActions: tfjsPolicy.actions,
96+
tfjsQValues: tfjsPolicy.qValues,
97+
});
98+
const row = {
99+
seed,
100+
maxAbsoluteError: comparison.maxAbsoluteError,
101+
tfjsActions: tfjsPolicy.actions,
102+
onnxActions: comparison.onnxActions,
103+
inputName: comparison.inputName,
104+
outputName: comparison.outputName,
105+
};
106+
process.stdout.write(`ONNX_COMPARE ${JSON.stringify(row)}\n`);
107+
expect(comparison.greedyMatch).toBe(true);
108+
expect(comparison.onnxActions).toEqual(tfjsPolicy.actions);
109+
rows.push(row);
110+
} finally {
111+
tfjsPolicy.runner.stop();
112+
tfjsPolicy.agent.dispose();
113+
}
114+
}
115+
process.stdout.write(`ONNX_COMPARE_ALL ${JSON.stringify(rows)}\n`);
116+
expect(rows).toHaveLength(EXPORT_MEASUREMENT.seeds.length);
117+
},
118+
300_000,
119+
);
120+
});

‎examples/skill-export-onnx.ts‎

Lines changed: 253 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,253 @@
1+
/**
2+
* Executable helpers for the ignition-export-onnx skill.
3+
* Declares measurement budget before any training or conversion.
4+
*/
5+
import { execFileSync } from 'node:child_process';
6+
import { existsSync, mkdtempSync, writeFileSync } from 'node:fs';
7+
import { createRequire } from 'node:module';
8+
import { tmpdir } from 'node:os';
9+
import { join, resolve } from 'node:path';
10+
import { DQNAgent, IgnitionEnvTFJS } from '@ignitionai/backend-tfjs';
11+
import {
12+
OnnxAgent,
13+
createOnnxSession,
14+
generateConversionScript,
15+
runInference,
16+
saveForOnnxExport,
17+
} from '@ignitionai/backend-onnx';
18+
import { CartPoleEnv } from '@ignitionai/environments';
19+
20+
/** Root vitest does not alias @tensorflow/*; resolve through the TF.js package. */
21+
const requireBackendTfjs = createRequire(resolve('packages/backend-tfjs/package.json'));
22+
const tf = requireBackendTfjs('@tensorflow/tfjs') as {
23+
tidy: <T>(fn: () => T) => T;
24+
tensor2d: (values: number[][]) => unknown;
25+
Tensor2D: { new (): { arraySync: () => number[][] } };
26+
disposeVariables: () => void;
27+
};
28+
29+
export function disposeTfjsVariables(): void {
30+
tf.disposeVariables();
31+
}
32+
33+
/** Declared before any run. Not a convergence budget. */
34+
export const EXPORT_MEASUREMENT = {
35+
seeds: [7, 13, 29] as const,
36+
trainingSteps: 32,
37+
hiddenLayers: [8] as const,
38+
batchSize: 4,
39+
memorySize: 64,
40+
/** Continuous Q-value match: |onnx - tfjs| <= abs + rel * |tfjs|. */
41+
absoluteTolerance: 1e-5,
42+
relativeTolerance: 1e-5,
43+
};
44+
45+
export const FIXED_OBSERVATIONS: number[][] = [
46+
[0, 0, 0, 0],
47+
[0.1, -0.2, 0.03, 0.4],
48+
[-0.4, 0.2, -0.1, -0.3],
49+
];
50+
51+
export function declareExportBudget(): string {
52+
return [
53+
'DECLARED BEFORE RUN',
54+
`seeds=${JSON.stringify([...EXPORT_MEASUREMENT.seeds])}`,
55+
`trainingSteps=${EXPORT_MEASUREMENT.trainingSteps} per seed`,
56+
`hiddenLayers=${JSON.stringify([...EXPORT_MEASUREMENT.hiddenLayers])}`,
57+
`batchSize=${EXPORT_MEASUREMENT.batchSize}`,
58+
`memorySize=${EXPORT_MEASUREMENT.memorySize}`,
59+
`observations=${JSON.stringify(FIXED_OBSERVATIONS)}`,
60+
`qValueTolerance=abs ${EXPORT_MEASUREMENT.absoluteTolerance} + rel ${EXPORT_MEASUREMENT.relativeTolerance} * |tfjs|`,
61+
'greedyActions=exact match',
62+
'not a CartPole convergence experiment',
63+
].join('\n');
64+
}
65+
66+
export function qValueWithinTolerance(onnx: number, tfjs: number): boolean {
67+
const difference = Math.abs(onnx - tfjs);
68+
const limit =
69+
EXPORT_MEASUREMENT.absoluteTolerance +
70+
EXPORT_MEASUREMENT.relativeTolerance * Math.abs(tfjs);
71+
return Number.isFinite(difference) && difference <= limit;
72+
}
73+
74+
export function conversionScriptFor(outputDir: string, opset = 13): string {
75+
const resolvedDir = resolve(outputDir);
76+
return generateConversionScript(resolvedDir, `${resolvedDir}_savedmodel`, `${resolvedDir}.onnx`, opset);
77+
}
78+
79+
export async function collectGreedyTfjsPolicy(seed: number): Promise<{
80+
seed: number;
81+
steps: number;
82+
actions: number[];
83+
qValues: number[][];
84+
agent: DQNAgent;
85+
runner: IgnitionEnvTFJS;
86+
}> {
87+
const runner = new IgnitionEnvTFJS(new CartPoleEnv());
88+
runner.train('dqn', {
89+
backend: 'cpu',
90+
hiddenLayers: [...EXPORT_MEASUREMENT.hiddenLayers],
91+
batchSize: EXPORT_MEASUREMENT.batchSize,
92+
memorySize: EXPORT_MEASUREMENT.memorySize,
93+
seed,
94+
});
95+
runner.stop();
96+
const agent = runner.agent;
97+
if (!(agent instanceof DQNAgent)) {
98+
throw new Error('Expected DQN');
99+
}
100+
for (let step = 0; step < EXPORT_MEASUREMENT.trainingSteps; step++) {
101+
await runner.step();
102+
}
103+
const qValues = tf.tidy(() => {
104+
const predicted = agent.getModel().predict(tf.tensor2d(FIXED_OBSERVATIONS) as never) as {
105+
arraySync: () => number[][];
106+
};
107+
return predicted.arraySync();
108+
});
109+
const actions: number[] = [];
110+
for (const observation of FIXED_OBSERVATIONS) {
111+
const action = await agent.getAction(observation, true);
112+
if (typeof action !== 'number') {
113+
throw new Error('Expected discrete greedy action');
114+
}
115+
actions.push(action);
116+
}
117+
return { seed, steps: runner.stepCount, actions, qValues, agent, runner };
118+
}
119+
120+
const CONVERSION_IMPORT_CHECK =
121+
'import tensorflowjs, tf2onnx, onnxruntime; assert tensorflowjs.__version__; assert tf2onnx.__version__';
122+
123+
function pythonCandidates(): string[] {
124+
const fromEnv = process.env.IGNITION_ONNX_PYTHON;
125+
const localVenv = resolve('.venv-onnx/bin/python');
126+
return [
127+
...(fromEnv ? [fromEnv] : []),
128+
...(existsSync(localVenv) ? [localVenv] : []),
129+
'python3.11',
130+
'python3',
131+
'python',
132+
];
133+
}
134+
135+
export function probePythonConversion(): { available: boolean; python?: string; reason: string } {
136+
const errors: string[] = [];
137+
for (const python of pythonCandidates()) {
138+
try {
139+
execFileSync(python, ['-c', CONVERSION_IMPORT_CHECK], { stdio: 'ignore' });
140+
return {
141+
available: true,
142+
python,
143+
reason: `${python} imports tensorflowjs, tf2onnx, and onnxruntime`,
144+
};
145+
} catch (error) {
146+
errors.push(`${python}: ${error instanceof Error ? error.message.split('\n')[0] : String(error)}`);
147+
}
148+
}
149+
return {
150+
available: false,
151+
reason:
152+
'No Python 3.11 venv with tensorflowjs, tf2onnx, and onnxruntime. Install with: uv python install 3.11 && uv venv .venv-onnx --python 3.11 && uv pip install --python .venv-onnx/bin/python -r packages/backend-onnx/examples/requirements-onnx.txt onnxruntime==1.16.3. Then export IGNITION_ONNX_PYTHON="$PWD/.venv-onnx/bin/python". ' +
153+
errors.join(' | '),
154+
};
155+
}
156+
157+
export function probeTfjsNodeFileHandler(): { available: boolean; reason: string } {
158+
try {
159+
requireBackendTfjs.resolve('@tensorflow/tfjs-node');
160+
requireBackendTfjs('@tensorflow/tfjs-node');
161+
return { available: true, reason: '@tensorflow/tfjs-node loaded (file:// save handler)' };
162+
} catch (error) {
163+
return {
164+
available: false,
165+
reason: `@tensorflow/tfjs-node not resolvable from @ignitionai/backend-tfjs (${error instanceof Error ? error.message : String(error)}). saveForOnnxExport uses file://.`,
166+
};
167+
}
168+
}
169+
170+
export async function compareOnnxToTfjs(params: {
171+
modelPath: string;
172+
actionSize: number;
173+
tfjsActions: number[];
174+
tfjsQValues: number[][];
175+
}): Promise<{
176+
greedyMatch: boolean;
177+
maxAbsoluteError: number;
178+
onnxActions: number[];
179+
inputName: string;
180+
outputName: string;
181+
}> {
182+
const session = await createOnnxSession(params.modelPath);
183+
const inputName = session.inputNames[0];
184+
const outputName = session.outputNames[0];
185+
const policy = new OnnxAgent({
186+
modelPath: params.modelPath,
187+
actionSize: params.actionSize,
188+
inputName,
189+
outputName,
190+
});
191+
await policy.load();
192+
let maxAbsoluteError = 0;
193+
const onnxActions: number[] = [];
194+
try {
195+
for (let row = 0; row < FIXED_OBSERVATIONS.length; row++) {
196+
const values = await runInference(
197+
session,
198+
new Float32Array(FIXED_OBSERVATIONS[row]),
199+
[1, FIXED_OBSERVATIONS[row].length],
200+
inputName,
201+
outputName,
202+
);
203+
if (values.length !== params.tfjsQValues[row].length) {
204+
throw new Error('ONNX output dimension mismatch');
205+
}
206+
for (let action = 0; action < values.length; action++) {
207+
const difference = Math.abs(values[action] - params.tfjsQValues[row][action]);
208+
maxAbsoluteError = Math.max(maxAbsoluteError, difference);
209+
if (!qValueWithinTolerance(values[action], params.tfjsQValues[row][action])) {
210+
throw new Error(`ONNX Q-value mismatch at row ${row}, action ${action}`);
211+
}
212+
}
213+
const onnxAction = await policy.getAction(FIXED_OBSERVATIONS[row]);
214+
onnxActions.push(onnxAction);
215+
if (onnxAction !== params.tfjsActions[row]) {
216+
throw new Error(`ONNX greedy action mismatch at row ${row}`);
217+
}
218+
}
219+
} finally {
220+
await session.release();
221+
policy.dispose();
222+
}
223+
return {
224+
greedyMatch: true,
225+
maxAbsoluteError,
226+
onnxActions,
227+
inputName,
228+
outputName,
229+
};
230+
}
231+
232+
export async function trySaveAndConvert(
233+
model: { save: (url: string) => Promise<unknown> },
234+
python: string,
235+
): Promise<{ modelPath: string; log: string }> {
236+
const output = mkdtempSync(join(tmpdir(), 'skill-export-onnx-'));
237+
const modelDir = join(output, 'model');
238+
const { conversionScript } = await saveForOnnxExport(model, modelDir);
239+
const scriptPath = join(output, 'convert.sh');
240+
writeFileSync(scriptPath, conversionScript);
241+
try {
242+
const log = execFileSync('bash', [scriptPath], {
243+
encoding: 'utf8',
244+
env: { ...process.env, PYTHON: python, TF_NUM_INTEROP_THREADS: '1', TF_NUM_INTRAOP_THREADS: '1' },
245+
});
246+
return { modelPath: `${modelDir}.onnx`, log };
247+
} catch (error) {
248+
const err = error as { stdout?: string; stderr?: string; status?: number; message?: string };
249+
const log = [err.stdout ?? '', err.stderr ?? '', err.message ?? ''].filter(Boolean).join('\n');
250+
process.stderr.write(`CONVERT_SH_FAILED exit=${err.status ?? 'unknown'}\n${log}\n`);
251+
throw error;
252+
}
253+
}

0 commit comments

Comments
 (0)