Skip to content

Commit ace3da2

Browse files
committed
feat(learning): persist learner configs and metrics
1 parent 42e02bb commit ace3da2

9 files changed

Lines changed: 730 additions & 19 deletions

File tree

‎packages/learning/README.md‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,41 @@ console.log(diagnostics.bestPolicy?.weights)
6161

6262
`LinearPolicySearchLearner` is a small non-neural continuous-control baseline. It learns a linear observation-to-action policy with seeded candidate perturbations, optional label-based warm starts, elite averaging, bounded actions and JSON checkpoints. It also exposes numeric learner metrics such as candidate index, population rewards, improvement flags and policy weight norms, plus label-aware diagnostics for inspecting the current, mean and best policy matrices. It is meant to prove the continuous learner/checkpoint/debugging path before PPO/SAC or native tensor backends exist.
6363

64+
## Configuration and Metrics
65+
66+
Learner hyperparameters are normalized through exported config helpers:
67+
68+
```ts
69+
import {
70+
defineTabularQConfig,
71+
learnerMetricSpecsForAlgorithm,
72+
} from "@ignitionrl/learning"
73+
74+
const learnerConfig = defineTabularQConfig({
75+
epsilon: 0.2,
76+
learningRate: 0.3,
77+
})
78+
79+
const metrics = learnerMetricSpecsForAlgorithm("tabular-q-learning")
80+
```
81+
82+
Default baseline configs are:
83+
84+
- `tabular-q-learning`: `learningRate: 0.2`, `discount: 0.95`, `epsilon: 0.1`, `initialQ: 0`, `observationPrecision: 2`, `seed: 0`;
85+
- `linear-policy-search`: `sigma: 0.2`, `actionNoise: 0.03`, `initialWeightScale: 0.05`, `populationSize: 6`, `eliteCount: 2`, `seed: 0`.
86+
87+
Default neural adapter cadences are:
88+
89+
- `dqn`: step updates every step after `warmupSteps: 1000`, with `batchSize: 64`;
90+
- `ppo`: rollout updates every `2048` steps, with `epochs: 10` and `minibatches: 32`;
91+
- `sac`: step updates every step after `warmupSteps: 1000`, with `batchSize: 64`.
92+
93+
Network, optimizer and replay-buffer hyperparameters are intentionally left to the future learner implementation while the adapter contract keeps their metrics and checkpoints stable.
94+
95+
Invalid hyperparameters throw during config normalization or learner initialization, before a Studio/SDK learner run is created. Learners also expose `getConfig()`, so `@ignitionrl/sdk` can persist the effective JSON config under `run.config.learnerConfig` for reproducibility.
96+
97+
Metric catalogs use the persisted run metric names that Studio panels and CI gates should depend on. Episode metrics are `totalReward`, `length`, `success`, `terminated` and `truncated`. Learner metrics are namespaced as `learner.*`, for example `learner.lastTdError`, `learner.epsilon` or `learner.tdLoss`.
98+
6499
## Checkpoints
65100

66101
```ts

‎packages/learning/src/index.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,12 @@ export {
1111
export {
1212
LINEAR_POLICY_SEARCH_ALGORITHM,
1313
LINEAR_POLICY_SEARCH_CHECKPOINT_VERSION,
14+
DEFAULT_LINEAR_POLICY_SEARCH_CONFIG,
1415
LinearPolicySearchLearner,
1516
assertCheckpointMatchesContinuousSpec,
1617
assertLinearPolicySearchCheckpoint,
1718
createLinearPolicySearchLearner,
19+
defineLinearPolicySearchConfig,
1820
normalizeLinearPolicySearchCheckpoint,
1921
type LinearPolicySearchActOptions,
2022
type LinearPolicySearchCandidateSummary,
@@ -28,12 +30,29 @@ export {
2830
type LinearPolicySearchPolicyWeights,
2931
} from "./linear-policy-search.js";
3032
export {
33+
DEFAULT_TABULAR_Q_CONFIG,
3134
TabularQLearner,
3235
createTabularQLearner,
36+
defineTabularQConfig,
3337
transitionFromStep,
3438
type SelectActionOptions,
3539
type TabularQOptions,
3640
} from "./tabular-q.js";
41+
export {
42+
EPISODE_METRIC_SPECS,
43+
LINEAR_POLICY_SEARCH_METRIC_SPECS,
44+
TABULAR_Q_METRIC_SPECS,
45+
assertLearnerMetricSpecs,
46+
learnerMetricSpecsForAlgorithm,
47+
learnerMetricSpecsFromNeuralAdapterContract,
48+
stableMetricNamesForAlgorithm,
49+
stableMetricNamesFromNeuralAdapterContract,
50+
type LearnerMetricCatalogOptions,
51+
type LearnerMetricDirection,
52+
type LearnerMetricReducer,
53+
type LearnerMetricScope,
54+
type LearnerMetricSpec,
55+
} from "./metrics.js";
3756
export {
3857
NEURAL_ADAPTER_CONTRACT_VERSION,
3958
NEURAL_CHECKPOINT_SCHEMA_VERSION,

‎packages/learning/src/linear-policy-search.ts‎

Lines changed: 33 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -108,14 +108,14 @@ type LinearPolicySearchMetricDefaults = {
108108
readonly bestWeights?: readonly number[];
109109
};
110110

111-
const DEFAULT_CONFIG: LinearPolicySearchConfig = {
111+
export const DEFAULT_LINEAR_POLICY_SEARCH_CONFIG: LinearPolicySearchConfig = Object.freeze({
112112
seed: 0,
113113
sigma: 0.2,
114114
actionNoise: 0.03,
115115
initialWeightScale: 0.05,
116116
populationSize: 6,
117117
eliteCount: 2,
118-
};
118+
});
119119

120120
type Candidate = {
121121
readonly weights: readonly number[];
@@ -158,13 +158,13 @@ export class LinearPolicySearchLearner implements Learner<number[]> {
158158
private lastImproved = 0;
159159

160160
constructor(options: LinearPolicySearchOptions = {}) {
161-
this.config = normalizeConfig(options);
161+
this.config = defineLinearPolicySearchConfig(options);
162162
this.metadata = options.metadata;
163163
this.rng = createSeededRng(this.config.seed);
164164
}
165165

166166
async init(spec: EnvironmentSpec, config: LearnerConfig = {}): Promise<void> {
167-
const nextConfig = normalizeConfig({
167+
const nextConfig = defineLinearPolicySearchConfig({
168168
...this.config,
169169
...config,
170170
});
@@ -327,6 +327,10 @@ export class LinearPolicySearchLearner implements Learner<number[]> {
327327
};
328328
}
329329

330+
getConfig(): LinearPolicySearchConfig {
331+
return cloneLinearPolicySearchConfig(this.config);
332+
}
333+
330334
getPolicyWeights(
331335
policy: LinearPolicySearchPolicyKind = "best",
332336
): LinearPolicySearchPolicyWeights {
@@ -481,6 +485,12 @@ export function createLinearPolicySearchLearner(
481485
return new LinearPolicySearchLearner(options);
482486
}
483487

488+
export function defineLinearPolicySearchConfig(
489+
options: Partial<LinearPolicySearchConfig> = {},
490+
): LinearPolicySearchConfig {
491+
return normalizeConfig(options);
492+
}
493+
484494
export function normalizeLinearPolicySearchCheckpoint(
485495
value: unknown,
486496
): LinearPolicySearchCheckpoint {
@@ -599,8 +609,14 @@ export function assertCheckpointMatchesContinuousSpec(
599609

600610
function normalizeConfig(value: Partial<LinearPolicySearchConfig>): LinearPolicySearchConfig {
601611
const config = {
602-
...DEFAULT_CONFIG,
603-
...value,
612+
seed: value.seed ?? DEFAULT_LINEAR_POLICY_SEARCH_CONFIG.seed,
613+
sigma: value.sigma ?? DEFAULT_LINEAR_POLICY_SEARCH_CONFIG.sigma,
614+
actionNoise: value.actionNoise ?? DEFAULT_LINEAR_POLICY_SEARCH_CONFIG.actionNoise,
615+
initialWeightScale: value.initialWeightScale
616+
?? DEFAULT_LINEAR_POLICY_SEARCH_CONFIG.initialWeightScale,
617+
populationSize: value.populationSize
618+
?? DEFAULT_LINEAR_POLICY_SEARCH_CONFIG.populationSize,
619+
eliteCount: value.eliteCount ?? DEFAULT_LINEAR_POLICY_SEARCH_CONFIG.eliteCount,
604620
...(value.initialWeightMap !== undefined
605621
? { initialWeightMap: cloneInitialWeightMap(value.initialWeightMap) }
606622
: {}),
@@ -611,6 +627,17 @@ function normalizeConfig(value: Partial<LinearPolicySearchConfig>): LinearPolicy
611627
return config;
612628
}
613629

630+
function cloneLinearPolicySearchConfig(
631+
config: LinearPolicySearchConfig,
632+
): LinearPolicySearchConfig {
633+
return {
634+
...config,
635+
...(config.initialWeightMap !== undefined
636+
? { initialWeightMap: cloneInitialWeightMap(config.initialWeightMap) }
637+
: {}),
638+
};
639+
}
640+
614641
function assertLinearPolicySearchConfig(
615642
value: unknown,
616643
): asserts value is LinearPolicySearchConfig {

0 commit comments

Comments
 (0)