diff --git a/README.md b/README.md index 0e10e23af..a893f9033 100644 --- a/README.md +++ b/README.md @@ -93,14 +93,17 @@ print(fixed_item_calibration.best) strict structured parsing. A judge result becomes an IRT row only through LLMJudgeResult.to_irt_row() with at least two criteria, followed by validate_irt_response_matrix() for a multi-item dichotomous or explicitly - categorized polytomous matrix. Equal-width score projection is experimental; - category-count and prompt-perturbation calibration are required. See - [ADR 0005](https://github.com/ContextualWisdomLab/contextual-orchestrator/blob/89bd5bf73319dd21f2be1f094eb2639bb8ead8f3/docs/planning/adrs/0005-irt-response-matrix-contract.md) and - [ADR 0006](https://github.com/ContextualWisdomLab/contextual-orchestrator/blob/89bd5bf73319dd21f2be1f094eb2639bb8ead8f3/docs/planning/adrs/0006-polytomous-llm-judge-bias-calibration.md) and - [ADR 0008](https://github.com/ContextualWisdomLab/contextual-orchestrator/blob/2b65d5c0f3d6bd64a9e05818f1f9286e98c334c1/docs/planning/adrs/0008-fast-judge-review-hardening.md). + categorized polytomous matrix. Equal-width direct K-way projection is + experimental; opt-in `category_method="cumulative_threshold"` evaluates each + ordered boundary with a strict Boolean vector and derives the category in the + adapter. Category-count and prompt-perturbation calibration remain required + for both methods. See + [ADR 0005](https://github.com/ContextualWisdomLab/contextual-orchestrator/blob/1b7dbd2a46533f41072def1fb94283147134cab5/docs/planning/adrs/0005-irt-response-matrix-contract.md), + [ADR 0006](https://github.com/ContextualWisdomLab/contextual-orchestrator/blob/1b7dbd2a46533f41072def1fb94283147134cab5/docs/planning/adrs/0006-polytomous-llm-judge-bias-calibration.md), and + [ADR 0008](https://github.com/ContextualWisdomLab/contextual-orchestrator/blob/1b7dbd2a46533f41072def1fb94283147134cab5/docs/planning/adrs/0008-fast-judge-review-hardening.md). Cross-repository exact-head review, structured Strix evidence, and merge - policy are recorded in [contextual-orchestrator ADR 0004](https://github.com/ContextualWisdomLab/contextual-orchestrator/blob/befa094784e37947841948fb42016de7e6b965ab/docs/planning/adrs/0004-pr-review-merge-loop.md) and - [ADR 0009 dependency cooldown](https://github.com/ContextualWisdomLab/contextual-orchestrator/blob/befa094784e37947841948fb42016de7e6b965ab/docs/planning/adrs/0009-supply-chain-dependency-cooldown.md). + policy are recorded in [contextual-orchestrator ADR 0004](https://github.com/ContextualWisdomLab/contextual-orchestrator/blob/1b7dbd2a46533f41072def1fb94283147134cab5/docs/planning/adrs/0004-pr-review-merge-loop.md) and + [ADR 0009 dependency cooldown](https://github.com/ContextualWisdomLab/contextual-orchestrator/blob/1b7dbd2a46533f41072def1fb94283147134cab5/docs/planning/adrs/0009-supply-chain-dependency-cooldown.md). - Standalone HTML reports for saved fit or dimensionality diagnostics. - Automated benchmark evidence reports from release-acceptance timing. - Release evidence index reports that tie dist artifact hashes, acceptance, diff --git a/crates/mlsirm-core/src/jmle_opt.rs b/crates/mlsirm-core/src/jmle_opt.rs index 10b387336..f1f2a00cf 100644 --- a/crates/mlsirm-core/src/jmle_opt.rs +++ b/crates/mlsirm-core/src/jmle_opt.rs @@ -170,6 +170,16 @@ where break; } + // L-BFGS needs a function-reduction stopping rule as well as a gradient + // rule. Near a well-resolved optimum, gradients can remain above a + // strict tolerance while the accepted objective improvement is already + // negligible at the problem's scale; continuing until the next Armijo + // search cannot find a representable decrease misclassifies numerical + // convergence as a line-search failure. This relative reduction mirrors + // the conventional L-BFGS-B `ftol` criterion. + let objective_scale = obj.abs().max(next_obj.abs()).max(1.0); + let relative_reduction = (obj - next_obj) / objective_scale; + let mut s = vec![0.0; n]; let mut y_delta = vec![0.0; n]; for i in 0..n { @@ -194,6 +204,10 @@ where loglik = next_loglik; trace.push(obj); loglik_trace.push(loglik); + if relative_reduction <= tolerance { + status = "converged".into(); + break; + } } Ok((x, trace, loglik_trace, status)) } @@ -344,6 +358,23 @@ mod tests { assert!((x[1] + 1.0).abs() < 1e-4); } + #[test] + fn lbfgs_accepts_relative_objective_convergence() { + let mut obj = |x: &[f64]| { + let d = x[0] - 1.0; + let value = 1_000_000.0 + 0.5 * d * d; + // Keep the gradient above the convergence tolerance after the + // first accepted step so this test exercises objective reduction, + // not the gradient-norm guard. + Ok((value, vec![0.5 * d], -value)) + }; + let (x, trace, _, status) = + lbfgs(&[0.0], &mut obj, 10, 1e-6, 5).expect("lbfgs"); + assert_eq!(status, "converged"); + assert_eq!(trace.len(), 2); + assert!((x[0] - 0.5).abs() < 1e-12); + } + #[test] fn adam_lbfgs_sequence_runs() { let mut obj = quadratic(); @@ -352,7 +383,7 @@ mod tests { .expect("seq"); assert!(n_iter >= 1); assert!(!t.is_empty()); - assert!(status == "converged" || status == "max_iter_reached" || status == "line_search_failed"); + assert!(status == "converged" || status == "max_iter_reached"); assert!((x[0] - 1.0).abs() < 0.1); } diff --git a/docs/adr/0013-continuous-execution-and-documentation-governance.md b/docs/adr/0013-continuous-execution-and-documentation-governance.md index 9c647d766..916f44953 100644 --- a/docs/adr/0013-continuous-execution-and-documentation-governance.md +++ b/docs/adr/0013-continuous-execution-and-documentation-governance.md @@ -65,6 +65,15 @@ A substantive contract change is documentation-complete only when every applicab Documentation does not replace implementation. Conversely, unresolved architecture ambiguity is a product defect and may be selected as the next executable work item when product branches are blocked. +## Current review finding and remediation + +An active review found that the documentation matrix called the Proposed +canonical PyO3/public-export registry the native-entrypoint source of truth +while protected main still used separate initializers and package export +paths. The matrix must state target architecture and protected-main behavior +separately until ADR-0011 is implemented; the active PR applies that wording +correction. + ## Consequences ### Positive diff --git a/docs/adr/0014-bounded-llm-judge-category-inputs.md b/docs/adr/0014-bounded-llm-judge-category-inputs.md new file mode 100644 index 000000000..9d0fc484d --- /dev/null +++ b/docs/adr/0014-bounded-llm-judge-category-inputs.md @@ -0,0 +1,378 @@ +# ADR-0014: Bounded LLM-judge category inputs and security-scan evidence + +Status: **Proposed** +Date: 2026-08-12 +Supersedes: none +Superseded by: none + +## Context + +The LLM-as-a-Judge adapter accepts category counts and category values at a +public Python boundary before projecting results into dichotomous or +polytomous IRT items. The current-head Strix run for PR #778 at +`0721e55ce5d889b7917aa7da2891367adf3430dc` reported a supposed integer +overflow in this path. Python's built-in integers do not wrap, so that claim +was not independently valid. The review did, however, expose a real adjacent +boundary issue: `isinstance(value, int)` admits an `int` subclass whose +comparison methods can return forged results, allowing the category-count +bound to be bypassed before a resource-amplifying operation. + +The same trust-boundary concern applies to category values accepted by the +deterministic IRT projection. A model response parsed by `json.loads` contains +built-in scalar types, but callers can construct `LLMJudgeResult` directly. +Security evidence must therefore distinguish a real code defect from a model +or provider failure and must never weaken the fail-closed gate. Review of this +active PR also found that an unhashable `category_method` could leak a raw +`TypeError` during set membership instead of the package-owned `ValueError`. + +## Decision drivers + +- Reject untrusted or adversarial runtime scalar objects before comparisons, + arithmetic, or allocations. +- Preserve the existing dichotomous/polytomous response-matrix contract. +- Keep LLM security findings advisory until independently reproduced by code, + tests, or a structured scanner artifact. +- Keep provider rate limits separate from source-security conclusions. + +## Ownership and dependency direction + +`fast-mlsirm` owns the judge result contract and its IRT projection. LLM +transport remains injected through `contextual-orchestrator`; neither a model +response nor a Strix narrative becomes numerical or merge authority. + +## Decision + +1. `category_count` accepts only an exact built-in Python `int` in + `2..MAX_JUDGE_CATEGORIES` (currently 64). Booleans, floats, subclasses, + negative values, and oversized integers fail before any derived list or + threshold array is materialized. +2. Direct category values accept only exact built-in `int` or `float` values; + booleans, subclasses, non-finite values, fractional values, and out-of-range + values fail closed. +3. Any cumulative-threshold result remains bounded by the validated category + count, and every IRT row still requires multiple criterion items and passes + the shared dichotomous/polytomous response-matrix validator. +4. A Strix/provider failure is recorded with its exact head SHA, run URL, + failed step, backend error, and structured-report availability. A model-only + claim is not called a confirmed vulnerability without independent evidence. +5. Required checks remain fail-closed. No scan failure is bypassed, and no + self-approval or protected-branch merge is manufactured. +6. `category_method` must be an exact built-in string before vocabulary + membership is checked; unsupported or unhashable values fail with the + package-owned `ValueError`. +7. Public score, mode, item-type, text, criterion-key, and usage-counter + boundaries reject runtime subclasses before invoking conversion hooks or + set membership. Oversized numeric JSON values become `JudgeFormatError` + rather than leaking `OverflowError`; non-built-in usage counters are + ignored rather than compared or accumulated. +8. `JudgeCriterion.criterion_id` accepts only an exact built-in `str` before + regex validation, hashing, dictionary-key construction, or category + template generation; runtime string subclasses fail with the package-owned + `ValueError`. +9. Strict structured-output failure remains a failed comparison. An identical + second completion through `contextual-orchestrator` is not a repair contract: + it may be measured, but the final response must pass the same parser and no + keyword, positional, or silent-drop fallback may convert failure into an + IRT category. + +## Invariants / acceptance evidence + +1. `tests/test_llm_judge.py::test_category_count_and_category_values_reject_runtime_subclasses` + rejects booleans, floats, values above 64, astronomically large integers, + and comparison-forging `int` subclasses. +2. `judge()` and `LLMJudgeResult.to_irt_row()` share the same bounded category + validation before cumulative threshold allocation or IRT projection. +3. `tests/test_llm_judge.py::test_judge_rejects_unknown_category_method` covers + unknown strings, lists, and dictionaries without leaking `TypeError`. +4. `tests/test_llm_judge.py::test_judge_rejects_overflowing_and_runtime_subclass_scores` + and `test_judge_text_and_usage_boundaries_reject_runtime_subclasses` cover + overflow, conversion-hook, unhashable-mode, item-type, text, and usage + boundary behavior. +5. `tests/test_llm_judge.py::test_judge_rejects_unhashable_criterion_id_before_category_template` + rejects an unhashable `str` subclass before it can become a category + template key or leak a raw `TypeError`. +6. PR #778 Strix run `31549881616` is immutable evidence for exact scan head + `c54706cc16c8452d603c22d9604e7d27ede6288f`: + `https://github.com/ContextualWisdomLab/fast-mlsirm/actions/runs/31549881616` + and job + `https://github.com/ContextualWisdomLab/fast-mlsirm/actions/runs/31549881616/job/93970141054`. + GitHub recorded no failed job step; `Run Strix (quick)` itself concluded + success. Inside that step, however, `gate-console.log` recorded NVIDIA NIM + HTTP 429 rate limiting, GitHub Models HTTP 410 retirement-brownout failures, + repeated fail-closed/no-report markers, and a generic report that admitted an + incomplete AST pass. Artifact `9124255508` (`strix-reports`) therefore + predates the trusted provenance contract and has no authoritative + `evidence-binding.json`. This is provider-degraded/incomplete evidence, not + a clean security result. +7. PR #778 Strix run `31552408884` is immutable evidence for exact scan head + `6c42d4a53d6d70cb1ae0127df624c3cc178ddd4b`: + `https://github.com/ContextualWisdomLab/fast-mlsirm/actions/runs/31552408884` + and job + `https://github.com/ContextualWisdomLab/fast-mlsirm/actions/runs/31552408884/job/93977765693`. + Again, GitHub recorded no failed job step and `Run Strix (quick)` concluded + success, while `gate-console.log` recorded NVIDIA NIM HTTP 429, GitHub Models + HTTP 410 retirement-brownout, fail-closed/no-report markers, and a generic + fallback report. Artifact `9125226971` (`strix-reports`) lacked + `evidence-binding.json`, and its successful `run.json` omitted authoritative + head metadata. It is inconclusive provider evidence, not a clean scan. +8. The central correction is owned by `ContextualWisdomLab/.github` PR #937, + exact active-PR head `2c6f4323ac864587d767824464379678ebfe888a`. + Its `.github/workflows/strix.yml` change removes provider-outage + neutral-success behavior, records the exact scan-start head, requires a + completed successful `run.json` plus non-empty report, rejects fail-closed + provider markers, and emits `evidence-binding.json` bound to the exact head + and report digest. Protected central `main` is still + `6eb06cdd08c79a06f7b390069d4ffa49e2eb7dba`, so #937 is a read-only external + prerequisite here rather than shipped central truth. After it integrates, + this PR must be rescanned on its then-current exact head; no predecessor + Strix result transfers. +9. The full Python and Rust test suites, targeted Ruff checks, and exact-head + required checks must pass before this hardening is considered integrated. + +10. A 2026-08-12 local 3B probe used the + `ContextualOrchestratorJudge -> contextual-orchestrator -> mlx-lm` route + with temperature `0`, disabled thinking, and 256 output tokens. Nine + direct K-way calls parsed but varied across K and framing: neutral + `(0.0000, 1.0000, 0.8333)`, liked `(0.0000, 1.0000, 0.9167)`, and + disliked `(0.0000, 0.7500, 1.0000)` for K=`(2,5,7)`. Four cumulative + threshold calls failed strict parsing, and one identical retry per call + recovered none. This is calibration evidence, not a quality or bias + conclusion. + +11. Central PR #937's latest green Strix job + `31555003423`/`93985504528` for head + `2c6f4323ac864587d767824464379678ebfe888a` did not execute the PR-head + workflow definition: its step list had no `Validate Strix report + provenance` step, its artifact had no `evidence-binding.json`, and its + log contained the old neutral provider-outage skip plus NVIDIA 429 and + GitHub Models 410 failures. This is base-workflow evidence, not proof of + the central PR change; after central integration, this PR must be rescanned + through the active trusted workflow at its exact head. +12. Central PR #937 then advanced to exact head `8726df15` with a separate + non-privileged `strix-workflow-contract.yml` data-only workflow and a + doctoring record for the base-workflow evidence boundary. All previous + central checks and review interpretations are stale for this new head. + The provider-backed provenance binding remains unproven until central + integration and a fresh exact-head run emit a binding manifest without + provider-failure markers. +13. A review regression showed that a caller-supplied `list` subclass in the + orchestration trace could execute overridden iteration/length hooks during + usage aggregation and trace-count reporting. The judge now accepts only + the exact built-in list for provider trace accounting; subclass traces + produce zero derived usage and zero trace steps without executing hooks. + +14. CodeRabbit's exact-head review of central PR #937 at `8726df15` found that + the new data-only Strix workflow contract trusted marker strings: a phrase + could be supplied only by a comment or by a statically unreachable + `if: false` step. The contract now parses the fetched workflow with + `Psych.safe_load`, follows the reachable `jobs.strix.steps` structure and + required order, and checks executable provenance commands plus the + fail-closed gate. Comment-only and unreachable fixtures are regression + cases. This is still pre-integration evidence; all prior central Strix + results remain stale until the exact head is reviewed and a post-integration + run emits a clean binding manifest. + +15. A live bearer-authenticated probe used the configured + `mlx-community/llama-3.2-3b-instruct-4bit` worker through + `ContextualOrchestratorJudge -> contextual-orchestrator -> mlx-lm`. On the + same answer and two criteria with `category_count=5`, direct judging + produced score `1.0000` (`4/4`, accepted) and cumulative-threshold judging + produced score `0.0000` (`0/0`, rejected). Both outputs passed the same + strict parser and retained one nested contextual trace step (568 and 570 + tokens respectively). This is paired method sensitivity, not a claim that + more choices cause positive bias; method, K, trace, usage, and parse status + remain part of the calibration denominator. + +16. The same live route exposed an operational boundary: with MLX + `prompt-concurrency=1`, abandoned large prompts remained queued after a + client timeout, the provider logged `BrokenPipe` while writing to the + disconnected client, and the supervised process restarted. A judge retry + must not multiply that device work. `contextual-orchestrator` therefore + defaults same-agent retries to zero for explicit local providers while + retaining remote retry behavior and an explicit local opt-in. Judge + benchmarks must record provider readiness separately from gateway + liveness, timeout/queue/restart events, prompt size, and every failed + comparison; no retry may repair a malformed judge result. + +17. The remote PR/evidence path also exposed a host-network boundary: + GitHub HTTPS failed with `LibreSSL SSL_connect: SSL_ERROR_SYSCALL` on the + VPN `utun12` route but succeeded over `en0`. This distinguishes a route, + MTU, or firewall failure from a bad certificate, repository, or Keyverse + credential. The judge and security evidence must therefore not be + collected through a TLS-verification bypass or silently changed credential + path. An authorized temporary interface-bound relay may unblock a remote + operation, but it is not part of the package or benchmark configuration + and must be terminated and rechecked afterward. + +18. The same protected coverage path recorded `trusted uv archive download + failed: HTTPError` for this PR before repository coverage executed. That is + an evidence blocker, not a model or test pass. The central CI downloader + retains its fixed HTTPS/no-proxy/no-redirect/checksum boundary and now + retries only transient HTTP/socket failures while preserving the HTTP + status in the terminal error; this repository must rerun exact-head + coverage after that central fix and must not treat a skipped or failed + coverage job as approval evidence. +19. A local full-suite run after the protected-main merge exposed two JMLE + recovery failures (`lbfgs` and `adam_lbfgs`) reporting + `line_search_failed` at finite objective reduction. Reproduction showed the + Rust source's bounded objective-convergence rule was present, but the local + validation command + `maturin develop --manifest-path crates/fast-mlsirm-py/Cargo.toml` built a + separate `fast_mlsirm_core` module while pytest loaded the stale + `python/fast_mlsirm/_core` binary. The canonical validation command is the + repository-root `maturin develop --release`, followed by the Rust and + Python recovery suites. The optimizer regression test now keeps the + gradient above tolerance after the first accepted step, proving the + objective-reduction guard rather than accidentally passing through the + gradient-norm guard. Stale extension artifacts are not evidence of a + source regression, and a failed exact-head build must never be hidden. +20. The exact-head Strix run for PR #778 at + `0fb9e466847325edfb32506d77bb615d3c65298f` (`31581202078`, job + `94064514313`, artifact `9136142983`) is not clean security evidence. + GitHub marked every job step successful because the trusted central `main` + workflow still neutralized a provider-unavailable gate exit. The console + evidence records NVIDIA NIM HTTP 429 rate limits, GitHub Models HTTP 410 + retirement-brownout failures, and repeated fail-closed/no-report markers; + the job had no `Validate Strix report provenance` step and emitted no + `evidence-binding.json`. The structured NVIDIA fallback report therefore + cannot override the provider failure. Central `.github` PR #937 contains + the minimal fail-closed/provenance repair; after it integrates, rerun this + PR at the same exact head and accept only a clean bound artifact. +21. The 2026-08-12 Zotero Local API review distinguishes evidence from + redistribution permission. Li et al. (2025, arXiv:2506.22316), Sharma et + al. (2023, arXiv:2310.13548), and Pezeshkpour and Hruschka (2024, ACL + Findings) expose CC BY 4.0 terms and their original PDFs are preserved in + `docs/papers/papers/` with source URLs and SHA-256 digests in + `docs/papers/oa-pdf-manifest.md`. Zheng et al. (2024, ICLR) and Samejima + (1969) remain citation-only because the locally available arXiv/ICLR and + Psychometric Society terms do not provide an explicit third-party + redistribution license. The literature supports option-ID/order and + scoring/sycophancy bias as risks; it does not establish that increasing K + necessarily creates positive bias. A K-effect claim requires paired, + randomized-order, multi-item, gold/human-anchored calibration with parse, + provider, and IRT-shape failures retained in the denominator. + +## Non-goals and claims not made + +- This decision does not claim that Python integers overflow or that a single + model-generated security report proves a CVSS score. +- Exact scalar typing does not establish that LLM judgments are unbiased. +- This decision does not authorize keyword matching, direct provider calls, or + a single scalar response as an IRT dataset. + +## Consequences and trade-offs + +### Benefits + +- Resource-amplifying category bounds cannot be bypassed through comparison + hooks or coercion protocols. +- Security and provider-availability evidence remains auditable and separate. +- The public judge path stays deterministic and compatible with the existing + multi-item IRT contract. + +### Costs / risks + +- Legitimate custom numeric subclasses are rejected and must be normalized by + the caller before crossing the boundary. +- Strix runs remain dependent on external model quotas and can require a + rerun after a provider outage. + +## Alternatives considered + +### Coerce every value with `int()` or `float()` + +Rejected because conversion hooks can execute caller code and can silently +change values at a security and measurement boundary. + +### Trust the Strix narrative as the vulnerability oracle + +Rejected because the observed run lacked a structured report, began with a +provider 429, and made a false claim about Python integer wraparound. + +### Ignore the finding because the primary claim was false + +Rejected because the same code admitted adversarial `int` subclasses that could +forge comparisons. The boundary is now hardened and regression-tested. + +### Blindly retry or repair invalid model output + +Rejected as a default because the local probe recovered no cumulative-threshold +failure after one identical contextual-orchestrator retry, and any lexical or +positional repair would violate the semantic judge boundary. A future +independent binary-threshold decomposition or stronger local judge must be +benchmarked with added cost, first/final parse status, held-out paired cases, +and the same fail-closed parser. + +### Iterate over arbitrary trace containers + +Rejected because a provider-controlled or caller-supplied list subclass can +execute code through `__iter__` or `__len__` while the result record is being +assembled. Exact built-in container typing keeps usage and trace metadata +bounded and side-effect free; malformed custom traces are treated as absent. + +## Failure, degraded, and recovery behavior + +Malformed category, score, text, mode, item-type, and criterion-key inputs raise +the package-owned `ValueError` or `JudgeFormatError` before model output is +accepted or an IRT row is returned. Invalid usage counters are deliberately +ignored by `_usage()` and contribute zero token totals; they are not promoted +to score, category, or merge evidence. +Malformed model JSON, missing structured scan evidence, provider rate limits, +and failed required checks remain fail-closed. After a code change, rerun the +security workflow against the exact pushed head and record the new result; +never reinterpret a stale run as evidence for a new head. + +## Security and privacy implications + +The change reduces coercion and resource-exhaustion risk without adding +credentials or retaining source text. Strix logs and provider errors may be +retained as CI evidence, but secrets must remain masked. The exact head SHA +and run URL are required for reproducibility; they do not grant merge +authority. + +## Compatibility, migration, and rollback + +The accepted JSON schema and valid category range are unchanged. Callers that +used custom numeric subclasses must pass built-in scalars. Rollback is allowed +only with a superseding ADR and equivalent boundary tests; reverting to +coercive validation is not an acceptable compatibility path. + +## Verification and release evidence + +- Run the targeted judge tests and Ruff on changed Python files. +- Run the full Python suite, Rust workspace tests, and binding tests. +- Re-run Strix and all required checks on the exact pushed PR head. +- Review current inline/general PR feedback and obtain any approval required by + the live repository policy before a protected merge. +- Preserve contextual-orchestrator routing for every LLM-as-a-Judge call. +- Keep the 3B retry probe and any future threshold-decomposition comparison in + the calibration denominator; do not promote a recovered subset to an IRT + release claim without paired gold/human evidence. + +## Research and standards basis + +Samejima, F. (1969). *Estimation of latent ability using a response pattern of +graded scores*. Psychometrika, 34(S1), 1–97. https://doi.org/10.1007/BF03372160. +The Psychometric Society provides an openly accessible reproduction at +https://www.psychometricsociety.org/sites/main/files/file-attachments/mn17.pdf. +Samejima's graded-response formulation uses ordered cumulative boundaries for +polytomous categories, which is the limited psychometric basis for representing +our cumulative threshold vector. It does not validate LLM prompts, rater +fairness, equal weighting, or the claim that more response options cause +positive bias; those remain empirical calibration questions. The local Zotero +record is parent `XR8LNVF5` with the 97-page PDF attachment `345PA99V` +(37,178,609 bytes; MD5 `e558448c14a2e400a947e19f16cbcb7e`). + +## Follow-ups + +- Re-run PR #778's Strix workflow after the central trusted-gate hardening and + compare the exact head SHA, provider status, structured artifact, and finding + intersection; require a current-head report/provenance binding. +- If a later scan reports a reproducible issue, create a superseding ADR or + append a new evidence record with the reproducer and regression test. + +## Reversal / supersession conditions + +Supersede this ADR only if the public judge contract adopts a different +bounded scalar representation or if structured security evidence demonstrates +that the current exact-type boundary is insufficient. diff --git a/docs/adr/README.md b/docs/adr/README.md index 83adc9353..0453d9467 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -30,6 +30,7 @@ A conversation, issue, PR body, design note, or paper summary is not an Accepted | [0011](0011-canonical-pyo3-public-export-registry.md) | Proposed | Future Rust-backed features converge on one reviewed PyO3/public-export registry instead of competing extension initializers/import rewrites. | | [0012](0012-purpose-limited-sensitive-data.md) | Accepted | Preserve valid measurement linkage through purpose-limited sensitive-data handling rather than blanket masking or raw-data proliferation. | | [0013](0013-continuous-execution-and-documentation-governance.md) | Proposed | Keep autonomous work work-conserving and enforce one canonical cross-cutting documentation writer with explicit maturity states. | +| [0014](0014-bounded-llm-judge-category-inputs.md) | Proposed | Bound LLM-judge category inputs to exact built-in scalars and keep model/provider security evidence fail-closed and independently verifiable. | ## ADR completeness rule diff --git a/docs/doctoring/fitstats_sx2_person_rust_required.md b/docs/doctoring/fitstats_sx2_person_rust_required.md index a15812902..cb4ba7fe3 100644 --- a/docs/doctoring/fitstats_sx2_person_rust_required.md +++ b/docs/doctoring/fitstats_sx2_person_rust_required.md @@ -15,4 +15,5 @@ Ordinary public S-X² and person-fit results are produced only by the compiled R ## Verification - `tests/test_fitstats_rust_ownership_failclosed.py` +- `tests/test_cov_c_fitstats.py` incomplete-core regression - `tests/test_fitstats.py` incomplete-core regression diff --git a/docs/doctoring/jmle_optimizer_recovery_evidence.md b/docs/doctoring/jmle_optimizer_recovery_evidence.md new file mode 100644 index 000000000..3454e8f68 --- /dev/null +++ b/docs/doctoring/jmle_optimizer_recovery_evidence.md @@ -0,0 +1,55 @@ +# JMLE Rust optimizer recovery evidence + +## Purpose + +PR #760 moved the result-affecting Adam, L-BFGS, and `adam_lbfgs` optimizer loops for public `backend="rust"` JMLE into `mlsirm-core`. Delegation and parity tests establish numerical ownership, but ownership alone does not establish that each public optimizer mode can recover a known generating model after the latent location/scale indeterminacy is identified. Issue #626 therefore retains a separate scientific acceptance gate. + +`tests/test_jmle_optimizer_recovery.py` adds deterministic public-surface evidence using one correctly specified unidimensional 2PL-generating sample (`gamma=0`) and the public MIRT/JMLE fit path. Each advertised optimizer is evaluated on the same data and must use the Rust CPU path, report convergence, reduce the objective, and satisfy explicit finite-sample bias, MAE, and RMSE bounds for item discrimination, item easiness, and person ability after one algebraically exact affine identification transform. + +## Identification boundary + +The public MIRT/JMLE predictor is `eta = a * theta + b`. Without an explicit location/scale constraint, raw `theta`, `a`, and `b` coordinates are not directly comparable across equivalent affine parameterizations. A recovery test that compares raw fitted item parameters to the generating scale can therefore report arbitrarily large error even when the fitted logits are unchanged. + +The recovery test identifies the fitted one-dimensional scale by + +`theta_aligned = q * (theta_est - mean_est) + mean_true`, with `q = sd_true / sd_est`. + +To preserve every fitted logit exactly, it applies the corresponding item transform + +`a_aligned = a_est / q` + +and + +`b_aligned = b_est + a_est * mean_est - a_aligned * mean_true`. + +The test explicitly verifies that the transformed and original fitted linear predictors agree to numerical precision before any recovery metric is accepted. The transform therefore removes only latent-coordinate indeterminacy; it cannot improve likelihood, convergence, fitted probabilities, or genuine parameter recovery. + +## Interpretation boundary + +This is optimizer-mode recovery evidence, not a claim that penalized joint maximum likelihood is asymptotically unbiased for item parameters. JMLE retains the incidental-parameter limitations of joint estimation. The test therefore uses deliberately broad finite-sample acceptance bounds and reports bias, MAE, and RMSE rather than correlation-only evidence. Marginal maximum likelihood remains the more appropriate consistency target for item-parameter recovery claims when its model assumptions apply. + +The initial 500-iteration evidence reached the configured iteration ceiling for L-BFGS and the hybrid mode rather than satisfying their convergence contract. The follow-up evidence expands the optimizer budget to 2,000 iterations without relaxing the `1e-5` tolerance or any recovery threshold; convergence remains a mandatory gate. + +The study does not authorize an end-to-end GPU optimizer claim. Existing GPU objective/gradient parity remains distinct from optimizer-state parity; profiling and dedicated parity/recovery evidence are required before such a path is advertised. + +## Evidence contract + +For each of `adam`, `lbfgs`, and `adam_lbfgs`: + +- exact public `fit(..., estimator="jmle", backend="rust", rust_device="cpu")` is exercised; +- convergence status and positive iteration count are required; +- the objective trace must remain finite and finish no worse than it starts; +- the fitted 2PL predictor must be invariant under the explicit affine identification transform; +- discrimination, easiness, and person ability expose aligned bias, MAE, and RMSE gates; +- no recovery or convergence threshold is relaxed merely because a mode reaches its iteration budget; and +- failure messages carry the complete metric dictionary so CI evidence is diagnostically useful rather than a binary pass/fail. + +## References + +Harwell, M., Stone, C. A., Hsu, T.-C., & Kirisci, L. (1996). Monte Carlo studies in item response theory. *Applied Psychological Measurement, 20*(2), 101–125. https://doi.org/10.1177/014662169602000201 + +Reckase, M. D. (2009). *Multidimensional item response theory*. Springer. https://doi.org/10.1007/978-0-387-89976-3 + +Kingma, D. P., & Ba, J. (2015). Adam: A method for stochastic optimization. *3rd International Conference on Learning Representations*. https://arxiv.org/abs/1412.6980 + +Liu, D. C., & Nocedal, J. (1989). On the limited memory BFGS method for large scale optimization. *Mathematical Programming, 45*, 503–528. https://doi.org/10.1007/BF01589116 diff --git a/docs/documentation_coverage.md b/docs/documentation_coverage.md index be0f094c4..61be2ca2d 100644 --- a/docs/documentation_coverage.md +++ b/docs/documentation_coverage.md @@ -30,6 +30,17 @@ This matrix answers whether GitHub can reconstruct the current `fast-mlsirm` pro - **REJECTED** — reviewed and intentionally excluded. - **OUT_OF_SCOPE** — outside the reusable measurement-core boundary. +For compatibility with review reports, the human-readable status shorthand is +also normative: **IMPLEMENTED**, **ACTIVE PR**, **PLANNED**, and +**DOWNSTREAM**. The phrase **IMPLEMENTED / PLANNED extensions** means that a +protected-main primitive exists while the explicitly named broader extension +remains planned or partial; **ACTIVE PR** never means protected-main truth. +The **Canonical PyO3/public-export registry** is the target source of truth +for native entrypoints. ADR-0011 remains **Proposed**: protected main still +uses its existing separate native initializers and package export paths, while +**ACCEPTED_ARCHITECTURE / PARTIAL** describes the target registry capability +that is not yet a protected-main completion claim. + ## Canonical documentation coverage | Documentation capability | Canonical target | State | Current fitness / maintenance rule | diff --git a/docs/papers/README.md b/docs/papers/README.md index d77880f4e..aedfb2691 100644 --- a/docs/papers/README.md +++ b/docs/papers/README.md @@ -1,7 +1,14 @@ # Reference Papers -Curated references that ground the estimation core. Store compact citations and -canonical links here instead of vendoring PDF copies into the repository. +Curated references that ground the estimation core and the contextual- +orchestrator-backed LLM-judge boundary. Citation-only references stay in this +directory; `papers/` contains only original PDFs whose source explicitly allows +redistribution (or whose repository/archive terms are clear). Zotero presence +or public downloadability alone is not treated as a redistribution license. + +The retained files and their source/license/hash checks are recorded in +[`oa-pdf-manifest.md`](oa-pdf-manifest.md). No PDF is fetched through a +provider credential, and no paper is used as a keyword-matching judgment rule. ## Wu et al. 2021 @@ -20,3 +27,36 @@ arXiv:2108.11579. https://arxiv.org/abs/2108.11579 kernels in `crates/mlsirm-core/src/gpu.rs`. The paper is the design reference for keeping GPU-accelerated IRT estimation numerically faithful to the CPU objective. + +## LLM-judge and option/category-bias references + +Li, Q., Dou, S., Shao, K., Chen, C., & Hu, H. (2025). *Evaluating scoring bias +in LLM-as-a-Judge*. arXiv:2506.22316. . +CC BY 4.0. The original PDF is preserved as +[`papers/li-2025-evaluating-scoring-bias-llm-as-a-judge.pdf`](papers/li-2025-evaluating-scoring-bias-llm-as-a-judge.pdf). + +Pezeshkpour, P., & Hruschka, E. (2024). Large language models sensitivity to +the order of options in multiple-choice questions. *Findings of ACL: NAACL +2024*, 2006–2017. . +ACL Anthology materials published in or after 2016 are CC BY 4.0. The original +PDF is preserved as +[`papers/pezeshkpour-hruschka-2024-option-order-sensitivity.pdf`](papers/pezeshkpour-hruschka-2024-option-order-sensitivity.pdf). + +Sharma, M., Tong, M., Korbak, T., et al. (2023). *Towards understanding +sycophancy in language models*. arXiv:2310.13548. +. CC BY 4.0. The original PDF is preserved as +[`papers/sharma-2023-sycophancy.pdf`](papers/sharma-2023-sycophancy.pdf). + +Zheng, C., Zhou, H., Meng, F., Zhou, J., & Huang, M. (2024). *Large language +models are not robust multiple choice selectors*. ICLR 2024. +. +This remains citation-only: the Zotero/arXiv record exposes arXiv's +non-exclusive license to distribute to arXiv, while the ICLR policy grants +distribution rights to ICLR; neither is an explicit third-party repository +redistribution license for this project. + +Samejima, F. (1969). *Estimation of latent ability using a response pattern of +graded scores*. *Psychometrika, 34*(Suppl. 1), 1–97. +. The Psychometric Society reproduction is +linked from ADR-0014 but remains citation-only because its accessible PDF does +not state a redistribution license. diff --git a/docs/papers/directive-irt-coverage.md b/docs/papers/directive-irt-coverage.md index 6a46c95f3..9d0fb2460 100644 --- a/docs/papers/directive-irt-coverage.md +++ b/docs/papers/directive-irt-coverage.md @@ -141,12 +141,15 @@ the Rust core and `serving.py`). patterns); Bayesian EAP/MAP with a proper prior is always finite. Assertion: all-0/all-1 vectors return finite `θ̂` and defined SE; MLE is flagged/undefined. -## Open-access full-text availability (for future PDF preservation) +## Open-access full-text availability -Preserve these OA copies under `docs/papers/papers/` if/when full-text archiving is -performed (no Git LFS; direct PDF or a citation-only stub per repository policy): +The repository preserves only OA originals with an explicit redistribution basis +in `docs/papers/papers/` (no Git LFS). Other accessible copies remain stable +citation-only links; an OA label does not by itself grant this repository a right +to redistribute a PDF: -- Chalmers (2012) — fully OA: +- Chalmers (2012) — fully OA: (not yet + vendored; confirm the journal's redistribution terms before adding a binary) - Bock & Aitkin (1981) — OA scan (UC Merced course reserves) - Bock, Gibbons & Muraki (1988) — OA (U. Minnesota Conservancy) - Bock & Mislevy (1982) — OA (U. Minnesota Conservancy) diff --git a/docs/papers/oa-pdf-manifest.md b/docs/papers/oa-pdf-manifest.md new file mode 100644 index 000000000..257e1e671 --- /dev/null +++ b/docs/papers/oa-pdf-manifest.md @@ -0,0 +1,24 @@ +# OA PDF manifest + +This manifest records the original PDFs intentionally preserved in this +repository. Hashes are SHA-256 of the committed bytes. Zotero attachment keys +are local provenance only; they are not credentials and are not required by CI. + +| File | Source | Reuse basis | Zotero attachment | Bytes | SHA-256 | +| --- | --- | --- | --- | ---: | --- | +| [`li-2025-evaluating-scoring-bias-llm-as-a-judge.pdf`](papers/li-2025-evaluating-scoring-bias-llm-as-a-judge.pdf) | [arXiv:2506.22316](https://arxiv.org/abs/2506.22316), [PDF](https://arxiv.org/pdf/2506.22316) | arXiv record states CC BY 4.0 | `TVZMTEB8` | 786,966 | `1ea7b239ff1341189bd3927bbab63b43b190289437156d89e0e251aca146b744` | +| [`pezeshkpour-hruschka-2024-option-order-sensitivity.pdf`](papers/pezeshkpour-hruschka-2024-option-order-sensitivity.pdf) | [ACL Anthology page](https://aclanthology.org/2024.findings-naacl.130/), [PDF](https://aclanthology.org/2024.findings-naacl.130.pdf) | ACL Anthology states materials published in or after 2016 are CC BY 4.0 | `S5KQCN97` | 439,433 | `4d0fecf2b1da9544112c286a699f8873b0c7a4ce2f38f400bf09db5f10659624` | +| [`sharma-2023-sycophancy.pdf`](papers/sharma-2023-sycophancy.pdf) | [arXiv:2310.13548](https://arxiv.org/abs/2310.13548), [PDF](https://arxiv.org/pdf/2310.13548) | arXiv record states CC BY 4.0 | `47VH4PC7` | 1,383,108 | `ee764bd30119f2146f2e130a099d6d313fca6c70ab07b17b7fdbde456d96be36` | + +## Citation-only records + +- Zheng et al. (2024), [ICLR page](https://proceedings.iclr.cc/paper_files/paper/2024/hash/54dd9e0cff6d9214e20d97eb2a3bae49-Abstract-Conference.html): the local arXiv record grants a non-exclusive distribution license to arXiv, and the ICLR policy describes a license granted to ICLR; neither is an explicit third-party repository redistribution license. +- Samejima (1969), [Psychometric Society reproduction](https://www.psychometricsociety.org/sites/main/files/file-attachments/mn17.pdf): publicly accessible and preserved in Zotero for local research, but no redistribution license is asserted here. + +## Interpretation boundary + +These papers motivate paired option-order, category-count, rubric-order, score-ID, +and sycophancy probes. They do not prove a universal positive effect of larger +`K`. The repository therefore retains K, method, prompt/order variant, parse +status, provider readiness, and human/gold anchor fields as calibration evidence; +it does not convert any of these papers into a keyword-matching judge. diff --git a/docs/papers/papers/li-2025-evaluating-scoring-bias-llm-as-a-judge.pdf b/docs/papers/papers/li-2025-evaluating-scoring-bias-llm-as-a-judge.pdf new file mode 100644 index 000000000..bb10b98d0 Binary files /dev/null and b/docs/papers/papers/li-2025-evaluating-scoring-bias-llm-as-a-judge.pdf differ diff --git a/docs/papers/papers/pezeshkpour-hruschka-2024-option-order-sensitivity.pdf b/docs/papers/papers/pezeshkpour-hruschka-2024-option-order-sensitivity.pdf new file mode 100644 index 000000000..3965eca88 Binary files /dev/null and b/docs/papers/papers/pezeshkpour-hruschka-2024-option-order-sensitivity.pdf differ diff --git a/docs/papers/papers/sharma-2023-sycophancy.pdf b/docs/papers/papers/sharma-2023-sycophancy.pdf new file mode 100644 index 000000000..c1981f7aa Binary files /dev/null and b/docs/papers/papers/sharma-2023-sycophancy.pdf differ diff --git a/python/fast_mlsirm/llm_judge.py b/python/fast_mlsirm/llm_judge.py index 48314549f..62382ec45 100644 --- a/python/fast_mlsirm/llm_judge.py +++ b/python/fast_mlsirm/llm_judge.py @@ -42,8 +42,7 @@ def _duplicate_free_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: def _category_count(value: Any) -> int: if ( - not isinstance(value, int) - or isinstance(value, bool) + type(value) is not int or not 2 <= value <= MAX_JUDGE_CATEGORIES ): raise ValueError( @@ -54,7 +53,7 @@ def _category_count(value: Any) -> int: def _category(value: Any, name: str, category_count: int) -> int: """Accept JSON integer values, including mathematically integral 1.0 forms.""" - if isinstance(value, bool) or not isinstance(value, (int, float)): + if type(value) not in (int, float): raise JudgeFormatError( f"{name} must be an integer in 0..{category_count - 1}" ) @@ -83,12 +82,12 @@ class JudgeCriterion: weight: float = 1.0 def __post_init__(self) -> None: - if not isinstance(self.criterion_id, str): - raise ValueError("criterion_id must be a string") # noqa: TRY004 + if type(self.criterion_id) is not str: + raise ValueError("criterion_id must be a string") if not _IDENTIFIER.fullmatch(self.criterion_id): raise ValueError("criterion_id must contain two or more snake_case words") - if not isinstance(self.description, str): - raise ValueError("criterion description must be a string") # noqa: TRY004 + if type(self.description) is not str: + raise ValueError("criterion description must be a string") if not self.description.strip() or len(self.description) > 2_000: raise ValueError("criterion description must be non-empty and <= 2000 characters") if type(self.weight) not in (int, float): @@ -123,6 +122,7 @@ class LLMJudgeResult: usage: Mapping[str, int] criterion_categories: Mapping[str, int] | None = None category_count: int | None = None + category_method: str = "direct" def to_irt_row( self, @@ -137,11 +137,11 @@ def to_irt_row( that equal-width score bins remove judge bias; category-count and prompt-perturbation calibration remains required. """ - if item_type not in {"dichotomous", "polytomous"}: + if type(item_type) is not str or item_type not in {"dichotomous", "polytomous"}: raise JudgeFormatError("item_type must be dichotomous or polytomous") if not isinstance(self.criterion_scores, Mapping): raise JudgeFormatError("criterion_scores must be an object") - if any(not isinstance(criterion_id, str) for criterion_id in self.criterion_scores): + if any(type(criterion_id) is not str for criterion_id in self.criterion_scores): raise JudgeFormatError("criterion_scores keys must be strings") criterion_ids = sorted(self.criterion_scores) if len(criterion_ids) < 2: @@ -154,7 +154,7 @@ def to_irt_row( if not isinstance(self.criterion_categories, Mapping): raise JudgeFormatError("criterion_categories must be an object") if any( - not isinstance(criterion_id, str) + type(criterion_id) is not str for criterion_id in self.criterion_categories ): raise JudgeFormatError("criterion_categories keys must be strings") @@ -251,11 +251,12 @@ def to_dict(self) -> dict[str, Any]: else None ), "category_count": self.category_count, + "category_method": self.category_method, } def _bounded_text(value: Any, name: str) -> str: - if not isinstance(value, str) or not value.strip(): + if type(value) is not str or not value.strip(): raise ValueError(f"{name} must be a non-empty string") normalized = value.strip() if len(normalized) > MAX_JUDGE_TEXT_CHARACTERS: @@ -277,7 +278,11 @@ def _criteria(values: Iterable[JudgeCriterion | Mapping[str, Any]]) -> tuple[Jud weight=value.get("weight", 1.0), ) else: - raise ValueError("criteria must contain JudgeCriterion or mapping values") + # The public criterion contract deliberately normalizes malformed + # inputs to ValueError for callers that validate user-supplied mappings. + raise ValueError( # noqa: TRY004 + "criteria must contain JudgeCriterion or mapping values" + ) normalized.append(criterion) if not 1 <= len(normalized) <= MAX_JUDGE_CRITERIA: raise ValueError(f"criteria must contain 1..{MAX_JUDGE_CRITERIA} values") @@ -328,9 +333,12 @@ def _response_object(raw: str, *, required_fields: set[str]) -> dict[str, Any]: def _score(value: Any, name: str) -> float: - if isinstance(value, bool) or not isinstance(value, (int, float)): + if type(value) not in (int, float): raise JudgeFormatError(f"{name} must be a number between 0 and 1") - normalized = float(value) + try: + normalized = float(value) + except (OverflowError, TypeError, ValueError): + raise JudgeFormatError(f"{name} must be a number between 0 and 1") from None if not math.isfinite(normalized) or not 0.0 <= normalized <= 1.0: raise JudgeFormatError(f"{name} must be a number between 0 and 1") return normalized @@ -338,7 +346,7 @@ def _score(value: Any, name: str) -> float: def _usage(trace: Any) -> dict[str, int]: totals = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} - if not isinstance(trace, list): + if type(trace) is not list: return totals for step in trace: usage = step.get("usage") if isinstance(step, dict) else None @@ -346,7 +354,7 @@ def _usage(trace: Any) -> dict[str, int]: continue for key in totals: value = usage.get(key) - if isinstance(value, int) and not isinstance(value, bool) and value >= 0: + if type(value) is int and value >= 0: totals[key] += value return totals @@ -357,7 +365,7 @@ class ContextualOrchestratorJudge: def __init__(self, orchestrator: Any, *, mode: str = "route", accept_threshold: float = 0.7) -> None: if not callable(getattr(orchestrator, "complete", None)): raise TypeError("orchestrator must provide complete(messages, mode=...)") - if mode not in {"auto", "route", "conduct"}: + if type(mode) is not str or mode not in {"auto", "route", "conduct"}: raise ValueError("mode must be auto, route, or conduct") self.orchestrator = orchestrator self.mode = mode @@ -371,8 +379,16 @@ def judge( criteria: Iterable[JudgeCriterion | Mapping[str, Any]], reference_answer: str | None = None, category_count: int | None = None, + category_method: str = "direct", ) -> LLMJudgeResult: """Return a strict JSON decision from the orchestrator-backed judge.""" + if ( + type(category_method) is not str + or category_method not in {"direct", "cumulative_threshold"} + ): + raise ValueError( + "category_method must be direct or cumulative_threshold" + ) task = _bounded_text(task, "task") answer = _bounded_text(answer, "answer") if reference_answer is not None: @@ -381,32 +397,61 @@ def judge( expected_ids = [criterion.criterion_id for criterion in normalized_criteria] if category_count is not None: category_count = _category_count(category_count) + if category_method == "cumulative_threshold" and category_count is None: + raise ValueError( + "cumulative_threshold requires an explicit category_count" + ) criterion_payload = [criterion.to_dict() for criterion in normalized_criteria] reference_block = reference_answer or "(none supplied)" category_instruction = "" if category_count is not None: - category_template = { - "score": 0.0, - "accepted": False, - "rationale": "brief evidence-based reason", - "criterion_categories": {criterion_id: 0 for criterion_id in expected_ids}, - } - category_instruction = ( - f" Use exactly {category_count} ordered categories indexed 0 through " - f"{category_count - 1}. Return criterion_categories as a JSON object " - f"with exactly these string keys: {json.dumps(expected_ids)}. " - f"Use only whole-number values from {list(range(category_count))}; " - "category values are JSON integers, never decimals such as 0.2 or 0.8; " - "never use numeric keys or an array. " - f"The exact JSON shape is {json.dumps(category_template, ensure_ascii=False)}. " - "Replace the example values and keep every key unchanged. Derive the overall score from those " - "categories. Category 0 means no credible evidence or complete failure; " - f"category {category_count - 1} means fully satisfies the criterion with accurate evidence. " - "Intermediate categories are ordered levels between those anchors. A strong answer that fully " - f"satisfies a criterion must use category {category_count - 1}. More categories add " - "resolution; they do not reverse the meaning of the anchors. Do not choose a higher category " - "merely because more categories exist." - ) + if category_method == "cumulative_threshold": + threshold_template = { + "score": 0.0, + "accepted": False, + "rationale": "brief evidence-based reason", + "criterion_thresholds": { + criterion_id: [False] * (category_count - 1) + for criterion_id in expected_ids + }, + } + category_instruction = ( + f" Use exactly {category_count} ordered categories indexed 0 through " + f"{category_count - 1}, but judge them with cumulative thresholds rather than one K-way choice. " + "Return criterion_thresholds as a JSON object with exactly these string keys: " + f"{json.dumps(expected_ids)}. Each value must be a JSON boolean array of exactly " + f"{category_count - 1} values. Array position j answers whether the evidence meets at least " + "ordered category j+1 for that criterion. Threshold arrays must be monotone: once false, " + "all later values must be false; never emit a higher true threshold after a lower false one. " + f"The exact JSON shape is {json.dumps(threshold_template, ensure_ascii=False)}. " + "Replace the example values and keep every key unchanged. Category 0 means no credible " + "evidence or complete failure; the highest category means fully satisfies the criterion " + "with accurate evidence. Derive the overall score from the number of true thresholds. " + "Do not reward answer length, agreement, or the presence of more categories." + ) + else: + category_template = { + "score": 0.0, + "accepted": False, + "rationale": "brief evidence-based reason", + "criterion_categories": {criterion_id: 0 for criterion_id in expected_ids}, + } + category_instruction = ( + f" Use exactly {category_count} ordered categories indexed 0 through " + f"{category_count - 1}. Return criterion_categories as a JSON object " + f"with exactly these string keys: {json.dumps(expected_ids)}. " + f"Use only whole-number values from {list(range(category_count))}; " + "category values are JSON integers, never decimals such as 0.2 or 0.8; " + "never use numeric keys or an array. " + f"The exact JSON shape is {json.dumps(category_template, ensure_ascii=False)}. " + "Replace the example values and keep every key unchanged. Derive the overall score from those " + "categories. Category 0 means no credible evidence or complete failure; " + f"category {category_count - 1} means fully satisfies the criterion with accurate evidence. " + "Intermediate categories are ordered levels between those anchors. A strong answer that fully " + f"satisfies a criterion must use category {category_count - 1}. More categories add " + "resolution; they do not reverse the meaning of the anchors. Do not choose a higher category " + "merely because more categories exist." + ) else: category_instruction = ( " Include criterion_scores as a JSON object with exactly one number " @@ -417,6 +462,7 @@ def judge( "answer": answer, "reference": reference_block, "criteria": criterion_payload, + "category_method": category_method, } messages = [ { @@ -447,9 +493,12 @@ def judge( raw = _bounded_text(completion.get("answer"), "judge answer") except ValueError as exc: raise JudgeFormatError(str(exc)) from exc - criterion_field = ( - "criterion_categories" if category_count is not None else "criterion_scores" - ) + if category_count is None: + criterion_field = "criterion_scores" + elif category_method == "cumulative_threshold": + criterion_field = "criterion_thresholds" + else: + criterion_field = "criterion_categories" parsed = _response_object( raw, required_fields={"score", "accepted", "rationale", criterion_field}, @@ -467,18 +516,43 @@ def judge( # Validate the redundant field's shape, but derive the accepted score # from the ordered category items below rather than trusting it. _score(parsed.get("score"), "score") - raw_categories = parsed.get("criterion_categories") - if not isinstance(raw_categories, Mapping) or set(raw_categories) != expected_id_set: - raise JudgeFormatError( - "criterion_categories must contain exactly the rubric criterion ids" - ) criterion_categories = {} - for criterion_id in sorted(expected_ids): - criterion_categories[criterion_id] = _category( - raw_categories[criterion_id], - f"criterion_categories.{criterion_id}", - category_count, - ) + if category_method == "cumulative_threshold": + raw_thresholds = parsed.get("criterion_thresholds") + if not isinstance(raw_thresholds, Mapping) or set(raw_thresholds) != expected_id_set: + raise JudgeFormatError( + "criterion_thresholds must contain exactly the rubric criterion ids" + ) + for criterion_id in sorted(expected_ids): + thresholds = raw_thresholds[criterion_id] + if not isinstance(thresholds, list) or len(thresholds) != category_count - 1: + raise JudgeFormatError( + "criterion thresholds must be a boolean array for every ordered boundary" + ) + if any(type(value) is not bool for value in thresholds): + raise JudgeFormatError( + "criterion thresholds must contain only boolean values" + ) + if any( + not thresholds[index] and thresholds[index + 1] + for index in range(len(thresholds) - 1) + ): + raise JudgeFormatError( + "criterion thresholds must be monotone" + ) + criterion_categories[criterion_id] = sum(thresholds) + else: + raw_categories = parsed.get("criterion_categories") + if not isinstance(raw_categories, Mapping) or set(raw_categories) != expected_id_set: + raise JudgeFormatError( + "criterion_categories must contain exactly the rubric criterion ids" + ) + for criterion_id in sorted(expected_ids): + criterion_categories[criterion_id] = _category( + raw_categories[criterion_id], + f"criterion_categories.{criterion_id}", + category_count, + ) criterion_scores = { criterion_id: criterion_categories[criterion_id] / (category_count - 1) for criterion_id in sorted(expected_ids) @@ -513,18 +587,19 @@ def judge( criterion_scores=criterion_scores, raw_output=raw, orchestration_mode=str(completion.get("mode", self.mode)), - trace_step_count=len(trace) if isinstance(trace, list) else 0, + trace_step_count=len(trace) if type(trace) is list else 0, usage=_usage(trace), criterion_categories=criterion_categories, category_count=category_count, + category_method=category_method, ) __all__ = [ "MAX_JUDGE_CATEGORIES", "MAX_JUDGE_CRITERIA", - "MAX_JUDGE_TEXT_CHARACTERS", "MAX_JUDGE_JSON_DEPTH", + "MAX_JUDGE_TEXT_CHARACTERS", "ContextualOrchestratorJudge", "JudgeCriterion", "JudgeFormatError", diff --git a/tests/test_cov_c_fitstats.py b/tests/test_cov_c_fitstats.py index b9720879c..fcac16df1 100644 --- a/tests/test_cov_c_fitstats.py +++ b/tests/test_cov_c_fitstats.py @@ -1,10 +1,9 @@ """Coverage tests (batch C) for ``fast_mlsirm.fitstats``. -These target NumPy-fallback bodies (reached by forcing ``_core_module`` to -return ``None``), input-validation guards, and edge branches in the -fit-statistics core. They assert real behaviour (the exact exception types the -guards raise, and NumPy/native structural agreement) rather than merely -executing lines. +These target fit-statistics validation and the fail-closed Rust ownership +boundary, input-validation guards, and edge branches in the fit-statistics +core. They assert real behaviour (the exact exception types the guards raise, +and native structural agreement) rather than merely executing lines. """ from __future__ import annotations @@ -48,7 +47,7 @@ def _pure_bh(p_values, q: float = 0.05): class _CoreWithFitStatsOnly: - """Stub core exposing only fit-statistics entrypoints for unrelated fallback tests.""" + """Stub core without the native S-X²/person-fit entrypoints.""" def chi2_sf(self, x, df): return _pure_chi2_sf(float(x), float(df)) @@ -184,7 +183,7 @@ def test_icc_grid_and_factorized_guards(): # --------------------------------------------------------------------------- -# S-X2 NumPy fallback body (core forced absent) +# S-X2 native ownership (incomplete core fails closed) # --------------------------------------------------------------------------- @@ -214,7 +213,7 @@ def test_sx2_numpy_fallback_spatial_and_dim_floors(monkeypatch): # --------------------------------------------------------------------------- -# person-fit / infit-outfit NumPy fallbacks (core forced absent) +# person-fit Rust ownership (incomplete core fails closed) # --------------------------------------------------------------------------- diff --git a/tests/test_fitstats_infit_outfit_allocations.py b/tests/test_fitstats_infit_outfit_allocations.py index 1f59554e7..f65052fd5 100644 --- a/tests/test_fitstats_infit_outfit_allocations.py +++ b/tests/test_fitstats_infit_outfit_allocations.py @@ -116,4 +116,3 @@ def infit_outfit_stat(self, *args, **kwargs): result = fitstats.infit_outfit(responses, factor_id, params, "mlsirm", mask=observed) assert core.calls == 1 np.testing.assert_array_equal(result["infit"], np.full(n_items, 0.9)) - diff --git a/tests/test_jmle_optimizer_recovery.py b/tests/test_jmle_optimizer_recovery.py new file mode 100644 index 000000000..304cc8763 --- /dev/null +++ b/tests/test_jmle_optimizer_recovery.py @@ -0,0 +1,191 @@ +"""True-parameter recovery evidence for Rust-owned public JMLE optimizers. + +The optimizer-ownership migration is not accepted scientifically from delegation +sentinels alone. This study generates one deterministic, correctly specified +unidimensional MIRT data set, estimates it through each advertised public JMLE +optimizer, and checks convergence plus identification-aligned bias/MAE/RMSE +recovery. The thresholds are intentionally broad finite-sample acceptance +bounds rather than claims of asymptotic unbiasedness; JMLE retains the usual +incidental-parameter limitations. + +References +---------- +Harwell, M., Stone, C. A., Hsu, T.-C., & Kirisci, L. (1996). Monte Carlo + studies in item response theory. *Applied Psychological Measurement, + 20*(2), 101-125. https://doi.org/10.1177/014662169602000201 +Reckase, M. D. (2009). *Multidimensional item response theory*. Springer. + https://doi.org/10.1007/978-0-387-89976-3 +""" + +from __future__ import annotations + +import numpy as np +import pytest + +from fast_mlsirm import FitConfig, MLS2PLMConfig, fit, simulate + + +def _bias(truth: np.ndarray, estimate: np.ndarray) -> float: + """Return signed mean error over identified parameter values.""" + return float(np.mean(np.asarray(estimate) - np.asarray(truth))) + + +def _mae(truth: np.ndarray, estimate: np.ndarray) -> float: + """Return mean absolute error over identified parameter values.""" + return float(np.mean(np.abs(np.asarray(estimate) - np.asarray(truth)))) + + +def _rmse(truth: np.ndarray, estimate: np.ndarray) -> float: + """Return root mean squared error over identified parameter values.""" + error = np.asarray(estimate) - np.asarray(truth) + return float(np.sqrt(np.mean(error * error))) + + +def _identify_1d_mirt(truth, estimate) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """Affine-align a 1-D JMLE fit to the generating ability location/scale. + + The 2PL predictor ``a * theta + b`` is invariant to an affine change of the + latent coordinate. For ``theta_aligned = q * (theta_est - mean_est) + + mean_true`` with ``q = sd_true / sd_est``, the algebraically equivalent + item parameters are ``a_aligned = a_est / q`` and + ``b_aligned = b_est + a_est*mean_est - a_aligned*mean_true``. This is an + identification transform only: it cannot improve the fitted logits or hide + optimizer error. + """ + truth_theta = np.asarray(truth.theta[:, 0], dtype=np.float64) + estimate_theta = np.asarray(estimate.theta[:, 0], dtype=np.float64) + truth_mean = float(truth_theta.mean()) + estimate_mean = float(estimate_theta.mean()) + truth_sd = float(truth_theta.std()) + estimate_sd = float(estimate_theta.std()) + assert truth_sd > 0.0 + assert estimate_sd > 0.0 + + q = truth_sd / estimate_sd + theta_aligned = q * (estimate_theta - estimate_mean) + truth_mean + a_aligned = np.asarray(estimate.a, dtype=np.float64) / q + b_aligned = ( + np.asarray(estimate.b, dtype=np.float64) + + np.asarray(estimate.a, dtype=np.float64) * estimate_mean + - a_aligned * truth_mean + ) + + original_eta = ( + np.asarray(estimate.a, dtype=np.float64)[None, :] * estimate_theta[:, None] + + np.asarray(estimate.b, dtype=np.float64)[None, :] + ) + aligned_eta = a_aligned[None, :] * theta_aligned[:, None] + b_aligned[None, :] + np.testing.assert_allclose(aligned_eta, original_eta, rtol=1e-11, atol=1e-11) + return theta_aligned, a_aligned, b_aligned + + +@pytest.fixture(scope="module") +def _jmle_recovery_data(): + """Return one deterministic, non-separated 2PL sample shared by optimizer modes. + + ``simulate`` intentionally spans easiness from 0 to 5 for broad simulation + coverage. The original 12-item recovery fixture corrected only item margins; + its deterministic response sample still contained 15 all-zero and 3 all-one + person patterns, so JMLE correctly encountered person-parameter separation + rather than an optimizer-recovery problem. Keep the generated abilities and + discriminations, use 40 balanced-easiness items, and resample a deterministic + response matrix whose item and person margins are both non-extreme. No fit, + convergence, or recovery threshold is relaxed by this fixture correction. + """ + data = simulate( + MLS2PLMConfig( + n_persons=160, + n_dims=1, + items_per_dim=40, + latent_dim=1, + gamma=0.0, + seed=62612, + ) + ) + rng = np.random.default_rng(62621) + data.truth.b = rng.permutation( + np.linspace(-1.5, 1.5, data.truth.b.size, dtype=np.float64) + ) + eta = ( + data.truth.a[None, :] * data.truth.theta[:, data.factor_id] + + data.truth.b[None, :] + ) + data.probabilities = 1.0 / (1.0 + np.exp(-eta)) + data.Y = rng.binomial(1, data.probabilities).astype(np.uint8) + + item_rates = data.Y.mean(axis=0) + person_rates = data.Y.mean(axis=1) + assert np.all((item_rates > 0.05) & (item_rates < 0.95)) + assert np.all((person_rates >= 0.05) & (person_rates <= 0.95)) + return data + + +@pytest.mark.parametrize("optimizer", ["adam", "lbfgs", "adam_lbfgs"]) +def test_rust_jmle_optimizer_modes_recover_known_parameters( + _jmle_recovery_data, + optimizer: str, +) -> None: + """Each Rust JMLE optimizer must recover after the required affine alignment.""" + data = _jmle_recovery_data + result = fit( + data.Y.astype(np.float64), + data.factor_id, + FitConfig( + model="MIRT", + estimator="jmle", + optimizer=optimizer, + latent_dim=1, + max_iter=2000, + n_restarts=1, + learning_rate=0.03, + tolerance=1e-5, + seed=62612, + backend="rust", + rust_device="cpu", + ), + ) + + theta_aligned, a_aligned, b_aligned = _identify_1d_mirt( + data.truth, + result.params, + ) + truth_theta = np.asarray(data.truth.theta[:, 0], dtype=np.float64) + metrics = { + "a_bias": _bias(data.truth.a, a_aligned), + "a_mae": _mae(data.truth.a, a_aligned), + "a_rmse": _rmse(data.truth.a, a_aligned), + "b_bias": _bias(data.truth.b, b_aligned), + "b_mae": _mae(data.truth.b, b_aligned), + "b_rmse": _rmse(data.truth.b, b_aligned), + "theta_bias": _bias(truth_theta, theta_aligned), + "theta_mae": _mae(truth_theta, theta_aligned), + "theta_rmse": _rmse(truth_theta, theta_aligned), + } + + evidence = { + "optimizer": optimizer, + "status": result.convergence_status, + "n_iter": result.n_iter, + **metrics, + } + + assert result.backend == "rust", evidence + assert result.rust_device == "cpu", evidence + assert result.convergence_status == "converged", evidence + assert result.n_iter > 0, evidence + assert result.objective_trace and np.all(np.isfinite(result.objective_trace)), evidence + assert result.objective_trace[-1] <= result.objective_trace[0], evidence + + # Finite-sample recovery gates with deliberately wide headroom. Bias, MAE, + # and RMSE are evaluated only after the exact 2PL affine-identification + # transform above; no threshold is relaxed to accommodate an arbitrary JMLE + # latent scale or a non-converged optimizer. + assert abs(metrics["a_bias"]) < 1.0, evidence + assert metrics["a_mae"] < 1.2, evidence + assert metrics["a_rmse"] < 1.5, evidence + assert abs(metrics["b_bias"]) < 1.0, evidence + assert metrics["b_mae"] < 1.2, evidence + assert metrics["b_rmse"] < 1.5, evidence + assert abs(metrics["theta_bias"]) < 0.15, evidence + assert metrics["theta_mae"] < 1.1, evidence + assert metrics["theta_rmse"] < 1.35, evidence diff --git a/tests/test_llm_judge.py b/tests/test_llm_judge.py index 470e04e9e..0d37abfc9 100644 --- a/tests/test_llm_judge.py +++ b/tests/test_llm_judge.py @@ -6,6 +6,7 @@ from dataclasses import replace import pytest +from fast_mlsirm.irt_contract import validate_irt_response_matrix from fast_mlsirm.llm_judge import ( ContextualOrchestratorJudge, JudgeCriterion, @@ -59,6 +60,18 @@ def _category_payload(): }) +def _threshold_payload(thresholds=None): + return json.dumps({ + "score": 0.0, + "accepted": True, + "rationale": "The ordered evidence supports separate cumulative thresholds.", + "criterion_thresholds": thresholds or { + "task_alignment": [True, True, True, True], + "factual_support": [True, False, False, False], + }, + }) + + def test_judge_uses_contextual_orchestrator_route_and_reports_usage() -> None: orchestrator = _FakeOrchestrator(_payload()) result = ContextualOrchestratorJudge(orchestrator).judge( @@ -94,6 +107,18 @@ def test_judge_rejects_malformed_decisions_and_derives_acceptance() -> None: assert result.accepted is True +@pytest.mark.parametrize("accepted", [0, 1, "true", None]) +def test_judge_rejects_non_boolean_advisory_acceptance(accepted) -> None: + with pytest.raises(JudgeFormatError, match="accepted must be a boolean"): + ContextualOrchestratorJudge( + _FakeOrchestrator(_payload(accepted=accepted)) + ).judge( + task="task", + answer="answer", + criteria=CRITERIA, + ) + + def test_judge_rejects_wrapped_or_fenced_json() -> None: for answer in ( f"prefix {_payload()}", @@ -183,6 +208,8 @@ def test_irt_projection_rejects_malformed_result_mappings() -> None: criterion_categories=[0, 1], category_count=2, ).to_irt_row() + with pytest.raises(JudgeFormatError, match="item_type"): + result.to_irt_row(item_type=[]) def test_criteria_limit_is_enforced_during_iteration() -> None: @@ -229,6 +256,91 @@ def test_category_judgment_derives_ordered_scores_and_irt_items() -> None: assert "category 4" in prompt +def test_cumulative_threshold_judgment_derives_monotone_polytomous_items() -> None: + orchestrator = _FakeOrchestrator(_threshold_payload()) + result = ContextualOrchestratorJudge(orchestrator).judge( + task="task", + answer="answer", + criteria=CRITERIA, + category_count=5, + category_method="cumulative_threshold", + ) + + assert result.category_method == "cumulative_threshold" + assert dict(result.criterion_categories) == { + "factual_support": 1, + "task_alignment": 4, + } + assert result.score == 0.625 + assert result.accepted is False + assert result.to_dict()["category_method"] == "cumulative_threshold" + row = result.to_irt_row() + assert row == (1, 4) + matrix = validate_irt_response_matrix([row], "polytomous", n_categories=5) + assert matrix.shape == (1, 2) + prompt = orchestrator.calls[0][0][0]["content"] + assert "criterion_thresholds" in prompt + assert "cumulative thresholds" in prompt + assert "must be monotone" in prompt + assert "K-way choice" in prompt + + +@pytest.mark.parametrize( + ("thresholds", "match"), + [ + ( + {"task_alignment": [True, False, True, False], "factual_support": [False] * 4}, + "monotone", + ), + ( + {"task_alignment": [True, 1, False, False], "factual_support": [False] * 4}, + "boolean", + ), + ( + {"task_alignment": [True, True], "factual_support": [False] * 4}, + "boolean array", + ), + ], +) +def test_cumulative_threshold_rejects_malformed_thresholds(thresholds, match) -> None: + with pytest.raises(JudgeFormatError, match=match): + ContextualOrchestratorJudge( + _FakeOrchestrator(_threshold_payload(thresholds)) + ).judge( + task="task", + answer="answer", + criteria=CRITERIA, + category_count=5, + category_method="cumulative_threshold", + ) + + +def test_cumulative_threshold_requires_explicit_category_count() -> None: + with pytest.raises(ValueError, match="explicit category_count"): + ContextualOrchestratorJudge(_FakeOrchestrator(_payload())).judge( + task="task", + answer="answer", + criteria=CRITERIA, + category_method="cumulative_threshold", + ) + + +@pytest.mark.parametrize("category_method", ["unknown", [], {}]) +def test_judge_rejects_unknown_category_method(category_method) -> None: + with pytest.raises(ValueError, match="category_method"): + ContextualOrchestratorJudge(_FakeOrchestrator(_payload())).judge( + task="task", + answer="answer", + criteria=CRITERIA, + category_method=category_method, + ) + + +def test_judge_rejects_unhashable_mode_before_membership() -> None: + with pytest.raises(ValueError, match="mode must be"): + ContextualOrchestratorJudge(_FakeOrchestrator(_payload()), mode=[]) + + def test_category_judgment_rejects_non_integral_categories() -> None: payload = json.dumps({ "score": 0.5, @@ -245,6 +357,40 @@ def test_category_judgment_rejects_non_integral_categories() -> None: ) +def test_category_count_and_category_values_reject_runtime_subclasses() -> None: + class _ForgedInt(int): + def __le__(self, other): + return True + + def __ge__(self, other): + return True + + judge = ContextualOrchestratorJudge(_FakeOrchestrator(_category_payload())) + for value in (True, 1.0, 65, 10**1000, _ForgedInt(10**1000)): + with pytest.raises(ValueError, match="category_count must be an integer"): + judge.judge( + task="task", + answer="answer", + criteria=CRITERIA, + category_count=value, + ) + + result = judge.judge( + task="task", + answer="answer", + criteria=CRITERIA, + category_count=5, + ) + with pytest.raises(JudgeFormatError, match="criterion_categories"): + replace( + result, + criterion_categories={ + "task_alignment": _ForgedInt(10**1000), + "factual_support": 1, + }, + ).to_irt_row() + + def test_category_judgment_rejects_malformed_top_level_score() -> None: payload = json.dumps({ "score": {"factual_support": 0.8}, @@ -261,6 +407,92 @@ def test_category_judgment_rejects_malformed_top_level_score() -> None: ) +def test_judge_rejects_overflowing_and_runtime_subclass_scores() -> None: + class _HookedFloat(float): + invoked = False + + def __float__(self): + type(self).invoked = True + return super().__float__() + + overflowing = json.dumps({ + "score": 10**1000, + "accepted": True, + "rationale": "unsupported numeric shape", + "criterion_scores": {"task_alignment": 0.8, "factual_support": 0.8}, + }) + with pytest.raises(JudgeFormatError, match="score must be a number"): + ContextualOrchestratorJudge(_FakeOrchestrator(overflowing)).judge( + task="task", + answer="answer", + criteria=CRITERIA, + ) + + subclass_score = _HookedFloat(0.8) + result = ContextualOrchestratorJudge(_FakeOrchestrator(_payload())).judge( + task="task", + answer="answer", + criteria=CRITERIA, + ) + with pytest.raises(JudgeFormatError, match="criterion_scores"): + replace( + result, + criterion_scores={ + "task_alignment": subclass_score, + "factual_support": 0.8, + }, + ).to_irt_row(item_type="dichotomous") + assert _HookedFloat.invoked is False + + +def test_judge_text_and_usage_boundaries_reject_runtime_subclasses() -> None: + class _HookedString(str): + invoked = False + + def strip(self, *args, **kwargs): + type(self).invoked = True + return super().strip(*args, **kwargs) + + class _ForgedInt(int): + invoked = False + + def __ge__(self, other): + type(self).invoked = True + return True + + with pytest.raises(ValueError, match="task must be"): + ContextualOrchestratorJudge(_FakeOrchestrator(_payload())).judge( + task=_HookedString("task"), + answer="answer", + criteria=CRITERIA, + ) + assert _HookedString.invoked is False + + forged = _ForgedInt(7) + result = _CompletionOrchestrator({ + "mode": "route", + "answer": _payload(), + "trace": [{ + "usage": { + "prompt_tokens": forged, + "completion_tokens": forged, + "total_tokens": forged, + } + }], + }) + judged = ContextualOrchestratorJudge(result).judge( + task="task", + answer="answer", + criteria=CRITERIA, + ) + assert dict(judged.usage) == { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + } + assert _ForgedInt.invoked is False + + def test_judge_rejects_missing_or_malformed_model_fields() -> None: cases = [ {}, @@ -320,6 +552,25 @@ def __float__(self): assert _HookedFloat.invoked is False +def test_judge_rejects_unhashable_criterion_id_before_category_template() -> None: + class _UnhashableStr(str): + __hash__ = None + + with pytest.raises(ValueError, match="criterion_id must be a string"): + ContextualOrchestratorJudge(_FakeOrchestrator(_payload())).judge( + task="task", + answer="answer", + criteria=[ + { + "criterion_id": _UnhashableStr("task_alignment"), + "description": "ok", + }, + CRITERIA[1], + ], + category_count=3, + ) + + def test_judge_criteria_reject_non_contract_values_with_value_error() -> None: """Arbitrary criterion elements must fail through the stable benign error contract.""" with pytest.raises(ValueError, match="JudgeCriterion or mapping"): @@ -372,4 +623,3 @@ def test_judge_accepts_bounded_json_nesting() -> None: criteria=CRITERIA, ) assert result.score == 0.8 - diff --git a/tests/test_llm_judge_description_boundary.py b/tests/test_llm_judge_description_boundary.py new file mode 100644 index 000000000..45e21c061 --- /dev/null +++ b/tests/test_llm_judge_description_boundary.py @@ -0,0 +1,70 @@ +"""Regressions for exact built-in LLM-judge trust-boundary values.""" + +import json + +import pytest + +from fast_mlsirm.llm_judge import ContextualOrchestratorJudge, JudgeCriterion + + +def test_criterion_description_rejects_runtime_string_subclass_before_hooks() -> None: + """Descriptions reject string subclasses before invoking caller hooks.""" + + class _HookedString(str): + invoked = False + + def strip(self, *args, **kwargs): + type(self).invoked = True + return super().strip(*args, **kwargs) + + with pytest.raises(ValueError, match="criterion description must be a string"): + JudgeCriterion("task_alignment", _HookedString("observable evidence")) + assert _HookedString.invoked is False + + +def test_trace_rejects_runtime_list_subclass_before_hooks() -> None: + """Provider trace subclasses cannot execute iteration or length hooks.""" + + class _HookedTrace(list): + invoked = False + + def __iter__(self): + type(self).invoked = True + return super().__iter__() + + def __len__(self): + type(self).invoked = True + return super().__len__() + + trace = _HookedTrace( + [{"usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}}] + ) + answer = json.dumps( + { + "score": 0.8, + "accepted": True, + "rationale": "bounded evidence", + "criterion_scores": {"task_alignment": 0.8, "factual_support": 0.8}, + } + ) + + class _Orchestrator: + def complete(self, messages, mode="auto"): + return {"mode": "route", "answer": answer, "trace": trace} + + result = ContextualOrchestratorJudge(_Orchestrator()).judge( + task="task", + answer="answer", + criteria=( + JudgeCriterion("task_alignment", "observable evidence"), + JudgeCriterion("factual_support", "supported claims"), + ), + ) + + assert result.trace_step_count == 0 + assert dict(result.usage) == { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + } + assert _HookedTrace.invoked is False