diff --git a/.adr-config.yml b/.adr-config.yml new file mode 100644 index 000000000..2341f3381 --- /dev/null +++ b/.adr-config.yml @@ -0,0 +1,6 @@ +project_slug: contextual-orchestrator +owner: ContextualWisdomLab +default_status: proposed +decision_id_format: NNNN +template_source: madr-v4 +last_decision_id: 0009 diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 7e53db87c..d1d3b446c 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -7,6 +7,8 @@ updates: day: monday time: "04:00" timezone: Asia/Seoul + cooldown: + default-days: 7 open-pull-requests-limit: 5 - package-ecosystem: pip @@ -16,4 +18,6 @@ updates: day: monday time: "04:30" timezone: Asia/Seoul + cooldown: + default-days: 7 open-pull-requests-limit: 5 diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index cfa62adad..79e479384 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -53,10 +53,11 @@ jobs: persist-credentials: false - name: Set up Python - # Atheris ships wheels/builds cleanly for 3.11 (Clang + libFuzzer). + # Atheris 3.1.0 dropped the cp311 wheel; use the same cp312-compatible + # pin that the central CPython 3.14 coverage-evidence image preflights. uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 with: - python-version: "3.11" + python-version: "3.12" - name: Install Atheris # Hash-pinned per OpenSSF Scorecard Pinned-Dependencies. Sources: @@ -72,16 +73,19 @@ jobs: fi - name: Fuzz request-body parser - run: python fuzz/fuzz_request_body.py -max_total_time=${FUZZ_SECONDS} -artifact_prefix=crash- fuzz/corpus/request_body + run: python fuzz/fuzz_request_body.py -max_total_time="${FUZZ_SECONDS}" -artifact_prefix=crash- fuzz/corpus/request_body - name: Fuzz agent-config parser - run: python fuzz/fuzz_agent_config.py -max_total_time=${FUZZ_SECONDS} -artifact_prefix=crash- fuzz/corpus/agent_config + run: python fuzz/fuzz_agent_config.py -max_total_time="${FUZZ_SECONDS}" -artifact_prefix=crash- fuzz/corpus/agent_config - name: Fuzz secret redaction - run: python fuzz/fuzz_redaction.py -max_total_time=${FUZZ_SECONDS} -artifact_prefix=crash- fuzz/corpus/redaction + run: python fuzz/fuzz_redaction.py -max_total_time="${FUZZ_SECONDS}" -artifact_prefix=crash- fuzz/corpus/redaction - name: Fuzz orchestration engine - run: python fuzz/fuzz_orchestration.py -max_total_time=${FUZZ_SECONDS} -artifact_prefix=crash- fuzz/corpus/orchestration + run: python fuzz/fuzz_orchestration.py -max_total_time="${FUZZ_SECONDS}" -artifact_prefix=crash- fuzz/corpus/orchestration + + - name: Fuzz model-judge response parser + run: python fuzz/fuzz_model_judge.py -max_total_time="${FUZZ_SECONDS}" -artifact_prefix=crash- fuzz/corpus/judge - name: Upload crash artifacts if: failure() diff --git a/CLAUDE.md b/CLAUDE.md index f893b5f7b..b9d0d840b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -44,8 +44,9 @@ python fuzz/fuzz_request_body.py -max_total_time=60 fuzz/corpus/request_body python -m contextual_orchestrator "your prompt" --agents examples/agents.mock.json # Serve the OpenAI-compatible API + /admin console -export CONTEXTUAL_ORCHESTRATOR_TOKEN="$(python -c 'import secrets; print(secrets.token_urlsafe(32))')" -python -m contextual_orchestrator --serve --agents examples/agents.mock.json --port 8000 +local_token="$(python -c 'import secrets; print(secrets.token_urlsafe(32))')" +python -m contextual_orchestrator --serve --agents examples/agents.mock.json --port 8000 \ + --auth-token "$local_token" # Loopback-only local dev server (auth disabled; loopback only) ./.superset/run.sh diff --git a/Dockerfile b/Dockerfile index c2e50c624..478b5356c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,10 +2,11 @@ # tree on a slim Python base. Runs the OpenAI-compatible server. # # Build: docker build -t contextual-orchestrator . -# Run : docker run --rm -p 8000:8000 \ -# -e CONTEXTUAL_ORCHESTRATOR_TOKEN=change-me \ -# -e OPENAI_API_KEY=sk-... \ -# contextual-orchestrator +# Run : seed CONTEXTUAL_ORCHESTRATOR_TOKEN and provider credentials into the KV +# registry first, then use: +# docker run --rm -p 8000:8000 contextual-orchestrator +# Runtime secrets are never passed through the container environment or argv; +# see docs/kv-credentials.md for the bootstrap flow. # Agents: defaults to the bundled mock pool; mount your own and set AGENTS_FILE: # -v ./agents.json:/app/agents.json -e AGENTS_FILE=/app/agents.json # python:3.12-slim @@ -26,4 +27,4 @@ HEALTHCHECK --interval=30s --timeout=3s --start-period=5s \ CMD ["python", "-c", "import urllib.request,os;urllib.request.urlopen(f'http://127.0.0.1:{os.environ.get(\"PORT\",\"8000\")}/healthz', timeout=2)"] # --allow-public-bind: 컨테이너 내부 0.0.0.0 바인딩 필요(외부 노출은 호스트 포트 매핑이 결정) -CMD ["sh", "-c", "python -m contextual_orchestrator --serve --agents \"$AGENTS_FILE\" --host 0.0.0.0 --port \"$PORT\" --allow-public-bind"] +CMD ["sh", "-c", "python -m contextual_orchestrator --serve --agents \"$AGENTS_FILE\" --host 0.0.0.0 --port \"$PORT\" --allow-public-bind --auth-token-key CONTEXTUAL_ORCHESTRATOR_TOKEN"] diff --git a/README.md b/README.md index 65f57dd4c..0ea85d04f 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,11 @@ Stdlib Python lab for a single API that routes, delegates, verifies, and synthesizes work across a configurable pool of OpenAI-compatible model agents. -This is not a Sakana AI product or a reproduction of their trained models. It is a small implementation of the public architecture pattern: expose one model-like interface while keeping the agent pool, routing, workflow, and verification logic behind it. +This is not a Sakana AI product or a reproduction of their trained models. It is +a small implementation of the public architecture pattern: expose one +model-like orchestration candidate while keeping the worker pool, routing, +workflow, and verification logic behind it. `contextual-orchestrator` is the +public control-plane model; it is not just an HTTP gateway. ## Quick Start @@ -17,8 +21,9 @@ python -m contextual_orchestrator "Summarize why model orchestration helps long Run the OpenAI-compatible subset: ```bash -export CONTEXTUAL_ORCHESTRATOR_TOKEN="$(python -c 'import secrets; print(secrets.token_urlsafe(32))')" -python -m contextual_orchestrator --serve --agents examples/agents.mock.json --port 8000 +local_token="$(python -c 'import secrets; print(secrets.token_urlsafe(32))')" +python -m contextual_orchestrator --serve --agents examples/agents.mock.json --port 8000 \ + --auth-token "$local_token" ``` Admin console: @@ -29,14 +34,15 @@ http://127.0.0.1:8000/admin ```bash curl -s http://127.0.0.1:8000/v1/chat/completions \ - -H "authorization: Bearer $CONTEXTUAL_ORCHESTRATOR_TOKEN" \ + -H "authorization: Bearer $local_token" \ -H 'content-type: application/json' \ -d '{"model":"contextual-orchestrator","messages":[{"role":"user","content":"Analyze this code review task and verify the answer."}]}' | jq . ``` HTTP serving is hardened for local lab use: -- `/admin`, `/admin/state`, `/api/v1/*`, and `/v1/chat/completions` require a Bearer token. Use `--admin-token` and `--inference-token` to separate operator and runtime access, or `--auth-token` / `CONTEXTUAL_ORCHESTRATOR_TOKEN` for one local-development token. +- `/admin`, `/admin/state`, `/api/v1/*`, and `/v1/chat/completions` require a Bearer token. Use `--admin-token-key` and `--inference-token-key` to resolve split tokens from the KV, or `--auth-token-key` for one token. Explicit `--auth-token`/split-token values are local-development escape hatches; the CLI no longer reads auth secrets from environment variables. +- A production deployment that uses the ecosystem identity plane must inject a reviewed `bearer_verifier` into `SecurityConfig` to validate Keyverse-issued OIDC tokens (issuer, audience, signature, expiry, and scope). The core does not hand-roll JWT parsing or hold Keycloak admin credentials; a static bearer token is not a Keyverse integration. - Binding to `0.0.0.0` or `::` requires `--allow-public-bind`. - JSON request bodies, chat message roles, orchestration modes, body sizes, request rate, and concurrent run counts are validated before orchestration runs. - Full orchestration traces are not returned by default. Set `include_orchestration_trace: true` per chat request or start with `--expose-trace-by-default` when the caller is trusted. @@ -60,6 +66,36 @@ Use real workers by replacing `mock://` agents with OpenAI-compatible endpoints. } ``` +For a local `mlx-lm` OpenAI-compatible server, use the explicit `mlx://` scheme. It is loopback-only, does not require a credential, and is translated to HTTP only after the loopback check: + +```json +{ + "agents": [ + { + "id": "local_fast_agent", + "model": "mlx-community/llama-3.2-3b-instruct-4bit", + "base_url": "mlx://127.0.0.1:8080/v1", + "provider_name": "mlx-lm", + "tags": ["reasoning", "coding", "verification"] + } + ] +} +``` + +The full local candidate registry is [examples/agents.local.json](examples/agents.local.json). +It contains the public `contextual-orchestrator` candidate, discovered MLX +worker models, and every discovered llama.cpp/LM Studio candidate. Discovery +does not decide governance state: seed candidates are enabled by default, while +`disabled` is reserved for an explicit operator/admin quarantine or a persisted +removal tombstone. The contextual-orchestrator record is excluded from internal +roles because this implementation has no bounded recursive self-call protocol; +that is a routing safety constraint, not a disabled candidate. The registry is +explicit; runtime discovery does not silently change the pool. + +Run an evaluation against that server with `--temperature 0` for repeatable judging. For reasoning-capable mlx models, pass `--chat-template-args '{"enable_thinking":false}'` when a short structured judge response is required. `--local-concurrency N` enables bounded concurrent local batch requests (`1..64`; the current measured starting point for this server is `8`); when serving HTTP, set `--max-concurrent-runs N` explicitly as well if the measured batch concurrency exceeds the secure default of `8`. Keep interactive route/conduct requests on the default sequential path. + +Model-based conduct verification requires `fast-mlsirm` in the same runtime and fails closed when it is absent or broken; fast-mlsirm sends its judge completion through this contextual-orchestrator gateway, so no direct provider fallback is used. “Same runtime” means that the exact interpreter used for the live run can import both packages: install both checkouts into one environment (prefer editable installs), or expose both source roots with `PYTHONPATH` during a source run. Before a live judge benchmark, run `python -m contextual_orchestrator check-fast-mlsirm` with that exact interpreter. It prints the interpreter, package version, transitive-import status, and contextual contract check, and exits nonzero on a missing dependency or contract mismatch. Do not run the preflight in one virtual environment and the judge in another. See [ADR 0001](docs/planning/adrs/0001-fail-closed-model-judgment.md). + The agent pool is manageable at runtime: `POST`/`PATCH`/`DELETE` on `/api/v1/agent_pools/default/worker_agents[/{id}]` add, govern, and remove model-group members. Pass `--agents-db PATH` (or `CONTEXTUAL_ORCHESTRATOR_AGENTS_DB`) to persist those changes to a stdlib sqlite file — stored changes overlay the seed agents file at startup, and removals write disabled tombstones so they survive restarts; without it the pool is in-memory as before. Seed the credential into the KV once at bootstrap: @@ -68,7 +104,13 @@ Seed the credential into the KV once at bootstrap: echo "$OPENAI_API_KEY" | python -m contextual_orchestrator register-credential --name OPENAI_API_KEY --value-stdin ``` -Non-mock providers must use `https://` URLs and a **resolvable KV credential** — a non-mock agent whose credential is missing raises `NotConfigured` rather than falling back to an environment variable. The runtime blocks loopback, private, link-local, multicast, and reserved provider addresses before sending a key. Set `CONTEXTUAL_ORCHESTRATOR_ALLOWED_PROVIDER_HOSTS` to a comma-separated host allowlist when only approved model gateways should be reachable. External calls use a timeout and default output token cap. +For a persistent KV-backed server token, seed a credential such as +`CONTEXTUAL_ORCHESTRATOR_TOKEN` and start with +`--auth-token-key CONTEXTUAL_ORCHESTRATOR_TOKEN`. The in-memory credential +backend is process-local and is suitable only for tests; production auth +registration and OIDC client secrets belong to the deployment/KV boundary. + +Non-mock providers must use `https://` URLs and a **resolvable KV credential** — a non-mock agent whose credential is missing raises `NotConfigured` rather than falling back to an environment variable. The runtime blocks loopback, private, link-local, multicast, and reserved provider addresses before sending a key. Pass `--allowed-provider-host HOST` once per approved gateway when an explicit host allowlist is required; it is bound at client construction and is not changed by request-time environment variables. External calls use a timeout and default output token cap. > The legacy `api_key_env` field is still accepted for back-compat, but its value is now treated as the **credential name** in the KV, not as an environment variable to read. This supersedes the old `api_key_env` env pattern. @@ -76,6 +118,7 @@ Non-mock providers must use `https://` URLs and a **resolvable KV credential** One public interface: +- `contextual-orchestrator` is the model-like control-plane candidate exposed to callers. `/v1/models` lists it first, followed by every configured worker candidate, including disabled candidates with their status. - `/v1/chat/completions` accepts normal chat messages, and `"stream": true` returns an OpenAI-compatible `text/event-stream` of `chat.completion.chunk` deltas terminated by `data: [DONE]`. In **route** mode the worker's tokens are streamed live as they arrive from the provider (real token streaming); in **conduct** mode the multi-step answer is produced then framed as deltas (a workflow can't honestly token-stream a synthesizer that hasn't run yet). - `TaskOrchestrator.complete()` decides whether to route to one worker or run a short workflow. - `TaskOrchestrator.compare_to_baseline(prompts, mode)` (CLI `--eval PROMPT...`) measures the orchestration engine against a single-worker baseline — per-prompt and aggregate latency plus a structural coverage delta (contributing steps + verifier-pass presence). It is a measured tradeoff report, not a human-quality claim. @@ -125,7 +168,7 @@ Local spend observability, aggregated from in-memory workflow runs. It is honest ```bash curl -s http://127.0.0.1:8000/api/v1/spend_analytics/latest \ - -H "authorization: Bearer $CONTEXTUAL_ORCHESTRATOR_TOKEN" | jq '.totals, .by_model, .budget' + -H "authorization: Bearer $local_token" | jq '.totals, .by_model, .budget' ``` - **Tokens.** `by_model[].output_tokens` uses the provider-reported `usage.completion_tokens` when a real worker returns it, and falls back to a `~4 chars/token` estimate otherwise. Each row carries `usage_source`: `reported` (all steps reported), `mixed`, or `estimated`. `estimated_output_tokens` is always the estimate, kept alongside for comparison. `measurement_status` is `local_runtime_estimate`, not production telemetry. @@ -192,7 +235,10 @@ is read from a **KV config store**, never `os.getenv`. backend (local in-process backend standalone), and records one usage-ledger row per original vector with the full attribution dimensions (service, team, group, company, provider) carried in `metadata`. -- **Health.** `GET /healthz` is an unauthenticated liveness probe. +- **Health.** `GET /healthz` is an unauthenticated liveness probe; it never + claims that an upstream chat worker is serving. Admins can use + `GET /api/v1/provider_readiness/latest?refresh=true` for one bounded, + non-retrying chat probe per enabled worker. - **Standalone + optional pg-llm-batch integration.** The hub runs standalone with the in-memory config store and local batch backend; wiring a Postgres DSN and an installed/deployed `pg_llm_batch` client activates the KV/secret stores, diff --git a/contextual_orchestrator/__main__.py b/contextual_orchestrator/__main__.py index 5f68c3b74..d30ece403 100644 --- a/contextual_orchestrator/__main__.py +++ b/contextual_orchestrator/__main__.py @@ -6,11 +6,124 @@ import json import os import sys +from dataclasses import replace -from .credentials import register_credential -from .orchestrator import ModelClient, TaskOrchestrator, load_agents +from .credentials import get_credential, register_credential +from .orchestrator import ( + CONTEXTUAL_ORCHESTRATOR_CONTRACT_V1, + MAX_LOCAL_CONCURRENCY, + ModelClient, + TaskOrchestrator, + load_agents, +) from .server import SecurityConfig, serve +DEFAULT_AUTH_TOKEN_KEY = "CONTEXTUAL_ORCHESTRATOR_TOKEN" +DEFAULT_ADMIN_TOKEN_KEY = "CONTEXTUAL_ORCHESTRATOR_ADMIN_TOKEN" +DEFAULT_INFERENCE_TOKEN_KEY = "CONTEXTUAL_ORCHESTRATOR_INFERENCE_TOKEN" + + +def _positive_int(value: str) -> int: + """Parse a strictly positive integer for an argparse option.""" + try: + parsed = int(value) + except ValueError as exc: + raise argparse.ArgumentTypeError("positive integer required") from exc + if parsed < 1: + raise argparse.ArgumentTypeError("positive integer required") + return parsed + + +def _local_concurrency(value: str) -> int: + """Parse a bounded local batch concurrency value.""" + parsed = _positive_int(value) + if parsed > MAX_LOCAL_CONCURRENCY: + raise argparse.ArgumentTypeError( + f"integer in 1..{MAX_LOCAL_CONCURRENCY} required" + ) + return parsed + + +def _json_object(value: str) -> dict[str, object]: + """Parse a JSON object for an argparse option, rejecting other JSON values.""" + try: + parsed = json.loads(value) + except json.JSONDecodeError as exc: + raise argparse.ArgumentTypeError("valid JSON object required") from exc + if not isinstance(parsed, dict): + raise argparse.ArgumentTypeError("JSON object required") + return parsed + + +def _resolve_auth_token(explicit: str, credential_name: str) -> str: + """Resolve a server bearer token from an explicit local value or the KV.""" + if explicit: + return explicit + token = get_credential(credential_name) + if not token: + raise ValueError(f"server auth credential '{credential_name}' is not configured in the KV") + return token + + +def _fast_mlsirm_runtime_status() -> tuple[dict[str, object], bool]: + """Report whether this interpreter can load the required judge contract.""" + status: dict[str, object] = { + "python": sys.executable, + "package": "fast-mlsirm", + } + try: + import fast_mlsirm + from fast_mlsirm import ( + CONTEXTUAL_ORCHESTRATOR_CONTRACT_V1 as fast_contract, + ContextualOrchestratorJudge, + JudgeCriterion, + JudgeFormatError, + ) + except ModuleNotFoundError as exc: + status.update( + { + "available": False, + "reason": "missing_dependency", + "missing_module": exc.name or "unknown", + } + ) + return status, False + except Exception as exc: # noqa: BLE001 - diagnostic command must fail closed + status.update( + { + "available": False, + "reason": "import_error", + "error_type": type(exc).__name__, + } + ) + return status, False + + checks = { + "judge_symbols": all( + callable(symbol) + for symbol in (ContextualOrchestratorJudge, JudgeCriterion, JudgeFormatError) + ), + "contextual_contract": fast_contract == CONTEXTUAL_ORCHESTRATOR_CONTRACT_V1, + } + available = all(checks.values()) + status.update( + { + "available": available, + "version": getattr(fast_mlsirm, "__version__", "unknown"), + "contract": fast_contract, + "checks": checks, + } + ) + return status, available + + +def _check_fast_mlsirm_command() -> None: + """Validate the same-interpreter fast-mlsirm integration boundary.""" + status, available = _fast_mlsirm_runtime_status() + print(json.dumps(status, ensure_ascii=False, sort_keys=True)) + if not available: + raise SystemExit(1) + def _register_credential_command(argv: list[str]) -> None: """Bootstrap: read a deploy-time secret and store it in the KV credential registry. @@ -60,6 +173,9 @@ def main() -> None: if len(sys.argv) > 1 and sys.argv[1] == "register-credential": _register_credential_command(sys.argv[2:]) return + if len(sys.argv) > 1 and sys.argv[1] == "check-fast-mlsirm": + _check_fast_mlsirm_command() + return parser = argparse.ArgumentParser(description="Route or conduct chat requests across model agents.") parser.add_argument("prompt", nargs="?", help="User prompt for CLI mode.") @@ -70,9 +186,15 @@ def main() -> None: parser.add_argument("--serve", action="store_true", help="Run the chat completions HTTP server.") parser.add_argument("--host", default="127.0.0.1") parser.add_argument("--port", type=int, default=8000) - parser.add_argument("--auth-token", default=os.environ.get("CONTEXTUAL_ORCHESTRATOR_TOKEN", "")) - parser.add_argument("--admin-token", default=os.environ.get("CONTEXTUAL_ORCHESTRATOR_ADMIN_TOKEN", "")) - parser.add_argument("--inference-token", default=os.environ.get("CONTEXTUAL_ORCHESTRATOR_INFERENCE_TOKEN", "")) + parser.add_argument("--auth-token", default="", help="Explicit local-development bearer token; prefer a KV token name.") + parser.add_argument("--admin-token", default="", help="Explicit local-development admin token; prefer a KV token name.") + parser.add_argument("--inference-token", default="", help="Explicit local-development inference token; prefer a KV token name.") + parser.add_argument("--auth-token-key", default=None, + help="KV credential name for the single server bearer token.") + parser.add_argument("--admin-token-key", default=None, + help="KV credential name for the admin bearer token.") + parser.add_argument("--inference-token-key", default=None, + help="KV credential name for the inference bearer token.") parser.add_argument("--allow-public-bind", action="store_true") parser.add_argument("--insecure-disable-auth", action="store_true", help="Deprecated; API auth is always required.") parser.add_argument("--expose-trace-by-default", action="store_true") @@ -82,8 +204,28 @@ def main() -> None: help="Optional sqlite path so runtime agent-pool changes (add/patch/remove) survive restarts.") parser.add_argument("--provider-ca-bundle", default=os.environ.get("CONTEXTUAL_ORCHESTRATOR_PROVIDER_CA_BUNDLE") or None, help="Path to a CA bundle used to verify provider TLS (e.g. a corporate gateway root).") - parser.add_argument("--insecure-skip-tls-verify", action="store_true", - help="Dev only: do not verify provider TLS certificates (insecure).") + parser.add_argument("--allowed-provider-host", action="append", dest="allowed_provider_hosts", default=None, + help="Explicit remote provider host allowlist; repeat for multiple hosts (default: unrestricted public hosts).") + parser.add_argument( + "--sampling-temperature", + "--temperature", + dest="sampling_temperature", + type=float, + default=0.2, + help="Default provider sampling temperature (default: 0.2; --temperature is a compatibility alias).", + ) + parser.add_argument("--max-output-tokens", type=int, default=2048, + help="Default provider output token cap (default: 2048).") + parser.add_argument("--local-concurrency", type=_local_concurrency, default=1, + help=f"Concurrent requests for explicit mlx:// local batch work (default: 1; maximum: {MAX_LOCAL_CONCURRENCY}).") + parser.add_argument("--max-concurrent-runs", type=_local_concurrency, default=8, + help=f"Maximum simultaneous HTTP orchestration runs (default: 8; maximum: {MAX_LOCAL_CONCURRENCY}).") + parser.add_argument("--route-text-length-threshold", type=_positive_int, default=None, + help="Auto-mode minimum prompt length that can trigger conduct instead of route.") + parser.add_argument("--conduct-hint-threshold", type=_positive_int, default=None, + help="Auto-mode hint-count minimum that can trigger conduct instead of route.") + parser.add_argument("--chat-template-args", type=_json_object, default={}, + help="JSON kwargs forwarded to local mlx-lm chat templates, e.g. '{\"enable_thinking\":false}'.") parser.add_argument("--budget-max-output-tokens", type=int, default=None, help="Refuse new runs once estimated/reported output tokens reach this cap (default: no cap).") parser.add_argument("--budget-max-cost-usd", type=float, default=None, @@ -94,7 +236,14 @@ def main() -> None: help="Measure orchestration vs a single-worker baseline on these prompts and print the report.") args = parser.parse_args() - client = ModelClient(ca_bundle=args.provider_ca_bundle, verify_tls=not args.insecure_skip_tls_verify) + client = ModelClient( + ca_bundle=args.provider_ca_bundle, + temperature=args.sampling_temperature, + max_output_tokens=args.max_output_tokens, + local_concurrency=args.local_concurrency, + chat_template_args=args.chat_template_args, + allowed_provider_hosts=args.allowed_provider_hosts, + ) orchestrator = TaskOrchestrator( load_agents(args.agents), client=client, @@ -105,28 +254,62 @@ def main() -> None: cache_ttl=args.cache_ttl, ) + if args.conduct_hint_threshold is not None or args.route_text_length_threshold is not None: + overrides: dict[str, int] = {} + if args.conduct_hint_threshold is not None: + overrides["conduct_hint_threshold"] = args.conduct_hint_threshold + if args.route_text_length_threshold is not None: + overrides["route_text_length_threshold"] = args.route_text_length_threshold + orchestrator.policy = replace(orchestrator.policy, **overrides) + if args.eval: print(json.dumps(orchestrator.compare_to_baseline(args.eval, mode=args.mode), ensure_ascii=False, indent=2)) return if args.serve: - if not (args.auth_token or args.admin_token or args.inference_token): + single_requested = bool(args.auth_token or args.auth_token_key) + split_requested = bool( + args.admin_token or args.inference_token or args.admin_token_key or args.inference_token_key + ) + if single_requested and split_requested: + parser.error("choose either --auth-token or the split --admin-token/--inference-token mode") + if split_requested and not ( + (args.admin_token or args.admin_token_key) and (args.inference_token or args.inference_token_key) + ): parser.error( - "--serve requires --auth-token, split --admin-token/--inference-token, " - "or matching CONTEXTUAL_ORCHESTRATOR_* environment variables" + "split token mode requires admin and inference tokens, " + "provided by --admin-token/--inference-token or " + "--admin-token-key/--inference-token-key" ) - if not args.auth_token and (args.admin_token or args.inference_token) and not ( - args.admin_token and args.inference_token - ): - parser.error("split token mode requires both --admin-token and --inference-token") + try: + auth_token = ( + _resolve_auth_token(args.auth_token, args.auth_token_key or DEFAULT_AUTH_TOKEN_KEY) + if not split_requested + else "" + ) + admin_token = ( + _resolve_auth_token(args.admin_token, args.admin_token_key or DEFAULT_ADMIN_TOKEN_KEY) + if split_requested + else "" + ) + inference_token = ( + _resolve_auth_token(args.inference_token, args.inference_token_key or DEFAULT_INFERENCE_TOKEN_KEY) + if split_requested + else "" + ) + except ValueError as exc: + parser.error(str(exc)) + if not (auth_token or admin_token or inference_token): + parser.error("--serve requires a KV auth credential or explicit local token") serve( orchestrator, host=args.host, port=args.port, security=SecurityConfig( - auth_token=args.auth_token, - admin_token=args.admin_token, - inference_token=args.inference_token, + auth_token=auth_token, + admin_token=admin_token, + inference_token=inference_token, + max_concurrent_runs=args.max_concurrent_runs, allow_public_bind=args.allow_public_bind, expose_trace_by_default=args.expose_trace_by_default, ), diff --git a/contextual_orchestrator/api_contract.py b/contextual_orchestrator/api_contract.py index fae9fba0b..364294e3a 100644 --- a/contextual_orchestrator/api_contract.py +++ b/contextual_orchestrator/api_contract.py @@ -71,6 +71,20 @@ "responses": {"200": {"description": "Active policy"}}, } }, + "/api/v1/provider_readiness/latest": { + "get": { + "operationId": "get_latest_provider_readiness", + "summary": "Read or explicitly refresh bounded provider chat readiness", + "security": [{"admin_bearer_auth": []}], + "parameters": [{ + "name": "refresh", + "in": "query", + "required": False, + "schema": {"type": "boolean", "default": False}, + }], + "responses": {"200": {"description": "Provider readiness report"}}, + } + }, "/api/v1/analytics_snapshots/latest": { "get": { "operationId": "get_latest_analytics_snapshot", diff --git a/contextual_orchestrator/batch_routing.py b/contextual_orchestrator/batch_routing.py index d07a48d25..1f4fdab2f 100644 --- a/contextual_orchestrator/batch_routing.py +++ b/contextual_orchestrator/batch_routing.py @@ -21,13 +21,13 @@ from __future__ import annotations import asyncio -from dataclasses import dataclass, field import hashlib import json import time -from typing import Any, Callable, Dict, List, Optional, Protocol import uuid - +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, field +from typing import Any, Callable, Dict, List, Optional, Protocol _ROUTING_CATEGORY = "routing" @@ -225,26 +225,36 @@ class LocalBatchBackend: name = "local" - def __init__(self, runner: Callable[[List[Dict[str, str]], str], Dict[str, Any]]) -> None: + def __init__( + self, + runner: Callable[[List[Dict[str, str]], str], Dict[str, Any]], + *, + max_concurrency: int = 1, + ) -> None: + if type(max_concurrency) is not int or max_concurrency < 1: + raise ValueError("max_concurrency must be a positive integer") self._runner = runner + self.max_concurrency = max_concurrency self._results: Dict[str, List[BatchResultItem]] = {} def submit(self, requests: List[BatchRequest], metadata: Optional[Dict[str, Any]] = None) -> BatchJob: """Run every request in-process and stash the results under a job id.""" job_id = f"localbatch_{uuid.uuid4().hex}" - items: List[BatchResultItem] = [] - for request in requests: + def run(request: BatchRequest) -> BatchResultItem: result = self._runner(request.messages, request.mode) answer = result.get("answer", "") - items.append( - BatchResultItem( - custom_id=request.custom_id, - answer=answer, - attribution=dict(request.attribution), - model=request.model, - mode=result.get("mode", request.mode), - ) + return BatchResultItem( + custom_id=request.custom_id, + answer=answer, + attribution=dict(request.attribution), + model=request.model, + mode=result.get("mode", request.mode), ) + if self.max_concurrency == 1 or len(requests) <= 1: + items = [run(request) for request in requests] + else: + with ThreadPoolExecutor(max_workers=min(self.max_concurrency, len(requests))) as pool: + items = list(pool.map(run, requests)) self._results[job_id] = items return BatchJob(job_id=job_id, backend=self.name, status="completed", request_count=len(requests)) diff --git a/contextual_orchestrator/cost_ledger.py b/contextual_orchestrator/cost_ledger.py index d3943c5be..537b9bc1b 100644 --- a/contextual_orchestrator/cost_ledger.py +++ b/contextual_orchestrator/cost_ledger.py @@ -553,6 +553,59 @@ def __len__(self) -> int: "cost_amount", "currency_code", ) +_DIMENSION_SELECT_SQL = { + "qmark": "SELECT 1 FROM cost_attribution_dimensions WHERE dimension_name = ?", + "pyformat": "SELECT 1 FROM cost_attribution_dimensions WHERE dimension_name = %s", +} +_DIMENSION_INSERT_SQL = { + "qmark": ( + "INSERT INTO cost_attribution_dimensions " + "(dimension_name, dimension_label, dimension_order) VALUES (?, ?, ?)" + ), + "pyformat": ( + "INSERT INTO cost_attribution_dimensions " + "(dimension_name, dimension_label, dimension_order) VALUES (%s, %s, %s)" + ), +} +_USAGE_INSERT_SQL = { + "qmark": ( + "INSERT INTO llm_usage_records " + "(usage_record_id, created_at, workflow_run_id, request_channel, " + "route_mode, provider_name, model_name, account_name, service_name, " + "upstream_api, team_name, group_name, company_name, prompt_tokens, " + "completion_tokens, total_tokens, cost_amount, currency_code) VALUES " + "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" + ), + "pyformat": ( + "INSERT INTO llm_usage_records " + "(usage_record_id, created_at, workflow_run_id, request_channel, " + "route_mode, provider_name, model_name, account_name, service_name, " + "upstream_api, team_name, group_name, company_name, prompt_tokens, " + "completion_tokens, total_tokens, cost_amount, currency_code) VALUES " + "(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)" + ), +} +_USAGE_SELECT_SQL = ( + "SELECT usage_record_id, created_at, workflow_run_id, request_channel, " + "route_mode, provider_name, model_name, account_name, service_name, " + "upstream_api, team_name, group_name, company_name, prompt_tokens, " + "completion_tokens, total_tokens, cost_amount, currency_code " + "FROM llm_usage_records" +) +_USAGE_QUERY_SQL = { + (style, has_start, has_end): query + for style, placeholder in (("qmark", "?"), ("pyformat", "%s")) + for has_start, has_end, query in ( + (False, False, _USAGE_SELECT_SQL), + (True, False, f"{_USAGE_SELECT_SQL} WHERE created_at >= {placeholder}"), + (False, True, f"{_USAGE_SELECT_SQL} WHERE created_at < {placeholder}"), + ( + True, + True, + f"{_USAGE_SELECT_SQL} WHERE created_at >= {placeholder} AND created_at < {placeholder}", + ), + ) +} class SqlLedgerStore: @@ -564,14 +617,13 @@ class SqlLedgerStore: """ def __init__(self, connection: Any, paramstyle: str = "qmark") -> None: + if paramstyle not in ("qmark", "pyformat"): + raise ValueError("paramstyle must be qmark or pyformat") self._conn = connection self._paramstyle = paramstyle self._create_schema() self._seed_dimension_catalog() - def _placeholder(self) -> str: - return "?" if self._paramstyle == "qmark" else "%s" - def _create_schema(self) -> None: cur = self._conn.cursor() for statement in SCHEMA_SQL.strip().split(";"): @@ -580,17 +632,15 @@ def _create_schema(self) -> None: self._conn.commit() def _seed_dimension_catalog(self) -> None: - ph = self._placeholder() cur = self._conn.cursor() for order, (name, label, _column) in enumerate(ATTRIBUTION_DIMENSION_CATALOG): cur.execute( - f"SELECT 1 FROM cost_attribution_dimensions WHERE dimension_name = {ph}", # nosec B608 - ph is a DB-API placeholder. + _DIMENSION_SELECT_SQL[self._paramstyle], (name,), ) if cur.fetchone() is None: cur.execute( - "INSERT INTO cost_attribution_dimensions " - f"(dimension_name, dimension_label, dimension_order) VALUES ({ph}, {ph}, {ph})", # nosec B608 - ph is a DB-API placeholder. + _DIMENSION_INSERT_SQL[self._paramstyle], (name, label, order), ) self._conn.commit() @@ -598,31 +648,25 @@ def _seed_dimension_catalog(self) -> None: def append(self, record: UsageRecord) -> None: """Insert a usage record row.""" row = record.as_dict() - ph = self._placeholder() - placeholders = ", ".join(ph for _ in _USAGE_COLUMNS) - columns = ", ".join(_USAGE_COLUMNS) cur = self._conn.cursor() cur.execute( - f"INSERT INTO llm_usage_records ({columns}) VALUES ({placeholders})", # nosec B608 - columns are fixed _USAGE_COLUMNS. + _USAGE_INSERT_SQL[self._paramstyle], tuple(row.get(column) for column in _USAGE_COLUMNS), ) self._conn.commit() def query(self, start: Optional[int] = None, end: Optional[int] = None) -> List[Dict[str, Any]]: """Return record rows in the optional half-open window.""" - ph = self._placeholder() - clauses: List[str] = [] params: List[Any] = [] if start is not None: - clauses.append(f"created_at >= {ph}") params.append(start) if end is not None: - clauses.append(f"created_at < {ph}") params.append(end) - where = f" WHERE {' AND '.join(clauses)}" if clauses else "" - columns = ", ".join(_USAGE_COLUMNS) cur = self._conn.cursor() - cur.execute(f"SELECT {columns} FROM llm_usage_records{where}", tuple(params)) # nosec B608 - columns and clauses are fixed. + cur.execute( + _USAGE_QUERY_SQL[(self._paramstyle, start is not None, end is not None)], + tuple(params), + ) return [dict(zip(_USAGE_COLUMNS, values)) for values in cur.fetchall()] diff --git a/contextual_orchestrator/cost_router.py b/contextual_orchestrator/cost_router.py index bfbe159db..b1ab5df2b 100644 --- a/contextual_orchestrator/cost_router.py +++ b/contextual_orchestrator/cost_router.py @@ -68,9 +68,15 @@ def __init__( build_token_counter(postgres_dsn) if postgres_dsn else HeuristicTokenCounter() ) self.policy = routing_policy or RoutingPolicy(self.config) - self.batch_backend: BatchBackend = batch_backend or LocalBatchBackend( - runner=lambda messages, mode: orchestrator.complete(messages, mode=mode) - ) + if batch_backend is None: + client = getattr(orchestrator, "client", None) + local_concurrency = getattr(client, "local_concurrency", 1) + self.batch_backend = LocalBatchBackend( + runner=lambda messages, mode: orchestrator.complete(messages, mode=mode), + max_concurrency=local_concurrency, + ) + else: + self.batch_backend = batch_backend self.embedding_batch_backend: EmbeddingBatchBackend = ( embedding_batch_backend or LocalEmbeddingBatchBackend(token_counter=self.token_counter) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 0097b722e..bdfdac6e6 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -3,13 +3,19 @@ from __future__ import annotations from collections import Counter, deque, OrderedDict +from collections.abc import Iterable, Mapping +from contextlib import contextmanager from contextvars import ContextVar +from concurrent.futures import ThreadPoolExecutor import copy from dataclasses import dataclass, replace from functools import wraps import hashlib +import http.client +import io import ipaddress import json +import math import os from pathlib import Path import random @@ -21,7 +27,7 @@ import time import uuid from typing import Any -from urllib.parse import urlparse +from urllib.parse import urlparse, urlunsplit import urllib.error import urllib.request @@ -30,6 +36,41 @@ ChatMessage = dict[str, str] +ProviderDestination = tuple[int, tuple[Any, ...]] +MAX_LOCAL_CONCURRENCY = 64 +DEFAULT_PROVIDER_PROBE_TIMEOUT = 5.0 +MAX_PROVIDER_PROBE_TIMEOUT = 30.0 +_SAFE_PROVIDER_PROBE_ERROR_TYPES = frozenset({ + "ConnectionError", + "HTTPError", + "OSError", + "RuntimeError", + "SSLError", + "TimeoutError", + "TypeError", + "UnknownError", + "URLError", + "ValueError", +}) + + +def _safe_provider_probe_error_type(exc: Exception) -> str: + """Keep provider diagnostics package-owned instead of echoing exception classes.""" + name = type(exc).__name__ + return name if name in _SAFE_PROVIDER_PROBE_ERROR_TYPES else "UnknownError" + + +def _validate_provider_probe_timeout(timeout: float) -> float: + """Validate the finite, bounded timeout used by explicit readiness probes.""" + if isinstance(timeout, bool) or not isinstance(timeout, (int, float)): + raise ValueError("provider probe timeout must be a finite number") + value = float(timeout) + if not math.isfinite(value) or not 0.1 <= value <= MAX_PROVIDER_PROBE_TIMEOUT: + raise ValueError( + f"provider probe timeout must be between 0.1 and {MAX_PROVIDER_PROBE_TIMEOUT:g} seconds" + ) + return value + class BudgetExceededError(RuntimeError): """Raised when an operator-configured spend budget is already exhausted.""" @@ -60,6 +101,136 @@ def estimate_tokens(text: str) -> int: ) DEFAULT_COMMERCIAL_TARGET_VALUE_KRW = 2_000_000_000 +MAX_MODEL_JUDGE_REPLY_CHARACTERS = 32_000 +CONTEXTUAL_ORCHESTRATOR_CONTRACT_V1 = "contextual-orchestrator-contract-v1" + + +@dataclass(frozen=True) +class FastMLSIRMJudgeComponents: + """Resolved fast-mlsirm symbols used by model verification.""" + + judge_cls: type[Any] + criterion_cls: type[Any] + format_error: type[Exception] + + +def _resolve_fast_mlsirm_components() -> FastMLSIRMJudgeComponents | None: + """Resolve the fast-mlsirm adapter symbols without importing unconditionally.""" + try: + from fast_mlsirm import ContextualOrchestratorJudge, JudgeCriterion, JudgeFormatError + except ModuleNotFoundError as exc: + if exc.name == "fast_mlsirm": + return None + raise + return FastMLSIRMJudgeComponents(ContextualOrchestratorJudge, JudgeCriterion, JudgeFormatError) + + +@dataclass +class _FastMLSIJudgeAdapter: + """Adapter that exposes `complete()` for `ContextualOrchestratorJudge`.""" + + orchestrator: "TaskOrchestrator" + text: str + judge: str + served_agent_id: str | None = None + mode: str = "auto" + + @property + def contextual_orchestrator_contract(self) -> str: + """Declare the versioned gateway boundary required by fast-mlsirm.""" + return CONTEXTUAL_ORCHESTRATOR_CONTRACT_V1 + + @property + def client(self) -> ModelClient: + """Expose the existing gateway client capability to fast-mlsirm.""" + return self.orchestrator.client + + def complete(self, messages: list[ChatMessage], mode: str | None = None) -> dict[str, Any]: + if mode is not None and (type(mode) is not str or mode not in {"auto", "route", "conduct"}): + raise ValueError("mode must be auto, route, or conduct") + output, served_id, usage = self.orchestrator._invoke(self._agent(), messages, text=self.text, role="verifier") + return self._completion_payload(output, served_id, usage, self.mode if mode is None else mode) + + def complete_structured( + self, + messages: list[ChatMessage], + mode: str | None = None, + *, + response_format: dict[str, Any], + ) -> dict[str, Any]: + """Route a Judge JSON-schema request through the existing gateway proxy.""" + if mode is not None and (type(mode) is not str or mode not in {"auto", "route", "conduct"}): + raise ValueError("mode must be auto, route, or conduct") + if not isinstance(response_format, dict): + raise TypeError("response_format must be a mapping") + agent = self._agent() + response = self.orchestrator.proxy_completion({ + "model": agent.model, + "messages": messages, + "temperature": self.orchestrator.client.temperature, + "max_tokens": self.orchestrator.client.max_output_tokens, + "response_format": response_format, + }) + output = ModelClient._response_content(agent, response) + usage = response.get("usage") if isinstance(response.get("usage"), dict) else None + return self._completion_payload(output, agent.id, usage, self.mode if mode is None else mode) + + def _completion_payload( + self, + output: str, + served_id: str, + usage: dict[str, Any] | None, + mode: str, + ) -> dict[str, Any]: + """Build the bounded adapter response shared by normal and structured calls.""" + self.served_agent_id = served_id + trace = [ + { + "id": 0, + "role": "verifier", + "agent_id": served_id, + "subtask": "LLM-as-a-Judge evaluation", + "output": output, + } + ] + if usage is not None: + trace[0]["usage"] = usage + return { + "answer": output, + "mode": mode, + "trace": trace, + } + + def _agent(self) -> ModelAgent: + return self.orchestrator._agent(self.judge) + + +def _parse_model_judge_reply(reply: str) -> tuple[str, str]: + """Parse one exact, duplicate-free model-judge verdict.""" + if not isinstance(reply, str) or len(reply) > MAX_MODEL_JUDGE_REPLY_CHARACTERS: + raise ValueError("judge response is missing or exceeds the maximum size") + + def reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ValueError("judge response contains duplicate object keys") + result[key] = value + return result + + try: + decision = json.loads(reply.strip(), object_pairs_hook=reject_duplicate_keys) + except (json.JSONDecodeError, RecursionError, TypeError) as exc: + raise ValueError("judge response is not valid JSON") from exc + if not isinstance(decision, dict) or set(decision) != {"decision", "reason"}: + raise ValueError("judge response must match the exact verdict schema") + decision_value = decision["decision"] + if not isinstance(decision_value, str) or decision_value not in {"ACCEPT", "REJECT"}: + raise ValueError("judge decision is not an allowed enum value") + reason = decision["reason"] + if not isinstance(reason, str) or not reason.strip(): + raise ValueError("judge reason is missing") + return decision_value, reason.strip() @dataclass(frozen=True) @@ -79,9 +250,16 @@ class ModelAgent: disabled: bool = False provider_name: str = "" provider_exclusions: tuple[str, ...] = () + # Explicit KV credential for an authenticated loopback gateway. Keep this + # separate from ``credential_key`` so mlx:// workers remain keyless. + local_credential_key: str = "" def __post_init__(self) -> None: require_object_name(self.id, "agent.id") + if type(self.local_credential_key) is not str: + raise TypeError("local_credential_key must be a string") + if self.local_credential_key and urlparse(self.base_url).scheme != "local": + raise ValueError("local_credential_key requires a local:// gateway URL") def to_config(self) -> dict[str, Any]: """Round-trippable agent configuration (from_dict(to_config(a)) == a).""" @@ -96,6 +274,7 @@ def to_config(self) -> dict[str, Any]: "disabled": self.disabled, "provider_name": self.provider_name, "provider_exclusions": list(self.provider_exclusions), + "local_credential_key": self.local_credential_key, } @property @@ -123,9 +302,33 @@ def from_dict(cls, value: dict[str, Any]) -> "ModelAgent": # pragma: no cover disabled=bool(value.get("disabled", False)), provider_name=value.get("provider_name", ""), provider_exclusions=tuple(value.get("provider_exclusions", value.get("provider_exclusion", ()))), + local_credential_key=value.get("local_credential_key", ""), ) +def _validate_batch_results( + requests: Mapping[str, list[ChatMessage]], + results: Mapping[str, Mapping[str, Any]], +) -> dict[str, dict[str, Any]]: + """Reject incomplete batch output before it can become an accepted run.""" + if not isinstance(results, Mapping): + raise TypeError("batch provider returned an invalid result map") + if set(requests) != set(results): + raise RuntimeError( + "batch provider returned an incomplete or unexpected result set " + f"(requested={len(requests)}, received={len(results)})" + ) + invalid_count = sum( + not isinstance(result, Mapping) or not isinstance(result.get("content"), str) + for result in results.values() + ) + if invalid_count: + raise RuntimeError( + f"batch provider returned {invalid_count} result(s) without assistant content" + ) + return {custom_id: dict(result) for custom_id, result in results.items()} + + @dataclass(frozen=True) class WorkflowStep: """One visible step in a conducted orchestration workflow.""" @@ -157,24 +360,29 @@ class OrchestrationPolicy: route_p95_seconds: float = 2.5 conduct_hint_threshold: int = 2 + route_text_length_threshold: int = 700 verifier_required: bool = True - verifier_positive_terms: tuple[str, ...] = ("verified", "accepted", "confirmed", "pass", "good", "ok") - verifier_negative_terms: tuple[str, ...] = ("reject", "disagree", "conflict", "unsafe", "fails", "error", "risky") # Conductor-style planning (arXiv:2512.04388): "generated" asks the planner model to # emit the workflow (subtasks, worker assignment, access lists); "template" keeps the # fixed 4-step plan. Generated plans that fail validation fall back to the template. workflow_planning: str = "template" max_workflow_steps: int = 6 - # Verifier verdict: "terms" (default) matches accept/reject vocabulary in the verifier - # report; "model" asks a verifier-selected model to reply ACCEPT/REJECT (fixes the - # known term-matching false negative on risk-vocabulary verifier outputs). - verifier_judge: str = "terms" + # Verifier verdicts are structured model judgments. Keyword matching is intentionally + # unsupported: it cannot handle negation, language, or a report that quotes a risk. + verifier_judge: str = "model" + + def __post_init__(self) -> None: + if self.verifier_judge != "model": + raise ValueError("keyword-based verifier_judge modes are unsupported; use 'model'") + if type(self.route_text_length_threshold) is not int or self.route_text_length_threshold < 1: + raise ValueError("route_text_length_threshold must be a positive integer") def as_dict(self) -> dict[str, Any]: """Return the API-safe policy snapshot for workflow records.""" return { "route_p95_seconds": self.route_p95_seconds, "conduct_hint_threshold": self.conduct_hint_threshold, + "route_text_length_threshold": self.route_text_length_threshold, "verifier_required": self.verifier_required, "workflow_planning": self.workflow_planning, "verifier_judge": self.verifier_judge, @@ -188,6 +396,261 @@ def as_dict(self) -> dict[str, Any]: # and the standard upstream/gateway failures. Everything else (400/401/403/404 ...) # is a caller or configuration error and must not be retried. TRANSIENT_HTTP_STATUS = frozenset({408, 409, 425, 429, 500, 502, 503, 504}) +LOCAL_PROVIDER_SCHEMES = frozenset({"mlx", "local"}) +LOCAL_PROVIDER_HOSTS = frozenset({"127.0.0.1", "::1", "localhost"}) + + +def _is_local_provider_url(base_url: str) -> bool: + """Return whether a provider uses the explicit loopback-only local scheme.""" + parsed = urlparse(base_url) + try: + parsed.port + except ValueError: + return False + return parsed.scheme in LOCAL_PROVIDER_SCHEMES and parsed.hostname in LOCAL_PROVIDER_HOSTS + + +def _is_direct_mlx_provider_url(base_url: str) -> bool: + """Return whether a loopback provider is the direct mlx-lm transport.""" + return _is_local_provider_url(base_url) and urlparse(base_url).scheme == "mlx" + + +def _provider_credential_name(agent: ModelAgent) -> str | None: + """Return the credential name allowed for this provider transport.""" + if not _is_local_provider_url(agent.base_url): + return agent.credential_name + # mlx-lm is intentionally keyless; only the explicit local:// gateway + # transport may opt into a separately named loopback bearer credential. + if urlparse(agent.base_url).scheme != "local": + return None + return agent.local_credential_key or None + + +def _provider_credential(agent: ModelAgent) -> str | None: + """Resolve the transport-specific credential from the KV registry.""" + name = _provider_credential_name(agent) + return get_credential(name) if name else None + + +class _LocalProviderState: + """Coordinate model switching and bounded concurrency for one local endpoint.""" + + def __init__(self) -> None: + self.condition = threading.Condition() + self.active_model: str | None = None + self.active = 0 + self.capacity = 1 + + +_LOCAL_PROVIDER_STATES: dict[str, _LocalProviderState] = {} +_LOCAL_PROVIDER_STATES_GUARD = threading.Lock() + + +def _local_provider_state(base_url: str) -> _LocalProviderState: + """Return the shared in-process coordinator for one loopback provider endpoint.""" + parsed = urlparse(base_url) + key = urlunsplit(("http", (parsed.netloc or base_url).lower(), parsed.path.rstrip("/"), "", "")) + with _LOCAL_PROVIDER_STATES_GUARD: + return _LOCAL_PROVIDER_STATES.setdefault(key, _LocalProviderState()) + + +@contextmanager +def _local_provider_slot( + agent: ModelAgent, + capacity: int, + timeout: float, +): + """Bound local requests and serialize model switches on a shared endpoint.""" + if not _is_local_provider_url(agent.base_url): + yield + return + + state = _local_provider_state(agent.base_url) + deadline = time.monotonic() + max(float(timeout), 0.0) + with state.condition: + while True: + if state.active == 0: + state.active_model = agent.model + state.capacity = capacity + elif state.active_model == agent.model: + state.capacity = min(state.capacity, capacity) + + if state.active_model == agent.model and state.active < state.capacity: + state.active += 1 + break + + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("local provider endpoint is busy past its request deadline") + state.condition.wait(remaining) + + try: + yield + finally: + with state.condition: + state.active -= 1 + if state.active == 0: + state.active_model = None + state.capacity = 1 + state.condition.notify_all() + + +def _responses_text(value: Any) -> str: + if isinstance(value, str): + return value + if not isinstance(value, list): + return "" + parts: list[str] = [] + for item in value: + if isinstance(item, str): + parts.append(item) + elif isinstance(item, dict) and isinstance(item.get("text"), str): + parts.append(item["text"]) + return "".join(parts) + + +def _responses_to_chat_payload(request: dict[str, Any]) -> dict[str, Any]: + # ADR 0002: keep Codex Responses compatibility at the public control-plane + # boundary; mlx-lm remains a local Chat Completions worker provider. + messages: list[dict[str, Any]] = [] + instructions = _responses_text(request.get("instructions")) + if instructions: + messages.append({"role": "system", "content": instructions}) + + raw_input = request.get("input", "") + if isinstance(raw_input, list): + items = raw_input + elif isinstance(raw_input, str): + items = [{"type": "message", "role": "user", "content": raw_input}] + else: + raise ValueError("local Responses input must be a string or item list") + for item in items: + if isinstance(item, str): + messages.append({"role": "user", "content": item}) + continue + if not isinstance(item, dict): + raise ValueError("local Responses input items must be objects") + item_type = item.get("type", "message") + if item_type == "message": + role = item.get("role", "user") + if role == "developer": + role = "system" + if role not in {"system", "user", "assistant"}: + raise ValueError(f"unsupported local Responses message role: {role}") + content = _responses_text(item.get("content")) + if content: + messages.append({"role": role, "content": content}) + elif item_type == "function_call_output": + messages.append({ + "role": "tool", + "tool_call_id": str(item.get("call_id", "")), + "content": _responses_text(item.get("output", item.get("content", ""))), + }) + elif item_type == "function_call": + messages.append({ + "role": "assistant", + "content": "", + "tool_calls": [{ + "id": str(item.get("call_id", "")), + "type": "function", + "function": { + "name": str(item.get("name", "")), + "arguments": str(item.get("arguments", "{}")), + }, + }], + }) + elif item_type in {"reasoning", "item_reference"}: + continue + else: + raise ValueError(f"unsupported local Responses input item: {item_type}") + + payload: dict[str, Any] = { + "model": request.get("model", "local-model"), + "messages": messages, + "stream": False, + } + for key in ( + "temperature", "top_p", "max_tokens", "stop", "seed", "presence_penalty", + "frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "user", + "parallel_tool_calls", "tool_choice", + ): + if key in request: + payload[key] = request[key] + if "max_output_tokens" in request and "max_tokens" not in payload: + payload["max_tokens"] = request["max_output_tokens"] + + tools: list[dict[str, Any]] = [] + for tool in request.get("tools", []): + if not isinstance(tool, dict) or tool.get("type") != "function": + continue + function = { + key: tool[key] + for key in ("name", "description", "parameters", "strict") + if key in tool + } + tools.append({"type": "function", "function": function}) + if tools: + payload["tools"] = tools + + tool_choice = payload.get("tool_choice") + if isinstance(tool_choice, dict) and tool_choice.get("type") == "function": + payload["tool_choice"] = { + "type": "function", + "function": {"name": tool_choice.get("name", "")}, + } + return payload + + +def _chat_to_responses_payload(data: dict[str, Any], request: dict[str, Any]) -> dict[str, Any]: + choice = (data.get("choices") or [{}])[0] + message = choice.get("message") if isinstance(choice, dict) else {} + message = message if isinstance(message, dict) else {} + content = message.get("content") + if not isinstance(content, str): + content = message.get("reasoning") if isinstance(message.get("reasoning"), str) else "" + + output: list[dict[str, Any]] = [] + if content or not message.get("tool_calls"): + output.append({ + "id": f"msg_{uuid.uuid4().hex}", + "type": "message", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": content, "annotations": []}], + }) + for tool_call in message.get("tool_calls", []): + if not isinstance(tool_call, dict): + continue + function = tool_call.get("function") or {} + output.append({ + "id": f"fc_{tool_call.get('id', uuid.uuid4().hex)}", + "type": "function_call", + "status": "completed", + "call_id": str(tool_call.get("id", uuid.uuid4().hex)), + "name": str(function.get("name", "")), + "arguments": str(function.get("arguments", "{}")), + }) + + usage = data.get("usage") if isinstance(data.get("usage"), dict) else {} + input_tokens = int(usage.get("prompt_tokens", 0) or 0) + output_tokens = int(usage.get("completion_tokens", 0) or 0) + response: dict[str, Any] = { + "id": f"resp_{data.get('id', uuid.uuid4().hex)}", + "object": "response", + "created_at": int(data.get("created", time.time())), + "model": data.get("model", request.get("model", "local-model")), + "output": output, + "output_text": content, + "status": "completed" if choice.get("finish_reason") != "length" else "incomplete", + "usage": { + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "total_tokens": int(usage.get("total_tokens", input_tokens + output_tokens) or 0), + }, + } + if isinstance(request.get("metadata"), dict): + response["metadata"] = request["metadata"] + return response def is_transient_error(exc: BaseException) -> bool: @@ -197,6 +660,11 @@ def is_transient_error(exc: BaseException) -> bool: # Network-level failures (DNS, connection reset, read timeout) are transient. if isinstance(exc, (urllib.error.URLError, TimeoutError, ConnectionError, socket.timeout)): return True + # A VPN/socket path can surface as an SSL EOF or SSL_ERROR_SYSCALL. Keep + # certificate verification failures non-transient so a bad trust boundary + # is never retried as if it were a network fault. + if isinstance(exc, ssl.SSLError): + return not isinstance(exc, ssl.SSLCertVerificationError) return False @@ -208,29 +676,46 @@ def __init__( timeout: int = 90, max_output_tokens: int = 2048, max_retries: int = 2, + local_max_retries: int = 0, retry_backoff: float = 0.5, retry_backoff_cap: float = 8.0, + temperature: float = 0.2, + local_concurrency: int = 1, + chat_template_args: dict[str, Any] | None = None, ca_bundle: str | None = None, verify_tls: bool = True, + allowed_provider_hosts: Iterable[str] | None = None, ) -> None: self.timeout = timeout self.max_output_tokens = max_output_tokens + if isinstance(max_retries, bool) or max_retries < 0: + raise ValueError("max_retries must be >= 0") self.max_retries = max_retries + if isinstance(local_max_retries, bool) or local_max_retries < 0: + raise ValueError("local_max_retries must be >= 0") + self.local_max_retries = int(local_max_retries) self.retry_backoff = retry_backoff self.retry_backoff_cap = retry_backoff_cap + self.temperature = temperature + if type(local_concurrency) is not int or not 1 <= local_concurrency <= MAX_LOCAL_CONCURRENCY: + raise ValueError( + f"local_concurrency must be an integer in 1..{MAX_LOCAL_CONCURRENCY}" + ) + self.local_concurrency = local_concurrency + self.chat_template_args = dict(chat_template_args or {}) + self.allowed_provider_hosts = self._normalize_allowed_provider_hosts(allowed_provider_hosts) # Seam so tests can observe/skip real sleeping during backoff. self._sleep = time.sleep # Per-thread usage from the most recent chat() (the server is threaded). self._local = threading.local() - # TLS trust for provider egress. Default verifies against the system trust store; - # ca_bundle points at a custom CA (corporate gateways); verify_tls=False is an - # explicit dev-only opt-out (insecure) for self-signed endpoints. - self._ssl_context = self._build_ssl_context(ca_bundle, verify_tls) + if not verify_tls: + raise ValueError("provider TLS verification cannot be disabled; configure a trusted ca_bundle") + # TLS trust for provider egress. The system trust store is the default; + # ca_bundle points at a custom CA for a reviewed corporate gateway. + self._ssl_context = self._build_ssl_context(ca_bundle) @staticmethod - def _build_ssl_context(ca_bundle: str | None, verify_tls: bool) -> ssl.SSLContext: - if not verify_tls: - return ssl._create_unverified_context() # nosec B323 - explicit dev-only provider TLS opt-out. + def _build_ssl_context(ca_bundle: str | None) -> ssl.SSLContext: if ca_bundle: if not os.path.isfile(ca_bundle): raise ValueError(f"provider CA bundle does not exist: {ca_bundle}") @@ -240,80 +725,308 @@ def _build_ssl_context(ca_bundle: str | None, verify_tls: bool) -> ssl.SSLContex raise ValueError(f"provider CA bundle could not be loaded: {ca_bundle}") from exc return ssl.create_default_context() + @staticmethod + def _normalize_allowed_provider_hosts(hosts: Iterable[str] | None) -> frozenset[str]: + """Normalize an explicit provider-host policy once at client construction.""" + if hosts is None: + return frozenset() + if isinstance(hosts, (str, bytes)): + raise ValueError("allowed_provider_hosts must be an iterable of host strings") + normalized: set[str] = set() + try: + values = iter(hosts) + except TypeError as exc: + raise ValueError("allowed_provider_hosts must be an iterable of host strings") from exc + for host in values: + if type(host) is not str: + raise ValueError("allowed_provider_hosts must contain only strings") + value = host.strip().lower() + if not value or any(character in value for character in "/?#"): + raise ValueError("allowed_provider_hosts entries must be bare host names") + normalized.add(value) + return frozenset(normalized) + def take_usage(self) -> dict[str, Any] | None: """Return and clear provider-reported usage from the most recent chat() on this thread.""" usage = getattr(self._local, "usage", None) self._local.usage = None return usage - def chat(self, agent: ModelAgent, messages: list[ChatMessage], temperature: float = 0.2) -> str: + def chat(self, agent: ModelAgent, messages: list[ChatMessage], temperature: float | None = None) -> str: """Send messages to a mock or OpenAI-compatible chat endpoint with retries.""" self._local.usage = None if agent.base_url.startswith("mock://"): return self._mock(agent, messages) - self._validate_provider(agent) # pragma: no cover - api_key = get_credential(agent.credential_name) # pragma: no cover - if not api_key: # pragma: no cover + destination = self._validate_provider(agent) # pragma: no cover + api_key = _provider_credential(agent) # pragma: no cover + credential_name = _provider_credential_name(agent) # pragma: no cover + if credential_name and not api_key: # pragma: no cover raise NotConfigured( - f"{agent.id} requires a resolvable credential '{agent.credential_name}' in the KV" + f"{agent.id} requires a resolvable credential '{credential_name}' in the KV" ) payload = { # pragma: no cover "model": agent.model, "messages": messages, - "temperature": temperature, + "temperature": self.temperature if temperature is None else temperature, "stream": False, "max_tokens": self.max_output_tokens, } - return self._send_with_retry(agent, payload) + if _is_direct_mlx_provider_url(agent.base_url) and self.chat_template_args: + payload["chat_template_kwargs"] = self.chat_template_args + with _local_provider_slot(agent, self.local_concurrency, self.timeout): + return self._send_with_retry(agent, payload, destination) + + def probe(self, agent: ModelAgent, *, timeout: float = DEFAULT_PROVIDER_PROBE_TIMEOUT) -> dict[str, Any]: + """Verify a local model registry, then run one bounded completion probe. + + ``/health`` and ``/v1/models`` only prove process/model-registry liveness; + this verifies the configured local model and deliberately exercises the + chat path with one output token. It never retries, so a stuck local queue + cannot be multiplied by the readiness check. + """ + probe_timeout = _validate_provider_probe_timeout(timeout) + started = time.monotonic() + self._local.usage = None + failure_code = "provider_probe_failed" + try: + if agent.base_url.startswith("mock://"): + content = self._mock(agent, [{"role": "user", "content": "Reply with exactly OK."}]) + usage = None + else: + destination = self._validate_provider(agent) + if _is_local_provider_url(agent.base_url): + registry_request = urllib.request.Request( + self._provider_url(agent, "/models"), + method="GET", + ) + with self._open_provider( + registry_request, destination, timeout=probe_timeout + ) as registry_response: + registry = json.loads( + registry_response.read().decode("utf-8") + ) + model_ids = { + item.get("id") + for item in registry.get("data", []) + if isinstance(item, dict) and type(item.get("id")) is str + } + if agent.model not in model_ids: + failure_code = "provider_model_not_registered" + raise RuntimeError( + f"provider {agent.id} model registry does not contain {agent.model!r}" + ) + payload: dict[str, Any] = { + "model": agent.model, + "messages": [{"role": "user", "content": "Reply with exactly OK."}], + "temperature": 0.0, + "stream": False, + "max_tokens": 1, + } + if _is_direct_mlx_provider_url(agent.base_url) and self.chat_template_args: + payload["chat_template_kwargs"] = self.chat_template_args + with _local_provider_slot(agent, self.local_concurrency, probe_timeout): + content = self._send(agent, payload, destination, timeout=probe_timeout) + usage = self.take_usage() + if not content.strip(): + failure_code = "provider_empty_probe_response" + raise RuntimeError(f"provider {agent.id} returned empty probe content") + return { + "agent_id": agent.id, + "model": agent.model, + "status": "ready", + "latency_ms": round((time.monotonic() - started) * 1000, 2), + "usage": usage, + } + except Exception as exc: # noqa: BLE001 - readiness reports failures, it does not serve them + return { + "agent_id": agent.id, + "model": agent.model, + "status": "not_ready", + "latency_ms": round((time.monotonic() - started) * 1000, 2), + "error_type": _safe_provider_probe_error_type(exc), + "failure_code": failure_code, + } - def _send_with_retry(self, agent: ModelAgent, payload: dict[str, Any]) -> str: + def _send_with_retry( + self, + agent: ModelAgent, + payload: dict[str, Any], + destination: ProviderDestination | None = None, + *, + timeout: float | None = None, + ) -> str: """Call the provider, retrying transient failures with exponential backoff + jitter.""" last_error: Exception | None = None - for attempt in range(self.max_retries + 1): + retry_limit = self._retry_limit(agent) + for attempt in range(retry_limit + 1): # pragma: no branch - retry limits are validated non-negative try: - return self._send(agent, payload) + return ( + self._send(agent, payload, destination) + if timeout is None + else self._send(agent, payload, destination, timeout=timeout) + ) except Exception as exc: # noqa: BLE001 - classify then decide last_error = exc - if attempt >= self.max_retries or not is_transient_error(exc): + if attempt >= retry_limit or not is_transient_error(exc): break self._sleep(self._backoff_delay(attempt)) - raise RuntimeError(f"provider {agent.id} request failed") from last_error + detail = f": {last_error}" if last_error else "" + raise RuntimeError(f"provider {agent.id} request failed{detail}") from last_error + + def _retry_limit(self, agent: ModelAgent) -> int: + """Return a retry budget without multiplying an expensive local queue by default.""" + return self.local_max_retries if _is_local_provider_url(agent.base_url) else self.max_retries def _backoff_delay(self, attempt: int) -> float: """Full-jitter exponential backoff, capped, so retries do not thundering-herd a provider.""" ceiling = min(self.retry_backoff_cap, self.retry_backoff * (2 ** attempt)) return random.uniform(0.0, ceiling) - def _send(self, agent: ModelAgent, payload: dict[str, Any]) -> str: + def _send( + self, + agent: ModelAgent, + payload: dict[str, Any], + destination: ProviderDestination | None = None, + *, + timeout: float | None = None, + ) -> str: """Perform one provider HTTP request (isolated so retry/backoff stays testable).""" - api_key = get_credential(agent.credential_name) or "" + api_key = _provider_credential(agent) + headers = {"content-type": "application/json"} + if api_key: + headers["authorization"] = f"Bearer {api_key}" request = urllib.request.Request( self._provider_url(agent, "/chat/completions"), data=json.dumps(payload).encode("utf-8"), - headers={ - "authorization": f"Bearer {api_key}", - "content-type": "application/json", - }, + headers=headers, method="POST", ) - with self._open_provider(request) as response: + opened = ( + self._open_provider(request, destination) + if timeout is None + else self._open_provider(request, destination, timeout=timeout) + ) + with opened as response: data = json.loads(response.read().decode("utf-8")) usage = data.get("usage") if isinstance(usage, dict): self._local.usage = usage - return data["choices"][0]["message"]["content"] + return self._response_content(agent, data) - def _open_provider(self, request: urllib.request.Request) -> Any: - """Open a provider request built from a validated provider URL.""" - return urllib.request.urlopen( # nosec B310 - request URL comes from _provider_url after provider validation. - request, - timeout=self.timeout, - context=self._ssl_context, + @staticmethod + def _response_content(agent: ModelAgent, data: dict[str, Any]) -> str: + """Extract text and explain provider responses that contain reasoning only.""" + choices = data.get("choices") + message = choices[0].get("message") if isinstance(choices, list) and choices else None + content = message.get("content") if isinstance(message, dict) else None + if isinstance(content, str): + return content + if isinstance(message, dict) and message.get("reasoning"): + raise RuntimeError( + f"provider {agent.id} returned reasoning without content; " + "for mlx-lm set chat_template_args={\"enable_thinking\": false} or increase max_output_tokens" + ) + raise RuntimeError(f"provider {agent.id} response did not contain assistant content") + + @staticmethod + def _connect_validated( + destination: ProviderDestination, timeout: float | None, source_address: tuple[str, int] | None + ) -> socket.socket: + """Connect to one already-resolved address without performing another DNS lookup.""" + family, sockaddr = destination + connection = socket.socket(family, socket.SOCK_STREAM) + try: + connection.settimeout(timeout) + if source_address is not None: + connection.bind(source_address) + connection.connect(sockaddr) + return connection + except Exception: + connection.close() + raise + + @staticmethod + def _resolve_addresses(hostname: str, port: int) -> list[ProviderDestination]: + try: + addresses = socket.getaddrinfo(hostname, port, type=socket.SOCK_STREAM) + except socket.gaierror as exc: + raise RuntimeError(f"provider host {hostname!r} could not be resolved") from exc + resolved = [(family, sockaddr) for family, _type, _proto, _canonname, sockaddr in addresses] + if not resolved: + raise RuntimeError(f"provider host {hostname!r} has no stream address") + return resolved + + def _open_provider( + self, + request: urllib.request.Request, + destination: ProviderDestination | None = None, + *, + timeout: float | None = None, + ) -> Any: + """Open a validated HTTP(S) request without generic URL-handler dispatch.""" + parsed = urlparse(request.full_url) + if ( + parsed.scheme not in {"http", "https"} + or not parsed.hostname + or parsed.username + or parsed.password + or parsed.fragment + ): + raise RuntimeError("provider request URL must be an HTTP(S) URL without userinfo or fragments") + try: + port = parsed.port or (443 if parsed.scheme == "https" else 80) + except ValueError as exc: + raise RuntimeError("provider request URL has an invalid port") from exc + if destination is None: + destination = self._resolve_addresses(parsed.hostname, port)[0] + connection_timeout = self.timeout if timeout is None else timeout + connection: http.client.HTTPConnection + if parsed.scheme == "https": + # The explicit verifying context is the security control for this reviewed API. + connection = http.client.HTTPSConnection( # nosemgrep: python.lang.security.audit.httpsconnection-detected.httpsconnection-detected + parsed.hostname, + port, + timeout=connection_timeout, + context=self._ssl_context, + ) + else: + connection = http.client.HTTPConnection(parsed.hostname, port, timeout=connection_timeout) + connection._create_connection = ( # type: ignore[attr-defined] + lambda _address, timeout, source_address: self._connect_validated( + destination, timeout, source_address + ) ) + target = urlunsplit(("", "", parsed.path or "/", parsed.query, "")) + try: + connection.request( + request.get_method(), + target, + body=request.data, + headers=dict(request.header_items()), + ) + response = connection.getresponse() + if response.status >= 400: + body = response.read() + status = response.status + reason = response.reason + headers = response.headers + response.close() + connection.close() + raise urllib.error.HTTPError( + request.full_url, + status, + reason, + headers, + io.BytesIO(body), + ) + return response + except Exception: + connection.close() + raise - def stream_chat(self, agent: ModelAgent, messages: list[ChatMessage], temperature: float = 0.2): + def stream_chat(self, agent: ModelAgent, messages: list[ChatMessage], temperature: float | None = None): """Yield content deltas from a mock or OpenAI-compatible streaming endpoint. Real token streaming: the provider is called with stream=true and its SSE deltas @@ -326,30 +1039,34 @@ def stream_chat(self, agent: ModelAgent, messages: list[ChatMessage], temperatur yield answer[start : start + 24] return - self._validate_provider(agent) # pragma: no cover + destination = self._validate_provider(agent) # pragma: no cover payload = { # pragma: no cover "model": agent.model, "messages": messages, - "temperature": temperature, + "temperature": self.temperature if temperature is None else temperature, "stream": True, "max_tokens": self.max_output_tokens, } - yield from self._stream_send(agent, payload) # pragma: no cover + if _is_direct_mlx_provider_url(agent.base_url) and self.chat_template_args: + payload["chat_template_kwargs"] = self.chat_template_args + with _local_provider_slot(agent, self.local_concurrency, self.timeout): # pragma: no cover + yield from self._stream_send(agent, payload, destination) # pragma: no cover - def _stream_send(self, agent: ModelAgent, payload: dict[str, Any]): + def _stream_send( + self, agent: ModelAgent, payload: dict[str, Any], destination: ProviderDestination | None = None + ): """Stream content deltas from a provider SSE response (real transport, testable).""" - api_key = get_credential(agent.credential_name) or "" + api_key = _provider_credential(agent) + headers = {"content-type": "application/json", "accept": "text/event-stream"} + if api_key: + headers["authorization"] = f"Bearer {api_key}" request = urllib.request.Request( self._provider_url(agent, "/chat/completions"), data=json.dumps(payload).encode("utf-8"), - headers={ - "authorization": f"Bearer {api_key}", - "content-type": "application/json", - "accept": "text/event-stream", - }, + headers=headers, method="POST", ) - with self._open_provider(request) as response: + with self._open_provider(request, destination) as response: for raw in response: line = raw.decode("utf-8").strip() if not line.startswith("data:"): @@ -377,39 +1094,59 @@ def proxy_send( """Passthrough a full request to one agent, returning the raw provider JSON.""" if agent.base_url.startswith("mock://"): return self._mock_raw(agent, endpoint, payload) - self._validate_provider(agent) # pragma: no cover - return self._send_raw_with_retry(agent, endpoint, payload) # pragma: no cover + destination = self._validate_provider(agent) # pragma: no cover + if endpoint.strip("/") == "responses" and _is_local_provider_url(agent.base_url): + chat_payload = _responses_to_chat_payload(payload) + chat_payload.setdefault("max_tokens", self.max_output_tokens) + if _is_direct_mlx_provider_url(agent.base_url) and self.chat_template_args: + chat_payload["chat_template_kwargs"] = self.chat_template_args + with _local_provider_slot(agent, self.local_concurrency, self.timeout): + chat_response = self._send_raw_with_retry( + agent, "chat/completions", chat_payload, destination + ) + return _chat_to_responses_payload(chat_response, payload) + with _local_provider_slot(agent, self.local_concurrency, self.timeout): # pragma: no cover + return self._send_raw_with_retry(agent, endpoint, payload, destination) def _send_raw_with_retry( - self, agent: ModelAgent, endpoint: str, payload: dict[str, Any] + self, + agent: ModelAgent, + endpoint: str, + payload: dict[str, Any], + destination: ProviderDestination | None = None, ) -> dict[str, Any]: # pragma: no cover """Passthrough transport with the same transient-failure retry policy as _send.""" last_error: Exception | None = None - for attempt in range(self.max_retries + 1): + retry_limit = self._retry_limit(agent) + for attempt in range(retry_limit + 1): try: - return self._send_raw(agent, endpoint, payload) + return self._send_raw(agent, endpoint, payload, destination) except Exception as exc: # noqa: BLE001 - classify then decide last_error = exc - if attempt >= self.max_retries or not is_transient_error(exc): + if attempt >= retry_limit or not is_transient_error(exc): break self._sleep(self._backoff_delay(attempt)) raise RuntimeError(f"provider {agent.id} passthrough request failed") from last_error def _send_raw( - self, agent: ModelAgent, endpoint: str, payload: dict[str, Any] + self, + agent: ModelAgent, + endpoint: str, + payload: dict[str, Any], + destination: ProviderDestination | None = None, ) -> dict[str, Any]: # pragma: no cover """One provider HTTP request returning the FULL provider JSON (for passthrough).""" - api_key = get_credential(agent.credential_name) or "" + api_key = _provider_credential(agent) + headers = {"content-type": "application/json"} + if api_key: + headers["authorization"] = f"Bearer {api_key}" request = urllib.request.Request( self._provider_url(agent, f"/{endpoint.lstrip('/')}"), data=json.dumps(payload).encode("utf-8"), - headers={ - "authorization": f"Bearer {api_key}", - "content-type": "application/json", - }, + headers=headers, method="POST", ) - with self._open_provider(request) as response: + with self._open_provider(request, destination) as response: return json.loads(response.read().decode("utf-8")) def _mock_raw( @@ -450,29 +1187,36 @@ def _mock_raw( "echo": echoed, } - def _validate_provider(self, agent: ModelAgent) -> None: - """Reject unsafe remote model endpoints before any egress happens.""" + def _validate_provider(self, agent: ModelAgent) -> ProviderDestination: + """Reject unsafe model endpoints and return the exact address to connect to.""" # Runtime secret must be resolvable from the KV — never an env var name, # never a silent os.getenv fallback. (Legacy api_key_env, if set, is used # only as the credential NAME; see ModelAgent.credential_name.) - if get_credential(agent.credential_name) is None: + if _is_local_provider_url(agent.base_url): + parsed = urlparse(agent.base_url) + if parsed.username or parsed.password or parsed.query or parsed.fragment: + raise RuntimeError(f"{agent.id} local provider URL must not contain credentials or query data") + addresses = self._resolve_addresses(parsed.hostname or "", parsed.port or 80) + if any(not ipaddress.ip_address(sockaddr[0]).is_loopback for _family, sockaddr in addresses): + raise RuntimeError(f"{agent.id} local provider resolves to a non-loopback address") + return addresses[0] + credential_name = _provider_credential_name(agent) + if credential_name and get_credential(credential_name) is None: raise NotConfigured( - f"{agent.id} requires a resolvable credential '{agent.credential_name}' in the KV " + f"{agent.id} requires a resolvable credential '{credential_name}' in the KV " "(this replaces the legacy api_key_env environment pattern)" ) parsed = urlparse(agent.base_url) if parsed.scheme != "https" or not parsed.hostname: raise RuntimeError(f"{agent.id} base_url must use https") - allowed_hosts = { - host.strip().lower() - for host in os.environ.get("CONTEXTUAL_ORCHESTRATOR_ALLOWED_PROVIDER_HOSTS", "").split(",") - if host.strip() - } + if parsed.username or parsed.password or parsed.query or parsed.fragment: + raise RuntimeError(f"{agent.id} base_url must not contain credentials, query data, or fragments") hostname = parsed.hostname.lower() - if allowed_hosts and hostname not in allowed_hosts: + if self.allowed_provider_hosts and hostname not in self.allowed_provider_hosts: raise RuntimeError(f"{agent.id} provider host is not allowlisted") - for address in socket.getaddrinfo(hostname, parsed.port or 443, type=socket.SOCK_STREAM): - ip_address = ipaddress.ip_address(address[4][0]) + addresses = self._resolve_addresses(hostname, parsed.port or 443) + for _family, sockaddr in addresses: + ip_address = ipaddress.ip_address(sockaddr[0]) if ( ip_address.is_private or ip_address.is_loopback @@ -481,15 +1225,22 @@ def _validate_provider(self, agent: ModelAgent) -> None: or ip_address.is_reserved ): raise RuntimeError(f"{agent.id} provider resolves to non-public address") + return addresses[0] def _provider_url(self, agent: ModelAgent, path: str) -> str: """Build a provider URL while rejecting urllib-supported local schemes.""" parsed = urlparse(agent.base_url) - if parsed.scheme not in {"http", "https"} or not parsed.hostname: + if _is_local_provider_url(agent.base_url): + if parsed.username or parsed.password or parsed.query or parsed.fragment: + raise RuntimeError(f"{agent.id} local provider URL must not contain credentials or query data") + base_url = urlunsplit(("http", parsed.netloc, parsed.path.rstrip("/"), "", "")) + elif parsed.scheme in {"http", "https"} and parsed.hostname: + base_url = agent.base_url.rstrip("/") + else: raise RuntimeError(f"{agent.id} base_url must be an http(s) provider URL") if not path.startswith("/") or path.startswith("//") or "\r" in path or "\n" in path: raise RuntimeError("provider path must be a single absolute URL path") - return f"{agent.base_url.rstrip('/')}{path}" + return f"{base_url}{path}" def _mock(self, agent: ModelAgent, messages: list[ChatMessage]) -> str: last = next((m["content"] for m in reversed(messages) if m.get("role") == "user"), "") @@ -506,7 +1257,7 @@ def batch_chat( self, agent: ModelAgent, requests: dict[str, list[ChatMessage]], - temperature: float = 0.2, + temperature: float | None = None, poll_interval: float = 5.0, poll_timeout: float = 3600.0, ) -> dict[str, dict[str, Any]]: @@ -517,12 +1268,35 @@ def batch_chat( using ``chat``. The mock path answers synchronously so tests and local runs work. """ if agent.base_url.startswith("mock://"): - return { + results = { custom_id: {"content": self._mock(agent, messages), "usage": None} for custom_id, messages in requests.items() } - self._validate_provider(agent) # pragma: no cover - return self._batch_run(agent, requests, temperature, poll_interval, poll_timeout) # pragma: no cover + elif _is_local_provider_url(agent.base_url): + results = self._local_batch_chat(agent, requests, temperature) + else: + destination = self._validate_provider(agent) # pragma: no cover + results = self._batch_run( # pragma: no cover + agent, requests, temperature, poll_interval, poll_timeout, destination + ) + return _validate_batch_results(requests, results) + + def _local_batch_chat( + self, + agent: ModelAgent, + requests: dict[str, list[ChatMessage]], + temperature: float | None, + ) -> dict[str, dict[str, Any]]: + """Run local OpenAI-compatible requests concurrently through mlx-lm.""" + def complete(custom_id: str, messages: list[ChatMessage]) -> tuple[str, dict[str, Any]]: + content = self.chat(agent, messages, temperature=temperature) + return custom_id, {"content": content, "usage": self.take_usage()} + + if self.local_concurrency == 1 or len(requests) <= 1: + return dict(complete(custom_id, messages) for custom_id, messages in requests.items()) + with ThreadPoolExecutor(max_workers=min(self.local_concurrency, len(requests))) as pool: + futures = [pool.submit(complete, custom_id, messages) for custom_id, messages in requests.items()] + return dict(future.result() for future in futures) def _batch_run( self, @@ -531,6 +1305,7 @@ def _batch_run( temperature: float, poll_interval: float, poll_timeout: float, + destination: ProviderDestination | None = None, ) -> dict[str, dict[str, Any]]: """Upload, create, poll, and parse one batch (isolated so the flow stays testable).""" lines = [ @@ -541,21 +1316,21 @@ def _batch_run( "body": { "model": agent.model, "messages": messages, - "temperature": temperature, + "temperature": self.temperature if temperature is None else temperature, "max_tokens": self.max_output_tokens, }, }, ensure_ascii=False) for custom_id, messages in requests.items() ] - input_file_id = self._batch_upload(agent, "\n".join(lines).encode("utf-8")) + input_file_id = self._batch_upload(agent, "\n".join(lines).encode("utf-8"), destination) batch_id = self._batch_json(agent, "POST", "/batches", { "input_file_id": input_file_id, "endpoint": "/v1/chat/completions", "completion_window": "24h", - })["id"] + }, destination)["id"] deadline = time.monotonic() + poll_timeout while True: - batch = self._batch_json(agent, "GET", f"/batches/{batch_id}") + batch = self._batch_json(agent, "GET", f"/batches/{batch_id}", destination=destination) status = batch.get("status") if status == "completed": break @@ -564,7 +1339,7 @@ def _batch_run( if time.monotonic() >= deadline: raise TimeoutError(f"batch {batch_id} still {status} after {poll_timeout}s") self._sleep(poll_interval) - raw = self._batch_raw(agent, f"/files/{batch['output_file_id']}/content") + raw = self._batch_raw(agent, f"/files/{batch['output_file_id']}/content", destination) results: dict[str, dict[str, Any]] = {} for line in raw.decode("utf-8").splitlines(): if not line.strip(): @@ -578,7 +1353,9 @@ def _batch_run( } return results - def _batch_upload(self, agent: ModelAgent, payload: bytes) -> str: + def _batch_upload( + self, agent: ModelAgent, payload: bytes, destination: ProviderDestination | None = None + ) -> str: """Upload a JSONL batch input via multipart/form-data; returns the file id.""" boundary = f"co-batch-{uuid.uuid4().hex}" api_key = get_credential(agent.credential_name) or "" @@ -596,10 +1373,17 @@ def _batch_upload(self, agent: ModelAgent, payload: bytes) -> str: }, method="POST", ) - with self._open_provider(request) as response: + with self._open_provider(request, destination) as response: return json.loads(response.read().decode("utf-8"))["id"] - def _batch_json(self, agent: ModelAgent, method: str, path: str, payload: dict[str, Any] | None = None) -> dict[str, Any]: + def _batch_json( + self, + agent: ModelAgent, + method: str, + path: str, + payload: dict[str, Any] | None = None, + destination: ProviderDestination | None = None, + ) -> dict[str, Any]: api_key = get_credential(agent.credential_name) or "" request = urllib.request.Request( self._provider_url(agent, path), @@ -610,17 +1394,17 @@ def _batch_json(self, agent: ModelAgent, method: str, path: str, payload: dict[s }, method=method, ) - with self._open_provider(request) as response: + with self._open_provider(request, destination) as response: return json.loads(response.read().decode("utf-8")) - def _batch_raw(self, agent: ModelAgent, path: str) -> bytes: + def _batch_raw(self, agent: ModelAgent, path: str, destination: ProviderDestination | None = None) -> bytes: api_key = get_credential(agent.credential_name) or "" request = urllib.request.Request( self._provider_url(agent, path), headers={"authorization": f"Bearer {api_key}"}, method="GET", ) - with self._open_provider(request) as response: + with self._open_provider(request, destination) as response: return response.read() @@ -811,7 +1595,6 @@ class TaskOrchestrator: "검증", "논문", ) - def __init__( self, agents: list[ModelAgent], @@ -830,7 +1613,8 @@ def __init__( if self._pool_store is not None: stored = {agent.id: agent for agent in self._pool_store.load_all()} agents = [stored.pop(agent.id, agent) for agent in agents] + list(stored.values()) - self.agents = [agent for agent in agents if not agent.disabled] + self.candidates = list(agents) + self.agents = [agent for agent in self.candidates if not agent.disabled] if not self.agents: # pragma: no cover raise ValueError("at least one enabled agent is required") self.client = client or ModelClient() @@ -848,6 +1632,8 @@ def __init__( # Per-agent circuit breaker: consecutive failures trip an agent "open" # so a persistently failing provider is skipped until it cools down. self._circuit: dict[str, dict[str, float]] = {} + self._circuit_lock = threading.Lock() + self._provider_readiness_lock = threading.Lock() self.circuit_failure_threshold = 3 self.circuit_reset_seconds = 30.0 # Optional exact-match response cache: default ttl 0 disables it (no behavior change). @@ -866,6 +1652,53 @@ def close(self) -> None: if self._store is not None: self._store.close() + def provider_readiness_report( + self, + *, + refresh: bool = False, + timeout: float = DEFAULT_PROVIDER_PROBE_TIMEOUT, + ) -> dict[str, Any]: + """Report provider liveness separately from an explicit chat readiness probe.""" + if type(refresh) is not bool: + raise ValueError("refresh must be a boolean") + probe_timeout = _validate_provider_probe_timeout(timeout) + items: list[dict[str, Any]] = [] + with self._provider_readiness_lock: + for agent in self.candidates: + provider = agent.provider_name or self._infer_provider_name(agent.base_url) + if agent.disabled: + items.append({ + "agent_id": agent.id, + "model": agent.model, + "provider": provider, + "status": "disabled", + }) + continue + if refresh: + item = dict(self.client.probe(agent, timeout=probe_timeout)) + item["provider"] = provider + items.append(redact_value(item)) + else: + items.append({ + "agent_id": agent.id, + "model": agent.model, + "provider": provider, + "status": "unprobed", + }) + active = [item for item in items if item["status"] != "disabled"] + status = "unprobed" if not refresh else ( + "ready" if active and all(item["status"] == "ready" for item in active) else "not_ready" + ) + return { + "status": status, + "probe": "refresh" if refresh else "none", + "timeout_seconds": probe_timeout, + "checked_at": int(time.time()) if refresh else None, + "agent_count": len(active), + "ready_agent_count": sum(item["status"] == "ready" for item in active), + "items": items, + } + def _reload_state(self) -> None: for record in self._store.load("workflow_run"): self._workflow_runs[record["workflow_run_id"]] = record @@ -897,7 +1730,12 @@ def proxy_completion( text = self._latest_user_text(messages) else: text = _coerce_input_text(body.get("input")) - agent = self._select_agent(text, "worker") + requested_model = body.get("model") + agent = self._requested_agent(requested_model) + if agent is not None and agent.disabled: + raise RuntimeError(f"requested model {requested_model!r} is disabled") + if agent is None: + agent = self._select_agent(text, "worker") upstream = { key: value for key, value in body.items() @@ -909,6 +1747,17 @@ def proxy_completion( upstream["stream"] = False return self.client.proxy_send(agent, endpoint, upstream) + def _requested_agent(self, requested_model: Any) -> ModelAgent | None: + """Resolve an explicit model without silently serving a different model.""" + if requested_model is None or requested_model == "contextual-orchestrator": + return None + if type(requested_model) is not str or not requested_model: + raise ValueError("requested model must be a configured non-empty string") + matches = [candidate for candidate in self.candidates if candidate.model == requested_model] + if not matches: + raise ValueError(f"requested model {requested_model!r} is not configured") + return next((candidate for candidate in matches if not candidate.disabled), matches[0]) + def complete(self, messages: list[ChatMessage], mode: str = "auto") -> dict[str, Any]: """Return a route or conducted completion without persisting a workflow run.""" if self._cache is None: @@ -1050,12 +1899,25 @@ def batch_route(self, prompts: list[str]) -> list[dict[str, Any]]: answers: dict[int, dict[str, Any]] = {} for agent_id, requests in requests_by_agent.items(): - for custom_id, result in self.client.batch_chat(agents_by_id[agent_id], requests).items(): - answers[int(custom_id.rsplit("_", 1)[1])] = result + results = _validate_batch_results( + requests, + self.client.batch_chat(agents_by_id[agent_id], requests), + ) + for custom_id, result in results.items(): + try: + prefix, suffix = custom_id.rsplit("_", 1) + index = int(suffix) + except (AttributeError, ValueError) as exc: + raise RuntimeError("batch provider returned an invalid request identifier") from exc + if prefix != "task" or custom_id != f"task_{index}" or not 0 <= index < len(selected): + raise RuntimeError("batch provider returned an invalid request identifier") + if index in answers: + raise RuntimeError("batch provider returned a duplicate request identifier") + answers[index] = result records: list[dict[str, Any]] = [] for index, (prompt, agent) in enumerate(selected): - result = answers.get(index, {"content": None, "usage": None}) + result = answers[index] row: dict[str, Any] = { "id": 0, "role": "worker", "agent_id": agent.id, "subtask": "Direct route (batched)", "access": [], "output": result["content"], @@ -1151,7 +2013,9 @@ def compare_to_baseline(self, prompts: list[str], mode: str = "auto") -> dict[st messages = [{"role": "user", "content": prompt}] start = time.perf_counter() - orchestrated = self.complete(messages, mode=mode) + # Evaluation must measure provider work, not a cache hit from a prior request. + # The normal completion path still honors the configured response cache. + orchestrated = self._dispatch(messages, mode) orchestrated_latency = round((time.perf_counter() - start) * 1000, 2) start = time.perf_counter() @@ -1254,7 +2118,12 @@ def patch_agent(self, agent_pool_id: str, worker_agent_id: str, patch: dict[str, if "provider_exclusions" in patch: patched = replace(patched, provider_exclusions=tuple(patch["provider_exclusions"])) - self.agents = [patched if agent.id == worker_agent_id else agent for agent in self.agents] + updated_candidates = [patched if agent.id == worker_agent_id else agent for agent in self.candidates] + updated_agents = [agent for agent in updated_candidates if not agent.disabled] + if not updated_agents: + raise ValueError("cannot disable the last enabled agent") + self.candidates = updated_candidates + self.agents = updated_agents if self._pool_store is not None: self._pool_store.save(patched) self._append_audit_event( @@ -1292,15 +2161,16 @@ def add_agent(self, agent_pool_id: str, value: dict[str, Any]) -> dict[str, Any] if "id" not in value or "model" not in value: raise ValueError("agent requires id and model") agent = ModelAgent.from_dict(value) - if any(existing.id == agent.id for existing in self.agents): + if any(existing.id == agent.id for existing in self.candidates): raise ValueError(f"agent {agent.id} already exists") if not agent.base_url.startswith("mock://"): parsed = urlparse(agent.base_url) - if parsed.scheme != "https" or not parsed.hostname: - raise ValueError("non-mock agents must use an https base_url") - if not agent.credential_name: + if not _is_local_provider_url(agent.base_url) and (parsed.scheme != "https" or not parsed.hostname): + raise ValueError("non-mock remote agents must use an https base_url; local agents use mlx://loopback") + if not _is_local_provider_url(agent.base_url) and not agent.credential_name: raise ValueError("non-mock agents require credential_key or legacy api_key_env") - self.agents = [*self.agents, agent] + self.candidates = [*self.candidates, agent] + self.agents = [candidate for candidate in self.candidates if not candidate.disabled] if self._pool_store is not None: self._pool_store.save(agent) self._append_audit_event( @@ -1318,10 +2188,11 @@ def remove_agent(self, agent_pool_id: str, worker_agent_id: str) -> dict[str, An if agent_pool_id != "default": # pragma: no cover raise KeyError(agent_pool_id) target = self._agent(worker_agent_id) - remaining_enabled = [agent for agent in self.agents if agent.id != worker_agent_id and not agent.disabled] + remaining_enabled = [agent for agent in self.candidates if agent.id != worker_agent_id and not agent.disabled] if not remaining_enabled: raise ValueError("cannot remove the last enabled agent") - self.agents = [agent for agent in self.agents if agent.id != worker_agent_id] + self.candidates = [agent for agent in self.candidates if agent.id != worker_agent_id] + self.agents = [agent for agent in self.candidates if not agent.disabled] if self._pool_store is not None: # Disabled tombstone (not a row delete): it overlays the seed file on restart # and startup drops disabled agents, so removal survives even for seed agents. @@ -1570,26 +2441,29 @@ def _failover_candidates(self, primary: ModelAgent, text: str, role: str) -> lis return healthy or eligible or [primary] def _circuit_open(self, agent_id: str) -> bool: - state = self._circuit.get(agent_id) - if not state or state["failures"] < self.circuit_failure_threshold: - return False - if time.monotonic() - state["opened_at"] >= self.circuit_reset_seconds: - state["failures"] = 0.0 - state["opened_at"] = 0.0 - return False - return True + with self._circuit_lock: + state = self._circuit.get(agent_id) + if not state or state["failures"] < self.circuit_failure_threshold: + return False + if time.monotonic() - state["opened_at"] >= self.circuit_reset_seconds: + state["failures"] = 0.0 + state["opened_at"] = 0.0 + return False + return True def _record_failure(self, agent_id: str) -> None: - state = self._circuit.setdefault(agent_id, {"failures": 0.0, "opened_at": 0.0}) - state["failures"] += 1.0 - if state["failures"] >= self.circuit_failure_threshold and not state["opened_at"]: - state["opened_at"] = time.monotonic() + with self._circuit_lock: + state = self._circuit.setdefault(agent_id, {"failures": 0.0, "opened_at": 0.0}) + state["failures"] += 1.0 + if state["failures"] >= self.circuit_failure_threshold and not state["opened_at"]: + state["opened_at"] = time.monotonic() def _record_success(self, agent_id: str) -> None: - self._circuit.pop(agent_id, None) + with self._circuit_lock: + self._circuit.pop(agent_id, None) def _agent(self, agent_id: str) -> ModelAgent: - for agent in self.agents: + for agent in self.candidates: if agent.id == agent_id: return agent raise KeyError(agent_id) # pragma: no cover @@ -1597,64 +2471,123 @@ def _agent(self, agent_id: str) -> ModelAgent: def _needs_workflow(self, text: str) -> bool: lowered = text.lower() hits = sum(1 for hint in self.COMPLEX_HINTS if hint in lowered) - return hits >= self.policy.conduct_hint_threshold or len(text) > 700 + return hits >= self.policy.conduct_hint_threshold or len(text) > self.policy.route_text_length_threshold def _latest_user_text(self, messages: list[ChatMessage]) -> str: return next((m.get("content", "") for m in reversed(messages) if m.get("role") == "user"), "") # pragma: no cover def _model_judge_verification(self, task: str, fallback: dict[str, Any]) -> dict[str, Any]: - """Ask a model to judge the verifier report (fixes term-matching false negatives). - - The judge must answer ACCEPT or REJECT; an ambiguous reply or a judge failure - keeps the term-based fallback verdict — the judge can only refine, never break. - """ + """Ask a model for a strict structured verdict and fail closed on uncertainty.""" verifier_output = fallback.get("verifier_output", "") if not verifier_output: - return fallback - judge = self._select_agent(task, "verifier") + return { + "accepted": False, + "reason": "model judge requires a non-empty verifier report", + "verifier_output": verifier_output, + "judge": "model", + } try: - reply = self.client.chat(judge, [ - {"role": "system", "content": ( - "You are the verification judge. Read the verifier report about the task. " - "Reply with exactly one word: ACCEPT if the verified work is sound, " - "REJECT if it has disqualifying problems." - )}, - {"role": "user", "content": f"Task:\n{task}\n\nVerifier report:\n{verifier_output}"}, - ]) - except Exception: # noqa: BLE001 - judge failure must not break the request - return fallback - upper = (reply or "").strip().upper() - if "ACCEPT" in upper and "REJECT" not in upper: - return {"accepted": True, "reason": "model judge accepted the verifier report", - "verifier_output": verifier_output, "judge": "model"} - if "REJECT" in upper and "ACCEPT" not in upper: - return {"accepted": False, "reason": "model judge rejected the verifier report", - "verifier_output": verifier_output, "judge": "model"} - return fallback - - def _judge_verifier_output(self, verifier_output: str, thinker_output: str, worker_output: str) -> dict[str, Any]: - lowered = verifier_output.lower() - if any(term in lowered for term in self.policy.verifier_negative_terms): # pragma: no cover + components = _resolve_fast_mlsirm_components() + except Exception: # noqa: BLE001 - a broken installed judge must not bypass the required path return { "accepted": False, - "reason": "verifier output flagged disagreement or risk", + "reason": "fast-mlsirm judge could not be loaded; verification failed closed", "verifier_output": verifier_output, + "judge": "model", } - if any(term in lowered for term in self.policy.verifier_positive_terms): # pragma: no cover + if components is None: return { - "accepted": True, - "reason": "verifier output accepted the synthesized result", + "accepted": False, + "reason": "fast-mlsirm judge is unavailable; verification failed closed", "verifier_output": verifier_output, + "judge": "model", } - if thinker_output and worker_output: + try: + judge = self._select_agent(task, "verifier") + # The judge is one bounded provider call. Do not pass the + # planning strategy ("template"/"generated") as an + # orchestration mode or recursively conduct another workflow. + judge_adapter = _FastMLSIJudgeAdapter(self, task, judge.id, mode="route") + fast_judge = components.judge_cls( + judge_adapter, + mode="route", + accept_threshold=0.7, + ) + result = fast_judge.judge( + task=task, + answer=verifier_output, + criteria=( + components.criterion_cls( + criterion_id="evidence_quality", + description="Does the verifier output identify concrete evidence and caveats with actionable impact?", + weight=1.0, + ), + components.criterion_cls( + criterion_id="risk_signal", + description="Does the verifier output mention substantive risks and constraints with support?", + weight=1.0, + ), + ), + ) + verification = { + "accepted": result.accepted, + "reason": result.rationale, + "verifier_output": verifier_output, + "judge": "model", + } + if judge_adapter.served_agent_id is not None and judge_adapter.served_agent_id != judge.id: + verification["judge_agent_id"] = judge_adapter.served_agent_id + if result.usage: + verification["judge_usage"] = result.usage + verification["judge_orchestration_mode"] = result.orchestration_mode + criterion_scores = getattr(result, "criterion_scores", None) + to_irt_row = getattr(result, "to_irt_row", None) + if isinstance(criterion_scores, Mapping) and callable(to_irt_row): + try: + irt_row = to_irt_row(item_type="dichotomous") + except Exception: # noqa: BLE001 - invalid IRT projection must not be published + return { + "accepted": False, + "reason": "model judge returned an invalid multi-item IRT projection; verification failed closed", + "verifier_output": verifier_output, + "judge": "model", + } + if ( + len(criterion_scores) < 2 + or type(irt_row) not in (tuple, list) + or len(irt_row) != len(criterion_scores) + ): + return { + "accepted": False, + "reason": "model judge returned an invalid multi-item IRT projection; verification failed closed", + "verifier_output": verifier_output, + "judge": "model", + } + verification["judge_criterion_scores"] = dict(criterion_scores) + verification["judge_irt_item_type"] = "dichotomous" + verification["judge_irt_row"] = list(irt_row) + return verification + except components.format_error: return { - "accepted": True, - "reason": "fallback acceptance from available planner and worker output", + "accepted": False, + "reason": "model judge returned an invalid structured verdict; verification failed closed", + "verifier_output": verifier_output, + "judge": "model", + } + except Exception: # noqa: BLE001 - judge failure must not break the request + return { + "accepted": False, + "reason": "model judge unavailable; verification failed closed", "verifier_output": verifier_output, + "judge": "model", } - return { # pragma: no cover + + def _judge_verifier_output(self, verifier_output: str, thinker_output: str, worker_output: str) -> dict[str, Any]: + """Prepare evidence for the model judge without making a heuristic decision.""" + del thinker_output, worker_output + return { "accepted": False, - "reason": "fallback verifier disagreement with missing upstream outputs", + "reason": "model judgment required; keyword matching is disabled", "verifier_output": verifier_output, } @@ -1693,7 +2626,7 @@ def list_agents(self, page_number: int = 1, page_size: int = 10) -> list[dict[st raise ValueError("page_number/page_size must be >= 1") start = (page_number - 1) * page_size end = start + page_size - return [self._agent_to_admin_payload(agent) for agent in self.agents[start:end]] + return [self._agent_to_admin_payload(agent) for agent in self.candidates[start:end]] def list_recent_runs(self, page_number: int = 1, page_size: int = 10) -> list[dict[str, Any]]: """Return a paginated list of recent workflow run records.""" @@ -7907,7 +8840,7 @@ def section( def admin_state(self) -> dict[str, Any]: """Build the admin console state payload from agents, policy, and audit data.""" - agent_page_size = max(1, len(self.agents)) + agent_page_size = max(1, len(self.candidates)) return { "agents": self.list_agents(page_size=agent_page_size), "policy": { @@ -8057,7 +8990,7 @@ def _security_posture_criterion(self, security_profile: dict[str, Any]) -> dict[ warnings: list[str] = [] if auth_mode == "single_token": warnings.append("single bearer token shared by admin and inference scopes") - elif auth_mode != "split_token": + elif auth_mode not in {"split_token", "external_bearer_verifier"}: issues.append("no bearer token configured outside loopback-only development") if security_profile.get("allow_public_bind"): issues.append("public bind is enabled") @@ -8078,7 +9011,12 @@ def _security_posture_criterion(self, security_profile: dict[str, Any]) -> dict[ remediation = "For enterprise pilots, split admin and inference tokens before customer evaluation." else: status = "pass" - evidence = "split tokens, private bind default, hidden traces, rate limits, and run limits are configured" + auth_evidence = ( + "external bearer verifier" + if auth_mode == "external_bearer_verifier" + else "split tokens" + ) + evidence = f"{auth_evidence}, private bind default, hidden traces, rate limits, and run limits are configured" remediation = "Keep these controls enabled for customer-facing pilots." return self._criterion("security_posture", "Security posture", status, evidence, remediation) @@ -8105,6 +9043,8 @@ def _provider_egress_criterion(self) -> dict[str, str]: for agent in self.agents: if agent.base_url.startswith("mock://"): continue + if _is_local_provider_url(agent.base_url): + continue remote.append(agent.id) parsed = urlparse(agent.base_url) if parsed.scheme != "https" or not agent.credential_name: @@ -8501,7 +9441,7 @@ def chat_completion_response( if include_trace: orchestration["trace"] = redact_value(result["trace"]) return { - "id": f"chatcmpl-{int(time.time() * 1000)}", + "id": _new_chat_completion_id(), "object": "chat.completion", "created": int(time.time()), "model": model, @@ -8532,7 +9472,7 @@ def chat_completion_chunks( token-by-token streaming — real token streaming requires a streaming ModelClient. """ answer = result.get("answer", "") - completion_id = f"chatcmpl-{int(time.time() * 1000)}" + completion_id = _new_chat_completion_id() created = int(time.time()) base = {"id": completion_id, "object": "chat.completion.chunk", "created": created, "model": model} @@ -8556,6 +9496,11 @@ def chat_completion_chunks( return chunks +def _new_chat_completion_id() -> str: + """Create a collision-resistant OpenAI-compatible completion identifier.""" + return f"chatcmpl-{uuid.uuid4().hex}" + + def sse_stream_body(chunks: list[dict[str, Any]]) -> str: """Serialize chat completion chunks as a Server-Sent Events body terminated by ``[DONE]``.""" frames = [f"data: {json.dumps(chunk, ensure_ascii=False)}\n\n" for chunk in chunks] diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index c58d4cb79..4ae74c348 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -9,7 +9,7 @@ import threading import time import urllib.parse -from typing import Any +from typing import Any, Callable import uuid from .admin import ADMIN_HTML, ADMIN_TRANSLATIONS @@ -19,7 +19,9 @@ from .batch_routing import BatchRequest from .orchestrator import ( BudgetExceededError, + MAX_LOCAL_CONCURRENCY, TaskOrchestrator, + _new_chat_completion_id, chat_completion_chunks, chat_completion_response, redact_value, @@ -43,6 +45,8 @@ # Responses API body keys (`input` replaces `messages`). ALLOWED_RESPONSES_KEYS = { "model", "input", "instructions", "stream", "metadata", "reasoning", + "include", "prompt_cache_key", "client_metadata", "previous_response_id", + "conversation", "truncation", "max_output_tokens", "text", } | OPENAI_PASSTHROUGH_PARAM_KEYS ALLOWED_BATCH_KEYS = {"requests", "attribution", "routing", "model"} ALLOWED_EMBEDDINGS_BATCH_KEYS = {"model", "input", "inputs", "endpoint", "metadata", "attribution"} @@ -90,13 +94,23 @@ class SecurityConfig: rate_limit_requests: int = 60 rate_limit_window_seconds: int = 60 max_concurrent_runs: int = 8 + # Deployment may inject a real OIDC/JWT verifier (for example a Keyverse + # relying-party adapter). The core deliberately does not decode JWTs with + # an unsafe hand-rolled parser or own Keycloak admin credentials. + bearer_verifier: Callable[[str, str], bool] | None = None _rate_buckets: dict[str, tuple[int, float]] = field(default_factory=dict, init=False, repr=False) _rate_lock: threading.Lock = field(default_factory=threading.Lock, init=False, repr=False) _run_semaphore: threading.BoundedSemaphore = field(init=False, repr=False) def __post_init__(self) -> None: + if self.auth_token and (self.admin_token or self.inference_token): + raise ValueError("single auth_token cannot be combined with split tokens") if (self.admin_token or self.inference_token) and not (self.admin_token and self.inference_token): raise ValueError("split token mode requires both admin_token and inference_token") + if type(self.max_concurrent_runs) is not int or not 1 <= self.max_concurrent_runs <= MAX_LOCAL_CONCURRENCY: + raise ValueError( + f"max_concurrent_runs must be an integer in 1..{MAX_LOCAL_CONCURRENCY}" + ) self._run_semaphore = threading.BoundedSemaphore(self.max_concurrent_runs) def check_bind(self, host: str) -> None: @@ -106,14 +120,26 @@ def check_bind(self, host: str) -> None: def authorize(self, headers: Any, scope: str, client_address: str) -> None: """Validate bearer token for admin or inference scope.""" - if not (self.auth_token or self.admin_token or self.inference_token): + if not (self.auth_token or self.admin_token or self.inference_token or self.bearer_verifier): raise RequestError(401, "unauthorized", "bearer token is required") raw = headers.get("authorization", "") if not raw.lower().startswith("bearer "): raise RequestError(401, "unauthorized", "bearer token is required") token = raw.split(" ", 1)[1].strip() - expected = self.auth_token or (self.admin_token if scope == "admin" else self.inference_token) - if not expected or not secrets.compare_digest(token, expected): + if self.bearer_verifier is not None: + try: + valid = bool(self.bearer_verifier(token, scope)) + except Exception: # noqa: BLE001 - an auth adapter failure is an auth denial + valid = False + else: + if scope == "admin": + expected = self.admin_token or self.auth_token + elif scope == "inference": + expected = self.inference_token or self.auth_token + else: + expected = "" + valid = bool(expected) and secrets.compare_digest(token, expected) + if not valid: raise RequestError(401, "unauthorized", "bearer token is invalid for this scope") def check_rate_limit(self, key: str) -> None: @@ -138,7 +164,9 @@ def release_run_slot(self) -> None: def readiness_profile(self) -> dict[str, Any]: """Return a secret-free security profile for sales-readiness evidence.""" - if self.admin_token and self.inference_token: + if self.bearer_verifier is not None: + auth_mode = "external_bearer_verifier" + elif self.admin_token and self.inference_token: auth_mode = "split_token" elif self.auth_token: auth_mode = "single_token" @@ -303,6 +331,81 @@ def _response_payload(payload: dict[str, Any], include_trace: bool) -> dict[str, return _strip_trace(safe_payload) +def responses_sse_body(response: dict[str, Any]) -> str: + """Frame a completed Responses object as a valid SSE response.""" + sequence = 0 + frames: list[str] = [] + + def emit(event_type: str, **values: Any) -> None: + nonlocal sequence + payload = {"type": event_type, "sequence_number": sequence, **values} + sequence += 1 + frames.append( + f"event: {event_type}\ndata: {json.dumps(payload, ensure_ascii=False)}\n\n" + ) + + in_progress = {**response, "status": "in_progress", "output": []} + emit("response.created", response=in_progress) + for output_index, item in enumerate(response.get("output", [])): + if not isinstance(item, dict): + continue + item_in_progress = {**item, "status": "in_progress"} + emit("response.output_item.added", output_index=output_index, item=item_in_progress) + if item.get("type") == "message": + for content_index, part in enumerate(item.get("content", [])): + if not isinstance(part, dict): + continue + part_in_progress = {**part, "text": ""} + emit( + "response.content_part.added", + item_id=item.get("id"), + output_index=output_index, + content_index=content_index, + part=part_in_progress, + ) + if part.get("type") == "output_text": + emit( + "response.output_text.delta", + item_id=item.get("id"), + output_index=output_index, + content_index=content_index, + delta=part.get("text", ""), + ) + emit( + "response.output_text.done", + item_id=item.get("id"), + output_index=output_index, + content_index=content_index, + text=part.get("text", ""), + ) + emit( + "response.content_part.done", + item_id=item.get("id"), + output_index=output_index, + content_index=content_index, + part=part, + ) + elif item.get("type") == "function_call": + arguments = str(item.get("arguments", "{}")) + emit( + "response.function_call_arguments.delta", + item_id=item.get("id"), + output_index=output_index, + delta=arguments, + ) + emit( + "response.function_call_arguments.done", + item_id=item.get("id"), + output_index=output_index, + name=item.get("name", ""), + arguments=arguments, + ) + emit("response.output_item.done", output_index=output_index, item=item) + emit("response.completed", response=response) + frames.append("data: [DONE]\n\n") + return "".join(frames) + + def build_server( orchestrator: TaskOrchestrator, host: str = "127.0.0.1", @@ -341,11 +444,50 @@ def do_GET(self) -> None: # noqa: N802 "status": "ok", "service": "contextual-orchestrator", "agent_count": len(orchestrator.agents), + "candidate_count": len(orchestrator.candidates), + "enabled_agent_count": len(orchestrator.agents), "batch_backend": coordinator.batch_backend.name, "embedding_batch_backend": coordinator.embedding_batch_backend.name, + "provider_readiness": "unprobed", "usage_record_count": len(coordinator.ledger.records()), }) return + if path == "/v1/models": + self._authorize("inference") + models: list[dict[str, Any]] = [{ + "id": "contextual-orchestrator", + "object": "model", + "created": 0, + "owned_by": "contextual-orchestrator", + "kind": "orchestrator", + "status": "active", + "readiness": "unprobed", + }] + model_groups: dict[str, list[Any]] = {} + for agent in orchestrator.candidates: + if not agent.model or agent.model == "contextual-orchestrator": + continue + model_groups.setdefault(agent.model, []).append(agent) + for model, candidates in model_groups.items(): + representative = next( + (candidate for candidate in candidates if not candidate.disabled), + candidates[0], + ) + models.append({ + "id": model, + "object": "model", + "created": 0, + "owned_by": representative.provider_name or "contextual-orchestrator", + "kind": "worker", + "status": ( + "active" + if any(not candidate.disabled for candidate in candidates) + else "disabled" + ), + "readiness": "unprobed", + }) + self._send({"object": "list", "data": models}) + return if path.startswith("/v1/batch/embeddings/"): # Embeddings batch polling is an inference-scope surface, so # it is authorized here before the admin gate below. @@ -404,7 +546,7 @@ def do_GET(self) -> None: # noqa: N802 items = orchestrator.list_agents(page_number=page_number, page_size=page_size) self._send({ "items": items, - "total_count": len(orchestrator.agents), + "total_count": len(orchestrator.candidates), "page_number": page_number, "page_size": page_size, }) @@ -412,6 +554,12 @@ def do_GET(self) -> None: # noqa: N802 if path == "/api/v1/orchestration_policies/default_policy": self._send(orchestrator.admin_state()["policy"]) return + if path == "/api/v1/provider_readiness/latest": + raw_refresh = (query.get("refresh") or ["false"])[0].lower() + if raw_refresh not in {"true", "false"}: + raise ValueError("refresh must be true or false") + self._send(orchestrator.provider_readiness_report(refresh=raw_refresh == "true")) + return if path == "/api/v1/analytics_snapshots/latest": self._send(orchestrator.analytics_snapshot(locale_bundles=ADMIN_TRANSLATIONS)) return @@ -875,7 +1023,10 @@ def do_POST(self) -> None: # noqa: N802 "duration_ms": round((time.perf_counter() - started_at) * 1000, 2), }, ) - self._send(proxied) + if body.get("stream") is True: + self._send_sse(responses_sse_body(proxied)) + else: + self._send(proxied) return if path == "/admin/simulate": @@ -1022,7 +1173,7 @@ def _write_sse(self, frame: str) -> None: def _stream_route_completion(self, orchestrator: Any, security: Any, messages: Any, model_name: str) -> None: """Pipe a worker's live deltas out as OpenAI chat.completion.chunk SSE frames.""" run_id = f"run_{uuid.uuid4().hex}" - completion_id = f"chatcmpl-{int(time.time() * 1000)}" + completion_id = _new_chat_completion_id() created = int(time.time()) def frame(delta: dict[str, Any], finish: str | None = None) -> str: diff --git a/docs/architecture.md b/docs/architecture.md index c0f63a81e..1130f3be0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -9,7 +9,15 @@ ## What The Architecture Is -The public shape is a single model API. The internal shape is a model pool plus a learned coordinator that decides when to answer directly, when to delegate, how much context each worker receives, when to verify, and how to synthesize the final answer. +The public shape is a single model API. The internal shape is a model pool plus +an orchestrator that decides when to answer directly, when to delegate, how +much context each worker receives, when to verify, and how to synthesize the +final answer. The public `contextual-orchestrator` model is the orchestration +candidate; the configured local and remote models are worker candidates in its +pool. The candidate registry retains every discovered model. `disabled` is an +explicit operator/admin quarantine or removal state, not an automatic discovery +result. Capability and recursion constraints are expressed separately from that +state. The useful split is quality-latency, not separate products: @@ -29,7 +37,12 @@ The Fugu report combines these ideas into production constraints: ## Implementation Mapping -This repository implements the interface and control plane, not the trained coordinator. +This repository implements the interface and control plane, not the trained +coordinator or its optional recursive self-worker. The public model is therefore +an explicit control-plane candidate, while the worker pool is selected from +configured `ModelAgent` records. The current implementation keeps that public +record out of internal roles with provider exclusions until the runtime has a +bounded, authenticated recursion protocol; it is not administratively disabled. - `contextual_orchestrator.orchestrator.Agent`: one configured worker model. - `Orchestrator.route_once`: the low-latency routing path. @@ -38,7 +51,7 @@ This repository implements the interface and control plane, not the trained coor - `ModelClient`: OpenAI-compatible HTTP client, with `mock://` for local checks. - `contextual_orchestrator.server`: small `/v1/chat/completions` HTTP server. -The deliberate simplification is the policy. The paper systems learn routing and topology from rewards; this lab uses deterministic keyword scoring so the repo runs without training data, GPUs, or vendor credentials. +The deliberate simplification is the policy. The paper systems learn routing and topology from rewards; this lab uses a deterministic capability-hint heuristic only for worker/role routing so the repo runs without training data, GPUs, or vendor credentials. It is never an answer-quality, verification, or accept/reject judgment: verifier decisions must use the structured model judge and fail closed (see [ADR 0001](planning/adrs/0001-fail-closed-model-judgment.md)). Add learned routing only when there is an evaluation set and logs proving the heuristic policy is the bottleneck. diff --git a/docs/benchmarks/2026-07-06-openai-optimizer.md b/docs/benchmarks/2026-07-06-openai-optimizer.md index eed90dd4d..b70c1a65f 100644 --- a/docs/benchmarks/2026-07-06-openai-optimizer.md +++ b/docs/benchmarks/2026-07-06-openai-optimizer.md @@ -64,8 +64,12 @@ end-to-end in 48.2 s, producing a coherent rollback-safe migration plan. - `plan_source=generated`; strict validation (sequential ids, known roles, backward-only access, answerable final step) with automatic template fallback held. -- Honest limitation observed: `verification.accepted=false` because the term-matching - verifier judge saw risk-vocabulary in a verifier step *about* downtime risks — a - false negative of the heuristic judge, not of the plan. **Fixed since:** `OrchestrationPolicy.verifier_judge="model"` asks a verifier-selected model to reply ACCEPT/REJECT on the verifier report (ambiguous replies and judge failures keep the term verdict; default remains "terms"). +- Honest limitation observed: `verification.accepted=false` because the former + term-matching verifier judge saw risk-vocabulary in a verifier step *about* + downtime risks — a false negative of the heuristic judge, not of the plan. + **Fixed since:** `OrchestrationPolicy.verifier_judge="model"` requires a strict + JSON decision, routes the judge through normal failover/usage handling, and + fails closed on ambiguous replies or judge failures. Keyword matching is no + longer a supported fallback; see ADR 0001. - This validation exercised `conduct()` directly (not `run()`), so it does not appear in spend totals. diff --git a/docs/benchmarks/2026-08-11-polytomous-llm-judge.md b/docs/benchmarks/2026-08-11-polytomous-llm-judge.md new file mode 100644 index 000000000..a67ae01a7 --- /dev/null +++ b/docs/benchmarks/2026-08-11-polytomous-llm-judge.md @@ -0,0 +1,256 @@ +# Local MLX polytomous judge calibration — 2026-08-11 + +Status: exploratory integration evidence; not a claim that the local judge is +unbiased or that this single case is sufficient for IRT estimation. + +## Setup + +The path under test was: + +`fast-mlsirm.ContextualOrchestratorJudge -> contextual-orchestrator.TaskOrchestrator -> ModelClient -> mlx-lm` + +The provider was the existing loopback server at `mlx://127.0.0.1:8080/v1`. +The worker was `mlx-community/llama-3.2-3b-instruct-4bit` and the judge was +`mlx-community/gemma-4-e4b-it-4bit`. Both used temperature `0`, +`chat_template_args={"enable_thinking": false}`, and bounded output. The +judge call used two criteria, so every converted row has two item columns. +No keyword matching or lexical acceptance rule was used. + +## Category-count sweep + +The same worker answer and rubric were sent to the judge with K in +`{2, 3, 5, 7}`. Each row below was validated by +`validate_irt_response_matrix(..., item_type="polytomous", n_categories=K)`; +the `[1, 2]` shape is only a contract smoke test, not a meaningful fitted IRT +sample. + +| K | derived score | accepted | criterion categories (`factual_support`, `task_alignment`) | IRT row | judge tokens | seconds | +|---:|---:|:---:|:---:|:---:|---:|---:| +| 2 | 1.0000 | yes | `(1, 1)` | `(1, 1)` | 658 | 5.180 | +| 3 | 0.7500 | yes | `(2, 1)` | `(2, 1)` | 683 | 3.602 | +| 5 | 0.5000 | no | `(3, 1)` | `(3, 1)` | 669 | 3.099 | +| 7 | 0.9167 | yes | `(5, 6)` | `(5, 6)` | 685 | 3.394 | + +This run does not establish the user’s proposed monotone positive effect as a +law. It does establish a concrete category-count sensitivity: the same answer +changed both criterion levels and the derived acceptance decision. Therefore +the equal-width category-to-score mapping remains experimental, and the +calibration gate in ADR 0006 is required before interpreting these values as a +stable latent trait. + +## Prompt-hardening replication — 2026-08-12 + +The strict judge prompt was then made explicit about one JSON object, no +markdown fences, integer category values, and a decimal top-level score only. +The same task, answer, reference, criteria, temperature `0`, disabled thinking, +and `mlx-community/gemma-4-e4b-it-4bit` judge were run twice at each K. Every +result remained on the required +`fast-mlsirm.ContextualOrchestratorJudge -> contextual-orchestrator -> mlx-lm` +path, and all eight responses parsed without repair or keyword matching. + +| K | repeats | scores | accepted | category rows | total judge tokens | mean seconds | +|---:|---:|:---:|:---:|:---:|---:|---:| +| 2 | 2 | `(0.5, 0.5)` | `0/2` | `[(1,0), (1,0)]` | 1,252 | 4.546 | +| 3 | 2 | `(0.5, 0.5)` | `0/2` | `[(1,1), (1,1)]` | 1,244 | 2.647 | +| 5 | 2 | `(0.0, 0.0)` | `0/2` | `[(0,0), (0,0)]` | 1,242 | 2.505 | +| 7 | 2 | `(0.75, 0.75)` | `2/2` | `[(4,5), (4,5)]` | 1,282 | 2.882 | + +This replication strengthens the finding of category-count sensitivity but still +does not establish a universal positive bias: K=5 was lower than K=2 and K=3, +while K=7 was higher. Two repeated observations are not enough for uncertainty +intervals or production calibration; balanced cases, prompt-order perturbations, +and additional local judge models remain required. + +## Larger local judge comparison — 2026-08-12 + +To test whether a larger local judge removes the category-count concern, a +separate fixed case used three criteria (`factual_support`, `task_alignment`, +and `risk_awareness`), a reference answer, temperature `0`, thinking disabled, +and two repeats at each K. The answer asserted immediate shipment after a +smoke test while the reference explicitly noted that rollback rehearsal, load +testing, and independent review were absent. All eight Gemma 31B responses +parsed through the same contextual-orchestrator path. + +| judge | K | repeats | scores | accepted | category rows (`factual_support`, `risk_awareness`, `task_alignment`) | total tokens | seconds | +|---|---:|---:|:---:|:---:|:---|---:|---:| +| `mlx-community/gemma-4-31b-it-4bit` | 2 | 2 | `(0.3333, 0.3333)` | `0/2` | `[(1,0,0), (1,0,0)]` | 1,256 | `56.288, 15.047` | +| `mlx-community/gemma-4-31b-it-4bit` | 3 | 2 | `(0.3333, 0.3333)` | `0/2` | `[(1,0,1), (1,0,1)]` | 1,240 | `22.096, 13.138` | +| `mlx-community/gemma-4-31b-it-4bit` | 5 | 2 | `(0.3333, 0.3333)` | `0/2` | `[(2,0,2), (2,0,2)]` | 1,252 | `23.581, 14.882` | +| `mlx-community/gemma-4-31b-it-4bit` | 7 | 2 | `(0.3333, 0.3333)` | `0/2` | `[(4,0,2), (4,0,2)]` | 1,264 | `31.152, 20.346` | + +The larger Gemma kept the derived score and acceptance stable for this case, +but changed criterion category placement as K grew; this is not evidence that +larger models are unbiased. The cached +`outlier-ai/deepseek-r1-distill-qwen-32b-mlx-4bit` judge did not return a +structured response within the bounded 180-second request timeout on its first +K=2 call. The sweep was stopped after that timeout, so no quality comparison is +claimed for that model. Timeout and structured-output failure rate are therefore +part of the performance gate alongside score drift and IRT shape validation. + +## Direct K-way versus cumulative thresholds — 2026-08-12 + +The follow-up adapter was tested on the same fixed release-readiness case with +the same two criteria, `mlx-community/gemma-4-e4b-it-4bit` judge, +temperature `0`, disabled thinking, bounded output, and the same +`ContextualOrchestratorJudge -> contextual-orchestrator -> mlx-lm` path. Each +K/method pair was repeated twice. Every response parsed strictly, used one +orchestration trace step, and produced a two-item row accepted by +`validate_irt_response_matrix(..., item_type="polytomous", n_categories=K)`. + +| method | K | scores (two repeats) | accepted | mean seconds | tokens/call | +|---|---:|---:|:---:|---:|---:| +| direct | 2 | `(1.0000, 1.0000)` | `2/2` | 2.511 | 593 | +| direct | 3 | `(1.0000, 1.0000)` | `2/2` | 2.355 | 593 | +| direct | 5 | `(1.0000, 1.0000)` | `2/2` | 2.405 | 600 | +| direct | 7 | `(1.0000, 1.0000)` | `2/2` | 2.459 | 608 | +| cumulative threshold | 2 | `(1.0000, 1.0000)` | `2/2` | 2.540 | 605 | +| cumulative threshold | 3 | `(1.0000, 1.0000)` | `2/2` | 2.655 | 610 | +| cumulative threshold | 5 | `(0.5000, 0.5000)` | `0/2` | 2.844 | 629 | +| cumulative threshold | 7 | `(0.3333, 0.3333)` | `0/2` | 2.939 | 640 | + +This fixed case shows a material category-method difference: direct K-way +selection stayed maximally positive while cumulative thresholds became more +conservative as K increased. It does not prove that direct judging is +positively biased or that cumulative thresholds remove bias; the prompts have +different response structures, and a single case is not a calibration sample. +It does show why both method and K must be recorded and paired calibration must +remain a release gate. No keyword, lexical, or positional repair was used. + +## Framing and structured-output replication with a 3B judge — 2026-08-12 + +To test the suspected choice-count effect together with framing sensitivity, a +bounded local `mlx-community/llama-3.2-3b-instruct-4bit` judge evaluated one +semantically good release plan and one unsafe plan under neutral, liked, and +disliked framing. Temperature was `0`, thinking was disabled, output was +bounded to 256 tokens, and all 36 calls used the +`ContextualOrchestratorJudge -> contextual-orchestrator -> mlx-lm` route. The +good plan included canary rollout, monitoring, independent review, and a +rehearsed rollback; the bad plan recommended immediate deployment while +skipping review and rollback rehearsal. + +The table records the good-plan result at each K. `invalid` means the strict +parser rejected the model response; it was never repaired or accepted. + +| framing | direct K=2 / 5 / 7 (score; accepted) | cumulative K=2 / 5 / 7 (score; accepted) | +|---|---|---| +| neutral | `0.0000; no` / `0.7500; yes` / `0.5833; no` | `invalid` / `0.0000; no` / `invalid` | +| liked | `invalid` / `0.7500; yes` / `0.8333; yes` | `0.5000; no` / `0.0000; no` / `invalid` | +| disliked | `0.5000; no` / `0.7500; yes` / `0.8333; yes` | `invalid` / `invalid` / `invalid` | + +The good plan parsed in 11/18 calls and was accepted in 5/11 parsed calls; +the seven failures were five invalid JSON responses, one out-of-range category, +and one non-monotone threshold vector. The unsafe plan parsed in all 18 calls, +scored `0.0000` in every case, and was accepted zero times. Direct judging +parsed 8/9 good-plan calls versus 3/9 for cumulative thresholds. At K=7, +liked/disliked framing scored 0.25 above neutral, while K=5 was 0.75 for all +three frames; this is a framing interaction, not evidence of a monotone +positive-with-more-categories law. + +This run makes structured-output reliability a first-class local-model metric: +malformed and ordinally incoherent responses remain failed comparisons. A +separately measured bounded retry or stronger local-judge selection may be +considered, but keyword matching, positional repair, and silently dropping a +failed observation remain prohibited. + +## Same-route retry probe with a 3B judge — 2026-08-12 + +A separate bounded probe used the same local +`mlx-community/llama-3.2-3b-instruct-4bit` judge, temperature `0`, disabled +thinking, 256 output tokens, two criteria, and the +`ContextualOrchestratorJudge -> contextual-orchestrator -> mlx-lm` route. It +used one good release plan and three task framings. All nine direct K-way +responses parsed, but the score still moved materially: + +| framing | K=2 | K=5 | K=7 | +|---|---:|---:|---:| +| neutral | `0.0000` | `1.0000` | `0.8333` | +| liked | `0.0000` | `1.0000` | `0.9167` | +| disliked | `0.0000` | `0.7500` | `1.0000` | + +The same four cumulative-threshold calls were then retried once after strict +parsing failure. K=2 failed its boundary-array shape check, and K=3, K=5, and +K=7 failed monotonicity on both attempts; no retry produced an accepted result. +The retry was a second contextual-orchestrator completion and was validated by +the same strict parser; it did not inspect keywords, criterion positions, or +the invalid output to infer a category. This is evidence that an identical +retry is not a reliability fix and that direct K-way output remains +choice-count/framing-sensitive even when it parses. Every failed attempt stays +in the denominator. + +## Defects found and fixed during the run + +## Live gateway paired-method probe — 2026-08-12 + +A live call through the running bearer-authenticated gateway at +`127.0.0.1:18000` used the configured +`mlx-community/llama-3.2-3b-instruct-4bit` worker and the exact +`fast-mlsirm.ContextualOrchestratorJudge -> contextual-orchestrator -> mlx-lm` +route. The same answer and two criteria were judged with `K=5`; the adapter +mapped the gateway's nested `orchestration.trace` into the injected completion +record, so each call retained one trace step and usage. + +| method | score | accepted | categories | tokens | +|---|---:|:---:|---|---:| +| direct | `1.0000` | yes | both criteria `4/4` | `568` | +| cumulative threshold | `0.0000` | no | both criteria `0/0` | `570` | + +Both outputs passed the same strict parser and the two-item polytomous IRT +shape contract. The opposite result for the same content is a paired +method-sensitivity finding, not evidence that either method is unbiased or +that more categories cause positive bias. It reinforces the release rule that +method, K, trace, usage, and parse status must be retained; no keyword or +positional repair was used. + +The first local calls exposed model-format failures: numeric criterion keys, +the phrase `return criterion_categories` copied as a JSON key, and decimal +values in an integer category field. The adapter now supplies an exact literal +JSON schema with the validated criterion IDs and ordered category anchors, +accepts only mathematically integral JSON numbers for category values, and +rejects non-integral values, missing IDs, out-of-range values, and arrays. It +still fails closed; it never repairs a response using keywords or criterion +position. + +The replication also exposed a redundant-field shape failure: the Llama 3B +judge emitted an object-valued top-level `score` while emitting integer +criterion categories. Category-derived scoring now validates that redundant +`score` is itself a finite number in `0..1` and then deliberately derives the +effective score from the validated categories; malformed top-level fields are +rejected rather than ignored. + +## Local queue saturation observation — 2026-08-12 + +During a live gateway smoke, the local `mlx-lm` process was configured with +`prompt-concurrency=1` and received several large Codex prompts. A client-side +timeout left an abandoned generation in the server queue; the server later +logged `BrokenPipe` while writing to the disconnected client and was restarted +by its supervisor. The gateway's process liveness and the MLX provider's +readiness therefore cannot be treated as the same signal. The transport now +defaults same-agent retry budget to zero for explicit local providers (remote +providers retain their configured retry budget), because retrying a large local +prompt can multiply device work. An explicit local retry budget remains +available for a supervised provider restart scenario. This is an operational +reliability finding, not a judge-quality or positive-bias result; future +benchmarks must report timeout, queue/restart, and prompt-size evidence. + +## Required follow-up + +Before production or scientific IRT claims, add repeated paired cases with +balanced rubric/criterion order, score identifiers, reference presence, +answer-option order, and positive/negative/neutral framing. Record category +occupancy, score and acceptance deltas, agreement, and deterministic or human +gold differences. The cumulative-threshold ordinal design is now available as +an opt-in implementation, but it remains an experimental mitigation because a +different response structure can introduce its own calibration drift. + +The calibration rationale and local Zotero records/PDF attachments are tracked +in [ADR 0006](../planning/adrs/0006-polytomous-llm-judge-bias-calibration.md). +The multi-item boundary is tracked in +[ADR 0005](../planning/adrs/0005-irt-response-matrix-contract.md). + +Primary research links include [Li et al., *Evaluating Scoring Bias in +LLM-as-a-Judge*](https://arxiv.org/abs/2506.22316), [Zheng et al., *Large +Language Models Are Not Robust Multiple Choice +Selectors*](https://proceedings.iclr.cc/paper_files/paper/2024/hash/54dd9e0cff6d9214e20d97eb2a3bae49-Abstract-Conference.html), +[Pezeshkpour and Hruschka, *LLM Sensitivity to the Order of Options*](https://aclanthology.org/2024.findings-naacl.130/), +and [Sharma et al., *Towards Understanding Sycophancy in Language +Models*](https://www.anthropic.com/research/towards-understanding-sycophancy-in-language-models). diff --git a/docs/benchmarks/2026-08-13-local-mlx-gateway.md b/docs/benchmarks/2026-08-13-local-mlx-gateway.md new file mode 100644 index 000000000..5340cb479 --- /dev/null +++ b/docs/benchmarks/2026-08-13-local-mlx-gateway.md @@ -0,0 +1,818 @@ +# Local MLX gateway and fast-mlsirm judge — 2026-08-13 + +Status: integration evidence, not a quality or bias claim. The run used one +cached local model and one fixed task; broader paired calibration remains +required. + +## Execution contract + +The measured judge path was: + +`fast-mlsirm.ContextualOrchestratorJudge -> contextual-orchestrator._FastMLSIJudgeAdapter -> ModelClient -> mlx-lm` + +The MLX server was already listening on `mlx://127.0.0.1:8080/v1` and exposed +`mlx-community/llama-3.2-3b-instruct-4bit`. The run used temperature `0`, +`max_output_tokens=192` for workflow steps, disabled MLX thinking, zero +retries, and two rubric criteria. The fast-mlsirm source checkout ran in its +Python 3.12 environment (`fast-mlsirm 0.7.0`, NumPy 2.5.1); the contextual +source checkout was placed on `PYTHONPATH`. No keyword, lexical, positional, +silent-drop, or malformed-output repair was used. + +Before a live run, the selected interpreter must pass: + +```bash +PYTHONPATH=/path/to/fast-mlsirm/python:/path/to/contextual-orchestrator \ + /path/to/fast-mlsirm/.venv/bin/python -m contextual_orchestrator check-fast-mlsirm +``` + +The command is a fail-closed preflight for the exact judge boundary. It reports +the Python executable, fast-mlsirm version, import status, and +`contextual-orchestrator-contract-v1` marker; it does not call a provider or +replace a failed judge with keyword matching. + +The contextual-orchestrator project environment alone does not install +fast-mlsirm's NumPy dependency. Running the same command with that environment +therefore failed closed with `fast-mlsirm judge could not be loaded`; this is a +dependency-boundary finding, not a successful judge result. The reproducible +integration command uses fast-mlsirm's declared runtime environment so the +injected judge is actually available. + +## Measurements + +| path | elapsed | provider usage | result | +| --- | ---: | ---: | --- | +| direct `route` worker | 23.002 s | 50 prompt + 48 completion = 98 | one trace step; route accepted by contract | +| isolated fast-mlsirm judge | 3.977 s | 392 prompt + 59 completion = 451 | strict JSON parsed; `accepted=true`; judge mode `route` | +| four-step `conduct` plus judge | 43.915 s | workflow 2,404 + judge 676 = 3,080 | four trace steps; fast judge parsed; `accepted=false` with evidence-based rationale; judge mode `route` | + +After the contract fix, the verification metadata also carries the two +criterion scores and `judge_irt_row` as a dichotomous two-item row. The row is +derived by fast-mlsirm's `to_irt_row`, not by lexical matching or a gateway +threshold heuristic. + +The four workflow step latencies were 10.089 s, 10.153 s, 8.697 s, and +10.225 s. The judge used a single bounded route call rather than recursively +starting another conduct workflow. The final rejection is a model-evaluation +result, not a keyword rule; malformed or unavailable judge output would remain +fail-closed. + +### Current-server polytomous judge probe + +On the same day, a fresh live probe used +`ContextualOrchestratorJudge -> contextual-orchestrator -> mlx-lm` with the +3B Llama model as judge, temperature `0`, disabled thinking, two criteria, +`max_output_tokens=256`, and one strict call at each K. Every call parsed and +produced a two-item row; no keyword, lexical, positional, or silent-drop +fallback was used. + +| K | score | accepted | criterion categories (`evidence_quality`, `risk_awareness`) | IRT row | total tokens | seconds | +|---:|---:|:---:|:---:|:---:|---:|---:| +| 2 | `0.5000` | no | `(1, 0)` | `(1, 0)` | 594 | 8.147 | +| 3 | `0.5000` | no | `(2, 0)` | `(2, 0)` | 600 | 3.972 | +| 5 | `0.5000` | no | `(4, 0)` | `(4, 0)` | 606 | 4.220 | +| 7 | `0.6667` | no | `(6, 2)` | `(6, 2)` | 612 | 4.122 | + +The criterion-level movement and K=7 score increase are fresh evidence of +category-count sensitivity, not proof of a universal positive-bias law. The +rows pass only the two-dimensional response-shape contract; one person is +insufficient for `validate_irt_experiment_readiness` or any IRT fit. The +result is therefore retained as a calibration observation, with K, method, +trace, usage, parse status, and readiness status kept separate. + +### Live cumulative-threshold semantic spot-check — 2026-08-13 + +A fresh two-criterion call used the same strict gateway path with +`category_count=5` and `category_method="cumulative_threshold"`. The answer +explicitly described canary rollout, monitoring, independent review, and +rollback rehearsal. The response parsed successfully and produced the valid +two-item polytomous row `(4, 0)`, with score `0.5), one trace step, and +`625` total provider tokens; the judge assigned +`evidence_quality=4` and `risk_awareness=0`. + +This is a semantic calibration miss, not a parser or transport failure: +structured output can be valid while an item-level judgment under-recognizes +evidence present in the answer. No keyword, lexical, positional, or +silent-drop repair is allowed. The result must remain in the calibration +denominator and be addressed with balanced held-out cases and human/gold +anchors before any quality or IRT interpretation. + +## Additional local model and batch-throughput probe + +Using the same loopback `mlx-lm` server, temperature `0`, disabled thinking, +one fixed deployment-risk prompt, and `max_tokens=96`, three cached models +returned valid Chat Completions after their load sample. The measured warm +latencies (three sequential requests; the first load sample is excluded) were: + +| model | prompt tokens | completion tokens | warm seconds | finish | +| --- | ---: | ---: | ---: | --- | +| `mlx-community/llama-3.2-1b-instruct-4bit` | 65 | 38 | `1.557, 2.120, 1.728` | stop | +| `mlx-community/llama-3.2-3b-instruct-4bit` | 65 | 27 | `2.211, 2.013, 1.820` | stop | +| `mlx-community/gemma-4-e4b-it-4bit` | 37 | 23 | `2.811, 2.267, 2.106` | stop | + +This is a transport/performance probe only; it is not a quality ranking. A +separate eight-request 3B local batch through `ModelClient.batch_chat` measured +the explicit concurrency knob: + +| `local_concurrency` | elapsed seconds | requests/second | non-empty outputs | +| ---: | ---: | ---: | ---: | +| 1 | `18.547` | `0.431` | `8/8` | +| 2 | `8.140` | `0.983` | `8/8` | +| 4 | `8.108` | `0.987` | `8/8` | + +This was an earlier eight-request snapshot. It is retained as historical +evidence, not as a universal tuning result: the later repeated probe below +used different request cardinality and warm-cache state. + +### Repeated warm-cache concurrency probe + +The follow-up used two trials per concurrency after one warm-up request per +model, temperature `0`, thinking disabled, `max_output_tokens=32`, and unique +short prompts. The 3B run used 16 requests; the 1B and Gemma 4B runs used +eight. Every cell completed with non-empty content for every request. + +| model | requests | c=1 median seconds (req/s) | c=2 median seconds (req/s) | c=4 median seconds (req/s) | c=8 median seconds (req/s) | +| --- | ---: | ---: | ---: | ---: | ---: | +| `llama-3.2-3b-instruct-4bit` | 16 | `25.585` (`0.625`) | `15.139` (`1.057`) | `10.956` (`1.460`) | `7.928` (`2.018`) | +| `llama-3.2-1b-instruct-4bit` | 8 | `2.825` (`2.832`) | — | `1.631` (`4.906`) | `1.524` (`5.251`) | +| `gemma-4-e4b-it-4bit` | 8 | `7.765` (`1.030`) | — | `5.003` (`1.599`) | `3.654` (`2.189`) | + +For the three-model comparison, `local_concurrency=8` is the fastest tested +cross-model setting. Keep interactive route/conduct paths sequential and keep +the library default at `1`; latency-tolerant batch callers should still +measure the target model before selecting a larger value. + +### Current 3B saturation probe + +A follow-up warm-cache probe on the same running service used the 3B model, +temperature `0`, disabled thinking, and short unique prompts. All requests +returned non-empty content. The 16-request trial compared concurrency through +`16`; the 32-request trial tested the higher settings after the c=16 result +was fastest. + +| requests | max output tokens | `local_concurrency` | elapsed seconds | requests/second | non-empty | +| ---: | ---: | ---: | ---: | ---: | ---: | +| 16 | 24 | 1 | `4.847` | `3.301` | `16/16` | +| 16 | 24 | 2 | `5.467` | `2.927` | `16/16` | +| 16 | 24 | 4 | `3.373` | `4.744` | `16/16` | +| 16 | 24 | 8 | `2.757` | `5.804` | `16/16` | +| 16 | 24 | 16 | `2.344` | `6.827` | `16/16` | +| 32 | 16 | 16 | `5.123` | `6.246` | `32/32` | +| 32 | 16 | 24 | `6.599` | `4.849` | `32/32` | +| 32 | 16 | 32 | `6.235` | `5.132` | `32/32` | + +For this specific 3B workload, `local_concurrency=16` is the best measured +setting; raising it to `24` or `32` reduced throughput. This is a bounded +transport result, not a universal hardware optimum or a quality ranking. +Use `--local-concurrency 16` only for latency-tolerant 3B batches after a +warm-cache check, and re-measure after changing the model, server flags, +prompt/output budgets, or device memory pressure. + +### 32-request saturation boundary + +The same warm service was probed again with 32 requests, the 3B model, +temperature `0`, disabled thinking, unique short prompts, and +`max_output_tokens=16`. All completed settings returned non-empty content; +`c=64` did not complete within the client timeout (`TimeoutError: [Errno 60]`) +even though the loopback `/v1/models` health request remained HTTP 200 after +the probe. + +| `local_concurrency` | elapsed seconds | requests/second | non-empty | +| ---: | ---: | ---: | ---: | +| 1 | `18.117` | `1.766` | `32/32` | +| 4 | `11.359` | `2.817` | `32/32` | +| 8 | `9.966` | `3.211` | `32/32` | +| 16 | `10.301` | `3.106` | `32/32` | +| 24 | `11.375` | `2.813` | `32/32` | +| 32 | `10.231` | `3.128` | `32/32` | +| 48 | `14.530` | `2.202` | `32/32` | +| 64 | timeout | — | not applicable | + +This boundary is provider saturation, not a LibreSSL failure: the requests +used loopback `lo0`, and the health endpoint stayed available. For this +short-output workload, `c=8` was the fastest stable setting; `c=48` already +collapsed and `c=64` is not an acceptable default. Failed saturation points +remain recorded rather than being hidden, and callers must re-measure after +changing model, prompt, output budget, server flags, or memory pressure. + +### Cost-routing integration smoke + +After the default batch-backend fix, an eight-request run through +`CostRoutingCoordinator.submit_batch()` and `retrieve_batch()` used the same +3B loopback service, `local_concurrency=8`, temperature `0`, disabled thinking, +and `max_output_tokens=16`. It completed in `4.371 s` (`1.830 req/s`), returned +`8/8` non-empty answers, and retained the submitted custom-ID order. This is +integration evidence for the concurrency handoff and result contract, not a +new cross-workload tuning recommendation. + +A repeated warm-cache smoke after the circuit-breaker lock fix completed in +`2.267 s` (`3.529 req/s`) with the same `8/8` non-empty, ordered result +contract. The difference from the first smoke is retained as warm-cache and +provider scheduling variance, not as a quality or universal throughput claim. + +### 2026-08-14 paired category-method probe + +To test the category-count concern against more than one semantic direction, a +fresh two-case probe used the same +`fast-mlsirm.ContextualOrchestratorJudge -> contextual-orchestrator -> mlx-lm` +path, the 3B Llama judge, temperature `0`, disabled thinking, two criteria, and +K in `{2, 5, 7}`. The cases were a release plan with canary monitoring, +independent review, load testing, and rollback rehearsal, and an unsafe plan +that explicitly omitted review and rollback rehearsal. Every valid output was +converted to a two-item polytomous row; malformed and non-monotone outputs were +not repaired. + +| case | method | K=2 | K=5 | K=7 | +|---|---|---:|---:|---:| +| safe release | direct score / categories | `0.5 / (1,0)` | `1.0 / (4,4)` | `1.0 / (6,6)` | +| unsafe release | direct score / categories | `0.0 / (0,0)` | `0.0 / (0,0)` | `0.3333 / (2,2)` | +| safe release | cumulative | parse failure | `0.0 / (0,0)` | monotonicity failure | +| unsafe release | cumulative | parse failure | `0.0 / (0,0)` | parse failure | + +Direct K-way output therefore moved materially with K for both semantic cases; +this is evidence of category-count sensitivity and a positive drift in this +sample, not a universal law. Cumulative output was not a reliable mitigation +in this run because four of six calls failed strict parsing or monotonicity. + +An opt-in `binary_threshold` follow-up asked one Boolean boundary question per +criterion. The safe case failed monotonicity at K=5 and K=7; the unsafe case +parsed at score `0.0`, using 8 calls/`2,606` tokens at K=5 and 12 +calls/`3,940` tokens at K=7. This makes the decomposition a useful +fail-closed calibration probe, not a production default or proof of unbiased +judgment. Its call count, latency, usage, semantic recall, and human/gold +agreement must be measured before any default change. + +### Binary-threshold bounded concurrency follow-up — 2026-08-14 + +The fast-mlsirm follow-up at exact commit `61e6be9` now reuses the injected +contextual-orchestrator `client.local_concurrency` for independent binary +boundary calls. It does not add a provider client or fallback transport; +generic injected orchestrators remain sequential, the request order in the +evidence record is deterministic, and all returned boundaries are still +validated for monotonicity before a result is produced. A bounded live probe +used the same 3B loopback path with `local_concurrency=8`: + +| case | K | result | elapsed | provider usage | +|---|---:|---|---:|---:| +| safe release | 5 | failed closed: non-monotone thresholds | `3.004 s` | not retained as a valid result | +| safe release | 7 | failed closed: non-monotone thresholds | `7.839 s` | not retained as a valid result | +| unsafe release | 5 | score `0.0`, categories `(0,0)` | `5.756 s` | `8` calls, `2,422` tokens | +| unsafe release | 7 | score `0.0`, categories `(0,0)` | `7.719 s` | `12` calls, `3,620` tokens | + +The lower serial time observed in a prior run is not treated as a causal +speedup because provider queue/cache state differed. The controlled contract +evidence is bounded concurrency, stable ordering, preserved trace/usage, and +fail-closed semantics; quality and bias remain unproven. The exact-source fast +tests passed `58`, and the rebuilt full suite passed `3630` with one skip and +two existing warnings. + +### Direct K-way default regression probe — 2026-08-14 + +A second concise semantic probe used the same 3B loopback model, two criteria, +temperature `0`, disabled thinking, and the same +`fast-mlsirm.ContextualOrchestratorJudge -> contextual-orchestrator -> mlx-lm` +route. Every direct response parsed, but the score changed with K even when +the answer and rubric were fixed: + +| case | K=2 | K=5 | K=7 | +|---|---:|---:|---:| +| safe evidence | `1.0000` | `1.0000` | `1.0000` | +| unsafe recommendation | `0.0000` | `0.5000` | `0.8333` | +| partial evidence | `0.0000` | `1.0000` | `0.0000` | + +The unsafe answer therefore became accepted at K=7 under the `0.7` +threshold, while the partial answer became accepted only at K=5. This is +direct evidence that implicit K-way selection is unsafe for an IRT item +producer; it is not a universal proof of monotone positive bias. + +The same safe and unsafe answers were then evaluated with explicit binary +thresholds at K=5 and K=7, using contextual-orchestrator local concurrency +`4`. Both cases returned score `0.0` at both K values. The safe result is a +semantic false negative, so binary decomposition is a fail-closed calibration +guard, not a claim of unbiased or high-recall judgment. + +| case | K | elapsed | provider usage | +|---|---:|---:|---:| +| safe | 5 | `4.12 s` | `8` calls, `2,380` tokens | +| safe | 7 | `5.38 s` | `12` calls, `3,617` tokens | +| unsafe | 5 | `3.57 s` | `8` calls, `2,379` tokens | +| unsafe | 7 | `6.23 s` | `12` calls, `3,647` tokens | + +The fast adapter now defaults to `category_method="binary_threshold"` when +`category_count` is supplied without an explicit method. Direct K-way output +remains available only as an explicit calibration method; no keyword, +positional, silent-drop, or malformed-output repair was added. + +### Actual adapter default smoke — 2026-08-14 + +The default-selection result was then verified on the real integrated path, +not only with an injected fake transport. Exact fast-mlsirm source was +`9d18f53`, contextual-orchestrator was `a0a354a`, the local 3B Llama worker was +used through `_FastMLSIJudgeAdapter`, and `category_count=5` was supplied with +no `category_method`. The gateway client used `local_concurrency=4`. + +| case | result | calls | tokens | seconds | +|---|---|---:|---:|---:| +| unsafe recommendation | score `0.0`, categories `(0,0)`, rejected | 8 | `2,379` | `3.73` | +| safe evidence | failed closed: non-monotone thresholds | 8 | not a valid result | `3.23` | + +The unsafe result proves the production adapter selected binary thresholds and +retained the contextual trace. The safe result is a semantic/calibration +failure, not a transport success; it remains in the denominator and is not +coerced into a category. This is integrated contract evidence only, not a +claim of high recall, unbiasedness, or sufficient IRT sample size. + +### Structured failure evidence follow-up — 2026-08-14 + +The same integrated safe-case failure was rerun after fast-mlsirm +`d1eca0c2fed89991e647802f0b27a91f0f6fe2bd` added bounded failure evidence to +`JudgeFormatError`. Through the real `_FastMLSIJudgeAdapter` and the same +contextual-orchestrator MLX route, the failure now records +`semantic_status=non_monotone`, `parse_status=passed`, `8/8` completed calls, +`8` trace steps, `2,639` provider tokens, and `2.88 s` elapsed time. The +evidence is retained as a failed calibration comparison; it is not converted +to an IRT category and is not a semantic-quality score. + +### Anchored judge model comparison — 2026-08-14 + +Using fast-mlsirm `dd44a95`, the same two-criterion K=5 anchored rubric was +executed through `_FastMLSIJudgeAdapter` and contextual-orchestrator's MLX +route. The results separate model capability from transport: + +| model | result | usage | elapsed | +|---|---|---:|---:| +| Gemma 4 e4b | strict `(4,4)`, score `1.0`, accepted | `3,031` tokens | `11.96 s` | +| Llama 3B | repeated safe false negative `(0,0)` and anchored non-monotone output | `2,497`–`3,119` tokens | `3.23`–`5.54 s` | +| Llama 1B | malformed JSON on all eight boundaries; failed closed | `4,764` tokens | `8.04 s` | + +Gemma 4 e4b is a measured candidate for quality/latency follow-up, not a +production conclusion. The Llama results remain in the denominator, and no +model is promoted to IRT use without balanced held-out gold recall, parse +success, category occupancy, and perturbation checks. + +The integrated path was then checked separately after an audit found that the +contextual `_FastMLSIJudgeAdapter` did not expose the gateway client capability +used by the fast judge. With contextual commit `d82e592` and exact fast judge +code from `61e6be9`, a two-criterion K=3 smoke made four boundary calls through +the adapter, reached peak concurrency `2` for `client.local_concurrency=2`, +and returned score `0.5`. This was a fake-provider integration contract smoke, +not a quality or MLX throughput claim; the original direct-injection evidence +must not be reused as evidence for the integrated path. + +### HTTP admission alignment follow-up — 2026-08-14 + +The gateway's secure default `max_concurrent_runs=8` is an independent +admission limit. A live loopback smoke used the same 3B MLX model, temperature +`0`, disabled thinking, `max_output_tokens=32`, and 16 simultaneous route +requests. With `local_concurrency=16`, changing only the gateway admission +setting produced: + +| `max_concurrent_runs` | HTTP 200 | HTTP 503 | elapsed | interpretation | +|---:|---:|---:|---:|---| +| `8` | `8` | `8` | `0.717 s` | secure default rejects excess simultaneous runs | +| `16` | `16` | `0` | `2.245 s` | explicit operator setting admits the measured batch width | + +The result verifies admission behavior, not a throughput or quality ranking; +the accepted requests necessarily changed provider queue pressure. The secure +default remains `8`, and operators must explicitly set +`--max-concurrent-runs` alongside a measured `--local-concurrency` value. + +### Current route throughput recheck — 2026-08-14 + +A fresh warm-cache route probe used the same 3B Llama worker, temperature `0`, +thinking disabled, `max_output_tokens=32`, and 16 concurrent route requests. +Every response was non-empty and the run used the real contextual-orchestrator +`TaskOrchestrator.route_once` path. + +| `local_concurrency` | successes | elapsed | requests/s | +|---:|---:|---:|---:| +| 1 | 16/16 | `8.166 s` | `1.959` | +| 4 | 16/16 | `2.608 s` | `6.136` | +| 8 | 16/16 | `2.534 s` | `6.315` | +| 16 | 16/16 | `2.760 s` | `5.797` | + +Under this exact workload, `local_concurrency=8` was the fastest measured +setting; this supersedes neither the earlier model/prompt-specific c=16 probe +nor the secure HTTP admission default. It is throughput evidence only, not a +quality or semantic-judge result. + +### Current exact-head anchored rerun and provider-readiness incident — 2026-08-14 + +The current fast-mlsirm source at `26b9ccc590a65cebf23537ce00f292f4d5f9e6f7` +and contextual-orchestrator source at +`18d8c3b63eba471f439dd50f36f0f1e395d202d7` were exercised through the real +`ContextualOrchestratorJudge -> _FastMLSIJudgeAdapter -> TaskOrchestrator -> +ModelClient -> mlx-lm` path. The candidate was Gemma 4 e4b, with two criteria, +K=5 complete category anchors, temperature 0, disabled thinking, client +`local_concurrency=4`, and an explicit server `prompt-concurrency=1` / +`decode-concurrency=1`. + +The first attempt failed closed at the provider boundary: `/health` and +`/v1/models` returned HTTP 200, but every `/v1/chat/completions` request +returned no bytes before a 15–20 second timeout. The eight judge boundaries +therefore recorded `0/8` completed calls, zero usage, and about 60.7 seconds +elapsed. The loopback mlx-lm process had accumulated many closed/CLOSE_WAIT +connections; it was restarted with the explicit Gemma 4 e4b model and bounded +server concurrency. A direct post-restart completion returned HTTP 200 in +0.316 seconds. This is provider-readiness evidence, not a LibreSSL or TLS +verification failure. + +The post-restart anchored rerun completed and parsed all eight Boolean calls in +14.118 seconds with `3,625` provider tokens and eight trace steps, but failed +the ordinal semantic gate: the evidence-quality boundaries were +`false,false,true,true` and the risk-signal boundaries were +`false,true,true,true`. Because a higher threshold became true after a lower +threshold was false, the result was rejected as `semantic_status=non_monotone`; +no category row, acceptance decision, or IRT observation was produced. Anchor +presence was true, but anchor binding and strict JSON parsing do not guarantee +semantic threshold consistency. Retain this complete failure in the +calibration denominator and do not promote Gemma 4 e4b, change verifier +priority, or repair the row without balanced held-out gold and perturbation +evidence. + +### Explicit provider readiness refresh — 2026-08-14 + +The new `TaskOrchestrator.provider_readiness_report(refresh=True)` path was +exercised against the live `mlx-lm` service through `ModelClient`, using +`mlx-community/gemma-4-e4b-it-4bit`, `temperature=0`, disabled thinking, a +three-second probe bound, and zero local retries. `/health` returned `{"status": +"ok"}` and the explicit one-token chat probe returned `ready` in `718.03 ms` +with `15` provider tokens (`14` prompt, `1` completion). The authenticated +`GET /api/v1/provider_readiness/latest?refresh=true` contract then returned +HTTP `200`, `status=ready`, and the same worker-specific usage shape (`250.55 +ms` on the warm second probe). + +This is serving-readiness evidence only: it proves a bounded chat completion +can pass now, not that the model is semantically calibrated or suitable for +LLM-as-a-Judge. The refresh is explicit, sequential per worker, and +non-retrying so the liveness endpoint cannot hide or multiply a stuck MLX +queue. + +### Anchored K=5 calibration after ordinal prompt hardening — 2026-08-14 + +The linked fast-mlsirm implementation was advanced to +`17e19ec90643a8dfcc464cd7dde0b63949539a32` and exercised through the exact +`ContextualOrchestratorJudge -> _FastMLSIJudgeAdapter -> TaskOrchestrator -> +ModelClient -> mlx-lm` path. The prompt was changed to state that each Boolean +call asks whether the answer meets *at least* the requested category (not +exactly that category), that stronger evidence remains true at lower +boundaries, and that criterion/task relevance is required. Generic intent, +unrelated detail, admissions that a control is missing, and rubric repetition +are not evidence. + +On a balanced six-case set with two criteria, K=5, complete per-criterion +anchors, temperature 0, disabled thinking, `local_concurrency=4`, and no local +retries, three complete reruns produced the following calibration-only +evidence: + +| prompt version | complete comparisons | failed closed | cell accuracy among complete rows | exact-case accuracy | provider tokens | +| --- | ---: | ---: | ---: | ---: | ---: | +| pre-hardening | `2/6` | `4/6` | `25.0%` | `0.0%` | `20,274` | +| minimum-boundary clarification | `5/6` | `1/6` | `40.0%` | `16.7%` | `22,925` | +| relevance clarification repeat | `4/6` | `2/6` | `37.5%` | `16.7%` | `25,528` | + +The first two token totals are sums of the per-case usage records retained in +the detailed run output. The third run used `25,528` provider tokens over +`77.907 s`. Complete rows still over-scored partial or +unsupported evidence (`partial_plan` and `evidence_without_safety`), while +non-monotone outputs for unsafe/irrelevant cases were rejected. The prompt +change therefore improves the ordinal contract's protection but does not +establish bias removal, model quality, or IRT readiness. Preserve all failed +comparisons and semantic over-scores in the denominator; require a larger +held-out human/gold set, prompt/order perturbations, and category occupancy +before any model promotion. + +The fast PR branch subsequently reconciled its GitHub pull ref at exact head +`2cd12090f6f4ef8188da15fc6a5704a6ad7063c7` (a documentation-only follow-up +recording the temporary branch/pull-ref drift). A later fast-mlsirm follow-up +advanced the linked PR to `ebd76b4664147c18a3e1cfcc3d689e916a2fff08`; it records +the validated `meets_threshold` Boolean in bounded non-monotone failure +evidence without retaining full model output. That push invalidates all +predecessor review/check evidence; the calibration implementation remains the +`17e19ec` ancestor, and the new exact head requires fresh review and checks. + +### Repeated local batch concurrency sweep — 2026-08-14 + +The live `mlx-lm` worker was measured through contextual-orchestrator's +`ModelClient.batch_chat` path with Gemma 4 e4b, eight identical short requests, +temperature `0`, `max_output_tokens=32`, disabled thinking, and zero local or +remote retries. The server was configured with `prompt-concurrency=1` and +`decode-concurrency=1`. Each client-concurrency setting was repeated twice; all +16 requests per setting completed and each run used 248 provider tokens. + +| client `local_concurrency` | mean throughput (req/s) | throughput stdev | mean elapsed (s) | +| ---: | ---: | ---: | ---: | +| 1 | `2.095` | `0.019` | `3.819` | +| 2 | `2.083` | `0.028` | `3.840` | +| 4 | `2.092` | `0.017` | `3.824` | +| 8 | `2.088` | `0.025` | `3.832` | + +The differences are within this small-run variance and show no benefit from +raising client concurrency while the worker's prompt/decode concurrency is +one. Keep the safe client default at `1`; tune the server-side queue separately +and repeat this workload after changing model, prompt budget, or server +concurrency. This is throughput evidence only, not judge-quality evidence. + +### Integrated two-item anchored smoke (pre-main-sync snapshot) — 2026-08-14 + +The pre-main-sync local source trees were exercised through the complete +`ContextualOrchestratorJudge -> _FastMLSIJudgeAdapter -> TaskOrchestrator -> +ModelClient -> mlx-lm` path with Gemma 4 e4b, temperature `0`, disabled +thinking, `max_output_tokens=192`, zero retries, `local_concurrency=1`, K=`3`, +and two complete anchored criteria. Four Boolean boundary calls completed in +`4.404 s` and used `1,797` provider tokens. The parsed categories were +`release_monitoring=2` and `rollback_safety=2`, producing the required +two-column polytomous IRT row `[2, 2]`. + +A separate one-criterion attempt was rejected at the IRT projection boundary +with `IRT output requires multiple criterion items; a scalar judge result is +invalid`. No scalar padding, keyword matching, positional repair, or silent +item synthesis was used. This is an integration and contract smoke, not proof +of semantic quality or model promotion. + +### Post-protected-main integrated two-item smoke — 2026-08-14 + +After fast-mlsirm was synchronized with protected `main` (source merge +`bbf5d0e1d1185d4a51fae24fa95c3c18a3ea2f23`; subsequent head +`c5727de` contains documentation only), the same complete +`ContextualOrchestratorJudge -> _FastMLSIJudgeAdapter -> TaskOrchestrator -> +ModelClient -> mlx-lm` path was rerun with Gemma 4 e4b, temperature `0`, +disabled thinking, `max_output_tokens=192`, zero retries, +`local_concurrency=1`, K=`3`, and two complete anchored criteria. Four +Boolean boundary calls completed in `6.023 s` and used `1,872` provider +tokens. The parsed categories remained `release_monitoring=2` and +`rollback_safety=2`, with `category_method="binary_threshold"`, producing +the required two-column polytomous row `[2, 2]` and score `1.0`. + +This confirms post-main synchronization integration and contract preservation, +not semantic accuracy, bias removal, IRT sufficiency, or model promotion. The +latency/token difference from the pre-sync snapshot is retained rather than +normalized away; future performance comparison must use repeated runs under +the same server and prompt configuration. + +### Paired option-count and framing controls — 2026-08-14 + +The new fast-mlsirm calibration controls at exact head +`5a072705c840ea70d87a73bf737d5b193ef428cb` were exercised through the same +`ContextualOrchestratorJudge -> _FastMLSIJudgeAdapter -> TaskOrchestrator -> +ModelClient -> mlx-lm` route with Gemma 4 e4b, temperature `0`, disabled +thinking, `max_output_tokens=192`, zero retries, `local_concurrency=1`, K=`3`, +two complete anchored criteria, and caller-declared `held_out` status. Each +case ran baseline, option-only/no-question, shuffled-option, and +distractor-replacement variants. Every successful result produced the +required two-column polytomous row; no keyword matching, repair, retry, or +positional category inference was used. + +| option count | variants | status | gold exact agreement | score range | paired score deltas | elapsed | provider tokens | +| ---: | ---: | --- | ---: | ---: | --- | ---: | ---: | +| 3 | 4 | `4 passed` | `4/4` | `1.0..1.0` | all `0.0` | `20.442 s` | `7,868` | +| 5 | 4 | `4 passed` | `4/4` | `1.0..1.0` | all `0.0` | `20.277 s` | `8,110` | + +This small held-out smoke did not show a positive option-count shift, but it +does not estimate or disprove a general LLM option-count effect. The report +retained contamination status, per-variant categories, IRT rows, trace-step +counts, and usage while excluding raw model output. Replication must expand +persons/items, correct-option positions, option counts, models, framing, and +human/gold anchors before any bias or IRT-readiness claim. + +### Model-size reliability comparison — 2026-08-14 + +Using the same held-out three-option case, K=`3`, two anchored criteria, +temperature `0`, disabled thinking, `local_concurrency=1`, and the four paired +variants, the real contextual-orchestrator route produced these exploratory +results: + +| model | passed variants | gold exact agreement | paired score deltas | elapsed | provider tokens | +| --- | ---: | ---: | --- | ---: | ---: | +| Llama 1B | `0/4` | not scored | not scored | `25.334 s` | first summary omitted usage; follow-up retained `2,302`–`2,681` tokens/variant | +| Llama 3B | `4/4` | `4/4` | all `0.0` | `24.139 s` | `7,960` | +| Gemma 4 e4b | `4/4` | `4/4` | all `0.0` | `39.731 s` | `7,860` | + +The 1B follow-up retained four complete boundary-failure records: all four +provider calls completed, JSON parsing failed on three or four boundaries per +variant, and the report preserved `2,302`–`2,681` provider tokens per variant +without retaining raw output. This is structured-output reliability evidence, +not a claim that the larger models are unbiased. Model promotion requires +more persons/items, balanced correct-option positions and counts, framing and +contamination controls, and human/gold recall. + +### Direct-versus-binary paired calibration follow-up — 2026-08-14 + +The current local `mlx-lm` process was configured for Gemma 4 e4b. Using the +real `fast-mlsirm.ContextualOrchestratorJudge -> contextual-orchestrator -> +mlx-lm` route, temperature `0`, disabled thinking, `max_output_tokens=192`, +zero retries, `local_concurrency=1`, K=`3`, two complete category anchors, and +caller-declared `held_out` status, the same four paired MCQ variants were run +with the explicit `direct` method and with the default method (which resolves +to independent `binary_threshold` calls). + +| method | passed | judge failures | gold exact agreement | rows among passed | elapsed | +| --- | ---: | ---: | ---: | --- | ---: | +| explicit `direct` | `2/4` | `2` | `1/2` | baseline `(2,1)`, replacement `(2,2)` | `12.204 s` | +| default `binary_threshold` | `4/4` | `0` | `4/4` | all `(2,2)` | `17.096 s` | + +The direct run's option-only and shuffled variants failed closed at the judge +format boundary; they were retained in the denominator. The binary result is +one anchored case, not evidence of universal accuracy or bias removal, but it +supports retaining binary thresholds as the implicit production polytomous +method and keeping direct K-way selection explicit calibration-only. No +keyword matching, positional inference, retry, repair, or silent drop was +used. + +The same default binary case was then repeated with the gateway's bounded +local concurrency capability. The server remained configured with prompt and +decode concurrency `1`; all four variants stayed at gold `4/4` with rows +`(2,2)`. + +| gateway `local_concurrency` | elapsed | status | gold exact agreement | +| ---: | ---: | --- | ---: | +| `1` | `17.096 s` | `4 passed` | `4/4` | +| `4` | `14.843 s` | `4 passed` | `4/4` | +| `8` | `14.816 s` | `4 passed` | `4/4` | + +Concurrency `4` is the smallest tested setting at the observed plateau; +raising it to `8` added no meaningful throughput. Keep the library default at +`1` for interactive or single-queue workloads, and let latency-tolerant batch +callers retune within the gateway bound after changing model, server queue, or +prompt budget. This is a workload-specific throughput result, not a quality +claim. + +### Zotero/OA literature audit and linked judge head — 2026-08-14 + +The local Zotero Desktop reports version `9.0.6` and exposes read-only Local +API reads. Jones--Loe item `CWY355RP` records `Open access` rights and has no +child attachment; Iannario item `MYPNHHWJ` records `Creative Commons +Attribution 4.0 International` rights and also has no child attachment. The +local Connector API successfully created the Cao et al. citation as item +`393S5NXZ`, but its all-rights-reserved record did not authorize copying the +PDF. Item PATCH and `/api/local/authorize` were unavailable, so no local file +upload was attempted. + +OpenAlex/Unpaywall/Crossref metadata support Jones--Loe CC BY gold OA and +Iannario CC BY 4.0. The official SAGE download returned anti-bot `403` and the +official De Gruyter download returned a WAF `202` with zero bytes. Those +responses, crawler text, archived bytes without independently verified +provenance, reconstructed PDFs, and unauthorized mirrors are not counted as +original OA attachments. This keeps the PDF requirement open without +misrepresenting retrieval failure as a licensing failure. + +The linked fast-mlsirm literature/calibration documentation is at exact head +`e6c457d36f483b7580e56e5825528c70506dd780`. Its evidence remains a bounded +calibration input, not an unbiasedness or IRT-readiness claim. + +### Current exact-head integrated smoke — 2026-08-14 + +Using contextual-orchestrator `ccfa292aafadc37b6a008ffa1fb3b1d4bc2e346e` +and fast-mlsirm `d1114e5e20c9aeb4c1cd7c8c8b46053db314ae4a`, the local +`mlx-community/gemma-4-e4b-it-4bit` server completed the fast-mlsirm +`ContextualOrchestratorJudge` route through `mlx://127.0.0.1:18083/v1`. +Two criteria produced four bounded K=`3` binary-threshold calls and a valid +polytomous row `[2,2]` in `12.569 s` with `1,956` total tokens. The run was +accepted with `criterion_categories={release_monitoring: 2, +rollback_safety: 2}` and `score=1.0`. + +This is current transport, parsing, trace, and multi-item shape evidence only. +It is not semantic gold agreement, evidence that the category boundaries are +unbiased, proof of the positive-option-count hypothesis, or sufficient data for +IRT estimation. The result remains in the calibration denominator and does +not override failed/non-monotone cases, category-occupancy requirements, or +the protected exact-head review and Merge gates. + +### Current authenticated gateway route sweep — 2026-08-15 + +The gateway's `/healthz` and authenticated `/v1/models` returned HTTP 200; the +worker's `/v1/models` also returned HTTP 200. The worker was Gemma 4 e4b with MLX `prompt-concurrency=4` and +`decode-concurrency=4`; the gateway used `local_concurrency=4` and +`max_concurrent_runs=4`. Four identical short route requests were issued for +each client parallelism level through the authenticated HTTP gateway. + +| client parallelism | successes | elapsed | requests/s | p50 / max latency | unique IDs | non-empty | tokens | +| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| `1` | `4/4` | `16.199 s` | `0.247` | `330.60 / 15,220.21 ms` | `4` | `4` | `88` | +| `2` | `4/4` | `0.920 s` | `4.346` | `459.94 / 472.30 ms` | `4` | `4` | `88` | +| `4` | `4/4` | `0.731 s` | `5.471` | `727.79 / 730.76 ms` | `4` | `4` | `88` | + +A warm serial repeat took `2.573 s` total with per-request latencies +`1,430.67`, `401.78`, `368.39`, and `372.00 ms`; the first serial sweep's +`15.220 s` outlier is retained as warm-up/operational evidence rather than +discarded. Under this exact short prompt, parallelism `4` maximized measured +throughput without overload responses. This is route/latency/response-integrity +evidence only; it does not establish semantic quality, judge accuracy, bias +absence, or IRT readiness. + +### Strict ordinal option-count calibration — 2026-08-15 + +After fast-mlsirm `ed62e1d1723d1274c1c0483dca4f46bb4eb81665` strengthened the +binary-threshold prompt, the exact local route +`ContextualOrchestratorJudge -> _FastMLSIJudgeAdapter -> TaskOrchestrator -> +ModelClient -> mlx-lm` evaluated 12 paired cases at K=`3,5,7`. Each case had +two criteria and four binary boundaries, for 48 bounded calls. The reference +answer was omitted in this run to avoid crediting requirements that appeared +only in the reference; no keyword, option-position, category repair, retry, or +silent drop was used. + +| option count | baseline / option-only / replacement | shuffled-options | status | +| ---: | --- | --- | --- | +| `3` | `[1,0]` / `0.25` for all three | `[1,0]` / `0.25` | `4/4` passed | +| `5` | `[1,0]` / `0.25` for all three | `[1,0]` / `0.25` | `4/4` passed | +| `7` | baseline and option-only `[1,0]` / `0.25`; replacement `[1,0]` / `0.25` | `[1,2]` / `0.75` | `4/4` passed | + +All 12 cases parsed and passed the ordinal gate, but the manually specified +gold row `[1,1]` had exact agreement `0/12`. The K=`7` shuffled control changed +only option order and produced a paired score delta of `+0.5`; this is a +descriptive sensitivity observation, not proof that score increases +monotonically with the number of options. It is sufficient to keep option +count/order as a live calibration factor and to block semantic/IRT promotion +until replicated held-out human/gold recall, non-ceiling category occupancy, +and perturbation stability are available. + +### Direct versus cumulative polytomous follow-up — 2026-08-15 + +The general strict-evidence prompt and cumulative-threshold contract were then +extended in fast-mlsirm `112b1956d9f19cdab20bbada6b596d65e8f5c827`. Using the +same contextual-orchestrator route, two criteria, K=`3,5,7`, four paired +variants, no reference answer, and the same manual gold row `[1,1]`: + +| method | passed | gold exact agreement among scored | observed result | +| --- | ---: | ---: | --- | +| explicit `direct` | `11/12` | `11/11` | all scored rows `[1,1]`, score `0.5`; one K=`7` replacement failed closed | +| explicit `cumulative_threshold` | `5/12` | `4/5` | K=`3`/`5` passed rows were `[1,1]`; one K=`7` shuffled row was `[2,2]`/`1.0`; seven outcomes were `JudgeFormatError` | + +The direct result is a useful candidate calibration improvement, not proof of +universal unbiasedness. Cumulative threshold remains calibration-only because +format/semantic failures are still common; every failure stayed in the +denominator and no array repair or category inference was applied. The +implicit production polytomous path remains binary-threshold, while direct and +cumulative methods require explicit opt-in and replicated held-out evidence. + +### Fresh warm transport and routed Judge recheck — 2026-08-15 + +The current checkouts (`contextual-orchestrator` `719d9cc83393c616f0a552adad0b41ae55d5b346`, +`fast-mlsirm` `e55a6c3e742e2688efe618267870e2007902857b`) were rechecked against +the long-running Gemma 4 e4b listener on `127.0.0.1:18083`. Transport requests +used temperature `0`, `max_tokens=32`, and the same short prompt; the gateway +used its authenticated `max_concurrent_runs=4` admission bound. + +| path | parallel width | statuses | wave | successful p50 | +| --- | ---: | --- | ---: | ---: | +| direct MLX | `1` | `1/1` HTTP 200 | `248 ms` | `248 ms` | +| direct MLX | `2` | `2/2` HTTP 200 | `325 ms` | `325 ms` | +| direct MLX | `4` | `4/4` HTTP 200 | `664 ms` | `660 ms` | +| direct MLX | `5` | `5/5` HTTP 200 | `830 ms` | `773 ms` | +| authenticated gateway | `1` | `1/1` HTTP 200 | `198 ms` | `198 ms` | +| authenticated gateway | `2` | `2/2` HTTP 200 | `354 ms` | `353 ms` | +| authenticated gateway | `4` | `4/4` HTTP 200 | `624 ms` | `621 ms` | +| authenticated gateway | `5` | `4/5` HTTP 200, `1/5` HTTP 503 | `582 ms` | `579 ms` | + +The direct worker queued its fifth request, while the gateway rejected the +fifth request explicitly with `concurrency_limit_exceeded`; both paths returned +non-empty content for every successful response. This supports the gateway's +bounded admission behavior and does not justify raising the limit or treating +the direct queue as a performance improvement. + +The same current-source pair then ran three hand-authored, two-criterion +polytomous Judge probes through the required route +`ContextualOrchestratorJudge -> _FastMLSIJudgeAdapter -> TaskOrchestrator -> +ModelClient -> mlx-lm`. The probes used K=`3`, two category anchors per +criterion, temperature `0`, disabled thinking, `max_output_tokens=128`, and +`local_concurrency=4`; no reference answer, keyword matching, option-position +inference, retry, category repair, or silent drop was used. + +| method | safe probe | partial probe | unsafe probe | +| --- | --- | --- | --- | +| implicit `binary_threshold` | `[2,2]`, `1.0`, `6.908 s` | `[0,1]`, `0.25`, `2.993 s` | `[0,0]`, `0.0`, `2.824 s` | +| explicit `direct` | `[2,2]`, `1.0`, `3.427 s` | `[0,0]`, `0.0`, `2.690 s` | `[0,0]`, `0.0`, `2.585 s` | +| explicit `cumulative_threshold` | fail-closed non-monotone, `4.191 s` | `[0,0]`, `0.0`, `3.246 s` | `[0,0]`, `0.0`, `3.209 s` | + +All completed rows were valid multi-item polytomous rows; the cumulative safe +case remained a semantic non-monotonicity failure rather than being repaired. +The direct path was cheaper in this three-case sample, but its stricter +partial result and the small sample do not establish superior recall or absence +of bias. Keep `binary_threshold` as the implicit production method and retain +direct/cumulative methods as explicit calibration-only choices until balanced +held-out human/gold evidence, category occupancy, perturbation stability, and +failure-rate targets are met. + +## IRT boundary + +The judge received two criteria, so its result can produce multiple +dichotomous items through `LLMJudgeResult.to_irt_row(item_type="dichotomous")`. +Explicit category-count runs must use the polytomous path and produce one +category item per criterion; a scalar or one-item row is rejected. These +measurements do not claim that the two-item smoke output is sufficient for IRT +estimation. Public fast-mlsirm fitters now enforce the same multi-item boundary, +while low-level diagnostic kernels retain their documented single-item use. + +## Interpretation and next gate + +The local 3B model is usable through the gateway after adapter-contract and +prompt-schema fixes, but it is latency-heavy for a four-step workflow and +structured-output reliability must remain in the denominator. Next calibration +should repeat balanced held-out cases across model size, category count, +option order, framing, direct versus cumulative-threshold methods, parse +status, and token/latency usage. No single K or this smoke run establishes a +universal positive-bias law. + +Related decisions: [ADR-0001](../planning/adrs/0001-fail-closed-model-judgment.md), +[ADR-0002](../planning/adrs/0002-explicit-local-mlx-evaluation.md), +[ADR-0005](../planning/adrs/0005-irt-response-matrix-contract.md), and +[ADR-0006](../planning/adrs/0006-polytomous-llm-judge-bias-calibration.md). diff --git a/docs/benchmarks/2026-08-14-local-mlx-verifier-routing.md b/docs/benchmarks/2026-08-14-local-mlx-verifier-routing.md new file mode 100644 index 000000000..1edc2394b --- /dev/null +++ b/docs/benchmarks/2026-08-14-local-mlx-verifier-routing.md @@ -0,0 +1,560 @@ +# Local MLX verifier routing calibration — 2026-08-14 + +Status: routing evidence; not a claim of unbiased judgment or production IRT +validity. + +## Execution contract + +Every judge call used the existing path: + +`fast-mlsirm.ContextualOrchestratorJudge -> contextual-orchestrator._FastMLSIJudgeAdapter -> TaskOrchestrator -> ModelClient -> mlx-lm` + +The initial live probe used `mlx://127.0.0.1:8080/v1`, temperature `0`, +disabled MLX thinking, `max_output_tokens=128`, zero local retries, two +criteria, three ordered categories, and the implicit `binary_threshold` method. +The later dedicated-port follow-ups in this document use +`mlx://127.0.0.1:18083/v1`; the port change is intentional because the 8080 +listener was not an exclusive MLX owner. Each result therefore produced a +two-column polytomous row when all four Boolean boundary calls were valid. The +safe and unsafe cases were judged separately; no retry, keyword matching, +positional inference, category synthesis, or silent repair was used. A parse or +monotonicity failure remains a failed comparison. + +The exact interpreter used for these runs also passed +`python -m contextual_orchestrator check-fast-mlsirm`, which verified the +fast-mlsirm import, required judge symbols, and +`contextual-orchestrator-contract-v1`. The contextual-orchestrator-only +environment intentionally fails this preflight with `missing_module: numpy`; +that is an integration-environment failure, not a judge result. + +A post-preflight warm smoke on the same e4b endpoint completed the four +Boolean boundary calls in `55.31 s`, returned categories +`{evidence_quality: 2, risk_signal: 2}`, score `1.0`, and row `[2,2]` with +`1,921` provider tokens. This is a successful gateway/contract run, but its +latency is high enough that it remains reliability evidence rather than a +promotion or quality claim; cold/warm distributions and held-out calibration +are still required. + +## Same-route model comparison + +| model | safe result | unsafe result | latency | provider tokens | +| --- | --- | --- | ---:| ---:| +| Llama 3B | passed, score `1.0`, row `[2,2]` | failed closed, `non_monotone`, `4/4` calls parsed | `6.82 s` / `2.56 s` | `1,855` / `1,854` | +| Gemma 4 e4b | passed, score `1.0`, row `[2,2]` | passed, score `0.5`, row `[1,1]` | `7.73 s` / `4.87 s` | `1,836` / `1,828` | +| Gemma 4 31B | failed closed, boundary failure, `1/4` calls completed | passed, score `0.25`, row `[1,0]` | `96.93 s` / `52.38 s` | `481` / `1,871` | +| DeepSeek R1 Qwen 32B | failed closed, boundary failure, `0/4` calls completed | failed closed, boundary failure, `0/4` calls completed | `100.04 s` / `100.06 s` | `0` / `0` | + +The e4b candidate is the best current verifier primary for this workload: +both balanced semantic cases returned bounded structured results, while the +larger candidates failed or timed out and the 3B case produced a non-monotone +unsafe comparison. This is a role-eligibility and service-reliability result, +not a quality ranking or proof that e4b is unbiased. The 3B remains an eligible +lower-priority fallback candidate for future calibration, and all four models +remain discoverable for non-verifier roles. + +## Routing decision + +The local registry excludes Gemma 4 31B, DeepSeek R1 Qwen 32B, and Llama 1B +from the `verifier` role. The 1B exclusion remains based on the earlier +all-boundary structured-output failure. Gemma 4 e4b is now selected as the +verifier primary by the existing priority/exclusion policy; no provider is +removed or silently disabled. Promotion requires a larger balanced held-out +calibration set with gold recall, false-positive/false-negative rates, +category occupancy, option-count/order perturbations, and non-ceiling rows. + +This evidence expands the active Goal and ADR acceptance boundary: model +selection must be rechecked after prompt, server, model, timeout, or output +budget changes, and a fast model cannot be promoted solely for throughput. + +## Runtime reliability follow-up + +The MLX process reported healthy `/health` and `/v1/models` responses while +real completion requests were timing out. Before the graceful restart, the +server had accumulated hundreds of request threads and macOS swap usage was +near capacity; this is a completion-path exhaustion signal, not proof that the +model or TLS stack is broken. After restart, direct e4b and Llama 3B +completions returned `OK`. + +The first full e4b judge run after a model switch still failed closed on the +safe case (`2/4` boundary calls completed within a 30-second request budget), +while the following unsafe case completed with polytomous row `[0,1]`. With +e4b warm and a 60-second request budget, the same safe case completed through +the full `fast-mlsirm -> contextual-orchestrator -> mlx-lm` path in `35.86 s`, +returned categories `{evidence_quality: 2, risk_signal: 2}`, and produced row +`[2,2]`. This separates cold-load/request-budget reliability from semantic +judgment evidence; it does not justify retries, keyword matching, or silent +repair. + +The gateway now bounds local requests per normalized loopback endpoint, +serializes requests that would switch the loaded model, preserves configured +same-model concurrency, and fails waiters when the request deadline expires. +Regression coverage includes a competing two-model endpoint and the full +contextual suite remains green (`384 passed`). Promotion still requires +separate cold/warm latency distributions, bounded completion success rates, +category occupancy, and balanced semantic calibration before changing the +verifier role. + +## Balanced K=3/K=7 edge-position follow-up — 2026-08-14 + +To test option-count and correct-position effects beyond the earlier K=`3`/K=`5` +smoke, the exact route +`fast-mlsirm.ContextualOrchestratorJudge -> _FastMLSIJudgeAdapter -> TaskOrchestrator -> ModelClient -> mlx-lm` +was run with contextual-orchestrator `d3480cc` and fast-mlsirm `dbbd41d`. The +Gemma 4 e4b worker used temperature `0`, disabled thinking, `max_output_tokens=128`, +zero retries, `local_concurrency=1`, two criteria, three ordered categories, and +implicit `binary_threshold`. Four held-out case groups crossed K=`3` and K=`7` +with the correct option at the first and last position; each group included +baseline, option-only, shuffled-option, and distractor-replacement variants. + +The run produced 16 paired outcomes (64 Boolean boundary calls) in `1,044.7 s`: +11 passed and 5 strict `JudgeFormatError` failures. The 11 valid rows all matched +the supplied gold `[2,2]` (`11/11`) and all observed categories were the maximum +category `2` for both criteria; the five failures remained in the denominator. +Only one case group had a complete baseline/control comparison, with score deltas +`0.0`. This is ceiling-saturated, incomplete calibration evidence: it neither +supports nor rejects a positive option-count bias, and it is not sufficient for +IRT interpretation or verifier promotion. No keyword matching, retry, positional +inference, category repair, or silent drop was used. + +## Dedicated-port non-ceiling follow-up — 2026-08-14 + +The first rerun correctly failed closed before model evaluation because the +temporary script used `http://127.0.0.1` instead of the explicit local-provider +scheme `mlx://127.0.0.1`. A subsequent readiness probe also showed the original +8080 endpoint was unsafe for this machine: an unrelated wildcard listener and +the MLX server shared the port, so `/health` could return 200 while a chat +completion returned zero bytes and timed out. Port 18080 was already occupied +by a Colima SSH forward. The MLX server was therefore restarted on dedicated +loopback port 18083 with prompt/decode concurrency 4; `ModelClient.probe()` +returned `ready` in `2.54 s` with 15 reported tokens. + +Using contextual-orchestrator `63451a0` and fast-mlsirm `3c2fecf`, the exact +route `ContextualOrchestratorJudge -> _FastMLSIJudgeAdapter -> TaskOrchestrator -> +ModelClient -> mlx-lm` evaluated four held-out groups: partial and unsupported +answers at K=`3` and K=`7`, with correct options at the first/last positions and +baseline, option-only, shuffled, and distractor-replacement variants. The +Gemma 4 e4b run used two anchored criteria, category_count=`3`, implicit +`binary_threshold`, gateway/server concurrency 4, and completed 16 outcomes +(128 boundary calls) in `202.781 s`: 15 passed and one strict non-monotone +`JudgeFormatError`. + +The 15 valid rows had conditional gold exact agreement `5/15` (`33.3%`). +Evidence-quality occupancy was `{0: 5, 1: 5, 2: 5}`; risk-awareness occupancy +was `{0: 7, 1: 0, 2: 8}`. Partial baseline rows were repeatedly over-scored as +`[2,2]`; option-only controls reduced evidence quality to `1` and raised +unsupported-answer evidence quality from `0` to `1` in both K strata. These +are semantic/control-sensitivity observations, not causal evidence of a +positive K law or IRT readiness. The non-monotone failure and every control +outcome remain in the denominator; no keyword matching, retry, repair, +positional inference, or silent drop was used. + +## Local readiness registry guard — 2026-08-14 + +The gateway readiness path now verifies the local `/v1/models` registry contains +the configured model before sending its bounded one-token completion probe. This +keeps a port-owner/configuration mismatch fail-closed before an expensive judge +run while preserving the existing no-retry and bounded-timeout contract. The +focused local transport suite passed `41` tests; the complete contextual suite +passed `387` tests. Against the dedicated MLX listener on port `18083`, the +registry-plus-completion probe returned `ready` in `5.48 s` with 15 provider +tokens. + +## Dedicated-port 3B non-ceiling follow-up — 2026-08-14 + +The same held-out control design was rerun with +`mlx-community/llama-3.2-3b-instruct-4bit` on the dedicated loopback listener +`mlx://127.0.0.1:18083/v1`. The exact route remained +`ContextualOrchestratorJudge -> _FastMLSIJudgeAdapter -> TaskOrchestrator -> +ModelClient -> mlx-lm`; contextual-orchestrator was at `62100d3` and +fast-mlsirm at `57795b1`. Temperature was `0`, thinking was disabled, +`max_output_tokens=128`, local/server concurrency was `4`, and the implicit +`binary_threshold` method produced two anchored criterion items with three +ordered categories. + +Two held-out groups were evaluated: a partial K=`3` answer with the correct +option first and an unsupported K=`7` answer with the correct option last. Each +group included baseline, option-only, shuffled-option, and +distractor-replacement variants. The run produced 8 outcomes (64 Boolean +boundary calls) in `67.454 s`: 5 passed and 3 strict `JudgeFormatError` +failures caused by non-monotone thresholds. Every valid row was saturated at +`[2,2]`; category occupancy was `{evidence_quality: {0:0, 1:0, 2:5}, +risk_awareness: {0:0, 1:0, 2:5}}`, and conditional gold exact agreement was +`0/5`. The four partial K=`3` variants over-scored the gold `[1,1]`, while the +unsupported K=`7` option-only variant over-scored the gold `[0,0]`; the other +three K=`7` variants failed closed. + +This is model-stratified saturation and reliability evidence, not a causal +positive-option-count estimate. Preserve all five valid rows and all three +failures in the denominator; keep 3B out of the verifier role until it passes +non-ceiling held-out gold calibration with bounded failure rates. No keyword +matching, retry, positional inference, category repair, or silent drop was +used. + +## K-stratified report follow-up — 2026-08-14 + +The updated fast-mlsirm calibration report was exercised against the same +dedicated Gemma 4 e4b listener at `mlx://127.0.0.1:18083/v1`, using contextual +head `b30697d06d1160b6a892fbdd26112316fb53a202` and fast head +`22596ab714e20e9b4d1aa7f50f621deec010f622`. The route remained +`ContextualOrchestratorJudge -> _FastMLSIJudgeAdapter -> TaskOrchestrator -> +ModelClient -> mlx-lm`; temperature was `0`, thinking was disabled, +`max_output_tokens=128`, local/server concurrency was `4`, and the implicit +`binary_threshold` method produced two criterion columns. + +Four held-out groups crossed K=`3` and K=`5` for partial `[1,1]` and +unsupported `[0,0]` gold anchors. Baseline, option-only, shuffled-option, and +distractor-replacement variants produced 16 valid outcomes and 64 boundary +calls in `221.505 s`; no provider, parse, IRT, or monotonicity failure occurred. +Conditional gold exact agreement was `7/16` (`43.75%`). Aggregate category +occupancy was evidence-quality `{0:8,1:0,2:8}` and risk-awareness +`{0:7,1:1,2:8}`. All partial rows were over-scored `[2,2]`; unsupported rows +were correctly `[0,0]` except the K=`5` shuffled control, which became +`[0,1]` (`score_delta=+0.25`). + +The new report exposed these strata directly: K=`3` and K=`5`, each variant's +status count, mean score, category occupancy, gold agreement, and an explicit +zero unstratified denominator. This is descriptive control evidence, not a +causal positive-K estimate; the K=`5` shuffled shift strengthens the requirement +to balance option position/order and retain non-ceiling human/gold anchors +before verifier promotion or polytomous IRT interpretation. No keyword +matching, retry, positional inference, category repair, or silent drop was +used. + +## HTTP gateway smoke and overload boundary — 2026-08-14 + +The dedicated worker was exposed through a live contextual-orchestrator HTTP +gateway on `127.0.0.1:18084`, authenticated with an explicit local test token, +and configured with `max_concurrent_runs=4`. The gateway agent targeted +`mlx://127.0.0.1:18083/v1` and the Gemma 4 e4b model. A single OpenAI-compatible +`/v1/chat/completions` request returned `200`, answer `OK`, and provider usage +`10/2/12` prompt/completion/total tokens. + +At the configured concurrency, four simultaneous HTTP requests completed +successfully with four distinct completion IDs; latency was p50 `1449.28 ms` +and maximum `1459.57 ms`. A fifth simultaneous request was rejected immediately +with structured `503 concurrency_limit_exceeded`. This is the intended bounded +overload behavior: it preserves an explicit failure rather than creating an +unbounded queue or silently dropping a judge item. + +The first smoke exposed a response-ID collision because completion IDs used the +current millisecond. The response, buffered-stream, and direct-stream paths now +share a UUID-based ID generator; the focused streaming suite passed `7` tests. +No keyword matching, retry, positional inference, category repair, or silent +drop was introduced. + +## Warm gateway throughput recheck — 2026-08-14T09:52Z + +The existing dedicated worker (`mlx://127.0.0.1:18083/v1`, Gemma 4 e4b) and +authenticated gateway (`127.0.0.1:18084`, `max_concurrent_runs=4`) were +re-measured with the same short `Reply with exactly OK.` request. Every `200` +response returned a unique completion ID and usage `10/2/12`. + +| simultaneous requests | result | wave time | successful p50 | max successful | +| ---: | --- | ---: | ---: | ---: | +| 1 | `1/1` HTTP 200 | `307.89 ms` | `306.94 ms` | `306.94 ms` | +| 2 | `2/2` HTTP 200 | `327.23 ms` | `326.96 ms` | `326.99 ms` | +| 4 | `4/4` HTTP 200 | `579.67 ms` | `576.15 ms` | `579.27 ms` | +| 5 | `4/5` HTTP 200, `1` HTTP 503 `concurrency_limit_exceeded` | `590.70 ms` | `587.53 ms` | `590.12 ms` | + +This warm sample supports the existing admission bound of four for this +server/model configuration: the fifth request is rejected explicitly and does +not create queue growth or silent loss. It is throughput evidence only, not a +judge-quality or general hardware-optimality claim; no concurrency default was +changed from this one workload recheck. + +## Cross-repository judge-contract regression — 2026-08-14 + +The live smoke below was executed at contextual-orchestrator `a07c11f` with +fast-mlsirm `3d42c0b`. The redaction-only fast follow-up `a536292`, checked +through the current contextual working tree `a9278d1`, also passed the exact +interpreter preflight with `available=true`, fast version `0.7.0`, and matching +`contextual-orchestrator-contract-v1` package-root exports. Before the +fast export fix, `ContextualOrchestratorJudge` itself could be imported and +called, but the same preflight returned `ImportError` because the package root +did not expose the versioned contract constant. That was an integration defect, +not evidence that the judge was unavailable; the constant is now public and a +fast-mlsirm regression test covers the export. + +The repaired route was smoke-tested against the dedicated Gemma 4 e4b listener +with temperature `0`, thinking disabled, `max_output_tokens=128`, two criteria, +three ordered categories, and `cumulative_threshold`. A partial answer returned +valid categories `{evidence_quality: 0, risk_awareness: 0}`, row `[0,0]`, score +`0`, and `accepted=false` in `21.205 s` with `641` provider tokens; an +unsupported answer returned the same row and score in `2.940 s` with `607` +provider tokens. These are descriptive integration observations only: the +semantic cases were not gold-calibrated, and no keyword matching, retry, +positional inference, category repair, or silent drop was used. + +## Current exact-head polytomous judge smoke — 2026-08-14 + +The current linked heads (`contextual-orchestrator` `474b667b576f8a019db51d892db41a605e3a0a85`, +`fast-mlsirm` `a536292cc05bd16287dab16431bc0c3fef74ba81`) were exercised against +the dedicated Gemma 4 e4b listener at `mlx://127.0.0.1:18083/v1`. The exact +route was `ContextualOrchestratorJudge -> _FastMLSIJudgeAdapter -> +TaskOrchestrator -> ModelClient -> mlx-lm`; two criteria, three anchored +categories, and four independent `binary_threshold` calls completed in +`19.354 s` with `2,163` provider tokens. The result was score `1.0`, +`accepted=true`, criterion categories `{evidence_quality: 2, risk_awareness: 2}`, +and the required two-column polytomous IRT row `[2,2]`. + +This is current-head integration and contract evidence, not semantic quality +promotion evidence. It confirms that the IRT output remains multi-item and +that the latest redaction-only fast-mlsirm change does not break the real +contextual route. No keyword matching, positional inference, category repair, +retry, scalar synthesis, or silent drop was used; balanced held-out gold and +perturbation calibration remain required. + +## Current exact-head K-stratified direct and threshold calibration — 2026-08-14 + +The current local pair (`contextual-orchestrator` `bc882c0e937bef1312b2e499bfb1fdd1b9076df5`, +`fast-mlsirm` `a536292cc05bd16287dab16431bc0c3fef74ba81`) was exercised against +the dedicated Gemma 4 e4b listener at `mlx://127.0.0.1:18083/v1`. Every call +used `ContextualOrchestratorJudge -> _FastMLSIJudgeAdapter -> TaskOrchestrator +-> ModelClient -> mlx-lm`, two criteria, explicit anchored categories, and no +keyword, positional, retry, category-repair, or silent-drop fallback. + +The explicit `direct` calibration sweep completed all 12 outcomes: + +| case | K | status | score | accepted | row | latency | total tokens | +| --- | ---: | --- | ---: | --- | --- | ---: | ---: | +| safe | 2 | complete | 1.0 | yes | `[1,1]` | 3.827 s | 701 | +| safe | 3 | complete | 1.0 | yes | `[2,2]` | 3.191 s | 729 | +| safe | 5 | complete | 1.0 | yes | `[4,4]` | 3.632 s | 772 | +| safe | 7 | complete | 1.0 | yes | `[6,6]` | 3.453 s | 801 | +| unsafe | 2/3/5/7 | complete | 0.0 | no | `[0,0]` | 2.377–2.896 s | 670–769 | +| partial | 2 | complete | 0.0 | no | `[0,0]` | 3.717 s | 714 | +| partial | 3 | complete | 0.5 | no | `[1,1]` | 3.361 s | 727 | +| partial | 5 | complete | 0.5 | no | `[2,2]` | 2.862 s | 739 | +| partial | 7 | complete | 0.5 | no | `[3,3]` | 2.967 s | 781 | + +This sample does not show monotone positive drift as K grows: safe and unsafe +were invariant, while partial changed once from K=2 to K=3 and then remained +stable. It is category-count sensitivity, not proof of neutrality or of the +user's positive-bias hypothesis. + +The production-default `binary_threshold` comparison completed K=`3` for all +three cases and K=`5` for unsafe; safe K=`5` and partial K=`5` failed closed +after all 8 boundary calls parsed but produced non-monotone vectors: + +| case | K | status | score/row | calls | latency | total tokens | +| --- | ---: | --- | --- | ---: | ---: | ---: | +| safe | 3 | complete | 1.0 / `[2,2]` | 4 | 5.395 s | 2,142 | +| safe | 5 | failed closed, `non_monotone` | no IRT row | 8 | 8.263 s | 4,384 | +| unsafe | 3 | complete | 0.0 / `[0,0]` | 4 | 4.237 s | 2,049 | +| unsafe | 5 | complete | 0.0 / `[0,0]` | 8 | 7.081 s | 4,163 | +| partial | 3 | complete | 0.0 / `[0,0]` | 4 | 4.145 s | 2,018 | +| partial | 5 | failed closed, `non_monotone` | no IRT row | 8 | 9.140 s | 4,167 | + +The two K=`5` failures had `parse_status=passed`, `completed_call_count=8`, +and `failed_call_count=0`; they are semantic ordinal failures, not transport +failures. Keep them in the denominator and do not repair them into an IRT row. +The binary path therefore remains the safer contract boundary but is not yet a +quality or unbiased-IRT claim; larger balanced gold, category occupancy, and +perturbation calibration remain required. + +## Current exact-head gateway and integrated Judge recheck — 2026-08-14 + +The current source pair (`contextual-orchestrator` +`8f922d806336fd41d8fd73585a7c225784249332`, `fast-mlsirm` +`47c5fbdde98b3550fe319d1de238a32cbaec8a1f`) was rechecked against the live +Gemma 4 e4b listener (`mlx://127.0.0.1:18083/v1`, prompt/decode concurrency +4) and authenticated gateway (`127.0.0.1:18084`, maximum concurrent runs 4). + +| concurrent width | statuses | wave | p50 successful | unique IDs | +| ---: | --- | ---: | ---: | ---: | +| 1 | `1x200` | 2,170.42 ms | 2,170.42 ms | 1 | +| 2 | `2x200` | 341.35 ms | 341.31 ms | 2 | +| 4 | `4x200` | 591.34 ms | 588.64 ms | 4 | +| 5 | `4x200, 1x503` | 595.39 ms | 592.41 ms | 4 | + +The first width-1 request is a cold/warm-up observation; subsequent widths are +not averaged with it. Successful short responses reported 10 prompt, 2 +completion, and 12 total tokens. The fifth request remained an explicit +`concurrency_limit_exceeded` overload rather than queue growth or silent loss. + +The same exact-head pair then ran one real two-criterion anchored +`binary_threshold` Judge through +`ContextualOrchestratorJudge -> _FastMLSIJudgeAdapter -> TaskOrchestrator -> +ModelClient -> mlx-lm`. Four boundary calls completed in 4.949 s with 1,923 +provider tokens, score `1.0`, accepted `true`, categories +`{evidence_completeness: 2, release_safety: 2}`, and IRT row `[2,2]`. +This is route/contract and throughput evidence, not semantic quality or bias +promotion evidence; the K-stratified failures and balanced gold requirements +remain unchanged. + +## Direct MLX versus gateway warm comparison — 2026-08-14T10:49:55Z + +The dedicated worker and gateway were re-measured with the same short request +(`Reply with exactly OK.`), after one warm-up request per endpoint. The direct +worker used `http://127.0.0.1:18083/v1/chat/completions`; the gateway used the +authenticated `http://127.0.0.1:18084/v1/chat/completions`. Both returned unique +completion IDs for successful responses. + +| path | width | statuses | wave | successful p50 | max successful | +| --- | ---: | --- | ---: | ---: | ---: | +| direct MLX | 1 | `1x200` | 202.69 ms | 202.52 ms | 202.52 ms | +| gateway | 1 | `1x200` | 204.12 ms | 203.96 ms | 203.96 ms | +| direct MLX | 2 | `2x200` | 323.97 ms | 323.74 ms | 323.74 ms | +| gateway | 2 | `2x200` | 321.26 ms | 321.01 ms | 321.13 ms | +| direct MLX | 4 | `4x200` | 596.54 ms | 593.77 ms | 596.12 ms | +| gateway | 4 | `4x200` | 562.70 ms | 559.07 ms | 562.18 ms | +| direct MLX | 5 | `5x200` | 899.16 ms | 837.65 ms | 898.60 ms | +| gateway | 5 | `4x200, 1x503` | 564.28 ms | 560.04 ms | 563.75 ms | + +At widths one through four, gateway latency was within this small warm-sample +measurement variation of direct MLX; there is no evidence that the gateway +should be bypassed for performance. Direct width five completed by queueing +against the worker's four-request configuration and took substantially longer, +while the gateway preserved the explicit four-request admission bound and +rejected the fifth request. Keep the bound at four for this model/server pair; +increasing it would hide queue latency rather than improve throughput. This is +transport evidence only and does not alter the semantic calibration or IRT +acceptance boundary. + +## Current exact-head integrated Judge recheck — 2026-08-14T10:53:08Z + +The current source pair (`contextual-orchestrator` `070d9297675ebc45e821808b532fb6af809cbbf2`, +`fast-mlsirm` `8f5d85ae58d462a552831c238fc3967476589934`) was run through the +same dedicated Gemma 4 e4b worker and authenticated gateway. The route remained +`ContextualOrchestratorJudge -> _FastMLSIJudgeAdapter -> TaskOrchestrator -> +ModelClient -> mlx-lm`, with two anchored criteria, three categories, and the +implicit `binary_threshold` method. + +All four boundary calls completed in `3.731 s` with `2,015` provider tokens +(`1,860` prompt and `155` completion). The result was score `1.0`, +`accepted=true`, criterion categories +`{evidence_completeness: 2, release_safety: 2}`, trace step count `4`, and +the required multi-item IRT row `[2,2]`. This confirms the current +cross-repository transport and IRT shape after the benchmark/ADR documentation +push; it is not semantic quality, bias, or production IRT promotion evidence. +Balanced non-ceiling gold, perturbation stability, category occupancy, and all +failure denominators remain required. + +## Warm direct MLX versus authenticated gateway recheck — 2026-08-14T11:56:58Z + +The dedicated Gemma 4 e4b worker and authenticated gateway were re-measured +after one warm-up request per endpoint with the identical short prompt +(`Reply with exactly OK.`), temperature `0`, and `max_tokens=32`. Each width +sent `4 * width` requests; the gateway remained configured with +`max_concurrent_runs=4`. + +| path | width | requests/statuses | wave | successful p50 | successful p95 | +| --- | ---: | --- | ---: | ---: | ---: | +| direct MLX | 1 | `4/4 x 200` | 0.837 s | 0.212 s | 0.220 s | +| direct MLX | 2 | `8/8 x 200` | 1.333 s | 0.334 s | 0.348 s | +| direct MLX | 4 | `16/16 x 200` | 2.599 s | 0.672 s | 0.694 s | +| direct MLX | 5 | `20/20 x 200` | 3.535 s | 0.919 s | 0.943 s | +| gateway | 1 | `4/4 x 200` | 0.921 s | 0.233 s | 0.236 s | +| gateway | 2 | `8/8 x 200` | 1.530 s | 0.386 s | 0.392 s | +| gateway | 4 | `16/16 x 200` | 2.560 s | 0.616 s | 0.737 s | +| gateway | 5 | `4/20 x 200`, `16/20 x 503` | 0.806 s | 0.802 s | 0.805 s | + +The gateway stays close to direct MLX through width `4` and explicitly rejects +excess admission at width `5`; it does not silently queue, drop, or repair +requests. This is warm transport/admission evidence only. It does not change +the multi-item Judge, semantic calibration, category-occupancy, or IRT +promotion gates, and the rejected requests remain in the overload denominator. + +## Current exact-head cross-repository Judge smoke — 2026-08-14T11:59:55Z + +The current source pair (`contextual-orchestrator` `f15ccb0ff53a0a2782438974f543bfc041cb1a69`, +`fast-mlsirm` `c9f2c280c4113e49486cb01e69daa40583f38127`) was run through the +same injected path: +`ContextualOrchestratorJudge -> _FastMLSIJudgeAdapter -> TaskOrchestrator -> +ModelClient -> mlx-lm`. The dedicated Gemma 4 e4b worker used temperature `0`, +disabled thinking, `max_output_tokens=128`, local concurrency `4`, two anchored +criteria, and category count `3`. + +All four binary boundary calls completed in `4.221 s` with `1,881` provider +tokens (`1,780` prompt and `101` completion). The result was +`accepted=true`, score `1.0`, categories +`{evidence_quality: 2, release_safety: 2}`, route mode, four trace steps, and +the required multi-item polytomous IRT row `[2,2]`. This proves current +transport, adapter, strict parsing, and response-shape integration only; it is +not a semantic-quality, bias, human/gold, or production-IRT promotion result. + +## Current exact-head Judge smoke after batch-integrity remediation — 2026-08-14T12:26:08Z + +The current source pair (`contextual-orchestrator` `cdca9d8e55f54b8b6ed67e146d73f7f32df93542`, +`fast-mlsirm` `c9f2c280c4113e49486cb01e69daa40583f38127`) was rerun through +`ContextualOrchestratorJudge -> _FastMLSIJudgeAdapter -> TaskOrchestrator -> +ModelClient -> mlx-lm` after the batch-result integrity fix. The dedicated +Gemma 4 e4b worker used temperature `0`, disabled thinking, +`max_output_tokens=128`, local concurrency `4`, two anchored criteria, and +category count `3`. + +All four binary boundary calls completed in `3.637 s` with `1,824` provider +tokens (`1,728` prompt and `96` completion). The result was `accepted=true`, +score `1.0`, categories `{evidence_quality: 2, release_safety: 2}`, four +trace steps, and the required two-item polytomous IRT row `[2,2]`. This is +current-head transport, adapter, strict-parse, and shape evidence only; it is +not semantic-quality, bias, human/gold, or production-IRT promotion evidence. +The batch-integrity change is orthogonal to this single route smoke; future +batch evaluations must retain incomplete-result failures rather than treating +them as successful observations. + +## Model-id and request-count correction — 2026-08-14T18:28:58Z + +An initial raw HTTP sweep was discarded because it sent the gateway's internal +model alias `mlx_e4b` to the direct MLX endpoint. The direct endpoint returned +`404` and attempted an invalid Hugging Face model lookup; this was a harness +configuration error, not a model or transport result. The same sweep also sent +only four requests at width `8`, so it did not test width-8 admission. + +The corrected sweep used the worker-advertised model id +`mlx-community/gemma-4-e4b-it-4bit` for both endpoints. Widths `1`, `2`, and +`4` sent four requests; width `8` sent eight requests. The authenticated +gateway retained `max_concurrent_runs=4`, and all completion IDs were unique +for successful responses. + +| path | width | requests/statuses | wave | attempt req/s | successful req/s | +| --- | ---: | --- | ---: | ---: | ---: | +| direct MLX | 1 | `4/4 x 200` | 0.959 s | 4.172 | 4.172 | +| direct MLX | 2 | `4/4 x 200` | 0.644 s | 6.215 | 6.215 | +| direct MLX | 4 | `4/4 x 200` | 0.604 s | 6.623 | 6.623 | +| direct MLX | 8 | `8/8 x 200` | 1.132 s | 7.066 | 7.066 | +| gateway | 1 | `4/4 x 200` | 0.819 s | 4.881 | 4.881 | +| gateway | 2 | `4/4 x 200` | 0.651 s | 6.141 | 6.141 | +| gateway | 4 | `4/4 x 200` | 0.564 s | 7.088 | 7.088 | +| gateway | 8 | `4/8 x 200`, `4/8 x 503` | 0.638 s | 12.539 | 6.269 | + +The gateway's width-4 result is the highest all-success candidate in this +small workload. Width 8 increases attempted request rate only by admitting +four explicit overload failures; it does not improve successful throughput. +This is transport/admission evidence, not semantic Judge or IRT evidence, and +does not justify raising the concurrency bound or changing fail-closed +overload behavior. + +## Authenticated structured Judge smoke — 2026-08-15 + +The free-form Judge path was first rechecked through the authenticated gateway: +all four binary boundary calls completed, but Gemma emitted prose/Markdown and +all four strict parses failed closed. This was a format/transport capability +finding, not a reason to add keyword matching, positional inference, retries, +or output repair. + +The corrected path used +`ContextualOrchestratorJudge -> _FastMLSIJudgeAdapter.complete_structured -> +TaskOrchestrator.proxy_completion -> ModelClient.proxy_send -> authenticated +local:// gateway -> mlx-lm`, with `local_credential_key=LOCAL_GATEWAY_TOKEN`. +The request carried a strict JSON Schema response format; the gateway bearer +credential was resolved from KV, and `chat_template_kwargs` was not forwarded +to the gateway. + +The live two-criterion, three-category run completed all four boundary calls and +returned: + +```json +{"ok":true,"score":1.0,"accepted":true, + "criterion_scores":{"evidence_completeness":1.0,"release_safety":1.0}, + "irt_row":[2,2],"category_method":"binary_threshold", + "trace_step_count":4, + "usage":{"prompt_tokens":2961,"completion_tokens":148,"total_tokens":3109}, + "served_agent_id":"mlx_judge"} +``` + +This validates authenticated local transport, structured response-format +delivery, strict parsing, trace/usage propagation, and the required +multi-criterion polytomous shape. It does not establish semantic accuracy, +option-count/order neutrality, absence of positive response-category bias, +human/gold agreement, IRT readiness, or production promotion. diff --git a/docs/kv-credentials.md b/docs/kv-credentials.md index 6860aeeec..88d6c1b73 100644 --- a/docs/kv-credentials.md +++ b/docs/kv-credentials.md @@ -20,11 +20,17 @@ register_credential("OPENAI_API_KEY", value) # writes into the KV The orchestrator resolves an agent's provider key through this seam only: -- `ModelClient.chat()` calls `get_credential(agent.credential_name)`. -- `ModelClient._send()` reads the key the same way for the outgoing request. +- Remote `ModelAgent` records use `get_credential(agent.credential_name)`. +- Direct `mlx://` workers are intentionally keyless and never receive a + provider credential. +- Authenticated loopback `local://` gateways may use the separate, + explicitly named `ModelAgent.local_credential_key`. +- `ModelClient._send()` resolves the transport-specific key before building + the outgoing request. - `ModelClient._validate_provider()` requires the credential to be **resolvable** - before any egress. A non-mock agent whose credential is missing raises - `NotConfigured` — it never silently falls back to `os.getenv`. + before any egress when that transport names a key. A non-mock agent whose + credential is missing raises `NotConfigured` — it never silently falls back + to `os.getenv`. Mock agents (`base_url` starting with `mock://`) early-return before any credential logic and stay keyless. @@ -44,6 +50,31 @@ string is treated as the **credential name** in the KV — it is *not* read as a environment variable. `ModelAgent.credential_name` returns `api_key_env` when present, otherwise `credential_key`. +### Direct MLX versus an authenticated local gateway + +These schemes have different credential contracts: + +```json +{ "id": "mlx_worker", "model": "mlx-community/gemma-4-e4b-it-4bit", + "base_url": "mlx://127.0.0.1:18083/v1" } +``` + +The direct `mlx://` transport is a loopback-only, keyless mlx-lm server. A +`credential_key` or remote `OPENAI_API_KEY` is never forwarded to it. A +`local://` URL instead denotes the contextual-orchestrator loopback gateway; +when that gateway requires bearer authentication, configure only its explicit +local token name: + +```json +{ "id": "mlx_gateway", "model": "mlx-community/gemma-4-e4b-it-4bit", + "base_url": "local://127.0.0.1:18084/v1", + "local_credential_key": "LOCAL_GATEWAY_TOKEN" } +``` + +The gateway owns worker template settings, so `chat_template_kwargs` is sent +only to direct `mlx://` workers. Missing local gateway credentials fail closed; +they do not fall back to an OpenAI credential or an unauthenticated request. + ## Backends Backends implement a tiny interface (`get(name)` / `set(name, value)`), selected @@ -147,6 +178,21 @@ principle **"No os.getenv, values from KV"**, that source moves to the KV: `api_key_env` is retained only as a back-compat *credential name* alias. +## Server authentication and Keyverse + +Provider credentials and gateway bearer authentication are separate concerns. +The CLI resolves named server tokens from this KV when `--auth-token-key`, +`--admin-token-key`, or `--inference-token-key` is used; it does not read the +legacy `CONTEXTUAL_ORCHESTRATOR_*TOKEN` environment variables at request time. +Explicit token flags remain local-development escape hatches. + +For production ecosystem access, construct `SecurityConfig` with a reviewed +`bearer_verifier` that validates Keyverse-issued OIDC tokens. The adapter must +own issuer/audience/signature/expiry/scope validation and key rotation; do not +decode JWTs with a string split or place Keycloak admin credentials in this +repository. Keyverse RP registration, desired-state reconciliation, and +confidential-client secret placement remain deployment-controller operations. + ## Gateway direction This credential seam is the durable first step of growing diff --git a/docs/planning/adrs/0001-fail-closed-model-judgment.md b/docs/planning/adrs/0001-fail-closed-model-judgment.md new file mode 100644 index 000000000..ce9002ad7 --- /dev/null +++ b/docs/planning/adrs/0001-fail-closed-model-judgment.md @@ -0,0 +1,156 @@ +--- +id: "0001" +title: "Fail-closed structured model judgment" +status: accepted +proposed_date: "2026-08-10" +accepted_date: "2026-08-11" +deciders: + - "repository maintainer" +consulted: + - "contextual-orchestrator runtime" + - "fast-mlsirm evaluation adapter" +informed: + - "contributors" +affected_components: + - "contextual_orchestrator/orchestrator.py" + - "tests/test_model_judge.py" + - "fast-mlsirm/python/fast_mlsirm/llm_judge.py" +effort: M +supersedes: null +superseded-by: null +related: + - path: "docs/planning/adrs/0002-explicit-local-mlx-evaluation.md" + relation: influences + - path: "docs/planning/adrs/0003-keyverse-authentication-boundary.md" + relation: informational +asr_triggers: + - kind: performance + evidence: "A judge adds a provider call and evaluation comparisons must measure real work." + note: "The extra call is explicit in the trace and evaluation bypasses the response cache." + - kind: maintainability + evidence: "Heuristic verdicts mix language interpretation with orchestration control flow." + note: "A strict JSON protocol gives one auditable decision boundary." +success_criteria: + - metric: "heuristic verifier decisions" + target: "zero keyword-based accept/reject decisions in production code" + measurement_window: "every test and review of the merged change" + source: "tests/test_model_judge.py and repository search" + - metric: "invalid or unavailable model verdicts" + target: "100% rejected without term fallback" + measurement_window: "every conducted workflow" + source: "structured judge parser and regression tests" +--- + +# Fail-closed structured model judgment + +## Context + +The verifier decision is a trust boundary: accepting a result changes the answer returned by a conducted workflow. Keyword matching is not a valid base judgment because it cannot reliably represent negation, quoted risks, Korean or other languages, or a report whose positive and negative evidence coexist. + +> OrchestrationPolicy.verifier_judge is configured as "model"; unsupported keyword modes raise ValueError. +> +> _judge_verifier_output records "model judgment required; keyword matching is disabled" and never accepts from thinker/worker presence. +> +> _model_judge_verification accepts only an explicit JSON decision enum and returns rejection when the judge is unavailable or malformed. + +## Decision Drivers + +* Do not allow a lexical accident to approve or reject model work. +* Keep routing heuristics explicitly separate from semantic judgment; a capability hint may select a worker, but it cannot decide answer quality or verification. +* Preserve failover, circuit-breaker, and provider usage accounting for judge calls. +* Keep evaluation latency honest when response caching is enabled. +* Make the decision auditable and testable without adding a provider SDK. + +## Considered Options + +* Keep positive/negative term matching as the default. +* Permit free-form model replies and search for ACCEPT/REJECT. +* Require a structured model verdict, route it through normal orchestration failure handling, and fail closed. + +## Decision Outcome + +Chosen option: "Require a structured model verdict and fail closed". + +| Driver | Term matching | Free-form keyword scan | Structured model verdict | +| --- | --- | --- | --- | +| Language/negation safety | poor | poor | explicit evidence-based assessment | +| Failure behavior | hidden fallback | ambiguous | rejected and observable | +| Runtime integration | cheap but bypasses semantics | extra call | normal _invoke path with usage | +| Evaluation truthfulness | cache-sensitive | cache-sensitive | cache-bypassed comparison | + +The judge returns exactly one bounded, duplicate-free JSON object with `{"decision":"ACCEPT"|"REJECT","reason":"brief evidence-based reason"}`. Wrapper text, extra fields, duplicate keys, missing fields, parser-stressing input, provider failure, or an empty verifier report reject the workflow. The contextual-orchestrator model-judge boundary requires the fast-mlsirm adapter; absence or import failure is fail-closed rather than a direct-jury fallback. The fast-mlsirm judge adapter calls an injected contextual-orchestrator object and never calls a provider directly. + +### Consequences + +* Good, because keyword matching is removed from the production decision path and its false-positive/false-negative class has regression coverage. +* Good, because judge failover and usage are recorded through _invoke. +* Bad, because an unavailable local model can now reject a workflow instead of silently accepting a worker output. +* Bad, because a conducted local run pays for one additional judge completion. + +### Confirmation + +Run python3 tests/test_model_judge.py, the full contextual test suite, and the fast-mlsirm judge adapter test. Search the merged tree for verifier_positive_terms, verifier_negative_terms, and "terms" verifier modes; no production judgment path may remain. + +## Pros and Cons of the Options + +### Keep term matching + +* Good, because it has no extra model call. +* Bad, because it misreads quoted risk language and language-dependent wording. +* Bad, because it makes a safety decision from substring presence rather than evidence. + +### Free-form keyword scan of a model reply + +* Good, because it is easy to retrofit. +* Bad, because explanations can contain both decisions and the scan is another heuristic. +* Bad, because malformed output is difficult to audit consistently. + +### Structured model verdict (chosen) + +* Good, because the protocol is small, strict, and observable. +* Good, because failure is explicit and fail-closed. +* Bad, because it requires a capable local judge and increases latency/token use. + +## Problem Register and Remediation Directions + +| Finding | Direction | State | +| --- | --- | --- | +| Keyword matching was language- and context-unsafe. | Delete term-based verdicts; use strict model JSON and fail closed. | Implemented | +| Architecture notes described deterministic keyword scoring without explicitly limiting it to routing, which could be mistaken for a judgment fallback. | Describe the mechanism as capability-hint routing only, link this ADR, and retain the structured model-judge regression tests as the acceptance boundary. | Implemented 2026-08-12 | +| The judge previously bypassed failover/circuit/usage handling. | Call the judge through _invoke. | Implemented | +| compare_to_baseline could measure cache hits instead of provider work. | Use _dispatch directly for both measured arms. | Implemented | +| Core orchestration has no gold-answer quality metric. | Keep structural latency metrics honest and inject fast-mlsirm for rubric quality; do not invent a lexical proxy. | Adapter implemented; benchmark gate ongoing | +| Judge prompt/output can be malformed or truncated. | Use bounded JSON extraction, actionable mlx template guidance, and fail closed. | Implemented | +| A model can wrap a verdict, add fields, duplicate keys, or send parser-stressing text. | Parse the complete bounded response with an exact duplicate-free schema and exercise the parser with Hypothesis and Atheris; never repair or keyword-match it. | Implemented | +| The strict judge parser had invalid-enum, empty/non-string-reason, and maximum-size boundaries that were only exercised indirectly. | Add direct regression tests for each fail-closed parser boundary so future schema changes cannot turn malformed model output into a decision. | Implemented in current local head; exact-head CI/review follow-up required | +| An environment toggle could bypass the fast-mlsirm judge adapter even though the Goal requires all model judgments to cross contextual-orchestrator through that adapter. | Always use the injected fast-mlsirm adapter for model judgments, remove the runtime environment bypass, and fail closed when the package is absent or broken; no direct contextual judge fallback is permitted. | Implemented in current local head; exact-head CI/review follow-up required | +| `ContextualOrchestratorJudge` passes `mode=` to its injected completion object, while the gateway's fast-mlsirm adapter accepted only `messages`, causing the real adapter path to fail closed before judging. | Accept and validate the mode keyword at the adapter seam and preserve it in the returned completion metadata; add a direct regression. | Implemented in current local head; exact-head CI/review follow-up required | +| A broken or absent fast-mlsirm import could bypass the required calibration boundary if it selected a different judge path. | Treat both absent and broken fast-mlsirm imports as fail-closed conditions; never fall back to a direct contextual judge, and add regressions for both states. | Implemented in current local head; exact-head CI/review follow-up required | +| The planning strategy values `template`/`generated` are not valid fast-mlsirm orchestration modes; passing one into the judge made the default conduct verification fail closed before a model call. | Keep the judge as one bounded `route` call through the gateway, independent of the workflow planning strategy, and record that mode in the trace. | Implemented in current local head; exact-head CI/review follow-up required | +| The gateway passed `accept_threshold` both to `ContextualOrchestratorJudge.__init__` and to `judge()`, but the public judge method accepts the threshold only at construction. | Match the injected fast-mlsirm public signature: configure the threshold once in the constructor and pass only the documented judge arguments. | Implemented in current local head; exact-head CI/review follow-up required | +| A missing fast-mlsirm installation still allowed contextual-orchestrator to issue a direct strict-JSON judge call, bypassing the required cross-repository calibration boundary. | Remove the direct fallback, require fast-mlsirm for every model judgment, and add a regression proving that absence does not call a provider or accept a verdict. | Goal expanded and implemented locally 2026-08-13; exact-head CI/review follow-up required | +| The two repositories' default virtual environments cannot import each other's source packages, so a live integration can fail before the first judge call even when both checkouts are healthy. | Make the exact interpreter the integration boundary: install both packages into that interpreter (prefer editable installs), or expose both checkout source roots with `PYTHONPATH` for a source run; execute `python -m contextual_orchestrator check-fast-mlsirm` with that interpreter before a live benchmark. Never repair this with a direct-provider fallback or a second interpreter. | Documented 2026-08-14; the isolated-import failure and same-interpreter preflight are covered by the current MLX recheck | +| Local model capacity can make four workflow steps too slow. | Benchmark route/conduct and expose concurrency/template controls; optimize only from measured traces. | Ongoing | + +## Risks and Mitigations + +| risk | likelihood | impact | mitigation | owner | +| --- | --- | --- | --- | --- | +| A small local model emits malformed JSON. | medium | high | Disable thinking when needed, raise output cap, reject malformed output, record the reason. | maintainer | +| Strict rejection lowers availability. | medium | medium | Keep route mode available and provide explicit model/provider failover. | maintainer | +| A judge agrees with a bad verifier report. | medium | high | Use fast-mlsirm rubric evaluation and curated regression prompts; never treat the judge as ground truth. | evaluation owner | + +## Rollback / Exit Strategy + +Revert the implementation commit if the structured protocol causes unacceptable availability or latency, but retain this ADR and the test that forbids keyword matching. A rollback may restore a prior orchestration behavior only behind a separately approved ADR; it must not reintroduce keyword matching as an implicit fallback. + +## Affected Components + +* contextual_orchestrator/orchestrator.py +* tests/test_model_judge.py +* fast-mlsirm/python/fast_mlsirm/llm_judge.py +* local mlx benchmark commands and traces + +## More Information + +The decision follows the structured-output and evaluator/optimizer patterns in the local agentic-evaluation guidance. It is deliberately provider-neutral so mlx-lm remains the runtime provider and fast-mlsirm remains the calibration/evaluation layer. diff --git a/docs/planning/adrs/0002-explicit-local-mlx-evaluation.md b/docs/planning/adrs/0002-explicit-local-mlx-evaluation.md new file mode 100644 index 000000000..ceb6e2b51 --- /dev/null +++ b/docs/planning/adrs/0002-explicit-local-mlx-evaluation.md @@ -0,0 +1,366 @@ +--- +id: "0002" +title: "Explicit local mlx transport and evaluation adapter" +status: accepted +proposed_date: "2026-08-10" +accepted_date: "2026-08-11" +deciders: + - "repository maintainer" +consulted: + - "mlx-lm runtime" + - "fast-mlsirm evaluation adapter" + - "Sakana Fugu Technical Report" + - "TRINITY: An Evolved LLM Coordinator" + - "Learning to Orchestrate Agents in Natural Language with the Conductor" +informed: + - "contributors" +affected_components: + - "contextual_orchestrator/orchestrator.py" + - "contextual_orchestrator/server.py" + - "contextual_orchestrator/__main__.py" + - "contextual_orchestrator/batch_routing.py" + - "contextual_orchestrator/cost_router.py" + - "examples/agents.mlx.json" + - "examples/agents.local.json" + - "tests/test_local_mlx.py" + - "tests/test_batch_routing.py" + - "tests/test_cost_router.py" + - "tests/test_openai_passthrough.py" +effort: M +supersedes: null +superseded-by: null +related: + - path: "docs/planning/adrs/0001-fail-closed-model-judgment.md" + relation: informational + - path: "docs/planning/adrs/0003-keyverse-authentication-boundary.md" + relation: informational +asr_triggers: + - kind: performance + evidence: "The local 3B mlx-lm model is available and orchestration multiplies provider calls." + note: "Temperature, token cap, template behavior, and bounded local concurrency are explicit controls." + - kind: security + evidence: "A local endpoint must never receive a remote provider credential." + note: "Only explicit loopback mlx/local URLs are keyless and translated to HTTP after validation." + - kind: maintainability + evidence: "Codex uses the Responses wire contract while mlx-lm exposes Chat Completions." + note: "Keep protocol conversion, SSE framing, and model discovery at the orchestrator boundary with focused regression tests." +success_criteria: + - metric: "local provider safety" + target: "loopback-only mlx/local URL, no Authorization header, remote HTTP rejected" + measurement_window: "every local transport test run" + source: "tests/test_local_mlx.py" + - metric: "judge integration" + target: "fast-mlsirm judge reaches an injected contextual-orchestrator only" + measurement_window: "every LLM-as-a-Judge run" + source: "fast-mlsirm/tests/test_llm_judge.py" + - metric: "Codex Responses compatibility" + target: "authenticated /v1/responses requests are adapted to local Chat Completions and return response.completed plus [DONE] when streamed" + measurement_window: "every passthrough regression run and local Codex smoke" + source: "contextual_orchestrator/orchestrator.py, contextual_orchestrator/server.py, tests/test_openai_passthrough.py" + - metric: "local model discovery" + target: "authenticated /v1/models returns contextual-orchestrator plus every configured worker candidate with governance status" + measurement_window: "every Codex provider startup and passthrough test run" + source: "contextual_orchestrator/server.py and tests/test_openai_passthrough.py" + - metric: "credential separation" + target: "ChatGPT/OpenAI authentication is selected only by the built-in OpenAI provider; no OpenAI credential is forwarded to mlx-lm" + measurement_window: "every local server startup and provider configuration review" + source: "contextual_orchestrator/orchestrator.py, contextual_orchestrator/server.py, local Codex profile configuration" +--- + +# Explicit local mlx transport and evaluation adapter + +## Context + +The Fugu technical report describes an orchestrator model that behaves as one +model to callers while selecting, delegating to, verifying with, and +synthesizing work from a swappable worker pool. It also permits the +orchestrator to be selected as a worker for recursive topologies. TRINITY +defines role contracts for thinker, worker, and verifier; Conductor defines +natural-language subtasks, worker identifiers, and access lists. + +This repository is a stdlib control-plane implementation of that public shape, +not a trained Fugu/Trinity/Conductor coordinator. Its `contextual-orchestrator` +model is therefore the public orchestration candidate, while `ModelAgent` +records are worker candidates. Every discovered record remains in the registry; +`disabled` is reserved for an explicit operator/admin quarantine or a persisted +removal tombstone, not for an automatic capability or availability judgment. + +mlx-lm exposes an OpenAI-compatible server, but local reasoning models may +return a reasoning-only message when thinking consumes the output budget. The +previous transport wording also treated local HTTP as a remote provider shape, +and the evaluation package had no provider-neutral boundary that guaranteed +contextual-orchestrator was used for LLM-as-a-Judge. + +Codex custom providers use the Responses wire contract, while the installed +mlx-lm server exposes an OpenAI-compatible Chat Completions endpoint. A direct +Codex-to-mlx-lm configuration therefore cannot preserve the Codex request and +streaming contract. The compatibility boundary belongs in +contextual-orchestrator, which is the authenticated public control-plane and +provider-egress boundary. + +> ModelClient accepts an explicit mlx:// or local:// loopback URL and maps it to HTTP only after validation. Direct mlx:// is keyless; authenticated local:// uses only an explicitly named local KV credential. +> +> Direct mlx:// requests can forward chat_template_kwargs, including {"enable_thinking": false}; the local:// gateway owns worker template settings and rejects unsupported template fields. Both report an actionable error when content is absent. +> +> ContextualOrchestratorJudge calls an injected contextual-orchestrator adapter and, when supported, sends a strict JSON Schema request through the gateway; it parses bounded rubric JSON without provider credentials. + +## Decision Drivers + +* Maximize useful local model output on the available Apple Silicon runtime. +* Keep local tests offline and free of provider SDK dependencies. +* Prevent credentials from being sent to loopback or arbitrary HTTP endpoints. +* Measure quality and latency separately instead of labeling structure as quality. +* Allow the existing ChatGPT Codex login to remain available without sending its credentials to a local model. + +## Considered Options + +* Treat all OpenAI-compatible endpoints identically. +* Add a direct mlx-specific provider dependency to both repositories. +* Make mlx-lm implement the Codex Responses API or fork its server transport. +* Treat contextual-orchestrator as only a thin gateway and omit it from the model candidate surface. +* Keep the core stdlib-only and add explicit loopback transport controls plus an injected evaluation adapter. + +## Decision Outcome + +Chosen option: "Explicit loopback local transport plus provider-neutral adapter". + +| Driver | Generic HTTP | Direct mlx dependency | Explicit loopback + injected adapter | +| --- | --- | --- | --- | +| Local safety | ambiguous | provider-specific | scheme/host/credential checks | +| Runtime footprint | small | larger | stdlib core, installed mlx executable | +| Judge composition | not enforced | couples packages | contextual-orchestrator boundary is testable | +| Performance controls | implicit | provider-specific | temperature/cap/template/concurrency knobs | + +The core accepts only mlx:// or local:// with loopback hosts and a valid port. Direct mlx:// traffic is keyless; authenticated local:// gateway traffic may use only an explicitly named local KV credential, never the remote provider credential. Local batch requests use a bounded thread pool; interactive paths remain sequential by default. fast-mlsirm receives an injected contextual-orchestrator adapter, strict criteria, bounded JSON parsing, and usage/trace metadata. + +For Codex, the public control plane accepts the Responses request, converts supported +message and function-tool items to the local Chat Completions shape, forwards +the configured direct mlx-lm chat-template arguments, converts the result back to a +Responses object, and emits a valid Responses SSE sequence for streaming. The +control plane exposes `contextual-orchestrator` followed by every configured +worker candidate at /v1/models, including explicit governance status. Discovery +does not set `disabled`: that field is reserved for operator/admin quarantine or +persisted removal tombstones. Recursive self-selection is constrained by +provider exclusions in the current untrained implementation, rather than by +disabled state. +The OpenAI/ChatGPT login remains a separate built-in Codex provider selected by +a Codex profile; its credential is never sent to the loopback mlx-lm endpoint. + +### Consequences + +* Good, because the existing mlx_lm.server can be benchmarked without adding a runtime dependency. +* Good, because local reasoning behavior is controlled by explicit template kwargs rather than silent content loss. +* Good, because fast-mlsirm cannot accidentally call a provider outside contextual-orchestrator. +* Good, because Codex can use the same authenticated gateway without changing mlx-lm or leaking ChatGPT credentials to it. +* Bad, because local conduct remains several sequential model calls and can be slow. +* Bad, because the adapter does not manufacture a ground truth; a rubric model is still a model. +* Bad, because the Responses-to-Chat conversion supports only the provider-neutral message/function subset; unsupported Codex namespaces and standalone web search are not forwarded to mlx-lm. +* Bad, because a small local model may not reliably follow the full Codex tool protocol even when the transport is valid. + +## Non-goals + +* Do not modify or fork mlx-lm to add a Responses endpoint. +* Do not forward ChatGPT/OpenAI auth material to any `mlx://` or loopback provider. +* Do not silently switch runtimes based only on process presence. Discovery may produce an explicit candidate registry; availability and model capability are runtime/provider facts, while `disabled` remains an explicit operator/admin governance action. +* Do not enable recursive contextual-orchestrator self-selection until recursion depth, authentication, and failure termination are explicit. +* Do not bind the local Codex bridge to a public interface or make inference unauthenticated. + +## Implementation Plan + +* `contextual_orchestrator/orchestrator.py`: keep the Responses-to-Chat and Chat-to-Responses conversion at `ModelClient.proxy_send`; validate loopback endpoints before sending, resolve only the explicit local gateway credential for `local://`, and preserve `chat_template_kwargs` only for direct `mlx://` workers. +* `contextual_orchestrator/batch_routing.py` and `contextual_orchestrator/cost_router.py`: reuse the bounded `ModelClient.local_concurrency` value for the default in-process batch backend; keep the standalone default at one, preserve request ordering, and propagate runner errors without fallback. +* `contextual_orchestrator/__main__.py`: keep the secure HTTP run-slot default at eight, but expose a separate bounded `--max-concurrent-runs` option so an operator can align the gateway admission limit with a measured local batch setting without changing the library default. +* `contextual_orchestrator/server.py`: authenticate `/v1/models` and `/v1/responses`, proxy Responses requests, and frame streamed responses with `response.completed` and `data: [DONE]`. +* `examples/agents.mlx.json`: keep the minimal selected MLX worker example visible in data, not code. +* `examples/agents.local.json`: keep the explicit candidate registry: public contextual-orchestrator and every discovered MLX, llama.cpp, and LM Studio candidate. Do not pre-disable entries as a discovery side effect. +* `tests/test_local_mlx.py`: verify direct MLX template arguments, authenticated local gateway credential separation, and fail-closed missing credentials. +* `tests/test_model_judge.py`: verify structured fast-mlsirm completion requests remain on the contextual gateway adapter. +* `tests/test_openai_passthrough.py`: verify the Responses SSE completion contract and model discovery endpoint. +* Local machine configuration: keep the ChatGPT login in Codex's normal auth cache, select the built-in `openai` provider through a profile when needed, and keep the local gateway bearer token in the OS credential store. + +## Verification + +* `PYTHONPATH=. .venv/bin/python -m pytest -q tests/test_local_mlx.py tests/test_openai_passthrough.py` passes in the repository test environment. +* `GET /healthz` and authenticated `GET /v1/models` succeed on the loopback control plane; model discovery includes the public orchestrator and the complete configured candidate registry. +* Authenticated streamed `POST /v1/responses` contains `response.completed` and `data: [DONE]` and reaches the configured mlx-lm model. +* A Codex local-provider smoke returns the requested exact sentinel response through contextual-orchestrator. +* Required exact-head CI, independent approval, zero unresolved threads, and final merge refetch are governed by ADR-0004; local verification cannot substitute for those gates. + +### Confirmation + +Run the local transport and passthrough tests, the real mlx route/conduct/judge benchmark, and the Codex smoke through `/v1/responses`. Confirm traces include provider usage, streamed responses terminate with `response.completed` and `[DONE]`, model discovery returns the configured model, and the judge is disabled-thinking or has enough output budget. + +## Pros and Cons of the Options + +### Treat all endpoints identically + +* Good, because the API surface is smaller. +* Bad, because it hides local security and template semantics. +* Bad, because reasoning-only responses become opaque provider failures. + +### Add direct mlx dependencies + +* Good, because provider-specific behavior could be wrapped deeply. +* Bad, because both repositories would become harder to install and test. +* Bad, because the installed mlx_lm.server already supplies the required transport. + +### Explicit loopback transport and injected adapter (chosen) + +* Good, because it reuses the existing OpenAI-compatible surface and stdlib. +* Good, because the trust boundary is visible in configuration and tests. +* Bad, because an external deployment still owns model lifecycle and OIDC integration. + +### Direct Codex-to-mlx-lm transport (rejected) + +* Good, because it has one fewer process. +* Bad, because the current Codex provider contract is Responses-only while mlx-lm serves Chat Completions. +* Bad, because it would require weakening Codex streaming/tool semantics or maintaining a second transport implementation in mlx-lm. + +## Problem Register and Remediation Directions + +| Finding | Direction | State | +| --- | --- | --- | +| Reasoning-only mlx responses hid the real failure. | Forward template kwargs and emit an actionable content error. | Implemented | +| Local URLs could be confused with remote egress. | Require explicit loopback scheme/host and strip credentials/query data. | Implemented | +| Unbounded local parallelism could exhaust memory, while non-integer values could be silently truncated into a different concurrency. | Accept only an exact positive built-in integer in `1..64` at the client and CLI boundaries; preserve sequential default and keep the measured batch tuning value explicit. | Implemented in current head; exact-head CI/review follow-up required | +| An earlier eight-request 3B snapshot favored `local_concurrency=2` and incorrectly suggested that `4` was the throughput ceiling for the service. | Treat that result as historical; preserve the sequential interactive/default path, and require repeated warm-cache measurements across request cardinality and model size before changing a tuning recommendation. | Superseded by the repeated probe below; no quality claim | +| The repeated 2026-08-13 loopback probe measured `local_concurrency=8` as fastest for the current mlx-lm service: 16-request 3B median `7.928 s` (`2.018 req/s`), eight-request 1B median `1.524 s` (`5.251 req/s`), and eight-request Gemma 4B median `3.654 s` (`2.189 req/s`); all tested requests returned non-empty content. | Recommend explicit `local_concurrency=8` for latency-tolerant local batches on this server, while retaining library/interactive default `1`; re-measure after model, server-flag, prompt-size, or memory changes and never infer quality from throughput. | Observed and recorded in `docs/benchmarks/2026-08-13-local-mlx-gateway.md`; current tuning evidence | +| A current warm-cache saturation probe on the same 3B service measured c=16 at `6.827 req/s` for 16 requests and `6.246 req/s` for 32 requests, while c=24 and c=32 fell to `4.849` and `5.132 req/s`; every response remained non-empty. | Keep the general multi-model c=8 observation as the conservative cross-model baseline, but recommend explicit c=16 for this measured 3B batch profile only; retain default/interactive c=1 and require re-measurement after model, prompt, server, or memory changes. | Observed and recorded in `docs/benchmarks/2026-08-13-local-mlx-gateway.md`; current 3B tuning evidence | +| The generic `CostRoutingCoordinator` defaulted to a sequential `LocalBatchBackend` even when its `ModelClient` had an explicit local concurrency, so latency-tolerant local batches did not use the measured throughput path. | Pass the existing bounded `ModelClient.local_concurrency` into the default in-process backend, preserve standalone/default concurrency `1`, keep result ordering and runner-error propagation without fallback, and cover the handoff plus a two-request barrier regression. | Implemented in current local head; targeted/full suites and an 8-request live coordinator-to-MLX smoke passed; exact-head CI/review follow-up required | +| The new concurrent coordinator batch path can exercise `TaskOrchestrator`'s shared circuit-breaker state from multiple worker threads; an unlocked failure counter could lose increments and delay provider isolation. | Protect circuit-breaker read/reset/failure/success transitions with one lock and cover concurrent failure recording; keep the lock narrow so provider I/O remains outside it. | Implemented in current local head; concurrent circuit regression added; exact-head CI/review follow-up required | +| The current 32-request 3B saturation probe completed through c=48 only after a throughput collapse and timed out at c=64, while `/v1/models` remained healthy. | Treat c=64 as an observed provider saturation failure rather than a transport or LibreSSL defect; retain the explicit bound for controlled experiments, document the failed point, and use measured c=8/c=16 profiles instead of raising the default or hiding provider timeouts. | Observed 2026-08-13; benchmark evidence and adaptive tuning remain required | +| A live fast-mlsirm cumulative-threshold call returned a valid two-item polytomous row `(4,0)` but assigned `risk_awareness=0` to an answer that explicitly included rollback rehearsal, while `evidence_quality=4`; strict parsing and transport therefore did not guarantee semantic item accuracy. | Keep the complete result and provider/trace metadata in the calibration denominator; add balanced held-out cases and human/gold anchors for item-level recall and severity, and never repair the miss with keywords, category position, or silent coercion. | Observed 2026-08-13; semantic calibration required | +| The 2026-08-14 same-route 3B direct K-way probe scored an unsafe answer `0.0`, `0.5`, `0.8333` and a partial answer `0.0`, `1.0`, `0.0` at K=`2,5,7`; explicit binary thresholds scored both safe and unsafe probes `0.0` at K=`5,7`, with a safe semantic false negative. | Keep the gateway transport neutral while making fast-mlsirm's omitted polytomous method resolve to bounded binary thresholds; retain direct K-way only for explicit calibration, record calls/tokens/latency and semantic misses, and require held-out human/gold recall before IRT use. Never keyword-match or repair. | Implemented in fast-mlsirm `608cfbd`; calibration and exact-head review/check remain required | +| The actual integrated `_FastMLSIJudgeAdapter` smoke with fast-mlsirm `9d18f53` and contextual-orchestrator `a0a354a` selected the binary default at K=5: unsafe output was a valid rejected `(0,0)` result in 8 calls/`2,379` tokens/`3.73 s`, while safe output failed monotonicity after 8 calls/`3.23 s`. | Preserve the gateway's provider-neutral role and record both valid and failed integrated results; use the default only as a fail-closed measurement guard, not a quality claim, and require semantic gold/recall calibration before IRT interpretation. | Observed 2026-08-14; integrated contract verified, calibration remains required | +| The integrated safe-case failure previously exposed only `criterion thresholds must be monotone`, losing per-boundary status and provider accounting. fast-mlsirm `d1eca0c2fed89991e647802f0b27a91f0f6fe2bd` now reports bounded failure evidence: `semantic_status=non_monotone`, `parse_status=passed`, `8/8` completed calls, `8` trace steps, and `2,639` tokens. | Preserve structured failure evidence in the calibration denominator and distinguish complete-but-invalid semantics from transport or parse failure; never coerce the comparison into an IRT row or repair it lexically/positionally. | Observed 2026-08-14; integrated evidence capture implemented, semantic calibration remains required | +| A fresh 16-request 3B route recheck completed `16/16` at c=`1,4,8,16`; throughput was `1.959`, `6.136`, `6.315`, and `5.797 req/s` respectively, with c=8 fastest for this exact workload. | Retain library/interactive default c=1, use explicit c=8 as the current latency-tolerant batch starting point, and re-measure after model, prompt, output budget, server, or memory changes; never infer judge quality from throughput. | Observed 2026-08-14; benchmark recorded, tuning remains workload-specific | +| An anchored K=5 judge comparison through the real MLX route found Gemma 4 e4b strict `(4,4)`/score `1.0` in `3,031` tokens and `11.96 s`, while Llama 3B repeatedly produced a safe `(0,0)` semantic false negative and Llama 1B failed JSON on all eight boundaries. | Keep model choice evidence-based and separate quality, parse reliability, latency, and token cost; treat Gemma 4 e4b as a candidate only, retain Llama failures in the denominator, and require held-out gold/perturbation calibration before changing verifier priorities or IRT claims. | Observed 2026-08-14; fast-mlsirm `dd44a95`, calibration remains required | +| The current exact-head anchored Gemma 4 e4b rerun completed and parsed all eight boundaries but produced `false,false,true,true` for evidence quality and `false,true,true,true` for risk signal, so the ordinal judge failed closed as non-monotone despite complete traces, anchors, and `3,625` provider tokens. | Preserve the complete semantic failure and its trace/usage in the calibration denominator; do not promote the model or repair threshold order with keywords, positions, retries, or coercion. Require balanced held-out gold recall, perturbation stability, and category occupancy before any verifier-priority change. | Observed 2026-08-14; current exact-head integrated evidence, calibration remains required | +| The mlx-lm process remained liveness-ready (`/health` and `/v1/models` returned 200) while all chat completions returned no bytes and timed out; the loopback process had accumulated closed/CLOSE_WAIT connections. | Keep provider readiness separate from process liveness, bound request timeouts/concurrency, record all failed boundaries, and let the single-port supervisor restart the stuck process after an explicit health/readiness diagnosis. Never classify this as a LibreSSL certificate failure or disable TLS verification. | Observed 2026-08-14; explicit process restart restored a direct HTTP 200 completion, lifecycle hardening remains required | +| fast-mlsirm binary-threshold calibration issued independent boundary calls serially even when the injected contextual-orchestrator client had a bounded `local_concurrency` setting. | Reuse only that existing gateway bound for binary boundary calls, keep generic injected orchestrators sequential, preserve deterministic request order and complete trace/usage, and retain fail-closed monotonicity validation; do not add direct provider transport. | Implemented in fast-mlsirm `61e6be9`; targeted/full evidence and live MLX probe recorded | +| The contextual `_FastMLSIJudgeAdapter` did not expose its existing gateway client, so fast-mlsirm's bounded binary concurrency capability was not discoverable on the actual integrated judge path. | Expose the existing client capability from the adapter without adding a provider path, add an integration regression, and keep the fast-mlsirm fallback sequential for generic injected transports. | Fixed locally; exact integrated-path test and review/check follow-up required | +| The HTTP gateway admitted only eight simultaneous orchestration runs even when a measured local MLX batch used `local_concurrency=16`, so the optimized batch setting could be hidden behind the server semaphore; direct `SecurityConfig` callers could also bypass the intended bound. | Preserve the secure default of eight, enforce the same `1..64` bound at the server API and CLI, expose an explicit bounded `--max-concurrent-runs` setting, and require operators to tune it separately with the local batch setting; never raise the default automatically or treat throughput as quality. | Implemented locally; API/CLI regressions and exact-head review/check follow-up required | +| Batch result errors could lose usage/IDs. | Preserve custom IDs and usage per local request. | Implemented | +| LLM-as-a-Judge could bypass the gateway. | Make fast-mlsirm depend on an injected contextual-orchestrator object, not a provider. | Implemented | +| The contextual-orchestrator base environment does not install fast-mlsirm's declared NumPy dependency, so an in-process source checkout can fail to import the judge even though both repositories are present. | Keep the core standalone, but make the integration runner install/use fast-mlsirm's declared environment, run an import preflight, and fail closed rather than silently changing the judge implementation. | Observed 2026-08-13; benchmark command and fail-closed path recorded; packaging/preflight automation remains required | +| Quality claims could be inferred from latency/step count. | Report structural metrics as structural and use rubric judgments for quality. | Implemented; benchmark ongoing | +| Codex sends Responses requests while mlx-lm accepts Chat Completions. | Convert the supported request/response subset at the authenticated gateway and test the SSE completion sequence. | Implemented | +| The Responses adapter treated any non-list input as a message payload, so a malformed object could be silently converted into an empty user message instead of being rejected at the trust boundary. | Accept only the supported string-or-list input contract, reject other JSON types, and cover message, developer, tool-result, function-call, ignored-item, tool-choice, reasoning, metadata, and malformed-input paths. | Fixed in current local head; exact-head CI/review follow-up required | +| A negative remote retry count was accepted and made the retry loop silently skip the provider call. | Validate `max_retries` at client construction with the same non-negative contract as local retries, and cover negative and boolean values before any request can start. | Fixed in current local head; exact-head CI/review follow-up required | +| Codex model discovery reached the gateway's missing `/v1/models` route. | Expose contextual-orchestrator plus the complete configured worker candidate registry through an authenticated OpenAI-compatible list response. | Implemented | +| Codex's large developer/tool payload exceeded the gateway's default 64 KiB body limit. | Keep the secure default; use an explicit 8 MiB limit only for the loopback, bearer-authenticated local Codex LaunchAgent. | Implemented | +| A local provider could accidentally receive ChatGPT/OpenAI credentials. | Keep built-in OpenAI auth and the local gateway bearer credential in separate Codex/provider boundaries; never attach OpenAI auth to `mlx://`. | Implemented | +| The public orchestrator was incorrectly described as a proxy and omitted from the candidate surface. | Treat contextual-orchestrator as the public model-like control plane; retain all discovered worker candidates in an explicit registry, reserve `disabled` for operator/admin governance, and constrain recursive self-selection until a bounded future protocol exists. | Implemented | +| Expanding the registry could silently change the meaning of the existing unauthenticated `/healthz` `agent_count` field from active workers to all candidates. | Preserve `agent_count` as the enabled worker count for compatibility, add explicit `candidate_count` for the full registry, and expose `enabled_agent_count` as a redundant named metric with a regression containing one disabled candidate. | Implemented | +| A live local run showed the mlx-lm `prompt-concurrency=1` queue continuing to process abandoned large prompts after the client timed out; the server logged `BrokenPipe` and was restarted, while a default same-agent retry would add another expensive queued request. | Keep remote retry policy unchanged, but default explicit `mlx://`/loopback local requests to zero same-agent retries; require an explicit local retry opt-in, bound prompt/output budgets, expose provider readiness separately from process liveness, and ensure the supervisor owns one process per port with stale-request cleanup. | Local retry isolation implemented; readiness, prompt-budget, and lifecycle evidence required | +| The local retry opt-in was applied to the loop bound but the stop condition still compared attempts with the remote `max_retries` field, silently capping an explicit local retry budget whenever it exceeded the remote default. | Compute one provider-specific retry limit and use it for both iteration and termination in normal and raw passthrough transports; retain zero local retries by default and regression-test an explicit local budget above the remote budget. | Fixed locally; exact-head CI/review follow-up required | +| An explicit unknown model ID silently fell back to a different worker, and duplicate IDs could report or select a disabled record before an enabled record. | Reject unknown/non-string explicit model IDs, resolve duplicate IDs to an enabled candidate when one exists, and aggregate `/v1/models` status across all records so discovery and execution agree. | Implemented with passthrough and model-list regressions | +| `/v1/models` returned governance-enabled candidates as `active` even when only one MLX model was loaded; a caller could mistake registry membership for provider readiness. | Preserve `status` as governance state for compatibility, add `readiness: "unprobed"` to model entries and `provider_readiness: "unprobed"` to liveness, never perform a blocking synchronous provider probe in discovery, and add a bounded readiness/refresh contract before claiming serving availability. | Explicit boundary implemented; readiness probe/refresh remains required | +| Liveness endpoints could not distinguish a responsive mlx-lm process from a chat path that was timing out, so operators had no safe way to refresh serving readiness; concurrent admin refreshes could also multiply a stuck local queue. | Keep `/healthz` and `/v1/models` non-blocking and `unprobed`; add an admin-authenticated `provider_readiness_report(refresh=true)` that performs one `max_tokens=1` chat probe per enabled worker with a 0.1–30 second bound, no retries, redacted bounded errors, and one narrow process-local lock around the refresh sequence. | Implemented and exercised against live Gemma 4 e4b on 2026-08-14; exact-head CI/review follow-up required | +| Three anchored K=5 Gemma 4 e4b calibration reruns after fast-mlsirm `17e19ec90643a8dfcc464cd7dde0b63949539a32` prompt hardening produced complete/failed-closed counts `2/6`, `5/6`, and `4/6`, with complete-row cell accuracy `25.0%`, `40.0%`, and `37.5%`; partial/unsupported answers were still over-scored and unsafe/irrelevant cases could remain non-monotone. | Treat the prompt change as ordinal-contract hardening only; preserve all semantic failures and over-scores, do not promote Gemma or infer positive-K bias removal, and require larger held-out human/gold recall, randomized order/framing perturbations, and category occupancy before IRT use. | Observed 2026-08-14 through contextual head `c2bb2b2f85b3eae1c0c0138dff7f4a39cd744cd0`; calibration remains open | +| The fast branch ref and GitHub PR #816 pull ref temporarily diverged after the calibration push (`17e19ec` versus predecessor `7605c154`), leaving no checks on the new commit until a normal subsequent named-branch push reconciled exact head `2cd12090f6f4ef8188da15fc6a5704a6ad7063c7`. | Treat branch refs, pull refs, checks, reviews, and rulesets as separate exact-head evidence; record the drift, do not force-push/cancel predecessor runs, and bind all future review/check claims to the reconciled fast head and the linked contextual head. | Resolved by normal push on 2026-08-14; governance follow-up remains required | +| fast-mlsirm PR #816 then advanced from `2cd12090f6f4ef8188da15fc6a5704a6ad7063c7` to `ebd76b4664147c18a3e1cfcc3d689e916a2fff08` after a real failure-evidence gap was found: parsed non-monotone Boolean boundaries were not retained in bounded evidence. | Keep only the validated `meets_threshold` Boolean per ordered boundary, never retain full model output or repair the ordinal result, and invalidate predecessor review/check evidence after the normal push; require fresh exact-head review/check evidence for the linked judge implementation. | Implemented in fast-mlsirm `ebd76b4`; focused `62 passed`, full Python `3654 passed`; exact-head review/check follow-up required | +| GitHub HTTPS from this macOS host failed with `LibreSSL SSL_connect: SSL_ERROR_SYSCALL` on the VPN `utun12` route, while the same endpoints returned HTTP 200 over `en0`; this is a path/MTU/firewall failure signal, not evidence of a bad repository, certificate, or Keyverse credential. | Diagnose the route and interface before changing credentials or TLS verification; retain certificate verification, do not set a global proxy or `GIT_SSL_NO_VERIFY`, and use only a temporary interface-bound relay when an authorized remote operation must proceed. Record the exact interface, endpoint, and cleanup state, then re-run remote checks after the network path recovers. | Observed 2026-08-12; temporary relay used for pushes/checks and must not become repository configuration | +| A later live check showed `curl 8.7.1` using SecureTransport/LibreSSL 3.3.6 returned HTTP 200 to GitHub, while the active Passepartout WireGuard route used `utun10`; therefore the historical `SSL_ERROR_SYSCALL` is not a reproducible LibreSSL installation/certificate defect. | Keep TLS verification enabled and compare VPN-on/off route, DNS, destination IP, socket reset, NAT/egress, endpoint, and MTU evidence. Do not set `GIT_SSL_NO_VERIFY`, replace certificates, or alter repository credentials; treat the error as a VPN path/endpoint failure until a controlled no-VPN reproduction proves otherwise. | Confirmed 2026-08-13; current LibreSSL transport healthy, VPN-path investigation retained | +| The provider host allowlist was read from `CONTEXTUAL_ORCHESTRATOR_ALLOWED_PROVIDER_HOSTS` at request time, which made runtime policy depend on mutable environment state outside the KV/config boundary. | Bind non-secret provider policy through `ModelClient(allowed_provider_hosts=...)` and the explicit `--allowed-provider-host` CLI option; retain environment variables only for bootstrap transport and prove request-time env changes do not alter policy. | Implemented in current local head; exact-head CI/review follow-up required | +| The classic branch-protection endpoint returned 404 even though GitHub's branch-rules endpoint exposed organization/repository pull-request and required-workflow rules; querying only classic protection would under-report the effective merge policy. | Query `/rules/branches/main` and PR aggregate state together, record approval/last-push/thread/check requirements, and keep merge fail-closed when the exact head is pending, `REVIEW_REQUIRED`, or lacks an independent approval. | Observed 2026-08-12; ruleset-aware verification required before every merge | +| A repeated live Gemma 4 e4b batch sweep through `ModelClient.batch_chat` used eight requests per run with server `prompt-concurrency=1`/`decode-concurrency=1`; two repetitions measured c=`1,2,4,8` at mean throughputs `2.095`, `2.083`, `2.092`, and `2.088` req/s respectively, with no errors and 248 provider tokens per run. | Keep client `local_concurrency=1` as the default for this single-queue server, do not raise it from a single warm-up result, and tune/re-measure server queue concurrency independently after model, prompt, token-budget, or server changes. Treat the result as throughput evidence, never judge-quality evidence. | Observed 2026-08-14; benchmark recorded, workload-specific retuning remains required | +| The current integrated anchored smoke used Gemma 4 e4b through `ContextualOrchestratorJudge -> _FastMLSIJudgeAdapter -> TaskOrchestrator -> ModelClient -> mlx-lm` with two criteria, K=`3`, c=`1`, and four boundary calls; it completed in `4.404 s`, used `1,797` provider tokens, and produced the two-column polytomous row `[2,2]`. A one-criterion attempt was rejected by the IRT projection contract. | Keep fast-mlsirm Judge traffic inside the contextual adapter, require multiple criteria for IRT output, and never synthesize a scalar second item; retain this as integration/contract evidence rather than semantic quality or promotion evidence. | Verified 2026-08-14; live semantic calibration remains required | +| After fast-mlsirm was synchronized with protected `main` at source merge `bbf5d0e1d1185d4a51fae24fa95c3c18a3ea2f23`, the same two-criterion Gemma 4 e4b integrated smoke completed four Boolean calls through the contextual adapter in `6.023 s`, used `1,872` provider tokens, retained `binary_threshold`, and produced `release_monitoring=2`, `rollback_safety=2`, and row `[2,2]`. | Treat this as post-main integration and contract evidence only. Retain the latency/token delta rather than averaging it into the earlier result; repeat paired runs before any performance claim, and continue to reject scalar output, keyword matching, positional repair, and silent synthesis. | Verified 2026-08-14; semantic calibration and exact-head remote review remain required | +| The first live integration attempt used contextual-orchestrator's own environment with the fast-mlsirm source on `PYTHONPATH`; fast-mlsirm's declared NumPy dependency was absent, so import failed closed even though both source trees were present. | Keep the standalone gateway dependency-light and keep missing/broken fast-mlsirm fail-closed, but add `python -m contextual_orchestrator check-fast-mlsirm` as a same-interpreter preflight that reports the missing transitive module, package version, required judge symbols, and exact contextual contract marker. Require this preflight before any live judge or IRT benchmark. | Implemented locally 2026-08-14; exact-head CI/review follow-up required | +| A fresh same-route model comparison used the real anchored binary-threshold judge with two criteria and baseline/option-only controls. Llama 1B failed both cases (`0/2` passed; all four boundary calls per case completed but strict parsing failed), while Llama 3B and Gemma 4 e4b each passed both cases with polytomous rows `[2,2]`, gold agreement `2/2`, and no provider failures. | Keep the 1B candidate available for fast non-verifier work but exclude it from the `verifier` role in the local registry; allow stronger eligible local agents to handle judge calls through the same gateway. Treat 3B/Gemma as candidates only, retain saturation and gold/perturbation evidence, and never use keyword or silent-repair fallback. | Observed 2026-08-14 through contextual-orchestrator `1614c7f40e5629c07dcfaa97d62b048a7bb459bf`; verifier exclusion implemented in the local registry, broader calibration remains required | +| A fresh same-route two-case comparison at K=`3` found Gemma 4 e4b passed both structured boundary comparisons (`[2,2]` safe, `[1,1]` unsafe) in `7.73 s`/`4.87 s`; Llama 3B passed the safe case but failed the unsafe case as non-monotone; Gemma 4 31B failed the safe boundary after `96.93 s`; DeepSeek R1 Qwen 32B completed no boundary in either `100 s` case. | Keep all models discoverable for non-verifier work, but exclude 31B and DeepSeek from `verifier` alongside the previously excluded 1B; select e4b as the current verifier primary and retain 3B as a lower-priority candidate. Treat this as workload-specific reliability evidence only, preserve every failure in the calibration denominator, and require larger balanced gold/perturbation calibration before promotion or IRT claims. | Observed 2026-08-14 through the contextual-orchestrator adapter; routing exclusions implemented, calibration remains required | + +| A local `mlx://127.0.0.1:8080/v1` endpoint shared the machine with an unrelated wildcard listener; `/health` remained HTTP 200 while chat completions returned zero bytes and timed out. Candidate port 18080 was also occupied by a Colima SSH forward. | Treat one-process-per-port ownership as part of local serving readiness: reserve a dedicated loopback port, verify the actual MLX model registry and one bounded completion before calibration, and let the local supervisor report a port-owner/configuration mismatch without terminating unrelated processes. Keep the explicit `mlx://` scheme, bounded concurrency, zero default local retries, and TLS verification rules. | Observed 2026-08-14; dedicated MLX port 18083 restored `ModelClient.probe()` readiness, lifecycle hardening remains required | +| `ModelClient.probe()` previously exercised only a completion after endpoint validation, so an incompatible local listener could produce an opaque timeout even when `/health` was 200. | For explicit local providers, verify `/v1/models` contains the configured model before the one-token completion probe; retain bounded timeouts, zero default local retries, and fail-closed error reporting. | Implemented 2026-08-14; focused `41 passed`, full `387 passed`, live registry-plus-completion probe ready on dedicated port 18083 | +| A warm authenticated gateway recheck at 09:52 UTC measured `1/1`, `2/2`, and `4/4` successful short completions at wave times `307.89 ms`, `327.23 ms`, and `579.67 ms`; a fifth simultaneous request was explicitly rejected as `503 concurrency_limit_exceeded`, and all successful completion IDs were unique. | Keep the measured `max_concurrent_runs=4` admission bound for this server/model configuration, preserve explicit overload failure and UUID IDs, and re-measure after model, prompt, token, server, or device changes. Do not promote one short workload to a global concurrency default or a quality claim. | Observed 2026-08-14; no new code change justified by this sample | +| The current exact-head integrated judge smoke used contextual-orchestrator `474b667b576f8a019db51d892db41a605e3a0a85` and fast-mlsirm `a536292cc05bd16287dab16431bc0c3fef74ba81` with Gemma 4 e4b, two criteria, K=`3`, and four `binary_threshold` calls; it completed in `19.354 s`, used `2,163` provider tokens, and produced the required two-column polytomous row `[2,2]`. | Retain this as current-head route/contract evidence only. Keep multiple criteria mandatory, preserve the model/category result and measured cost, and require balanced held-out gold, perturbation stability, and category occupancy before semantic or IRT claims. Continue to reject keyword matching, positional inference, category repair, scalar synthesis, and silent drop. | Verified 2026-08-14; no new code change justified, semantic calibration and protected PR gates remain required | +| The gateway classified DNS/connection/timeout failures as transient but did not classify `ssl.SSLEOFError` or `ssl.SSLSyscallError`, so a VPN-path `SSL_ERROR_SYSCALL` could fail without the existing bounded retry policy; certificate verification errors are a different trust-boundary failure. | Treat TLS socket EOF/SYSCALL errors as transient network failures, keep `SSLCertVerificationError` non-transient, retain certificate verification, and cover both branches. Never use `GIT_SSL_NO_VERIFY`, a global proxy, or certificate replacement as a retry fix. | Fixed in contextual-orchestrator `1b22ff0`; targeted `14 passed`, full `388 passed`, exact-head CI/review follow-up required | + +| The current exact-head MLX/gateway recheck used context `8f922d806336fd41d8fd73585a7c225784249332` and fast `47c5fbdde98b3550fe319d1de238a32cbaec8a1f`: width 1 had a 2,170.42 ms cold/warm-up response, width 2 had 341.35 ms wave latency, width 4 had 591.34 ms, and width 5 produced `4x200` plus bounded `1x503`; all successful response IDs were unique. A real two-criterion binary-threshold Judge completed through the contextual adapter in 4.949 s with 1,923 tokens and row `[2,2]`. | Keep the first request separate as warm-up evidence, retain gateway concurrency 4 as the measured plateau for this listener, preserve the explicit overload denominator, and treat the Judge result as route/contract evidence only. Do not infer semantic quality, unbiasedness, or IRT readiness from throughput or one safe case; rerun balanced gold and perturbation strata after model/server changes. | Verified 2026-08-14; Goal/ADR expanded, semantic calibration and protected PR gates remain open | +| A fresh warm comparison of the same worker versus the authenticated gateway found near-equivalent p50 latency at widths 1/2/4 (direct `202.52/323.74/593.77 ms`; gateway `203.96/321.01/559.07 ms`). Direct width 5 completed only by queueing to `899.16 ms`, while the gateway preserved its four-run admission bound and returned `4x200` plus bounded `1x503` in `564.28 ms`. | Keep the gateway in the performance path and retain `max_concurrent_runs=4` for this worker/server pair; increasing the bound would hide provider queue latency rather than improve throughput. Re-measure after model, prompt, output-budget, server, or device changes, and never infer judge quality from this transport result. | Verified 2026-08-14; no code change justified, benchmark recorded | +| The latest source pair `070d929`/`8f5d85a` completed a real two-criterion, three-category binary-threshold Judge through the contextual adapter in `3.731 s` with `2,015` provider tokens and IRT row `[2,2]`. | Retain the same-interpreter contract and multi-item IRT projection, but treat this as route/shape evidence only; require balanced non-ceiling gold, perturbation stability, category occupancy, and preserved provider/parse/semantic failure denominators before any verifier or IRT promotion. | Verified 2026-08-14; semantic calibration and protected PR gates remain open | + +| A fresh warm comparison at current source heads `e9935d763d267bf20abb0bec069070c94a838369`/`b4121d2e2071a02b1f497b7228b0ecde061fbb45` used direct MLX and the authenticated gateway at widths `1,2,4,5`; direct returned `5/5` HTTP 200 by queueing width 5, while the gateway returned `4/5` HTTP 200 plus an explicit `concurrency_limit_exceeded` HTTP 503. Three two-criterion K=`3` Judge probes through the required contextual route preserved binary/direct/cumulative boundaries; cumulative safe output failed closed on non-monotonicity. | Keep `max_concurrent_runs=4` and explicit overload failure for this worker; do not convert direct queueing into hidden gateway work or raise the bound from this sample. Preserve every semantic/format failure, keep binary as the implicit production method and direct/cumulative calibration-only, and require balanced human/gold, occupancy, perturbation, and failure-rate evidence. | Verified 2026-08-15; no code change justified, semantic calibration and protected exact-head gates remain open | + +| A live `compare_to_baseline` run through `mlx://127.0.0.1:18083/v1` with Gemma 4 e4b and one short conduct prompt measured orchestrated latency `20,348.22 ms` for four steps versus baseline latency `4,721.64 ms` for one step, a `15,626.58 ms` overhead; the structural proxy reported three additional steps and `verified=false`. | Keep explicit conduct/orchestration for tasks that warrant its verification cost, preserve the fast route for latency-sensitive work, and do not interpret step count, answer length, or this single `verified=false` result as human quality. Re-measure paired latency and held-out semantic outcomes before changing automatic routing or verifier priorities. | Observed 2026-08-15 on contextual head `2acb3a4`; no code change justified, performance/semantic calibration remains open | + +| The central OpenCode coverage image preflight failed on contextual head `775ea133` before executing repository tests because `fuzz/requirements-atheris.txt` pinned unavailable `atheris==3.0.0` while the image runs CPython 3.14; OpenCode consequently issued a false `CHANGES_REQUESTED` despite the repository's own coverage check succeeding. | Keep the hash lock, optional extra, fuzz workflow, and central preflight on the same available Atheris 3.1.0 / Python 3.12+ contract; regenerate hashes from the recorded compile command and require the exact-head central coverage job to execute before accepting an OpenCode verdict. | Fixed in this head; focused dependency/workflow validation and the next exact-head OpenCode run remain required | + +| OpenCode review run `31855666848` issued `CHANGES_REQUESTED` for contextual head `badcf287ec0dc905b6ce839b59a99523430e35e7` because it observed missing/failed coverage evidence, while the same-head check API records `coverage-source-tree` and `coverage-evidence` successful in run `31853174688` (`94932900714`/`94933075809`). | Treat a review result whose referenced coverage run disagrees with the authoritative same-head check-run API as stale timing/provenance evidence, not approval or a code finding. Re-request a review against a newly pushed exact head and bind the result to the exact coverage job; never hide the mismatch, infer quality from the local suite, or self-approve. | Observed 2026-08-15; ADR evidence follow-up, fresh exact-head review/check cycle required | +| On current head `6a411bee9acf7d1ca6094fc662023baf507417d2`, Noema workflow `31856680677`/job `94942650274` minted its repository-scoped reviewer token but logged `Current head does not have a primary OpenCode approval; Noema review skipped.` | Keep Noema fail-closed and dependent on an authenticated current-head OpenCode approval; a successful token exchange or placeholder check is not a review. Repair/activate the central dispatch path first, then rerun both reviewers against the same head. | Observed 2026-08-15; protected review gate remains open | + +| An authenticated `local://` gateway request initially returned `401` because the client treated every loopback URL as keyless and discarded the gateway bearer credential; the direct `mlx://` worker must remain keyless. | Add `ModelAgent.local_credential_key` as a separate KV name used only by `local://`, fail closed when it is missing, and never reuse `credential_key`/`OPENAI_API_KEY` for either local transport. | Fixed in current local head; focused local/KV tests and live authenticated Judge route passed, exact-head CI/review follow-up required | +| The same gateway rejected `chat_template_kwargs` with HTTP `400 unknown_fields`, although direct mlx-lm accepts the provider-specific template option. | Forward template kwargs only to direct `mlx://` workers; configure template behavior at the mlx-lm worker behind `local://`, and cover the distinction in transport tests. | Fixed in current local head; focused tests and live gateway smoke passed, exact-head CI/review follow-up required | +| Free-form Gemma Judge calls completed but emitted prose/Markdown for the strict rubric, so all four binary boundary parses failed closed despite healthy transport. | Let fast-mlsirm request the exact JSON Schema through the existing contextual adapter's gateway proxy when available, keep the old injected `.complete()` fallback for generic test transports, and preserve all parse failures in calibration denominators. | Implemented in current local heads; focused contextual `73 passed` and fast Judge `48 passed`, live structured route passed, semantic calibration remains open | +| A Gemma 4 e4b Judge request with caller `max_output_tokens=64` returned HTTP `200` but emitted fenced/truncated JSON (`finish_reason=length`) through both the authenticated gateway and direct MLX worker; `response_format` was advisory rather than a grammar guarantee. At `max_output_tokens=256`, direct K=`2,4,6` calibration parsed `12/12`, but option shuffling produced `[0,0]` at K=`2` and `[4,3]` at K=`6` versus `[4,4]` baselines; cumulative parsed only `6/12` and failed closed for the other six. | Keep the gateway's strict structured transport and fail-closed parser; do not treat HTTP success or a schema request as semantic validity. Use a measured output budget of at least `256` for this workload without silently overriding caller configuration, retain every format/semantic failure, and keep direct/cumulative calibration-only until balanced gold, occupancy, option-count/order replication, and human review support an IRT/verifier decision. | Verified 2026-08-15; transport/budget and bias-calibration evidence recorded, semantic calibration and protected Merge remain open | +| The linked fast-mlsirm exact-head Strix run `31836188815`/job `94882896174` returned a terminal success and `Vulnerabilities 0`, but its unbound artifact `9233137440` contained a report identifying symlink arbitrary-file-read paths in the Judge/IRT package's bounded JSON, CSV, NPY/NPZ, and params readers. The report digest was `81b718964554fb447e313a9e8f3679d0e57b1618d7ffa3d0780efdfdb45f1025`; no structured repository/head/run/job/report binding was present. | Treat the linked security result as a real source finding and a non-clean dependency gate. Require fast-mlsirm to pin reads to `O_NOFOLLOW`/regular descriptors, preserve NpzFile ownership safely, add symlink regressions, and rerun with structured exact-head evidence before contextual-orchestrator or fast-mlsirm Merge. Never route around a linked security finding or treat an unbound zero-finding line as clean. | Observed 2026-08-15; fast source fix in progress, linked exact-head security/review/Merge remain open | + +| Contextual head `d2072faa6525364c3ae8da98b7d16c69a0cd91d6` produced Strix run `31856873610`/job `94943244722`, artifact `9239422201`, and a zero-finding report (`SHA-256 37f3d8d920ab9f30948fbe606dae28ecf5300e076400136bb391cda90ecea944`), but the gate console (`SHA-256 d892c5c5db82768164b8c3f558d7a7bf0b4ee9345ff0106d22ee5799a404d331`) emitted provider/failure markers while the job was green. The artifact had no `evidence-binding.json`, and `run.json` (`SHA-256 06ef34efac3200f1d16af8402d4e58f9d4bbe2423187f440bceecb5d8fdedbc6`) bound only an ephemeral target path. | Preserve the report as non-clean provider/content evidence; do not accept a green check when the gate log says fail-closed or when repository/head/run/job/report binding is absent. Require the central trusted workflow fix and a fresh structured exact-head run before security or Merge acceptance. | Observed 2026-08-15; central binding/gate repair and protected review remain open | + +| Current contextual head `63d9abf70ad89cf5149aaf145ed6c0be539e127b` produced Strix run `31857573504`/job `94945113536`, artifact `9239733659`, and a report claiming a critical PostgreSQL credential SQL-injection fix (`SHA-256 5951d808941af8f7cc88f1d48f5961427890052263600b34a463a0e03786eb7d`). The claim is not source-backed: `PostgresCredentialBackend.get` binds passphrase/name through `%s` parameters at `credentials.py:151-155`, and `set` binds name/value/passphrase at `credentials.py:167-172`; no interpolated credential-name query or `get_credential`/`set_credential` methods exist. The artifact has no `evidence-binding.json`; its run metadata is not repository/head/job bound, and the gate console (`SHA-256 ae6e6fc23027791a8b87b72ecb75e36c158d09ffe52a7e293f13a938e0282f18`) contains fail-closed provider/failure markers despite a green job. | Treat this as an unbound Strix content/provider false positive, not as permission to invent a patch or weaken SQL/KV controls. Require future security reports to cite repository-relative file/line evidence and an exact-head diff before accepting a “fixed” finding; preserve parameterized queries, add no keyword/positional repair, and require trusted structured binding plus a clean rerun before security or Merge acceptance. | Observed 2026-08-15; source verification completed, no code change justified, protected review remains open | +| Current contextual head `83b17bd54508c6b27d02385231b0658423230e97` produced Strix run `31858536309`/job `94947596777`, artifact `9239945601`, and a zero-finding report (`SHA-256 67b21edfeb2ee6652dcfa73153c3d7e963236a425d96d40752b075b34a88b4b6`). The gate console (`SHA-256 8188acd3810c1d2d0946ddabfada29f8ef2b109a063709dfe47a4e8801f12613`) had no failure-marker output, but the artifact still had no `evidence-binding.json`; `run.json` (`SHA-256 e1f6a91486ed9bb9434d232d5ddd05715d7fcd3a4f5ed082e6dce3889725f46b`) bound only an ephemeral target path. | Preserve the zero-finding report as bounded provider/content evidence, not a clean exact-head security gate. Require trusted repository/head/run/job/report binding from the central workflow and a fresh exact-head review before security or Merge acceptance; do not infer security or semantic quality from an unbound green job. | Observed 2026-08-15; no source change justified, ADR evidence updated, protected review remains open | + +## Risks and Mitigations + +| risk | likelihood | impact | mitigation | owner | +| --- | --- | --- | --- | --- | +| A malicious config labels a remote endpoint as local. | low | high | Scheme, loopback host, port, credential/query validation and tests. | maintainer | +| Concurrent mlx requests exceed device memory. | medium | high | Default concurrency 1 and bounded user control; measure before raising it. | local-runtime owner | +| A small judge model produces invalid JSON. | medium | medium | Disable thinking, cap prompt/output, strict parse, fail closed. | evaluation owner | +| Responses-to-Chat conversion loses a future Codex item or tool type. | medium | high | Reject unknown request fields, forward only supported function tools, add a focused regression for every newly supported item type, and fail closed on malformed provider output. | control-plane owner | +| The local model emits a syntactically valid but operationally unusable tool call. | medium | medium | Keep the model/tool capability explicit in `agents.mlx.json`, use a capable local model for tool-heavy work, and retain exact Codex smoke plus tool-call tests. | local-runtime owner | +| A discovered candidate is installed but not a usable chat worker. | medium | high | Keep it in the registry without silently changing governance state; provider capability checks and failover determine whether a request can use it, while an operator may explicitly quarantine it. | local-runtime owner | +| Recursive self-selection can loop or re-enter the same authenticated server indefinitely. | medium | high | Keep the contextual-orchestrator self-worker candidate out of internal roles with provider exclusions until recursion depth, internal auth, and termination behavior have focused tests. | control-plane owner | + +## Security evidence follow-up + +The subsequent exact-head Strix runs were terminal success but still lacked trusted provenance. Fast head 1d05d785a3e5c4e0eecc96b807e3a88786cb8b1d produced run 31839153059, job 94892122092, artifact 9233769170, report SHA-256 05e3e48c1ebc475bdd62759970375268067872748947b3db35e6d4c2c2bfb2fc, and run.json SHA-256 10f4e33bc9dbe752444ba05063bbd9a02c9a58b07838e699d7ee5dcfad5aa768, but no evidence-binding.json. Contextual head 64b6d56a31f17721019d47d0f82945c722e1eb10 produced run 31839154460, job 94892129029, artifact 9234044840, report SHA-256 4f036e7f21c14920ce7fd95575e9e9228d35f88291cd5fdddac62bce3ab01a29, and run.json SHA-256 97b7d9218f86016ceb8b57d8c484cd50c21aad4314ac4f3560c7a2063df2bb43, also without evidence-binding.json. Keep both results as non-clean provider/content evidence until a trusted binding is published. + +The current contextual head `e0413fe16ddd7b47f736bfc5e3ea91921736af0d` likewise produced terminal Strix run `31840661126`, job `94896593729`, artifact `9234622284`, report SHA-256 `06193ebbf64e8ed47011d8a01f471c8dee5c1b426c9ed8e15ab3940191d6111f`, and run.json SHA-256 `90e757b30e01a98a324332f9eb5fce933bb90725a441f3aef2f269783153428e`, with no `evidence-binding.json`. The zero-finding report is therefore not a trusted exact-head security pass. A later OpenCode review claimed a coverage-evidence failure for this same head but referenced Actions run `31844382551`, which is not retrievable from the current run API; the current check-run API instead records coverage job `94899211701` as successful. When review evidence and current check evidence disagree or the referenced run is unavailable, discard the stale decision and require a fresh exact-head review; do not silently convert either snapshot into approval. + +The current contextual head `68002d0a260e4084452077eee7f37fd98270e55f` produced terminal-success Strix run `31845665320`, job `94911468155`, artifact `9236148265`, report SHA-256 `19d522787ac31cd9e4e46fdd9b406e8d1255b6242b04c1c78b4fd934e0dec8e4`, and run.json SHA-256 `8cdc067aaabf5e233bbdbaca45e4df89702cff18e4cd193910d995f9257d5502`. The report found no exploitable authentication bypass or other critical weakness, but the artifact still contains no `evidence-binding.json`; its run metadata records only a local temporary target path and does not bind repository, head, workflow job, and report digest. Treat this as provider/content evidence only, never as a clean protected gate. The exact-head review/check snapshot was invalidated by the subsequent ADR update, so require a new terminal run with trusted structured binding and a fresh independent review before Merge. + +The next exact-head contextual run for `d5236bb6f7e292fa8af9e82c7862ae66f5c47105` was run `31847116067`, job `94915707782`, artifact `9236521645`, report SHA-256 `8bea0f0f546a722b00ca1a580d5cc044ecc7d3c8c8ec6f452368461fec7115ef`, run.json SHA-256 `db8bd357d66518fe25692452fa71430b195f7f6447ed26efd9bf311ddf02b321`, and gate-console SHA-256 `2eef0b47688bcfd67ed3eb9061f8beef5eb6e40b22483d2b0f6db8052f27727b`. Although the Actions job was marked successful and the report said zero vulnerabilities, the gate console explicitly emitted `Strix report artifacts emitted warning/fatal/denied/timeout output; failing closed`, `Strix run emitted provider infrastructure or failure-signal output; failing closed`, and `Strix scan failed after provider infrastructure or failure-signal output; failing closed`. The artifact again had no `evidence-binding.json`, and run.json contained only a temporary target path with no repository/head/commit binding. Treat this as non-clean provider/failure evidence; the green status is not sufficient for Merge. The trusted workflow must make these failure markers fail closed and publish a structured binding before any exact-head result can be accepted. + +The subsequent exact-head run for `de4d95a478215c2665c8c1f5ba60bdb82c618738` was run `31847996163`, job `94918215754`, artifact `9236803993`, report SHA-256 `dc2bf085d2a96282efbdca4474398df3f7af464369dea4f771f173f15957847e`, and run.json SHA-256 `17de42263f89146b3f84440a0fa4a24d3a0af5901e42f359ae5b4a9c205a5515`. It completed with zero reported vulnerabilities and no failure-marker text, but still had no `evidence-binding.json`; run.json again had null repository/head/commit fields and only a local temporary target. Preserve the zero-finding result as bounded provider/content evidence, but do not call it a clean exact-head gate until the trusted structured binding is present. + +## Rollback / Exit Strategy + +The linked fast-mlsirm security finding is fixed in exact head +`8195434de6eb166a44dbda1f8bd4f2ca5086240a`; its focused IO/security suite +passed `305` tests and its full suite passed `3726` tests with 2 warnings. +Keep this as a non-clean dependency until fresh exact-head Strix evidence is +structured and independently reviewed. + +Remove the explicit local adapter and use the mock path if the local server is unavailable; retain remote HTTPS validation unchanged. Revert concurrency to one and keep the output-content guard. Do not broaden local URL matching as a convenience fix. + +## Affected Components + +* contextual_orchestrator/orchestrator.py +* contextual_orchestrator/server.py +* contextual_orchestrator/__main__.py +* examples/agents.mlx.json +* examples/agents.local.json +* tests/test_local_mlx.py +* tests/test_openai_passthrough.py +* fast-mlsirm/python/fast_mlsirm/llm_judge.py +* fast-mlsirm/tests/test_llm_judge.py + +## More Information + +The public projects are [contextual-orchestrator](https://github.com/ContextualWisdomLab/contextual-orchestrator) and [fast-mlsirm](https://github.com/ContextualWisdomLab/fast-mlsirm). This ADR intentionally does not add a provider SDK, a second orchestration runtime, or a Responses implementation inside mlx-lm. + +On 2026-08-12 the Codex compatibility path was implemented in +`ModelClient.proxy_send` and the HTTP server. The local checks covered the +transport adapter, Responses SSE framing, and `/v1/models`; a live Codex smoke +returned the requested sentinel through the loopback control plane. The machine's +ChatGPT login remains available through Codex's built-in `openai` provider +profile, while the local server uses a separate OS-keychain bearer token. + +The Fugu report was then re-read on 2026-08-12. Its distinction between the +single model-like orchestrator and the swappable worker pool, plus its optional +recursive orchestrator-as-worker topology, is the reason this ADR keeps a full +candidate registry while constraining recursive self-selection in the current +untrained stdlib implementation. `disabled` remains an operator decision, not a +discovery decision. diff --git a/docs/planning/adrs/0003-keyverse-authentication-boundary.md b/docs/planning/adrs/0003-keyverse-authentication-boundary.md new file mode 100644 index 000000000..506e29277 --- /dev/null +++ b/docs/planning/adrs/0003-keyverse-authentication-boundary.md @@ -0,0 +1,154 @@ +--- +id: "0003" +title: "Keyverse authentication boundary and KV credential placement" +status: accepted +proposed_date: "2026-08-11" +accepted_date: "2026-08-11" +deciders: + - "repository maintainer" +consulted: + - "Keyverse deployment boundary" + - "contextual-orchestrator security surface" +informed: + - "contributors" +affected_components: + - "contextual_orchestrator/server.py" + - "contextual_orchestrator/__main__.py" + - "contextual_orchestrator/credentials.py" + - "docs/kv-credentials.md" +effort: L +supersedes: null +superseded-by: null +related: + - path: "docs/planning/adrs/0004-pr-review-merge-loop.md" + relation: informational + - path: "docs/kv-credentials.md" + relation: informational +asr_triggers: + - kind: security + evidence: "The gateway is an ecosystem relying party and bearer tokens authorize admin/inference scopes." + note: "Authentication must fail closed and secrets must remain in approved KV/deployment boundaries." + - kind: maintainability + evidence: "OIDC discovery, JWKS rotation, claims, and client registration are deployment concerns." + note: "The stdlib gateway exposes an adapter seam instead of embedding an unsafe protocol implementation." +success_criteria: + - metric: "auth secret runtime source" + target: "CLI auth secrets come from explicit local flags or named KV entries, never legacy token env defaults" + measurement_window: "every server startup and auth regression run" + source: "contextual_orchestrator/__main__.py and test suite" + - metric: "external verifier failure handling" + target: "verifier exceptions and wrong scopes return unauthorized" + measurement_window: "every protected request" + source: "SecurityConfig authorization tests" +--- + +# Keyverse authentication boundary and KV credential placement + +## Context + +The repository originally treated gateway authentication as a static bearer-token comparison and read CLI token defaults from environment variables. The ecosystem identity plane is broader: Keyverse documents contextual-orchestrator as an OIDC relying party, keeps RP registration/client secrets in the IdP DB/KV, and requires deployment-controlled reconciliation and acceptance evidence. + +> Keyverse states that ecosystem applications, including contextual-orchestrator, are OpenID Connect relying parties. +> +> Keyverse states that RP client registrations and secrets live in the IdP database/KV, not in an RP environment. +> +> SecurityConfig.bearer_verifier now accepts a deployment-injected verifier and denies when that verifier errors or rejects scope. + +## Decision Drivers + +* Recognize Keyverse as the production identity boundary instead of pretending a static token is OIDC. +* Keep client secrets and provider credentials in KV/deployment systems. +* Avoid an unsafe hand-written JWT decoder in the stdlib core. +* Preserve offline/local tests and explicit local-development authentication. + +## Considered Options + +* Keep static bearer comparison as the only production authentication. +* Embed Keycloak admin calls and a custom JWT/JWKS implementation in this repository. +* Keep static local auth for development and inject a reviewed Keyverse/OIDC bearer verifier at the deployment boundary. + +## Decision Outcome + +Chosen option: "Deployment-injected Keyverse/OIDC verifier with KV-backed token naming". + +| Driver | Static token only | Embedded identity implementation | External verifier boundary | +| --- | --- | --- | --- | +| Keyverse compatibility | none | coupled/private | explicit OIDC RP seam | +| Secret safety | weak env temptation | high blast radius | KV/deployment ownership | +| Local operability | simple | heavy | simple explicit token or mock | +| Protocol correctness | incomplete | hard to maintain | owned by reviewed auth adapter | + +SecurityConfig.bearer_verifier(token, scope) is the only production integration point. The adapter must validate issuer, audience, signature, expiry, key rotation, and scope using an approved library or trusted Keyverse/WAF boundary. The core does not decode JWTs, call Keycloak Admin REST, or store RP client secrets. CLI token flags are explicit local escape hatches; named token flags resolve from the KV. + +### Consequences + +* Good, because the repository now records the Keyverse dependency and has a safe injection boundary. +* Good, because auth adapter failures are denials, not accidental access. +* Good, because runtime provider/auth secrets no longer use the legacy CLI environment defaults. +* Bad, because a complete production OIDC adapter still requires deployment-specific issuer, audience, JWKS, scopes, TLS, and acceptance evidence. +* Bad, because callers using SecurityConfig directly must choose an explicit token or verifier. + +### Confirmation + +Run the external-verifier security test and inspect readiness_profile()["auth_mode"]. In deployment, record Keyverse RP desired-state digest, convergence receipt, client UUID, controlled authorization-code/PKCE result, refresh/logout result, and rollback reference without recording bearer or client-secret bytes. + +## Pros and Cons of the Options + +### Static token only + +* Good, because it works offline. +* Bad, because it is not OIDC and does not express issuer/audience/claims. +* Bad, because legacy environment defaults encourage secret leakage and rotation drift. + +### Embed identity implementation + +* Good, because the service owns more of the flow. +* Bad, because custom cryptography/protocol code is a high-risk expansion. +* Bad, because it would cross Keyverse's deployment-controller trust boundary. + +### External verifier boundary (chosen) + +* Good, because Keyverse/OIDC correctness stays with a reviewed identity component. +* Good, because the gateway remains stdlib/local-test friendly. +* Bad, because production wiring is an explicit deployment task and cannot be simulated by a unit test alone. + +## Problem Register and Remediation Directions + +| Finding | Direction | State | +| --- | --- | --- | +| Keyverse dependency was not visible in the gateway docs/architecture. | Record it here and in KV/auth docs; require deployment acceptance evidence. | Implemented | +| Static auth was mistaken for ecosystem identity. | Expose bearer_verifier; label static tokens local-only. | Implemented | +| CLI had legacy token environment defaults. | Resolve named auth tokens from KV; remove token env defaults from the Python CLI. | Implemented | +| RP registration and client secret placement were absent. | Add a deployment-controller integration using Keyverse preflight/reconcile and approved secret storage; never put secrets in this repo. | Required follow-up | +| JWT validation library/issuer/JWKS contract is deployment-specific. | Select and review one adapter, including rotation, claims, TLS, clock skew, and negative tests before production. | Required follow-up | +| Partial or mixed CLI token modes could trigger an unrelated KV lookup before reporting the configuration error. | Treat explicit `--admin-token-key`/`--inference-token-key` as split-mode selectors and reject mixing/incompleteness before resolving any KV entry. | Implemented | +| Container startup passed `CONTEXTUAL_ORCHESTRATOR_TOKEN` as secret argv/env material, bypassing the KV boundary. | Pass `--auth-token-key CONTEXTUAL_ORCHESTRATOR_TOKEN`; let the Keyverse/KV deployment adapter resolve the value at runtime. | Implemented | +| Public API may be reachable without the identity edge. | Keep auth mandatory; deny when no static token or external verifier is configured. | Implemented | +| Strix run `31819952638` found a critical authorization bypass: `SecurityConfig.authorize` selected `auth_token` before the requested scope, and direct `SecurityConfig` construction allowed single and split token modes to be combined. | Reject mixed single/split configurations at the shared security boundary; select the single token only in single-token mode and select the exact `admin`/`inference` token in split mode, with a regression test. | Implemented locally; exact-head Strix rerun, independent review, and protected Merge remain required | +| `SecurityConfig` is a mutable dataclass, so a post-construction `auth_token` mutation could reintroduce precedence ambiguity if authorization trusted initialization alone. | Resolve the requested scope on every authorization call, give `admin_token`/`inference_token` precedence, reject unknown scopes, and retain a mutation regression test. | Implemented locally; exact-head Strix rerun, independent review, and protected Merge remain required | + +## Risks and Mitigations + +| risk | likelihood | impact | mitigation | owner | +| --- | --- | --- | --- | --- | +| Deployment injects a verifier that only decodes JWTs. | medium | critical | Require signature/issuer/audience/expiry/scope tests and code review; no decode-only adapter accepted. | security owner | +| Keyverse is unavailable during startup. | medium | high | Fail closed, keep liveness separate, and use explicit readiness/rollback evidence. | deployment owner | +| Local developers put a token in process arguments. | medium | medium | Document only for local use; prefer KV-backed names and external verifier in deployed environments. | maintainer | + +## Rollback / Exit Strategy + +For local rollback, use an explicit static token with the same mandatory auth gate. For production rollback, remove the external verifier only as part of a controlled deployment rollback; never silently downgrade a public deployment to unauthenticated or environment-default auth. + +## Affected Components + +* contextual_orchestrator/server.py +* contextual_orchestrator/__main__.py +* contextual_orchestrator/credentials.py +* docs/kv-credentials.md +* Keyverse deployment-controller/RP registration integration (follow-up) + +## More Information + +* [Keyverse repository](https://github.com/ContextualWisdomLab/keyverse) +* [Keyverse relying-party onboarding](https://github.com/ContextualWisdomLab/keyverse/blob/main/docs/rp-onboarding.md) +* [Keyverse architecture](https://github.com/ContextualWisdomLab/keyverse/blob/main/ARCHITECTURE.md) diff --git a/docs/planning/adrs/0004-pr-review-merge-loop.md b/docs/planning/adrs/0004-pr-review-merge-loop.md new file mode 100644 index 000000000..c63152d94 --- /dev/null +++ b/docs/planning/adrs/0004-pr-review-merge-loop.md @@ -0,0 +1,368 @@ +--- +id: "0004" +title: "Auditable PR review, remediation, and merge loop" +status: accepted +proposed_date: "2026-08-11" +accepted_date: "2026-08-11" +deciders: + - "repository maintainer" +consulted: + - "repository CI workflows" + - "repository security policy" +informed: + - "contributors" +affected_components: + - ".github/workflows/tests.yml" + - ".github/workflows/security.yml" + - "repository branches and pull requests" + - "ContextualWisdomLab/.github central merge scheduler" + - "docs/planning/adrs/" +effort: M +supersedes: null +superseded-by: null +related: + - path: "docs/planning/adrs/0003-keyverse-authentication-boundary.md" + relation: informational + - path: "AGENTS.md" + relation: informational +asr_triggers: + - kind: maintainability + evidence: "Changes span two repositories and require repeated review/test feedback." + note: "The loop makes each remediation and verification result traceable." + - kind: security + evidence: "Merge must not bypass required checks, current-head review, or secret boundaries." + note: "No credential or branch protection bypass is part of the loop." +success_criteria: + - metric: "PR verification" + target: "every named required check-run passes on the exact PR head SHA before merge; local commands are supplementary" + measurement_window: "each PR lifecycle" + source: "GitHub Actions checks and local reproducible commands" + - metric: "review remediation" + target: "every actionable security or correctness comment is fixed and revalidated; only non-blocking risk may be explicitly accepted before merge" + measurement_window: "each review iteration" + source: "PR conversation, diff, and follow-up commit history" +--- + +# Auditable PR review, remediation, and merge loop + +## Context + +The requested work changes a gateway and its evaluation companion, so a local green test run is not enough. The change must survive a current-head review, CI/security checks, and a merge decision that does not bypass repository protections. If a review finds a new issue, the Goal expands and the issue is recorded in an ADR before the next iteration. + +> .github/workflows/tests.yml runs the full unit and contract suite on pull requests targeting main. +> +> Repository instructions require security checks and prohibit leaking authentication tokens or provider secrets. +> +> This task explicitly requires PR creation, review-response iteration, and Merge rather than stopping at a local patch. + +## Decision Drivers + +* Make the final state reproducible and reviewable from commits, checks, and ADRs. +* Continue through actionable review findings instead of treating the first PR as final. +* Merge only the exact tested head and never bypass branch protection or unresolved security concerns. +* Keep the same discipline for contextual-orchestrator and fast-mlsirm. + +## Considered Options + +* Apply local edits and stop after local tests. +* Open one PR and merge immediately after the first green local run. +* Use a repeatable branch → test → PR → review → remediate → re-test → exact-head merge loop. + +## Decision Outcome + +Chosen option: "Repeatable exact-head review and merge loop". + +| Driver | Stop after local tests | Immediate merge | Exact-head review loop | +| --- | --- | --- | --- | +| Review quality | unknown | shallow | actionable findings iterated | +| CI/security evidence | absent | partial | required checks observed | +| Traceability | working tree only | one commit | ADR + commits + PR conversation | +| Safety | easy to miss regressions | protection pressure | merge only after evidence | + +The maintainer creates a codex/ branch, commits coherent changes, pushes a PR, inspects the diff and current checks, records/replies to actionable review findings, applies fixes, reruns tests and checks, and merges only when the PR head is the verified commit. If the platform disallows self-approval, the maintainer must not fake approval; it waits for or requests an authorized reviewer while continuing all safe local verification. + +### Exact merge-gate contract + +The required check-run names are read from each protected `main` branch and must be +green on one recorded `verified_head_sha` immediately before merge. For +`contextual-orchestrator`, the required contexts are `Hypothesis property tests`, +`Atheris coverage-guided`, `CodeQL analysis`, `Python supply chain`, +`dependency-review`, `osv-scan`, `trivy-fs`, `scorecard`, `coverage-evidence`, +`opencode-review`, `strix`, and `scan-pr-queue`. For `fast-mlsirm`, they are +`Analyze (actions)`, `close-empty`, `scan-pr-queue`, `dependency-review`, +`osv-scan`, `trivy-fs`, `scorecard`, `strix`, `required-workflow-bootstrap`, +`coverage-evidence`, `opencode-review`, `python`, `rust`, `package`, and `fuzz`. +The repository-local job names are defined in `.github/workflows/`; central +contexts remain required even when their workflow file is outside the repository. +Local pytest, Ruff, package, and fuzz commands are reproducible supporting +evidence only; they never replace a required GitHub check-run. + +The maintainer records the PR head SHA and the SHA attached to every required +check-run. If the PR head, any check SHA, or the reviewed diff changes, the merge +stops and the complete gate is re-evaluated on the new head. A documentation-only +note cannot resolve an actionable security or correctness finding; it remains a +merge blocker until code/test remediation and revalidation are complete. A +non-blocking risk may be accepted only with an owner, rationale, tracking issue, +and expiry date. + +Protected merge readiness additionally requires branch protection to report +`requiredApprovals >= 1` and `enforce_admins=true`, GitHub's aggregate +`reviewDecision=APPROVED`, an independent current-head approval, zero active +unresolved threads, terminal successful required checks, structured same-head +Strix evidence, and a final re-fetch immediately before any merge mutation. +Branch protection and the central scheduler must each reject direct and auto +merge when any control is absent or non-passing. A repository whose protection +does not yet enforce these settings remains a required follow-up and cannot use +an operational checklist as substitute merge evidence. + +### Consequences + +* Good, because every new concern becomes a reviewable code/test/ADR item. +* Good, because merge is tied to exact-head checks rather than an earlier green commit. +* Good, because user-requested autonomy is bounded by repository protection and secret safety. +* Bad, because the workflow takes longer and may require an external authorized reviewer. +* Bad, because GitHub permissions, branch protection, or CI availability can prevent autonomous merge; the evidence and remediation work still remain useful. + +### Confirmation + +For each repository, record branch, commit, PR URL, review result, check result, remediation commits, and merge SHA. Re-open the merged diff and rerun the smallest relevant local tests after merge. Do not report Merge unless the platform confirms it. + +## Pros and Cons of the Options + +### Stop after local tests + +* Good, because it is fast. +* Bad, because no remote CI, review, or merge evidence exists. + +### Immediate merge + +* Good, because it reduces elapsed time. +* Bad, because it discards the requested review iteration and can merge an unreviewed defect. +* Bad, because it encourages bypassing protections. + +### Exact-head review loop (chosen) + +* Good, because it handles review findings until the current head is green. +* Good, because it preserves a clear audit trail. +* Bad, because external reviewer/CI state remains outside the local process. + +## Problem Register and Remediation Directions + +| Finding | Direction | State | +| --- | --- | --- | +| Local tests alone cannot validate PR integration. | Push a PR and observe required CI/security checks. | Required for completion | +| Review feedback may reveal new quality/security problems. | Apply every actionable comment, extend Goal/ADR, rerun tests, and repeat; documentation alone never clears a security/correctness blocker. | Required for completion | +| A stale green commit can be merged accidentally. | Check exact PR head and merge only that SHA. | Required for completion | +| Required contexts can drift between local workflow files and protected-branch rules. | Read the protected-branch context list, record each check SHA, and treat local commands as supplementary. | Required for completion | +| CLI key-only split-token mode can silently select the single-token path. | Treat explicit `--admin-token-key`/`--inference-token-key` as split-mode selectors and test KV resolution before startup. | Required for completion | +| Partial split-token CLI errors can omit the supported KV-key flags and mislead operators about the accepted credential forms. | Name both explicit token and `--admin-token-key`/`--inference-token-key` paths in the validation error and regression-test the message. | Implemented in current head | +| CLI parsing allowed non-positive `--local-concurrency` values and non-object `--chat-template-args` to reach runtime construction. | Validate both options at the argparse boundary with strict positive-integer and JSON-object types, then assert invalid values fail with `parser.error` before `ModelClient` construction. | Implemented in current head | +| DNS can change between provider validation and connection. | Return the validated sockaddr and connect directly to it while preserving hostname-based TLS SNI and Host semantics. `ModelClient._validate_provider` now returns the resolved destination, `_open_provider` pins the socket connection to it, and `test_open_provider_uses_validated_destination_without_dns_relookup` covers the regression. | Implemented in current head; retain regression coverage | +| Model-judge output can contain wrappers, extra fields, duplicate keys, or parser-stressing input. | Require one bounded duplicate-free JSON object with exactly `decision` and `reason`; fail closed and cover Hypothesis/Atheris paths. | Required for completion | +| Container startup documentation could expose a bearer/provider secret through env/argv even though the image CMD uses the KV boundary. | Pass only credential names (`--auth-token-key`), remove misleading runtime-secret examples from the Dockerfile, and point operators to the KV bootstrap flow. | Implemented in current head; deployment still requires a seeded KV | +| External Keyverse/OIDC verification can be marked unavailable by sales readiness. | Treat the explicit external bearer verifier mode as authenticated while preserving fail-closed scope checks. | Required for completion | +| Self-approval may be disallowed or misleading. | Never fabricate approval; use authorized review or leave the PR unmerged with an explicit reason. | Required for completion | +| Two repositories can drift. | Use linked PRs/commits and run contextual + fast-mlsirm tests before each merge. | Required for completion | +| Secrets can leak through PR logs or ADRs. | Run secret scans, redact outputs, and keep Keyverse/KV credentials outside commits. | Required for completion | +| Central Strix can fail before producing a report when its external model provider is rate-limited or unavailable (observed NVIDIA NIM 429 and GitHub Models 410 brownout). | Keep the security gate fail-closed; record the provider/model error, retry the same verified HEAD after provider recovery, and never treat missing reports as a clean scan. | Required for completion | +| The two repositories' central Strix gate versions classified the same provider outage differently: `fast-mlsirm` reported a neutral pass without a structured report while `contextual-orchestrator` failed closed. | Align the trusted gate contract across repositories; until it is aligned, treat every neutral/no-report result as a blocking failure and merge only after a structured report proves the scan completed. This ADR defines no security-owner override for missing evidence; retry the same verified HEAD after provider recovery. | Required follow-up | +| The central `.github` Strix workflow explicitly converted provider outage/no-report exit code `1` into a neutral success, allowing missing security evidence to satisfy a required status. | Remove the neutralization branch and fail the required workflow closed for every nonzero Strix gate result; publish only failure/inconclusive status for failed or incomplete evidence. Central head `58561518e486d3230874c346220be96ca0a41e30` implements this with workflow-contract and provider-fallback regression tests; retain the exact-head review, structured-report, and re-fetch gates before merge. | Implemented on central working branch; required follow-up | +| Branch protection permits `requiredApprovals=0`, and the central scheduler merged fast-mlsirm PR #733 at `914127b` while its review decision remained `CHANGES_REQUESTED` and Strix was a neutral/no-report pass. | Treat `CHANGES_REQUESTED`/`REVIEW_REQUIRED` and neutral, cancelled, or no-report Strix states as hard scheduler blockers; require an independent current-head approval plus structured Strix evidence before either linked PR can merge, regardless of branch-protection approval count. Audit the central scheduler before contextual-orchestrator merge. | Required follow-up | +| Central scheduler `inspect_pr()` (trusted source `ContextualWisdomLab/.github` at `6eb06cdd`) gates only the latest OpenCode review state before direct/auto merge and does not independently reject GitHub's aggregate `reviewDecision` of `CHANGES_REQUESTED`/`REVIEW_REQUIRED`. | Add a live aggregate-review gate before every merge mutation: reject `CHANGES_REQUESTED`, `REVIEW_REQUIRED`, missing review data, and stale/non-current-head approvals; require `reviewDecision=APPROVED`, an independent current-head approval, zero active unresolved threads, required checks, and structured same-head Strix evidence. Add scheduler self-tests for each aggregate-review state and keep this PR on hold until the trusted scheduler is fixed or an equivalent protected rule is active. | Required follow-up | +| fast-mlsirm PR #742 merged as `933ce6c` at `2026-08-11T16:55:05Z` while its exact head `ad5600d` had no formal reviews (`reviewDecision` empty), the required `strix` check was still `IN_PROGRESS`, and branch protection allowed zero approvals with `enforce_admins=false`. | Treat this as a merge-protection incident, not compliant completion. Require the central aggregate-review fix, a completed structured same-head Strix result, an independent non-author review, and a final re-fetch immediately before mutation. Independently harden `main` with admin enforcement and at least one required approval, and add scheduler tests for in-progress required checks and empty review state; do not revert valid code without a reviewed replacement. | Required follow-up | +| The aggregate-review fix alone still left a merge path that could reach direct/auto merge after an exact-head OpenCode approval while another check was `IN_PROGRESS` or same-head Strix evidence was absent; this is the failure mode observed in fast-mlsirm PR #742. | Gate every approved-head merge mutation on terminal status contexts and completed same-head Strix evidence. The central `.github` follow-up now adds `running_status_checks()` plus an explicit Strix-completion gate and regression tests for running/missing evidence; keep the change unmergeable until it receives authorized review, exact-head CI, and a final re-fetch, then retain the rule in future scheduler tests. | Implemented on central working branch; required follow-up | +| The central scheduler's GraphQL status rollup exposed CheckRun status/conclusion/workflow but not the CheckSuite commit SHA, so a completed Strix or required check from an older PR head could be mistaken for current-head evidence. | Query `CheckSuite.commit.oid`, carry REST `head_sha` into the same shape, select only CheckRuns bound to `headRefOid`, and treat stale/unbound-only groups as a running blocker; add mixed-head, REST-shape, and scheduler self-test coverage. Central head `9a05f03f` implements the fail-closed binding. | Implemented on central working branch; requires exact-head review, terminal CI, structured Strix evidence, and final re-fetch | +| Trusted central OpenCode review dispatch run `31514989573` for exact head `dd8f59d` failed closed: coverage evidence failed, the model pool outcome was empty, and `OPENCODE_REVIEW_IDENTITY_UNAVAILABLE` prevented publication because the configured OpenCode App identity was unavailable. No formal review was posted. | Treat a successful/no-op `opencode-review` check as insufficient evidence of approval. Never publish a review with a GitHub Actions or PAT identity; block merge until the authorized App identity, coverage evidence, model pool, and structured same-head review are available. Add dispatch/scheduler tests for identity-unavailable, empty-pool, coverage-failure, and no-formal-review states, then retry the same head after recovery. | Required follow-up | +| The central scheduler follow-up advanced from `dd8f59d` to exact head `12e3d1f5` after adding the empty-string aggregate-review regression test; all earlier central review/security evidence is stale for the new head. | Invalidate prior central evidence on every push. Re-fetch `12e3d1f5`, obtain a new authorized non-author review and structured same-head security report, rerun required checks, and only then reconsider the linked contextual PR. | Required follow-up | +| The central `.github` repository's full local suite initially failed on this macOS arm64 host: five tests reached the intentional Linux x86_64 trusted-uv guard, while thirteen model-pool tests actually stopped before their fake provider because GNU `timeout` was unavailable. | Keep the production Linux x86_64 guard and Linux CI as the authoritative deployment path; make unit fixtures explicitly emulate the supported uv platform, add a stdlib-only signal-forwarding timeout fallback for hosts without GNU `timeout`, and cover both child success and timeout exit `124`. Do not classify platform/tooling failures as provider evidence. | Implemented on central working branch; `980 passed, 16 subtests passed` after the fix; retain the portability tests | +| The fast-mlsirm full suite exposed an avoidable `RuntimeWarning: overflow encountered in exp` in the NumPy marginal reference `_log_sigmoid`: `np.where` evaluated both mathematically stable branches. | Keep the Rust-equivalent branch-stable formulation (`-np.logaddexp(0, -x)`), retain an extreme-value regression test with overflow promoted to an error, and rerun the complete fast suite before merge. | Already implemented on fast main with `tests/test_marginal_log_sigmoid_stability.py`; the follow-up PR was correctly closed as an empty/superseded diff after exact-main reconciliation | +| GitHub Dependabot still reports five open central `.github` alerts (#5–#9) for `cryptography`/`aiohttp`, although both lock files pin the published patched versions (`cryptography==50.0.0`, `aiohttp==3.14.3`). | Do not dismiss security alerts merely because the lock appears fixed. Central head `249ba9864e9aa2309f40f947ba61fd1d126d31af` retains regression assertions covering both lock files and the patched exact pins; keep the alert-state discrepancy visible and close alerts only after GitHub refreshes manifest evidence. | CI exact-pin assertion implemented; Dependabot refresh still required | +| Exact-head required CI has remained queued or unassigned across the linked PRs. | For contextual-orchestrator PR #109 at `088eeed` (runs `31517356929`, `31517356932`, `31517356942`, `31517356967`, and `31517357009`), required jobs still show `QUEUED` with no runner allocation. For central `.github` PR #937 at `12e3d1f5` (runs `31517291040`, `31517291061`, `31517291080`, `31517291239`, `31517291412`, and `31517291784`), some jobs completed but required child jobs remain `QUEUED` and `strix` remains `IN_PROGRESS`; the platform status page reports Actions operational now, so no outage cause is inferred. Treat queued, in-progress, missing-runner, and unreported states as non-passing evidence; keep both PRs open, re-fetch after runner recovery, rerun the same exact head, and add/retain scheduler tests that prevent merge on these states. | Required follow-up | +| Latest exact-head re-fetch (2026-08-11 19:08 UTC) shows contextual-orchestrator PR #109 at `77c21869220c79ecb66d9c59ee1d28633e32de4d` with an empty aggregate review decision and `15 QUEUED / 7 SKIPPED / 1 SUCCESS`, while central `.github` PR #937 remains at `170b98e453292dd0eb63be5ec99160504252ed21` with an empty aggregate review decision and `1 IN_PROGRESS / 18 QUEUED / 13 SKIPPED / 1 SUCCESS`. | Keep both PRs blocked; every push invalidates earlier remote evidence. Do not interpret local green tests, CodeRabbit `COMMENTED` or `SUCCESS`, queued/in-progress checks, or missing formal review as approval. Re-fetch exact-head check SHAs, obtain an authorized independent review and structured Strix result, then evaluate the aggregate gate again immediately before any merge mutation. | Required follow-up | +| GitHub Status API observed a minor `GraphQL API Requests` incident in `monitoring` while exact-head checks remained queued; the status update says degradation was mitigated but does not establish that it caused the repository queues. | Record the external status as temporal context only, never as a check or review substitute, and re-fetch exact-head checks after recovery. If queues persist, investigate runner/org capacity through authorized GitHub controls; keep the merge gate fail-closed throughout. | Required follow-up | +| A review-evidence comment was once sent through an interpolating shell string, so Markdown backticks were executed as shell command substitutions and the posted evidence lost its exact SHA/field text. | Pass multi-line PR evidence through a body file or a shell quoting form that cannot perform command substitution; after posting, fetch the comment and verify the exact SHA, test evidence, and redaction before retaining it. Delete and replace malformed maintainer comments; never treat an unverified comment as merge evidence. | Implemented in current iteration; retain post-publication verification | +| Central `.github` PR #937 exact head `ed666c7c` exposed two real gate defects: the Python quality job passed 981 tests but failed its 100% coverage threshold because subprocess-only timeout tests and several exact-head scheduler branches were unmeasured, while the trusted-base Strix smoke still required the PR workflow's removed NVIDIA provider marker. | Keep the Strix outer workflow fail-closed, retain a non-executable provider marker only as a temporary trusted-base smoke compatibility contract, add in-process plus subprocess timeout tests, cover stale/current scheduler branches, and correct contradictory provider-outage documentation. Central commit `775025378b6685ab71e737e6cda1ee5ff36a7eca` implements the remediation; re-fetch its exact-head required checks and review evidence before merge. | Implemented on central follow-up branch; required review/check follow-up | +| fast-mlsirm PR #772 was force-pushed from reviewed head `7ccef2c` to `2ceeaeac` immediately before merge; its only review was CodeRabbit `COMMENTED`, all required check-runs were cancelled, branch protection reported `requiredApprovingReviewCount=0` and `enforce_admins=false`, and squash merge commit `c91ae21` was created. | Classify #772 as a non-compliant merge-protection incident, not completed review evidence. Preserve valid code but carry the unresolved accepted-type regression and stale-contract fixes in independent follow-up PR #778 at exact head `ccedd00a`; require one independent approval, terminal exact-head checks, aggregate approval, and final re-fetch before any future merge. Do not self-approve, bypass, force-push, or silently revert without a reviewed replacement. | Incident recorded; #778 required follow-up | +| The three protected `main` branches did not share the ADR contract: fast-mlsirm had `requiredApprovingReviewCount=0` and `enforce_admins=false`, while contextual-orchestrator and central `.github` also allowed zero required approvals. | Preserve every existing required check and enable `enforce_admins=true` plus `required_approving_review_count=1` on all three protected branches; keep stale-review dismissal, last-push approval, force-push prohibition, deletion prohibition, and conversation resolution intact. Re-read the protection response after each mutation and treat the new policy as necessary but not sufficient without exact-head CI and independent review. | Implemented on protected branches 2026-08-12; retain verification | +| CodeRabbit's exact-head review of central `.github` PR #937 found six remaining acceptance gaps: absent `reviewDecision` was not tested separately from null, Strix `COMPLETED` evidence did not require `conclusion=SUCCESS`, new timeout helpers lacked docstrings, background `$!` tracked a shell-function subshell instead of the timeout launcher and its child process group, the duplicate-check fixture did not exercise equal timestamps, and the Strix lock test docstring contradicted its exact-pin assertion. | Treat every current-head review finding as a Goal/ADR expansion. Fix the root behavior and regression contracts, re-run the complete central suite with 100% statement/branch coverage, answer and resolve only the exact-head review threads, then invalidate all evidence on every later push. Central head `249ba9864e9aa2309f40f947ba61fd1d126d31af` implements and locally verifies these six remediations; retain the independent approval, terminal exact-head checks, structured Strix evidence, protection verification, and final re-fetch gates before merge. | Implemented on central follow-up branch; required review/check follow-up | +| Contextual-orchestrator PR #109 remained a draft while it was being used as the linked implementation/review vehicle, preventing normal independent review from starting. | Treat draft state as a hard merge blocker; mark the PR ready only after direct-head coverage, structured same-head Strix evidence, and reviewable implementation criteria are met. A ready transition is not approval and never bypasses protection. The code-bearing implementation head is `8fee52b647fa656a2c1a325c8574f126ffad1d53`; subsequent tips only refresh this ADR evidence, so each such tip still requires a fresh exact-head check re-fetch. The local exact-source tree passes 363 tests with 90% total coverage and 93% `orchestrator.py` coverage, while remote checks remain queued, Strix has no qualifying structured report yet, and independent approval is absent. Do not mark it Ready until those conditions are repaired. | Current blocker; exact-head coverage, trusted Strix provenance, and independent review remain required | +| A full-tree coverage percentage can obscure whether newly changed security/correctness paths are measured, while unchanged legacy and live-integration paths keep the aggregate below an artificial 100% target. | Keep the full suite mandatory, report the origin/main baseline separately, measure changed production statements/branches directly, and require binding tests for every changed trust-boundary path. The current local working tree passes 363 tests with 90% total coverage and 93% `orchestrator.py` coverage; all added production statements and branches in `orchestrator.py` and `cost_ledger.py` are directly measured. Remote exact-head evidence must remain separate and be regenerated after every push. | Implemented locally; exact-head CI/review follow-up required | +| A stale CodeRabbit review comment still reproduced Ruff S608 in the SQL ledger and identified missing pyformat boundary coverage after earlier remediation. | Re-test every actionable stale comment against the current source; fix any still-valid root issue, add the smallest binding-faithful regression, record the finding here and in the domain ADR, then invalidate all remote evidence after the follow-up push. The complete SQL literals and four-window pyformat regression are now implemented locally; exact-head review/check follow-up remains required. | Implemented locally 2026-08-12; required exact-head review/check follow-up | +| GitHub exposed multiple same-head `scan-pr-queue` runs, including cancelled historical runs alongside a later success; `gh pr checks --required` displayed the historical cancellation as `fail`. | Evaluate one latest terminal check per required context for the exact PR head, preserve cancelled history as audit evidence, and never treat a historical duplicate as the current result. Reconcile the CLI listing with the protected-context result and final re-fetch before merge; retain the fail-closed rule for contexts with no latest success. | Observed on central PR #937 2026-08-11; latest same-head run succeeds; scheduler/merge-gate verification remains required | +| Current-head review found that the repository-security metadata test searched raw ADR text and did not assert the single `verified_head_sha` binding or re-evaluation on PR-head/check-SHA/reviewed-diff changes. | Normalize ADR whitespace before checking policy controls, require the exact-head binding and re-evaluation clauses as executable metadata contracts, and keep the test fail-closed if either policy sentence is removed. | Implemented locally 2026-08-12; exact-head review/check follow-up required | +| A maintainer review reply transcribed the pushed full SHA incorrectly, creating an inaccurate audit reference even though the fix and tests were correct. | Derive the SHA directly from `git rev-parse HEAD` immediately before composing evidence, verify the published reply through the API, and post a correction before retaining the thread as evidence. The incorrect reply was corrected with authoritative HEAD `895eb22d837074648e77e2e4bc5f2ab58458b627`; future evidence must use generated SHA values. | Corrected 2026-08-12; retain exact-head publication verification | +| The 2026-08-12 local Llama 3B judge sweep had seven strict-format/ordinal failures in 18 good-plan calls, so a green subset could overstate local judge quality. | Treat parse failure, timeout, and non-monotone output as failed comparisons in the Goal denominator; require any retry or model-selection mitigation to remain on the contextual-orchestrator route, add its own paired evidence, and never use keyword or positional repair. | Recorded in ADR 0006/0008; calibration and exact-head review remain required | +| The exact-head contextual Strix run `31547979688`/job `93964423546` produced a `SUCCESS` CheckRun while its gate artifact recorded NVIDIA NIM `429`, GitHub Models `410` brownout, fail-closed/no-report markers, and only a generic fallback report. The strict provider-signal path could also be bypassed by below-threshold short-circuit returns. | Fix the shared central gate at both primary and fallback below-threshold returns so strict provider-signal mode cannot bypass them; add a regression contract. Make the required workflow validate a completed successful `run.json`, a non-empty structured report, the absence of fail-closed/provider-infrastructure markers, and a trusted head/run/report hash binding before publishing evidence. Treat this finding as a Goal expansion and do not accept the old green status as security evidence. | Root fix implemented in central `.github` commit `ab2a1ae7`; exact-head review/check follow-up required | +| fast-mlsirm PR #778 exact-head Strix run `31549881616`/job `93970141054` repeated the same false-green pattern: its `SUCCESS` status coexisted with NVIDIA NIM `429`, GitHub Models `410` retirement-brownout, fail-closed/no-report markers, and a generic report admitting an incomplete AST pass. | Keep #778 blocked until the trusted central gate is reviewed/merged, then rerun this exact fast head through contextual-orchestrator's shared gate and require clean structured same-head provenance. Record the provider outage and generic report as inconclusive evidence, not as a vulnerability or approval. | New cross-repository evidence; central fix `ab2a1ae7` is pushed, fast rescan pending | +| A successful Strix status did not itself prove that the uploaded report belonged to the current PR head: `run.json` did not carry the PR head SHA, and the artifact had no binding sidecar. | Publish `evidence-binding.json` from the trusted workflow with the PR head SHA, run ID, report path, and report SHA-256; require the provenance-validation step as part of the required check and keep scheduler evidence current-head-bound. | Provenance validation implemented in central `.github` commit `ab2a1ae7`; exact-head review/check follow-up required | +| Central `.github` PR #937 head `ab2a1ae75a4275ec799d9b53a2bea78a5340e7a1` produced Strix run `31550939848`/job `93973322130` with a green CheckRun, but its downloaded artifact had no `evidence-binding.json` and `gate-console.log` recorded NVIDIA NIM `429`, GitHub Models `410` retirement-brownout, repeated fail-closed/no-report markers, and a generic fallback report. | Treat the result as stale/base-workflow or otherwise inconclusive evidence, not as validation of `ab2a1ae7`. Keep the central PR and linked implementation PRs unmergeable until the trusted provenance step is executed for the exact head (post-merge or an equivalent same-head dispatch), rejects the provider markers, and publishes a binding manifest; never repair this with keyword matching or a status-only override. | New exact-head evidence recorded 2026-08-12; central provenance fix remains unproven until a trusted workflow run | +| CodeRabbit's exact-head review of central `.github` PR #937 at `ab2a1ae75a4275ec799d9b53a2bea78a5340e7a1` found two remaining trust-boundary gaps: provenance validation could accept a successful `run.json` without binding its metadata (or scan-stage identity) to the current head, and fatal OpenCode cleanup could rediscover descendants after the launcher exited, leaving a TERM-ignoring child alive during model failover. | Require every candidate Strix report to match the exact current head, using only the scan-stage recorded SHA when the report metadata is absent; reject conflicting or non-string metadata; capture process-group IDs before TERM and reuse them for delayed KILL, with a regression that proves a TERM-ignoring child is gone before the next candidate starts. Central commits `e6c6d128` and `2c6f4323` implement the fixes; local verification is `992 passed, 16 subtests passed`, 100% statement/branch coverage, 100% docstrings, `actionlint`, and `test_strix_quick_gate: PASS`. All remote evidence remains invalidated until a trusted exact-head review/check cycle completes. | Implemented on central working branch; protected-branch review and trusted exact-head evidence required | +| fast-mlsirm PR #778 exact head `6c42d4a53d6d70cb1ae0127df624c3cc178ddd4b` later produced Strix run `31552408884`/job `93977765693` with a green CheckRun but no `evidence-binding.json`; the successful `run.json` omitted head metadata and the gate log contained NVIDIA NIM `429`, GitHub Models `410` brownout, fail-closed/no-report markers, and a generic fallback report. | Keep the fast PR unmergeable and treat the result as inconclusive. After the central provenance/process cleanup is reviewed and merged, rerun the exact fast head through the trusted central workflow, require a current-head binding manifest and no provider-failure markers, then repeat the independent-review and protected-merge gates. | New exact-head evidence recorded 2026-08-12; trusted rescan pending | +| The central full quality gate exposed one missing docstring in the nested signal-forwarding callback of the stdlib timeout fallback (`portable_timeout.py`), leaving the configured 100% docstring gate at 99.8%. | Document every executable callback required by the central quality contract, rerun the full suite, coverage, docstring, compile, and shell gates, and treat any future quality-gate shortfall as a Goal/ADR finding rather than an accepted warning. | Implemented in central `.github` commit `ab2a1ae7`; retain the 100% gate | +| Central PR #937's latest green Strix CheckRun `31555003423`/job `93985504528` ran the protected-base workflow rather than the PR-head workflow definition: the job had no `Validate Strix report provenance` step, logged a neutral provider-outage skip, and uploaded no `evidence-binding.json` despite head `2c6f4323ac864587d767824464379678ebfe888a`. | Treat the result as stale/base-workflow evidence, never as proof that the PR-head workflow is safe. Keep the central PR and all linked PRs blocked until an authorized review permits normal merge; after central integration, rerun the exact linked heads through the newly active trusted workflow and require the binding manifest, clean provider evidence, and final re-fetch. Add a protected-base/PR-head workflow-version assertion to the contract so future workflow changes cannot be accepted from a status-only pre-merge green. | New exact-head evidence recorded 2026-08-12; central workflow fix unproven until post-integration trusted run | +| Central PR #937 advanced to exact head `8726df15` after adding the separate non-privileged `strix-workflow-contract.yml` data-only check and recording the base-workflow evidence boundary. | Invalidate every prior central check, review, and Strix interpretation; obtain fresh exact-head contract/quality/security checks and an independent review. The provider-backed provenance binding remains unproven until the workflow is integrated and a post-integration run emits a current-head `evidence-binding.json` without provider-failure markers. Do not merge contextual or fast-mlsirm before that central sequence completes. | Goal expanded 2026-08-12; required follow-up | +| CodeRabbit's exact-head review of central PR #937 at `8726df15` found that the new data-only contract still trusted marker strings: a required phrase could appear only in a comment or an unreachable `if: false` step. | Parse the PR workflow as YAML with a safe standard-library loader; inspect the structured `jobs.strix.steps` graph, reject statically unreachable jobs/steps, require the fail-closed gate before collection/provenance/upload, and require executable provenance commands rather than marker-only echoes. Add adversarial comment-only and unreachable fixtures, then invalidate all remote evidence after the fix. | Root fix implemented locally after `8726df15`; exact-head review/check follow-up required | +| GitHub's classic branch-protection endpoint returned 404 for the three public repositories while `/rules/branches/main` exposed overlapping pull-request and required-workflow rules; relying on the classic endpoint would hide the effective approval and last-push policy. | Treat the rules endpoint plus the PR aggregate state as authoritative evidence: central currently exposes a two-approval/last-push rule, contextual-orchestrator an overlapping one-approval/last-push rule, and linked PRs still report `REVIEW_REQUIRED`/`BLOCKED`. Re-fetch rules, exact-head checks, threads, and approvals immediately before any normal merge; never self-approve or bypass a rule. | Observed 2026-08-12; merge remains blocked pending independent current-head approval and terminal exact-head evidence | +| GitHub HTTPS failed with `LibreSSL SSL_connect: SSL_ERROR_SYSCALL` only on the VPN `utun12` route, while `en0` reached GitHub successfully; a TLS bypass would erase the distinction between network-path failure and authentication failure. | Keep TLS verification and Keyverse boundaries unchanged, compare route/interface probes, use a temporary interface-bound relay only for authorized remote operations, clean it up afterward, and record the result as environmental transport evidence rather than a code or credential fix. | Observed 2026-08-12; no global proxy/TLS configuration changed | +| A follow-up probe on the active VPN `utun10` route (MTU 1400) succeeded with GitHub HTTP 200, certificate verification result `0`, and a valid `git ls-remote` response after the earlier `SSL_ERROR_SYSCALL`. | Treat LibreSSL `SSL_ERROR_SYSCALL` as an intermittent path-level TCP/TLS handshake interruption until packet loss, route/NAT, MTU, or firewall evidence proves otherwise; do not call it a certificate, repository, Keyverse, or application defect. Preserve TLS verification and compare VPN/interface probes before any authorized temporary relay. | Rechecked 2026-08-14; Goal/ADR expanded, no TLS or credential bypass performed | +| Same-head Strix runs for contextual-orchestrator PR #109 (`c7b5dbc25962817e458c345659c45cfeb6404977`, run `31673635738`/job `94363342167`) and fast-mlsirm PR #816 (`e2480e76dfa2139ab23f8372013681dd2cead46a`, run `31672477151`/job `94359841727`) reported `SUCCESS`, but their artifacts recorded NVIDIA NIM `429`, GitHub Models `410` retirement brownout, no-report/fail-closed markers, and incomplete provider evidence. | Reject both green statuses as security evidence. The trusted central workflow must propagate every non-zero Strix gate result; central `.github` PR #965 (`1d7ed8a42411395933a327f23f34b964b299a1ee`) removes the provider-outage neutralization and adds regression coverage. Complete its normal review/check/merge loop first, then rerun both linked exact heads and require clean structured same-head evidence before merge. | Observed 2026-08-13; central fix PR open, linked PRs remain blocked | +| The trusted Strix gate could theoretically exit `0` while its console still said `failing closed` or `incomplete evidence`, which would recreate a green-but-unaudited status through a different path. | Treat contradictory log markers as a hard failure even when `PIPESTATUS[0]` is zero. Central `.github` commit `849cee5c564ebb2c8224bca71e335a543b045891` adds this CWE-754 guard and regression contract; preserve it through the exact-head PR #965 cycle. | Implemented on central PR branch; exact-head checks/review/merge remain required | +| A successful `run.json` could be present without proving that its report came from the current PR head; independent metadata fields could also disagree, and a fatal OpenCode launcher could leave a TERM-ignoring descendant alive during fallback. | Record the scan-start SHA, require a completed successful run plus report, reject absent/conflicting/non-string metadata unless the scan-stage identity is the sole fallback, emit `evidence-binding.json` with report SHA-256, and capture process groups before termination. Central PR #965 heads `c7ca26dbb1339b67678a3d401ac8be77eebd7c62`/`520f639426e0c40d3c064ea3ab5af03de8592d06` carry the implementation and exact-pin regression; rerun trusted Strix and review every later head. | Implemented on central PR branch; not merge evidence until current-head workflow execution proves it | +| GitHub Dependabot continues to show alerts #5–#9 for `aiohttp` and `cryptography` even though live alert metadata says the vulnerable ranges end below the exact lock pins (`aiohttp==3.14.3`, `cryptography==50.0.0`). | Do not dismiss or suppress the alerts. Keep both requirements files exact-pinned, assert those pins in the central test suite, inspect the alert's `first_patched_version`/manifest after each dependency refresh, and close only after GitHub recomputes manifest evidence or a documented security-owner decision. | Exact-pin test added in central PR #965; alert refresh remains required | +| The first central exact-head Strix run after the fail-closed wrapper change stopped in its bounded required-path smoke test because the wrapper cleanup had removed the literal `Nvidia_nimException` provider-contract marker, although the trusted gate classifier still recognized it. | Keep executable smoke contracts synchronized with the workflow contract; restore the marker contract, run the smoke test before every push, and treat self-test failures as hard blockers rather than provider noise. Central PR #965 remediation commit `c7d8d234` restores the contract; all earlier evidence is invalidated and the exact head must be re-run. | Fixed on central PR branch; exact-head review/check/Strix evidence required | +| The zero-exit contradictory-log guard initially matched only spaced `failing closed`/`incomplete evidence` text, so hyphenated `fail-closed`/`incomplete-evidence` output could still be misclassified as clean evidence. | Match both spaced and hyphenated fail-closed/incomplete-evidence spellings, retain the non-zero gate propagation, and add regression assertions for each spelling. Central PR #965 commit `5489c510` implements this; invalidate all earlier evidence and re-run the exact head. | Fixed on central PR branch; exact-head review/check/Strix evidence required | + +| The central Actions queue showed a pre-start cancellation for Strix run `31685988529`, followed by same-head rerun `31686400315` attempt 2 remaining queued for more than ten minutes with no runner; unrelated Strix runs were also queued while the GitHub Status API reported Actions operational. | Classify pre-start cancellation and runner starvation as CI-capacity evidence, not a LibreSSL, provider, vulnerability, or code result. Preserve the exact run, inspect authorized runner/org capacity, avoid blind cancellation or status-only retries, and rerun the same verified head only after runner recovery; queued or unassigned required checks remain merge blockers. | Observed 2026-08-13; active blocker and Goal expansion | +| Fresh central PR #965 exact head `8da91d041d04d1cc52ff8a0bf169099995c29fe9` created Strix run `31693151287`, but its job remained queued with no runner while the repository runner API returned zero runners and GitHub Status reported `All Systems Operational`. | Preserve the exact run as capacity evidence, do not retry by status-only mutation or cancel a live required run, and keep central, contextual, and fast PRs blocked until runner recovery yields terminal exact-head checks. Re-fetch the same head, structured Strix artifact, reviews, and branch protection immediately before any normal merge. | Observed 2026-08-13; active blocker and Goal expansion | +| The central OpenCode approval helper `current_head_manual_strix_success_status` still had a fallback that accepted any same-head `repository_dispatch` run reporting `completed/success`, without proving a structured artifact binding. | Remove the unbound run-status fallback; accept supersession only from the explicit structured status whose trusted workflow has validated `evidence-binding.json` against the exact head, run ID, report path, and digest. Add static and runtime-contract regression tests, record the defect in central doctoring guidance, and invalidate every prior central Strix result after the follow-up push. | Goal expanded 2026-08-13; central PR #965 follow-up pending exact-head checks, structured Strix evidence, and independent review | +| Central provenance still treated a completed successful `run.json` with no `head_sha`/`commit_sha` metadata as current-head evidence by substituting the scan-start SHA. Central PR #965 head `b8695c534cf15a2227d92f942dcce3c653276393` exposed and removed this unbound-report path; the preceding run was cancelled when the head advanced. | Require the report itself to carry a matching string head/commit SHA (including nested `scan_results` metadata); skip metadata-less or conflicting candidates, retain the scan-start SHA only for mismatch diagnostics, and invalidate the cancelled run plus every earlier Strix/review/check result. Re-run all exact-head checks and obtain a fresh structured artifact and independent approval before any merge decision. | Goal expanded 2026-08-13; central fix is on the PR head, exact-head checks/review/structured evidence remain required | +| Live collaborator inventory for central `.github`, contextual-orchestrator, and fast-mlsirm contains only the PR author; the effective rulesets require two independent approvals for central and one for each linked implementation repository. | Keep self-approval prohibited, do not add a bypass actor or weaken the ruleset, and leave the exact heads open while requesting an authorized independent maintainer review through the normal GitHub flow. If no independent authority becomes available, report the precise governance blocker; local tests, bot comments, queued checks, and the author's own approval cannot satisfy it. | Observed 2026-08-13; independent review authority is required before normal merge | +| fast-mlsirm PR #816 exact-head Strix run `31695131004` again ended with a green job even though its artifact had only failed `run.json` files, no structured report/binding, and NVIDIA NIM `429`, GitHub Models `410` brownout, and context-window provider errors. | Preserve the artifact as inconclusive infrastructure evidence, not a clean scan; require the linked central fail-closed workflow to run on the current exact head, reject failed or metadata-less reports, and accept only a same-head structured report binding after provider recovery. A status-only green result, retry, keyword signal, or generic fallback report cannot clear this gate. | Goal expanded 2026-08-13; newly recorded evidence, central remediation and exact-head rescan remain required | +| Central PR #965 exact head `b8695c534cf15a2227d92f942dcce3c653276393` produced Strix run `31696985802`/job `94436969831` with a green job but no provenance-validation step or `evidence-binding.json`; one completed report lacked head metadata, three reports failed, and the log contained NVIDIA NIM `429`, GitHub Models `410`, fail-closed, and no-report markers. | Classify the result as inconclusive trusted-base evidence caused by the `pull_request_target` workflow boundary. Do not use the green status for merge; require a post-integration default-branch dispatch with exact-head structured binding and clean provider evidence, and preserve this incident in the central doctoring record. | Goal expanded 2026-08-13; central documentation follow-up and exact-head rescan remain required | +| The latest linked exact-head re-fetch after contextual-orchestrator `173288ca` and fast-mlsirm `fdbf62d` still found zero organization/repository Actions runners; contextual Tests/Fuzz/Security/SAST/Security-Scan runs `31706438869`, `31706438815`, `31706438829`, `31706438769`, `31706438790` and fast security/OpenCode runs `31704830521`, `31704830571`, `31704827103` remained queued. Central PR #965 also retained Strix run `31702234021` in progress with `coverage-evidence` queued. | Treat this as current CI-capacity evidence, not code, LibreSSL, provider, or security evidence. Preserve the exact SHAs and run IDs, do not cancel or status-retry live required runs, and keep every linked PR unmergeable until runner recovery produces terminal exact-head checks, structured Strix evidence, and independent current-head approval; re-fetch all of them immediately before normal merge. | Goal/ADR expanded 2026-08-13; active follow-up | +| Central PR #965 exact head `4d7267b3bf5a90a1fd5a64368bb5c9af33f12234` later produced Strix run `31702234021`/job `94453926612` with a green job but only failed `run.json` files, no `evidence-binding.json`, no provenance-validation step, and provider failures (`429`, GitHub Models `410`, context-window overflow) followed by the trusted-base `Treating as a neutral skip` warning. | Classify the result as a false-green trusted-base execution, never as clean security evidence. The central wrapper must reject neutral-skip log markers even on exit `0`; after normal protected merge, require a default-branch dispatch with exact-head structured binding and clean provider evidence before clearing the linked merge gate. | Goal/ADR expanded 2026-08-13; central remediation pushed locally, exact-head post-merge proof and independent approval remain required | +| fast-mlsirm PR #816 exact head `fdbf62dba3beb7fb06768214b241fe568b6b1f48` produced Strix run `31704826853`/job `94462628877` with a green job, but its artifact contained one failed NVIDIA NIM `429` attempt, a fallback zero-finding `run.json` without head metadata, no `evidence-binding.json`, and no provenance-validation step because `pull_request_target` used the trusted base workflow. | Treat this as inconclusive provider/base-workflow evidence, not a clean security result; keep the linked PR blocked until the central fix is integrated and a post-integration run produces a matching structured binding with clean provider evidence and current-head approval. | Goal/ADR expanded 2026-08-13; fast ADR updated, central `e1cfbed8` pushed, post-merge rescan remains required | +| After central `e1cfbed8`, Strix run `31708982141` for that exact head was cancelled at `2026-08-13T14:16:42Z` before producing evidence; contextual `823ce91` and fast `5b35852` required jobs remained queued while the contextual, fast, and central runner APIs each reported `0 total / 0 online / 0 busy`. | Record this as CI-capacity/cancellation evidence, not a source, LibreSSL, provider, or vulnerability result. Do not status-retry or cancel additional live runs; preserve the exact heads and require fresh terminal checks, structured Strix binding, and independent approval before any normal merge. | Goal/ADR expanded 2026-08-13; active infrastructure follow-up | +| The live branch-rules endpoint after the latest pushes reports `required_approving_review_count=0` for contextual-orchestrator, while fast-mlsirm reports `1` and central `.github` reports `2`; contextual PR #109 nevertheless remains `CHANGES_REQUESTED` and its current required checks are pending. | Treat the contextual zero-approval rule as governance drift, not permission to self-approve or bypass review. Restore one independent approval for contextual-orchestrator, preserve stale-review dismissal/last-push/thread requirements, and keep the PR blocked until current-head review and exact checks are clean. | Observed 2026-08-13; policy repair direction added, no ruleset bypass performed | +| A fresh exact-head cycle for contextual-orchestrator PR #109 at `73c6c98ec176b8796d481e645519bc794bc03ce6` ended with every non-skipped required CheckRun cancelled around `2026-08-13T14:55Z`; fast PR #816 remained queued and central PR #965's required runs were also cancelled, while all three runner APIs reported `0 total / 0 online / 0 busy`. | Classify cancellation, queueing, and zero-runner observations as CI-capacity evidence only. Do not convert them to pass/fail, status-retry them, or merge on local green tests; after runner recovery, re-run and re-fetch the same exact heads, terminal required contexts, structured same-head Strix binding, current independent approval, unresolved-thread state, and effective ruleset immediately before normal merge. | Goal/ADR expanded 2026-08-13; active infrastructure follow-up | +| The next contextual-orchestrator PR #109 exact head `0f61128bfe43285e2e9532bea5ab2c78a9ed67d5` created required runs `31718202448`, `31718202615`, `31718199862`, and `31718199670`, but all non-skipped jobs were cancelled within about one minute before producing terminal evidence; fast PR #816 exact head `61e6be97c10f497d5dbbc0b6110bae1f4133743c` still has queued runs `31716699852`, `31716699730`, and `31716697140`. Both repository runner APIs remain `0 total / 0 online / 0 busy`. | Preserve the exact run IDs as cancellation/runner-capacity evidence, not code or security results. Do not retry by mutating status, cancel further live runs, or merge; after runner recovery, rerun the current exact heads and require terminal checks, structured same-head Strix evidence, current independent approval, zero unresolved threads, and a final ruleset/ref before normal merge. | Goal/ADR expanded 2026-08-14; active infrastructure follow-up | +| After the polytomous default hardening, contextual-orchestrator PR #109 is exact head `d8c1b730f889719fc5e5be2c1fd6610d8aac1cf5` and fast-mlsirm PR #816 is exact head `608cfbd39983f485cebe76518c80375e7ff636dd`. All non-skipped required checks for both heads are `QUEUED`, both repository runner APIs report `0 total / 0 online / 0 busy`, contextual remains `CHANGES_REQUESTED` from a stale review, and fast remains `REVIEW_REQUIRED` with no current-head approval. | Preserve these exact SHAs and the queued/no-runner/review state as active merge-gate evidence. Do not treat local suites, issue-comment review requests, bot `COMMENTED` reviews, or queued checks as approval; obtain a fresh independent current-head review, terminal exact-head checks, structured same-head Strix evidence, zero unresolved threads, and final ruleset/refetch evidence before normal merge. | Goal/ADR expanded 2026-08-14; active infrastructure/review follow-up | +| The next re-fetch after the failure-evidence push found contextual PR #109 at `618810e1ef9d94532f463ea974b33e9f9abcd2ef` with `CHANGES_REQUESTED`, no review requests, and all non-skipped required checks queued; fast PR #816 is at `d1eca0c2fed89991e647802f0b27a91f0f6fe2bd` with `REVIEW_REQUIRED`, no review requests, and all non-skipped required checks queued. Both repository runner APIs report `0 total / 0 online / 0 busy`. | Treat the exact-head review/check/runner state as a hard merge blocker. Request fresh independent reviews, require terminal same-head checks plus structured Strix evidence, and re-fetch after every subsequent push; never self-approve, admin-merge, or infer approval from local tests, bot comments, or queued status. | Goal/ADR expanded 2026-08-14; active infrastructure/review follow-up | +| The latest re-fetch after the anchored-judge and throughput evidence push found contextual PR #109 at `f8cac364bcb90613ab777b512b39dac56bdc04f8` with stale `CHANGES_REQUESTED`, no review requests, and 15 non-skipped required checks queued; fast PR #816 is at `dd44a95deeb1b44f3ae6bf0cb44806f7854fbfeb` with `REVIEW_REQUIRED`, no review requests, and 11 non-skipped required checks queued. Both runner APIs remain `0 total / 0 online / 0 busy`. | Preserve the exact heads as the current merge-gate evidence, request authorized independent reviews, require terminal exact-head checks and structured Strix artifacts, and re-fetch after every push. Do not self-approve, admin-merge, or treat local full-suite/queued checks as approval. | Goal/ADR expanded 2026-08-14; active infrastructure/review follow-up | +| After fast-mlsirm PR #816 fell behind protected `main`, it was synchronized through merge head `bb1a285` and an ADR-only evidence update produced exact head `26b9ccc590a65cebf23537ce00f292f4d5f9e6f7`; contextual PR #109 remains at `6422a20425e52de734944fbde2fc3973f4fcb014`. The current aggregate states are contextual `CHANGES_REQUESTED`, fast `REVIEW_REQUIRED`, no review requests, and both repository runner APIs `0 total / 0 online / 0 busy`. | Treat every predecessor check/review as stale after the fast push. Request a fresh authorized independent review for `26b9ccc590a65cebf23537ce00f292f4d5f9e6f7`, re-fetch exact required contexts when runners recover, require structured same-head Strix evidence and zero unresolved threads, and keep both PRs on the normal protected flow; never self-approve, admin-merge, or infer mergeability from local green suites. | Goal/ADR expanded 2026-08-14; active infrastructure/review follow-up | +| After the integrated MLX readiness/semantic-failure evidence push, contextual PR #109 is exact head `e0000adb7215d89aa514a20089afe4d33749cf23` and fast PR #816 is exact head `7605c15456750810e54594b0e2f1ade5ebda6c7d`. Contextual remains `CHANGES_REQUESTED`, fast remains `REVIEW_REQUIRED`, both have no review requests, all currently materialized required contexts are `QUEUED`, and both repository runner APIs remain `0 total / 0 online / 0 busy`. | Preserve this as the current merge gate: obtain fresh authorized independent reviews against these exact heads, wait for runner recovery and terminal required checks, require structured same-head Strix evidence and zero unresolved threads, then re-fetch effective rules and refs immediately before normal merge. Do not self-approve, admin-merge, status-retry queued checks, or treat local test/live-MLX evidence as remote approval. | Goal/ADR expanded 2026-08-14; active infrastructure/review follow-up | +| Contextual PR #109 advanced to `c2bb2b2f85b3eae1c0c0138dff7f4a39cd744cd0` for bounded provider readiness, while linked fast-mlsirm PR #816 advanced to `17e19ec90643a8dfcc464cd7dde0b63949539a32` for ordinal-threshold prompt hardening and three-run calibration evidence. The new fast push invalidates all predecessor checks/reviews; contextual remains `CHANGES_REQUESTED`, fast remains `REVIEW_REQUIRED`, required contexts are queued, and both repository runner APIs remain `0 total / 0 online / 0 busy`. | Request fresh independent current-head reviews for both exact SHAs, rerun terminal required checks and structured same-head Strix evidence after runner recovery, and re-fetch rules/refs before any normal merge. Keep semantic calibration failures in the denominator and never treat local full suites, live MLX success, queued checks, or author comments as approval. | Goal/ADR expanded 2026-08-14; active cross-repository review/calibration follow-up | +| Fast PR #816's branch/pull-ref drift was reconciled by a normal named-branch push; its current exact head is `2cd12090f6f4ef8188da15fc6a5704a6ad7063c7`, while contextual PR #109 remains `2f904a274492fc09367a8d258f6e5f3c2eeeb4cb`. | Re-fetch both PR APIs, pull refs, required contexts, independent reviews, structured Strix evidence, rulesets, and runner capacity after every push; treat the reconciled fast head as the only current review target and keep both PRs unmergeable until all protected conditions are terminal and independently approved. | Resolved ref drift; active exact-head review/check gate | +| fast-mlsirm PR #816 advanced normally to exact head `ebd76b4664147c18a3e1cfcc3d689e916a2fff08` to retain parsed threshold values in failure evidence; contextual PR #109 remains at `bd4c1a3ffe4cc4353ac62f44288051d7960bc3bc`. | Invalidate all predecessor fast review/check evidence after this push, request fresh authorized reviews for both linked exact heads, re-fetch terminal required contexts and structured same-head Strix evidence after runner recovery, and keep the normal protected merge gate closed. | Observed 2026-08-14; active cross-repository exact-head follow-up | +| A contextual exact-head review request initially published a manually transcribed incorrect SHA; an API-verified correction then published the authoritative full head `fe8437dfd22f4d0696e27b552424cf90841165c8`. | Treat the malformed request as non-evidence, verify every published review/evidence comment through the GitHub API, derive the SHA directly from `git rev-parse HEAD`, and post a correction before retaining the thread; never use a manually copied short or guessed SHA. | Corrected 2026-08-14; current head still requires fresh exact-head review/check evidence | +| The effective protection re-audit returned overlapping pull-request rules: contextual's classic protection required one approval and last-push approval, while fast-mlsirm's classic protection reported zero approvals and no last-push approval; `/rules/branches/main` exposed both zero-approval and one-approval policies for both repositories without identifying their source. | Treat classic protection, ruleset responses, PR aggregate state, and required contexts as separate evidence; remediate fast-mlsirm to the one-independent-approval/last-push contract, preserve stale-review dismissal and thread resolution, and do not merge until the effective policy is re-read and unambiguous. | Observed and corrected 2026-08-14: fast classic protection now reports one approval and last-push approval; the overlapping rules response remains an audit caveat | +| Representative current-head jobs for contextual (`31733227437`/job `94558502060`, `31733227877`/job `94558503198`, `31733225324`/job `94558494309`) and fast (`31733222529`/job `94558486829`, `31733230722`/job `94558518535`) remain queued with `labels=["ubuntu-latest"]` and an empty `runner_name`; repository runner APIs report zero, while the GitHub status endpoint reports Actions operational. | Do not attribute this queue solely to missing repository self-hosted runners, LibreSSL, or the MLX provider. Diagnose hosted-runner/org queue capacity from job labels, runner assignment, workflow/run state, and status evidence; keep required checks fail-closed, do not cancel or status-retry live runs, and re-fetch exact heads after hosted capacity recovers. | Observed 2026-08-14; active CI-capacity follow-up | +| The latest exact-head re-audit found contextual PR #109 at `a7de9f6f9a3dc3a7949b6dd3689d775abe68f4ed` and fast PR #816 at `e407d818ed4693cf2a725024ea505f0f1a82c695`, with branch refs and pull refs agreeing. Contextual still reports aggregate `CHANGES_REQUESTED` from the stale OpenCode review at `216177f2c3524a145b24e6b9eafa3e8ca86306f5`; fast reports `REVIEW_REQUIRED`; current required CheckRuns are not terminal (contextual 15 queued, fast 11 queued), and representative hosted jobs have `ubuntu-latest` with an empty `runner_name` while both repository runner APIs report zero and GitHub Actions reports operational. | Invalidate predecessor review/check evidence on every push, request a fresh authorized independent review against each exact head, and keep both PRs unmergeable until all required contexts are terminal and successful, aggregate review is `APPROVED`, an independent current-head approval exists, structured same-head Strix evidence is present, and a final rules/refetch passes. Keep hosted queue evidence separate from local MLX readiness and LibreSSL/VPN transport diagnosis. | Observed 2026-08-14; active exact-head review and CI-capacity follow-up | +| After the literature-only pushes, contextual PR #109 is exact head `9d4562f1d702a82db40b457b9fa8f06a919c3ea9` and fast PR #816 is exact head `3c3ce1bce9145ecdb088b88bc7b676b753c06137`; each named branch and pull ref agrees. Contextual remains `CHANGES_REQUESTED`, fast `REVIEW_REQUIRED`; contextual has 15 queued required CheckRuns and fast 11, while both repository runner APIs report zero runners. The first automated review-request comments lost the SHA through shell command substitution and were corrected in place via the GitHub API (`5285501539`, `5285501551`). | Treat only the corrected API-verified comments and full SHAs as evidence; invalidate all predecessor reviews/checks after every push, request independent current-head reviews, wait for terminal required checks and structured same-head Strix evidence, and keep the normal protected merge gate closed until final refs/rules/approvals are re-fetched. Quote review-request payloads safely and never accept a comment whose SHA is absent or not equal to the branch and pull refs. | Observed and corrected 2026-08-14; active exact-head review/CI-capacity follow-up | +| A subsequent review-request attempt used abbreviated SHAs (`435c3ef`, `97b14a6`) before API correction to the full heads (`435c3efd9e244964af710917a31010e9f8de980e`, `97b14a6c73c668fae061762d85f162d83e756325`). | Treat abbreviated-SHA requests as non-evidence, correct them in place through the GitHub API, and require an exact full-SHA equality check against both the named branch and pull ref before any review or merge conclusion. | Corrected 2026-08-14; process control remains required | +| The exact-head audit at contextual PR #109 `fa20fe32beaccb3e5eaea808275c4687da635fd5` and linked fast PR #816 `a57ef506812cb54abe40d44494dd5a8a1028eb2e` found branch/pull refs equal, classic protection requiring one approval plus last-push approval and strict checks, but contextual aggregate `CHANGES_REQUESTED`, fast `REVIEW_REQUIRED`, 15/11 required checks pending, and zero repository runners. The subsequent fast evidence fix and docs pushes invalidate those predecessor heads. | Record each full SHA as historical gate evidence, re-request review only for the new exact heads, and require terminal same-head checks, structured Strix evidence, independent approval, zero unresolved threads, and a final rules/refetch before normal merge. Keep hosted-runner queue evidence separate from local MLX and LibreSSL/VPN diagnosis; never self-approve, Admin-merge, or treat local tests/queued checks as approval. | Observed 2026-08-14; active cross-repository exact-head review/CI-capacity follow-up | +| The current pre-update audit had contextual PR #109 at `2f42a1b5e19e4a54cacbf163214f7c3e388c4410` and fast PR #816 at final evidence head `5a072705c840ea70d87a73bf737d5b193ef428cb`; both branch/pull refs agreed, contextual aggregate was `CHANGES_REQUESTED`, fast `REVIEW_REQUIRED`, required checks were non-terminal, and no independent current-head approval existed. | Treat the next contextual docs push as invalidating this predecessor evidence; request fresh full-SHA reviews for both final heads, wait for terminal checks and structured same-head Strix evidence, and perform final protection/rules/refetch before normal merge. | Observed 2026-08-14; active protected-merge follow-up | +| A reported `LibreSSL SSL_connect` failure must not be classified from the error string alone. On 2026-08-14, `/usr/bin/curl` identified `SecureTransport` with `LibreSSL/3.3.6`; HTTP/2, HTTP/1.1, and TLS 1.2 requests to `api.github.com` all returned HTTP 200 with certificate verification result `0` while the default route and DNS used VPN `utun10` (`mtu 1400`, resolver `10.6.0.1`). | First identify the actual TLS backend and compare VPN-on/off route, DNS, destination IP, HTTP version, and client. The historical correlation with the full-tunnel VPN therefore points first to an intermittent egress/TCP reset or blackhole, not a broken LibreSSL installation or certificate verification. Never disable TLS verification; keep hosted CI queue evidence separate from this transport diagnosis. An explicit TLS 1.3 option failure on this SecureTransport build is a client/backend feature limitation until a real handshake trace proves otherwise. | Evidence recorded 2026-08-14; re-run the matrix when the original failure is captured | +| The latest exact-head audit after the final evidence pushes found contextual PR #109 at `d83a0294a292a7131a75d541f15f88bedda058bb` and fast PR #816 at `5a072705c840ea70d87a73bf737d5b193ef428cb`; branch and pull refs agree, contextual is `CHANGES_REQUESTED` from a stale OpenCode review, fast is `REVIEW_REQUIRED`, contextual has 15 non-skipped required checks queued, fast has 11, and both repository runner APIs report zero runners. | Preserve these heads as the only current review targets. Request an independent current-head approval, require terminal exact-head checks and structured same-head Strix evidence with zero unresolved threads, and re-fetch protection/rules/refs immediately before normal merge. Do not self-approve, Admin-merge, status-retry, or treat local green tests, bot comments, or queued checks as approval. | Observed 2026-08-14; active protected-merge and CI-capacity follow-up | +| Linked fast-mlsirm PR #816 advanced normally from `5a072705c840ea70d87a73bf737d5b193ef428cb` to `ac72dac12f5168f562990c51f158d230e473f0c2` for bounded paired-judge concurrency; contextual PR #109 remains at `d1f97a2ea7662e56d223edd47d854065d76dd9f9`. The fast branch and pull ref agree, its aggregate review is `REVIEW_REQUIRED`, its 11 non-skipped required checks are queued, and both repository runner APIs report zero runners. | Treat the new fast head as the only linked evidence target, invalidate all predecessor fast reviews/checks, and request fresh exact-head reviews for both repositories after any subsequent push. Require terminal protected checks, structured same-head Strix evidence, one independent last-push approval, zero unresolved threads, and final rules/refetch before normal merge. | Observed 2026-08-14; active cross-repository protected-merge/CI-capacity follow-up | +| Linked fast-mlsirm PR #816 advanced normally to `2f68747be079dd1a7790980f179210c4ef315750` for the lint-clean paired-calibration concurrency implementation; contextual PR #109 remains at `c460270ebc8eb2b0b393c23cdcb4e6f748b57f9b`. The fast branch and pull ref agree, aggregate review is `REVIEW_REQUIRED`, materialized required jobs are queued with additional required contexts absent from the rollup, and the repository runner APIs remain empty. | Treat `2f68747...` as the only linked fast evidence target, invalidate all predecessor reviews/checks, and request fresh exact-head reviews after every push. Require complete terminal protected contexts, structured same-head Strix evidence, one independent last-push approval, zero unresolved threads, and final rules/refetch before normal merge. | Observed 2026-08-14; active cross-repository protected-merge/CI-capacity follow-up | + +| The final docs-only contextual head `36b3d2aa90f8668b9c278f7f244392500bb476ed` created required pull-request runs that ended `CANCELLED` within roughly two minutes without a `cancelled_by` actor or assigned job/runner; the repository runner API returned zero runners. The linked fast head `6fc2259bafb22befab3ec6e9e272fa5dd2d2b92b` remained `QUEUED` across its required contexts, with the same zero-runner observation. Contextual still aggregates `CHANGES_REQUESTED` only from the stale OpenCode review at `216177f2c3524a145b24e6b9eafa3e8ca86306f5`; fast is `REVIEW_REQUIRED` with no exact-head approval. | Classify pre-start cancellation, hosted queueing, and zero runner assignment as CI-capacity/scheduler evidence, never as LibreSSL, local-MLX, provider, security, or source evidence. Preserve both exact heads and run IDs, do not status-retry or cancel additional live runs, and keep normal protected merge closed until runner recovery yields terminal exact-head checks, structured same-head Strix evidence, one independent current-head last-push approval, zero unresolved threads, and a final rules/refetch audit. | Observed 2026-08-14; Goal/ADR expanded, active cross-repository infrastructure follow-up | +| After the fast literature correction, contextual PR #109 is exact head `237ba2db1344eb66d267310e6caf80ab2ae5af1b` and fast PR #816 is exact head `e6c457d36f483b7580e56e5825528c70506dd780`. Contextual's materialized required workflows (`31745186854`, `31745188130`, `31745188206`, `31745186875`, `31745188160`) ended `CANCELLED` around `2026-08-13T21:20Z` before terminal evidence; its aggregate remains `CHANGES_REQUESTED` from the stale OpenCode review. Fast has new non-terminal `opencode-review`, `strix`, package, Rust, fuzz, GPU/Python, and related checks; its aggregate remains `REVIEW_REQUIRED`. Both repository runner APIs report `0 total / 0 online / 0 busy`, and neither exact head has an independent approval. | Treat cancellation, queueing, and in-progress hosted checks as CI-capacity/scheduler evidence only, not code, LibreSSL, local-MLX, provider, or security evidence. Invalidate predecessor reviews/checks after each push; request fresh independent reviews against the full SHAs, wait for terminal required checks plus structured same-head Strix evidence and zero unresolved threads, then re-fetch effective rules and refs immediately before a normal protected merge. Do not self-approve, Admin-merge, status-retry, or infer approval from local green suites, bot comments, or non-terminal checks. | Observed 2026-08-14; Goal/ADR expanded, active cross-repository review/CI-capacity follow-up | +| The effective-protection re-audit found contextual ruleset `18259551` requiring one approval, last-push approval, and thread resolution with no bypass actors, while fast ruleset `18259552` has the same requirements but exposes an `OrganizationAdmin` `always` bypass; fast's classic branch-protection endpoint simultaneously reported `0` approvals and `require_last_push_approval=false`, contradicting the active ruleset. | Treat classic endpoints and active rulesets as separate evidence until they converge. Never use the OrganizationAdmin bypass, self-approval, or an Admin merge; reconcile fast's overlapping policy so the effective one-approval/last-push/thread-resolution gate is unambiguous, then re-fetch rules, refs, reviews, and checks before normal merge. | Observed 2026-08-14; governance drift added to Goal/ADR, repair direction required | +| The fast governance repair was applied through the protected API: ruleset `18259552` now has `bypass_actors=[]` and `current_user_can_bypass=never`; classic `main` protection now reports one approval, stale-review dismissal, last-push approval, strict required checks, and conversation resolution. | Preserve the no-bypass policy and re-fetch both the active ruleset and classic protection after any future policy or default-branch change; require exact-head independent approval and terminal checks before normal merge. | Repaired and verified 2026-08-14; PR #816 exact-head review/check/Strix gate remains required | +| After the category-occupancy follow-up, contextual PR #109 is exact head `c722fa8291b696bfd7a8c88c4fe07ffb258e51f0` and fast PR #816 is exact head `186a98a1ea9776dd30deecafb0994fa9cc2e2f11`; the PR APIs report branch/pull refs aligned, `MERGEABLE` but `BLOCKED`, contextual aggregate `CHANGES_REQUESTED` from the stale OpenCode review, fast `REVIEW_REQUIRED`, and newly materialized required contexts `QUEUED`. Auto-merge remains enabled but has not merged either PR. | Treat the new full SHAs as the only review/check targets; invalidate predecessor evidence after this docs push, request fresh independent formal review, wait for terminal exact-head checks and structured same-head Strix evidence, and re-fetch rules, refs, approvals, threads, and auto-merge state before any normal protected merge. Never self-approve, Admin-merge, or use the queued checks, bot comments, or local full-suite/live-MLX evidence as approval. | Observed 2026-08-14; Goal/ADR expanded, active cross-repository protected-merge follow-up | +| Fast PR #816 then became behind protected `main`; after a clean-worktree merge of `fb67ced09d8ee00542c05d56374537a9a7239751`, the named PR ref now points to `c723d19f4ce0bb8840e82a9e2907745e9855ca4d` with base/head refs aligned and `MERGEABLE` but `BLOCKED`. Contextual remains `f3d54b18ee192d1f41c74ee936af402a05435365`; both PRs have new non-terminal required checks, and neither has an exact-head independent approval. | Treat `c723d19f4ce0bb8840e82a9e2907745e9855ca4d` as the only fast review/check target, invalidate all predecessor evidence, request fresh full-SHA reviews for both linked PRs, and require terminal exact-head checks, structured same-head Strix evidence, one independent last-push approval, zero unresolved threads, and final rules/refetch before normal protected merge. Keep the local PyO3 full-suite result separate from hosted approval and never self-approve, Admin-merge, or status-retry queued checks. | Observed 2026-08-14; Goal/ADR expanded, active cross-repository synchronization/review follow-up | +| On current exact heads contextual `f3d54b18ee192d1f41c74ee936af402a05435365` and fast `c723d19f4ce0bb8840e82a9e2907745e9855ca4d`, newly materialized required runs remain queued/pending and both repository runner APIs report `total_count=0`; exact-head formal approval is still absent. | Classify this as hosted Actions scheduler/capacity evidence, not a LibreSSL, local-MLX, provider, source, or security result. Do not cancel or status-retry queued runs, change required gates, self-approve, or merge around protection; after runner recovery, re-fetch the same full heads, require terminal checks plus structured same-head Strix evidence and an independent last-push approval, then perform the final rules/refetch audit. | Observed 2026-08-14; Goal/ADR expanded, active hosted-capacity/normal-merge follow-up | +| Fast PR #816 at exact head `c723d19f4ce0bb8840e82a9e2907745e9855ca4d` was still Draft after synchronization; attempting normal auto-merge correctly failed with `enablePullRequestAutoMerge`, so it was explicitly marked Ready and auto-merge was then re-enabled. | Treat `isDraft=false` as a precondition for independent review and normal auto-merge, re-fetch it after every branch/base synchronization, and keep approval/check/Strix/protection gates independent. A Ready transition is not approval and must never be used to bypass required review. | Repaired and verified 2026-08-14; active exact-head hosted-capacity/review follow-up | +| Contextual PR #109 advanced to exact head `adc4f800ca13c67854baa86865b0ff0926285f4b` for evidence-based exclusion of the 1B MLX model from the verifier role; fast PR #816 remains `c723d19f4ce0bb8840e82a9e2907745e9855ca4d`. Contextual has 15 new required contexts queued, both PRs remain `MERGEABLE` but `BLOCKED`, auto-merge is enabled, repository runner APIs report zero runners, and no exact-head independent approval exists. | Treat the new contextual full SHA as the sole review/check target, invalidate predecessor evidence, request fresh independent review, and wait for terminal exact-head checks plus structured Strix evidence and final rules/refetch. Keep the measured local `383 passed` result separate from hosted approval; never self-approve, Admin-merge, or status-retry the queued runs. | Observed 2026-08-14; Goal/ADR expanded, active evidence-based routing and protected-merge follow-up | +| After the verifier-role exclusion was recorded, contextual PR #109 advanced to exact head `67ed1284f33f79666195c1db9a66c259beef5b35`; fast PR #816 remains `c723d19f4ce0bb8840e82a9e2907745e9855ca4d`. Contextual's required hosted checks are still queued and its aggregate review state still reflects the stale OpenCode `CHANGES_REQUESTED`; fast has terminal security successes but remaining required checks are non-terminal, both PRs remain `MERGEABLE` but `BLOCKED`, auto-merge is enabled, exact-head independent approval is absent, and both repository runner APIs report zero runners. | Treat these full SHAs as the only valid review/check targets, invalidate predecessor evidence after the docs push, request fresh independent formal review, and wait for every required check plus structured same-head Strix evidence to terminate before the final rules/refetch and normal merge. Keep local suites and live MLX results separate from hosted approval; never self-approve, Admin-merge, cancel/retry queued checks by status mutation, or merge around protection. | Observed 2026-08-14; Goal/ADR expanded, active cross-repository hosted-capacity/review follow-up | +| The latest local-MLX verifier calibration routed every judge call through the contextual-orchestrator adapter and found Gemma 4 e4b valid on both K=`3` safe/unsafe cases; Llama 3B was non-monotone on the unsafe case, while Gemma 4 31B and DeepSeek R1 Qwen 32B failed the bounded boundary contract. Contextual head `490dfd8e11dc5eee1eb121b6a1a0c99eef9698f3` and fast head `21d3dbf1c8c765a684bbf4fc6f282cd23d51f456` now contain the routing/docs evidence; fast focused tests passed `452` with `84` skipped after the current Rust extension build, and contextual's prior full suite passed `383`. | Keep e4b verifier-eligible for this measured workload, retain larger models for non-verifier discovery, and preserve all failed/ceiling evidence until a larger balanced calibration proves readiness. Treat both new full heads as the only review/check targets: invalidate predecessors, obtain independent last-push approval, require terminal exact-head checks and structured same-head Strix evidence, resolve threads, and re-fetch protection/refs immediately before normal merge. Never use local green tests, keyword matching, bot comments, or queued checks as approval. | Observed 2026-08-14; Goal/ADR expanded, active model-readiness and protected-merge follow-up | +| A protection re-audit found fast-mlsirm ruleset `18259552` and classic `main` protection had drifted back to zero approvals, no last-push approval, and no thread-resolution requirement despite the preceding repair record; the effective contextual ruleset still required one approval. | Restore fast ruleset and classic protection through the GitHub API with one independent approval, last-push approval, conversation resolution, strict required checks, and `bypass_actors=[]`; verify both endpoints after every policy mutation. Treat the organization-level zero-approval overlay as governance drift, never as permission to self-approve or Admin-merge, and keep the PR blocked until the exact-head review/check gate is independently satisfied. | Repaired and verified 2026-08-14 for fast; Goal/ADR expanded, final exact-head review/check/refetch remains required | +| Final exact-head audit before this ADR follow-up found contextual PR #109 at `cc413a410ea9ba0e1ca8074f6820e20a0cfdf7aa` and fast PR #816 at `21d3dbf1c8c765a684bbf4fc6f282cd23d51f456`; both branch/pull refs agree and remain `MERGEABLE` but `BLOCKED`, auto-merge is enabled, context has `15` queued required checks, fast has `16` queued and `2` successful required checks, both repository runner APIs report `0 total / 0 online / 0 busy`, and neither head has an independent formal approval. Current CodeRabbit threads are resolved with `0` unresolved threads, but bot comments are not approvals and context still carries a stale aggregate `CHANGES_REQUESTED`. | Preserve these full SHAs as the only valid predecessors, invalidate review/check evidence after the follow-up push, request fresh independent full-SHA review, and require all required checks to terminate successfully, structured same-head Strix evidence, zero unresolved threads, the repaired protection rules, and a final refetch immediately before normal merge. Keep the auto-merge request armed; never self-approve, Admin-merge, status-retry, or infer approval from local suites, bot comments, or queued checks. | Observed 2026-08-14; Goal/ADR expanded, active final exact-head review/CI-capacity follow-up | +| MLX `/health` and `/v1/models` remained responsive while real completion calls timed out after the local server accumulated request threads and near-capacity swap; a restarted server recovered direct completions, but the first cold e4b judge run still failed closed on `2/4` boundary calls at a 30-second budget. A warm e4b run through `fast-mlsirm -> contextual-orchestrator -> mlx-lm` then completed with a valid polytomous `[2,2]` row in `35.86 s`. | Treat health/model-registry liveness as insufficient readiness evidence. Bound local requests per endpoint, serialize model switches, preserve bounded same-model concurrency, fail waiters at their request deadline, and record cold/warm latency and completion-failure distributions separately. Do not add keyword matching, retries that multiply a stuck queue, category repair, or TLS-verification bypass; require a warmup/readiness probe plus semantic/category calibration before verifier promotion. | Implemented in local follow-up with regression test; Goal/ADR expanded 2026-08-14, exact-head PR review/check/merge evidence remains required | +| Contextual PR #109 at exact head `281614044855be56dc487818e31bf90fc7f6b429` was Ready with auto-merge enabled even though its 178-commit line is divergent from prerequisite PR #96 head `c24614f1df4e0b0b6f7d1aaaf80006c409e19b9c` (`178` commits ahead / `111` behind) and its own description required #96 to reach protected `main` first. PR #96 has all 32 materialized exact-head checks successful but no independent current-head approval; #109 was therefore converted back to Draft and auto-merge was disabled. | Treat #109 as a non-authoritative evidence branch until #96 is protected-main integrated. Then create or refresh a successor from the exact protected base and selectively reconcile only the local-MLX, contextual-judge, polytomous-IRT, runtime-preflight, and directly supporting documentation/test changes; re-run the complete exact-head review/check/Strix/protection gate before marking Ready or re-enabling auto-merge. Never allow stale Ready/auto-merge metadata to outrun the dependency graph. | Observed and corrected 2026-08-14; Goal expanded, prerequisite integration and successor exact-head review remain required | + +| After the held-out K=`3`/K=`7` calibration docs push, contextual PR #109 was exact head `6373d98d8c91bf6a0c9f3bc3dcd1c67e85feba7a`, Draft with auto-merge off, `MERGEABLE` but `BLOCKED`, and its current check-run snapshot was 11 successful, 1 in progress, 6 queued, plus 7 skipped. Linked fast PR #816 was exact head `2fcc19c8647298b4430bd0ecfd01cb1657c5929e`, Ready with auto-merge on but `BLOCKED`, with 12 successful, 6 in progress, 4 queued, plus 7 skipped; neither exact head had an independent formal approval. Prerequisite contextual PR #96 remained at `c24614f1df4e0b0b6f7d1aaaf80006c409e19b9c` with 32 terminal checks but a stale `CHANGES_REQUESTED` and no current approval. | Preserve the exact-head evidence, keep #109 Draft and non-authoritative until #96 reaches protected main, request fresh independent review after this ADR follow-up, wait for all required contexts and structured same-head Strix evidence to terminate, and perform a final rules/refetch before normal merge. Never self-approve, Admin-merge, status-retry, or treat local green suites, queued checks, or bot comments as approval. | Observed 2026-08-14; Goal/ADR expanded, active cross-repository exact-head review/check follow-up | + +| The live MLX calibration discovered a machine-level port collision: an unrelated wildcard listener occupied 8080 beside the MLX process, and `/health` was therefore weaker than completion readiness; a second candidate port was occupied by a Colima SSH forward. | Keep runtime-conflict diagnosis separate from code/CI review, record the dedicated-port startup and bounded `ModelClient.probe()` evidence, and add a supervisor/port-ownership check before future live calibration. Do not terminate unrelated listeners, classify this as LibreSSL, or treat liveness as model readiness. | Observed and mitigated 2026-08-14; Goal/ADR expanded, dedicated port 18083 live evidence recorded | +| After the dedicated-port 3B calibration docs push, contextual PR #109 was exact head `bd2515fd6b141ebb12a52bdaef7347bc0945912d` (Draft, auto-merge off, `MERGEABLE`/`BLOCKED`, aggregate `CHANGES_REQUESTED`) and fast PR #816 was exact head `d8fa7347bfdec71f58a17b808ffe41ef2ed33ecf` (Ready, auto-merge on, `MERGEABLE`/`BLOCKED`, aggregate `REVIEW_REQUIRED`). Branch and pull refs agreed; contextual had 23 completed, 5 in-progress, and 1 queued check-runs, while fast had 25 completed and 7 in-progress check-runs, with no failures and no formal review on either exact head. | Invalidate predecessor reviews/checks after this push, request fresh independent current-head approvals, wait for terminal required checks and structured same-head Strix evidence, resolve threads, and re-fetch the active one-approval/last-push/thread-resolution rules and exact refs immediately before normal protected merge. Keep #109 non-authoritative until prerequisite #96 reaches protected main. Never self-approve, Admin-merge, status-retry, or infer approval from local suites, bot comments, or non-terminal checks. | Observed 2026-08-14; Goal/ADR expanded, active exact-head review/check/merge follow-up | +| Contextual PR #109 head `b36878caf3e6a207c585ce1f9485218890eda9af` produced a green Strix CheckRun (`31766642251`/job `94663866011`) and a `strix-reports` artifact, but `run.json` had no `head_sha` or `commit_sha`, no `evidence-binding.json` was present, and the workflow's same-head publish step was skipped. The report's KV finding was zero, but the artifact was not cryptographically bound to the PR head. The required workflow correction exists only on central `.github` PR #965 head `c3b65ac31f465aca50e27cd1ceeada50a9cb3e57`; protected `.github` `main@6eb06cdd08c79a06f7b390069d4ffa49e2eb7dba` still serves the older workflow. | Classify the green result as provider/content evidence only, not clean exact-head security evidence. Keep the PR blocked until central PR #965 reaches protected main, the trusted workflow emits a structured binding containing the target repository, full PR head, run/job IDs, report path, and digest, validates it before publishing the required status, and a post-integration run has no provider/no-report/neutral-skip markers. Record this as a Goal expansion; never accept a green status, fallback report, or unbound artifact as a merge gate. | Observed 2026-08-14; false-green provenance gap remains an active central-workflow dependency | + +| The exact-head Strix runs for contextual PR #109 (`31767405631`, head `29e138850bd196c671c046a10146f4d6ffb47446`) and fast PR #816 (`31767406097`, head `50db85391500c7f0ba9f82f6577338500426b013`) remained `IN_PROGRESS` for roughly nine hours with `updated_at` frozen near startup and no terminal conclusion. | Treat stale hosted execution as neither pass nor failure. Cancel only the stale run through the authorized Actions API, rerun the same exact head, and invalidate the old attempt while preserving its cancellation as audit evidence. Do not change source, weaken the required Strix gate, or classify this scheduler condition as a LibreSSL, MLX, provider, or judge-quality result. | Observed and remediated 2026-08-14; rerun attempts require fresh terminal structured evidence and independent review | +| fast-mlsirm's calibration layer previously kept option counts only in free-form metadata, weakening K-stratified evidence for the positive-choice-count hypothesis. | Keep the contextual-orchestrator route and binary-threshold semantics unchanged, while fast-mlsirm validates and reports option-count/variant strata, category occupancy, gold agreement, and unstratified outcomes. Use the result only as descriptive calibration evidence; never introduce keyword matching, positional repair, retries, or causal claims from score deltas. | Implemented in the current fast-mlsirm follow-up; focused tests pass, exact-head review/check follow-up required | +| Fast PR #816 exact-head Strix run `31769580669` (`ead9deaeb5eacb292f477cceeb0dfa4f16503e4a`, job `94672610472`) completed successfully and uploaded artifact `9207851993`, but its raw `run.json` contained null repository/head/digest metadata and the artifact had no `evidence-binding.json`; the run used protected `.github/main@6eb06cdd08c79a06f7b390069d4ffa49e2eb7dba`, not the unmerged central hardening head `c3b65ac31f465aca50e27cd1ceeada50a9cb3e57`. | Classify this as unbound provider/content evidence, not a clean security gate. Keep central PR #965 prerequisite and fast PR #816 blocked until the hardened workflow is protected-main integrated, validates target repository/full PR head/run/job/report digest, publishes same-head status only after validation, and a post-integration exact-head run emits a verified binding without provider/no-report/neutral markers. | Reproduced 2026-08-14; Goal expanded, central provenance dependency remains open | +| Contextual PR #109 exact-head Strix run `31770605499`/job `94675616668` had a first NVIDIA NIM attempt fail after `851s` with `Internal server error`/`MidStreamFallbackError`; a same-model retry then produced a zero-finding report and overall success after `2461s`. The artifact retained both attempt logs, but the final raw `run.json` still had null repository/head/digest metadata, no `evidence-binding.json` existed, and same-head publication was skipped. | Preserve every provider attempt and its latency/failure class; treat the recovered report as provider/content evidence only, not a clean security gate. Bound and surface retry budgets, require a structured head/run/job/report-digest binding after the final attempt, and keep the PR blocked until provider-failure markers and provenance validation both pass. Do not classify this provider incident as LibreSSL, local MLX, or a source vulnerability. | Reproduced 2026-08-14; Goal expanded, provider reliability and central provenance remain open | +| The exact-head review-request comments for contextual PR #109 and fast PR #816 were initially created with shell command-substitution stripping the full SHAs and literal markers; the malformed comments were API-verified and corrected in place before being used as evidence. | Treat any review/evidence comment as non-authoritative until its stored body is fetched from GitHub and its full SHA equals the branch and pull refs. Use structured API payloads or safe literal transport for future comments; never rely on terminal-rendered text or a manually interpolated comment. | Reproduced and corrected 2026-08-14; Goal/ADR expanded, exact-head review control remains required | +| Organization ruleset `18156473` (`CWL Central required workflows`) currently applies to contextual-orchestrator and fast-mlsirm default branches with an `OrganizationAdmin` `always` bypass and a pull-request rule requiring zero approvals with `require_last_push_approval=false`; repository rulesets separately require one approval and last-push approval. The overlap creates contradictory effective evidence and an administrative bypass path that the merge contract forbids. | Remove the organization-level bypass actor and align its pull-request rule to one independent approval, `require_last_push_approval=true`, stale-review dismissal, and conversation resolution while preserving required workflows. Verify the organization ruleset, both repository rulesets, classic `/rules/branches/main`, refs, reviews, checks, and auto-merge state after the mutation. Keep `.github`'s separate two-approval policy explicit, and never use the bypass, self-approve, or Admin-merge. | Observed 2026-08-14; Goal/ADR expanded, organization-policy repair and post-mutation exact-head re-audit required | +| The organization ruleset repair was applied and re-fetched: `18156473` now has `bypass_actors=[]`, one required approval, stale-review dismissal, last-push approval, and thread resolution. Effective rules for both contextual-orchestrator and fast-mlsirm now show the aligned one-approval/last-push/thread gate; classic `main` protection independently reports one approval, strict required checks, and last-push approval. The ruleset still excludes `.github`, whose separate two-approval policy remains explicit. | Keep the repaired policies under the final exact-head audit; require one independent last-push approval, terminal exact-head checks, structured same-head Strix provenance, zero unresolved threads, and final refs/ruleset refetch before normal merge. Treat current contextual head `446592223912e55704922ff3442a86474b5e37ec` as a new review/check target and keep Draft/blocked while the prerequisite and review gates remain unmet. | Repaired and verified 2026-08-14; Goal expanded, post-mutation exact-head review/check/merge follow-up required | +| Review-request comments `5290085273` (contextual PR #109) and `5290085260` (fast PR #816) were initially posted with incorrect 40-character strings instead of the actual full heads; the stored bodies were fetched, corrected in place by API PATCH, and re-fetched to match `da849d380768007fe298a1bc3c60d8b2c18cc6db` and `a533607d6c4d90e1ea034ed4996bad5d6ab3017e`. | Derive every review/evidence identity from a verified PR head or `git rev-parse`, then compare the stored GitHub body with both branch and pull refs before using it as evidence. Treat mismatches as incidents to correct and record; never treat a comment as approval or use synthetic, abbreviated, or unverified SHAs. | Reproduced and corrected 2026-08-14; Goal/ADR expanded, new docs heads require another exact-head review/check audit | +| Central `.github` PR #965 review-request comment `5290382381` was initially posted with a non-matching SHA for the then-current branch, then fetched from GitHub, patched to the actual PR head `3a2be84e983f44f4ad584a650f9721223621b52b`, and API-verified against the pull ref. It was never used as approval evidence. | Make every review/evidence comment non-authoritative until its stored body is fetched and compared with the branch and pull refs. Use exact, safe literal/structured payloads, correct mismatches in place, and retain independent review, terminal checks, provenance, and protected merge as the only acceptance gates. | Reproduced and corrected 2026-08-14; Goal/ADR expanded, exact-head identity control remains active | +| Contextual PR #109 exact head `27aa4ad3dcfbd94ec85fbce40a77955361b877c4` ran Strix `31775265809`/job `94689345852` for `884s` and failed with `agents.exceptions.ModelBehaviorError: Tool execute not found in agent strix` from NVIDIA NIM Nemotron; no vulnerability report was produced and publication was skipped. The gate correctly failed closed. Central PR #965 head `b27bdaaf07059490a4effa117c2b069c2aa92b83` adds a traceback-bound provider/model tool-contract classifier and distinct fallback routing; its local full suite passed `978` tests plus `16` subtests. A hosted central run `31776384905` then produced a zero-finding report, but artifact `9210207198` still lacked `evidence-binding.json` because pull-request-target used the protected base workflow. | Preserve the provider/model failure and no-report denominator; do not classify it as LibreSSL, a source vulnerability, or a clean scan. Keep the central fix subject to independent review, protected-main integration, and a post-integration bound artifact containing repository, full head, run/job, report path, and digest. Never turn an unbound success or fallback report into approval; retain the normal one-approval/last-push/thread-resolution and no-bypass gates. | Observed and remediated in central PR #965 2026-08-14; Goal expanded, central provenance and protected Merge remain open | +| Fast PR #816 exact head `03004b8ca54a6f821109afbc02bca5e7e3f94391` produced Strix run `31777428325`/job `94695759332` with a successful zero-finding report and artifact `9210847280`; the raw `run.json` had null repository/head/digest metadata and the artifact had no `evidence-binding.json`. | Keep the result as provider/content evidence only, not a clean exact-head security gate. Require central protected-main provenance binding and a post-integration run that binds repository, full head, run/job, report path, and digest before using Strix as merge evidence; preserve all provider and no-report failures in the denominator. | Observed 2026-08-14; Goal expanded, fast exact-head review/check/provenance follow-up remains required | +| Central `.github` PR #965 exact head `3a2be84e983f44f4ad584a650f9721223621b52b` produced Strix run `31777570466`/job `94696182267` with a successful zero-finding report, but the scope was seven changed CI/workflow files, the report described the scanning infrastructure rather than an application target, and artifact `9210803173` had no `evidence-binding.json` or repository/head/digest metadata. | Classify this as bounded provider/content and changed-file scope evidence only. Do not call it a clean security result or use it to satisfy a protected merge gate; central protected-main integration and a matching structured binding remain prerequisites for both linked PRs. | Observed 2026-08-14; Goal expanded, central provenance and normal protected Merge remain open | +| Contextual PR #109 exact head `b2b3d8e4603b144b60da1924dbbb7295029e3435` produced Strix run `31777428202`/job `94695703397` with a completed zero-finding report after approximately 30 minutes; artifact `9211049328` had no `evidence-binding.json`, and raw `run.json` had null repository/head/digest metadata. | Preserve the report as provider/content evidence only, not exact-head security proof. Require the central trusted workflow on protected `main` to emit and validate a matching repository/full-head/run/job/report-digest binding before using the result for merge; keep #109 Draft/blocked while independent review, prerequisite integration, and provenance remain unmet. | Observed 2026-08-14; Goal expanded, exact-head provenance/review/check/merge follow-up remains required | + +| CodeRabbit's exact-head review of central `.github` PR #965 identified four boundary defects: structured Strix status validation accepted description/URL substrings without resolving the referenced run, artifact evidence lacked an explicit minimum-disclosure scrub before upload, gate-marker matching could be triggered by untrusted scan text, and OpenCode timeout attempts could share the parent shell process group. | Require exact status description plus the configured server/repository Actions URL, resolve and validate the run as the same-head `.github/workflows/strix.yml` `repository_dispatch` success, redact credentials and allowlisted operational identifiers from every `strix_runs/` file before provenance binding/upload, emit run-scoped gate markers, and launch each model attempt in a dedicated POSIX session/process group. Keep the explicit download-stdin/cleanup-return and requirements-path regression contracts, plus executable spoofing regressions for description, URL, workflow, and head mismatches. Central PR #965 heads `e76b24a3b35841a03ccbd625b04fd3288d9e1ee4` and follow-up `b09e8b8f751614dc9f6802cb64c6247fef62658f` implement the fixes; local validation is `981 passed, 16 subtests passed`, `test_strix_quick_gate: PASS`, Bash syntax, workflow lint, diff check, and new redaction/process/contract tests. | Fixed and pushed 2026-08-14; CodeRabbit marked the prior findings addressed, but central PR #965 remains `BLOCKED` pending terminal exact-head hosted checks, independent approval, resolved threads, protected-main integration, and post-integration structured Strix evidence | +| The central PR #965 review-request comment for follow-up head `b09e8b8f751614dc9f6802cb64c6247fef62658f` was initially posted with a different, unverified SHA, then fetched and patched through the GitHub API before being used as review evidence. | Derive the full SHA from `git rev-parse HEAD`, compare branch and pull refs, publish the literal value, and API-fetch the stored comment body; if any mismatch occurs, patch or supersede it and record the incident. A review request is never approval or merge authority. | Reproduced and corrected 2026-08-14; Goal/ADR expanded, exact-head comment identity remains a hard control | +| Central `.github` PR #965 head `b09e8b8f751614dc9f6802cb64c6247fef62658f` produced Strix run `31782907101`/job `94712410703` with `success` and artifact `9213003174`, but the run's `publish-manual-pr-evidence-status` job was skipped and no structured `strix` commit status was present on the exact head; artifact contents could not be inspected during the GitHub API rate-limit window. | Treat this as provider/content success plus an unbound artifact, never as clean merge evidence. Require the protected-main default-branch `repository_dispatch` workflow to validate `evidence-binding.json` (exact head, scan-start SHA, run ID, artifact, report path, digest, and metadata-bearing `run.json`) before publishing the structured status; preserve the success/artifact IDs and rate-limit limitation as audit evidence. | Observed 2026-08-14; Goal/ADR expanded, central protected-main integration and post-integration structured Strix evidence remain required | +| A later review automation comment requested CodeRabbit to review stale central head `e76b24a3b35841a03ccbd625b04fd3288d9e1ee4` after PR #965 had advanced to `b09e8b8f751614dc9f6802cb64c6247fef62658f`; the contextual review comment likewise still named superseded head `e99b0978dda923beaf020f33d8127de476173db2` after the ADR push advanced it to `abfb45a86e1d5d1eeb145e013465910178054e48`. | Treat every review request as stale after any push. Patch the stored GitHub comment through the API to the current full branch/pull SHA, re-fetch it, and keep it as request-only evidence; never let a stale bot request, status, or comment authorize approval. | Reproduced and corrected 2026-08-14; Goal/ADR expanded, exact-head comment identity remains a release gate | +| While patching the contextual review comment, a shell payload containing Markdown backticks triggered command substitution and stored an empty-head request before immediate correction; the shell also emitted `command not found` for the SHA. | Never interpolate Markdown backticks or untrusted text into a shell command. Use a backtick-free literal/structured JSON payload, API-fetch the stored body, compare it with `git rev-parse HEAD` and both refs, and correct any malformed request before treating it as evidence. | Reproduced and corrected 2026-08-14; Goal/ADR expanded, safe comment transport is mandatory | +| A live HTTP gateway smoke on the dedicated MLX worker accepted four concurrent requests, but the pre-fix response wrapper generated the same millisecond-based `chatcmpl-1786698103742` ID for two distinct successful requests; the fifth request was correctly rejected as `503 concurrency_limit_exceeded` by the configured bound. | Generate non-streaming, buffered-streaming, and direct streaming completion IDs from one UUID-based helper. Keep bounded overload as an explicit `503` failure rather than queue growth or silent loss, and retain the overload response in benchmark denominators. The regression suite must prove unique IDs; the post-fix four-request gateway run returned `4/4` successes with `4` unique IDs (`p50 1449.28 ms`, max `1459.57 ms`), while the fifth remained an explicit `503`. | Defect fixed 2026-08-14; Goal/ADR expanded, live gateway evidence and exact-head review/check follow-up required | +| The current-head CodeRabbit request returned a rate-limit message (`21 minutes` remaining) while the status context changed to `pass`; no new review comments or approval were produced. A newline-bearing API payload also stored literal `\\n` text before correction. | Treat bot rate-limit/pass status as no-review evidence. Preserve required OpenCode/Noema and independent human review as separate gates, and fetch every stored comment body before using it. For review requests, use a backtick-free one-line payload or a transport that demonstrably preserves newlines, then compare the stored body with the exact branch and pull SHA. | Observed and corrected 2026-08-14; Goal/ADR expanded, current-head review and terminal checks remain required | +| After the latest linked pushes, contextual-orchestrator PR #109 is Ready at `19c3e88f78197595bf8759fe4d95797971689313` but `CHANGES_REQUESTED`/`BLOCKED`; fast-mlsirm PR #816 is at `3d42c0b2c7222f6f958ef851558aa94553ddfe78` with `REVIEW_REQUIRED`/`BLOCKED`. All non-skipped required contexts are pending, both repository runner APIs report `0 total / 0 online / 0 busy`, and prerequisite central `.github` PR #965 remains Draft at `0bdf042fa0bed9a293f2b8d3738595f4a20964bf`. | Preserve the exact heads and states as active merge-gate evidence. Request fresh independent current-head reviews, obtain terminal checks and structured same-head Strix evidence after runner/prerequisite recovery, resolve all threads, and re-fetch protection immediately before any normal merge. Do not convert local green tests, bot status, Draft/Ready state, or queued checks into approval; never self-approve, admin-bypass, force-push, or merge around the prerequisite. | Observed 2026-08-14; Goal/ADR expanded, protected Merge remains open | +| After the evidence-redaction remediation, contextual-orchestrator PR #109 is at `c0c2ecbdbd52d8902ab7c2a2da093a0bd5acd5fa` with `CHANGES_REQUESTED`/`BLOCKED`, and fast-mlsirm PR #816 is at `a536292cc05bd16287dab16431bc0c3fef74ba81` with `REVIEW_REQUIRED`/`BLOCKED`. The latest local suites are green (`388 passed` and `3714 passed` with 2 warnings), but all non-skipped hosted required contexts remain pending. The job API shows queued jobs targeting `ubuntu-latest` with an empty `runner_name`; the repository `actions/runners` endpoint reports only `0` self-hosted runners and does not establish a hosted-runner outage. Central `.github` PR #965 remains Draft at `0bdf042fa0bed9a293f2b8d3738595f4a20964bf`. | Keep local results as supplementary evidence only. Use job-level status, runner assignment, workflow queue/concurrency, and provider status to diagnose the pending cause; do not call a self-hosted-runner count a hosted-runner outage. Re-request current-head reviews after this final documentation push, then require terminal same-head checks, structured Strix, zero unresolved threads, an independent authorized approval, prerequisite central integration, and a final protection/refetch audit. Do not self-approve, bypass, force-push, or merge on queued checks or stale aggregate review state. | Corrected evidence interpretation 2026-08-14; Goal/ADR expanded, protected Merge remains open | +| The current exact-head queue audit found context PR #109 at `0fb8fdb3b182b070ddec95276a48ac235494b136` with 10 queued workflow runs and fast PR #816 at `e04e10938d63b4a71e3c5a1551c9bbb236b839ed` with 8 current queued workflow runs; the fast repository had 65 queued runs in total, with the oldest created at `08:56:09Z`. Representative bootstrap jobs on both PRs require `ubuntu-latest`, have `runner_id=0`, and have no `runner_name`; the official GitHub status page simultaneously reported All Systems Operational and Actions Operational. | Record this as hosted Actions queue/capacity evidence, not a public GitHub outage, LibreSSL, MLX, provider, judge-quality, or code failure. Keep every pending context non-passing; do not cancel, status-retry, self-approve, Admin-bypass, or merge around the queue. After capacity recovery, re-fetch both full heads, require terminal checks, structured same-head Strix, independent last-push approval, zero unresolved threads, and final rules/refetch evidence. | Observed 2026-08-14; Goal/ADR expanded, queue recovery and protected Merge remain open | +| The first queue-capacity review comments for the new heads were posted with transcribed SHA strings that did not match `git rev-parse HEAD` or the PR head (`f0f30f3b...` vs `f0f30f3256d939e30ede6740eb391b8f1206a089`, and `8349af6c...` vs `8349af60d7250f3c9cfefb4043dc2cb75463fed9`). The mismatch was caught before any review/approval use and both comments were API-patched and re-fetched with the exact full SHA. | Derive the SHA programmatically immediately before publishing, compare local, remote, and PR refs, fetch the stored body, and treat every mismatch as non-authoritative until corrected. Never use a hand-transcribed comment as review, approval, or merge evidence. | Reproduced and corrected 2026-08-14; Goal/ADR expanded, exact-head identity remains a hard gate | +| Contextual PR #109 exact head `53f47a661650c46e27c78bee55f2a9f12891c265` produced a successful Strix job `94747100680` and artifact `9217445216`, but `run.json` had null repository/head/commit/digest fields, no `evidence-binding.json` existed, and the same-head publish step was skipped. The report text claimed zero findings, but the artifact was not attributable to this PR head. | Classify the result as unbound provider/content evidence rather than a clean security gate. Keep Merge closed until the trusted central workflow emits and validates repository, full head, run/job, report path, digest, and provider/no-report binding on the exact head; do not infer approval from the green job or report prose. | Observed 2026-08-14; Goal/ADR expanded, structured same-head provenance and normal protected Merge remain required | +| `ModelClient.probe()` returned bounded-but-arbitrary provider exception text in the admin provider-readiness report. Secret-pattern redaction could not guarantee that provider response/error bodies were safe, and the public report shape had no stable failure code. | Return only package-owned failure codes (`provider_probe_failed`, `provider_model_not_registered`, `provider_empty_probe_response`) and an allowlisted exception type; omit arbitrary `error` text. Add a sentinel regression proving provider exception text cannot enter the serialized readiness report. | Fixed in current head; exact-head review/check/Strix evidence remains required | +| A fresh warm Gemma 4 e4b comparison at the dedicated MLX port showed direct MLX completing width-5 `20/20`, while the authenticated gateway completed width-5 `4/20` and returned `503` for the 16 excess requests; widths 1–4 were fully successful on both paths and gateway latency stayed close to direct MLX. | Keep the gateway admission bound at four for this model/server pair and preserve explicit overload responses in benchmark denominators. Do not increase the bound merely to hide queue latency, silently queue/drop requests, or treat transport throughput as Judge quality or IRT evidence; re-measure after model, server, prompt, output-budget, or device-memory changes. | Verified 2026-08-14; current source behavior is covered by local tests, exact-head review/check/Strix evidence remains required | +| The current exact-head cross-repository smoke (`contextual-orchestrator` `f15ccb0ff53a0a2782438974f543bfc041cb1a69`, `fast-mlsirm` `c9f2c280c4113e49486cb01e69daa40583f38127`) completed four strict binary boundary calls through `ContextualOrchestratorJudge -> _FastMLSIJudgeAdapter -> TaskOrchestrator -> ModelClient -> mlx-lm` in `4.221 s`, yielding two criterion items and IRT row `[2,2]`. | Retain this as transport/adapter/shape evidence only. Keep multi-item output, strict parse, balanced gold, category occupancy, perturbation stability, and failure denominators as separate gates; do not promote one successful row to Judge quality, unbiasedness, or production IRT evidence. | Verified 2026-08-14; exact-head review/check/Strix evidence remains required | +| `batch_route()` previously converted a missing Batch API result or a `content=null` result into an accepted route record, silently corrupting evaluation and spend denominators. | Validate the result-key set and assistant-content type at the shared `ModelClient.batch_chat()` boundary and again before route persistence for custom clients; reject incomplete, unexpected, duplicate, or malformed identifiers without keyword matching, positional repair, or silent drop. Keep the whole batch failed and retain the failure in the evaluation denominator. | Implemented in the remediation commit; exact-head review/check/Strix evidence remains required | +| After the batch-integrity remediation, the exact source pair `contextual-orchestrator` `cdca9d8e55f54b8b6ed67e146d73f7f32df93542` and `fast-mlsirm` `c9f2c280c4113e49486cb01e69daa40583f38127` completed the injected Gemma Judge smoke in `3.637 s` with `1,824` tokens and IRT row `[2,2]`. | Keep this as current-head transport/adapter/shape evidence only. The single route smoke does not validate batch completeness, semantic quality, bias, human/gold agreement, IRT sufficiency, or protected merge; preserve the separate batch failure-denominator and calibration gates. | Verified 2026-08-14; new contextual push invalidates predecessor remote checks/reviews and requires fresh exact-head evidence | +| After contextual-orchestrator advanced to exact head `018f6effb8d27b2acf1f7d41edd4a2a704a2e1fa`, all current non-skipped required contexts were still `QUEUED`; fast-mlsirm exact head `c9f2c280c4113e49486cb01e69daa40583f38127` had Strix run `31797316247` / job `94757198671` still `IN_PROGRESS` since `2026-08-14T12:04:29Z`, with no artifact or terminal conclusion, and neither PR had a current-head formal approval. | Treat queueing, in-progress execution, absent artifacts, and missing approval as non-passing evidence. Preserve the exact run/job/head identities, do not cancel/status-retry/self-approve/Admin-bypass/merge around them, and require terminal exact-head checks, structured same-head Strix binding, zero unresolved threads, independent last-push approval, and a final rules/refetch audit before normal Merge. | Observed 2026-08-14; active hosted-capacity/review/provenance follow-up | +| The same fast-mlsirm Strix run `31797316247` / job `94757198671` later completed `success` for exact head `c9f2c280c4113e49486cb01e69daa40583f38127` and uploaded artifact `9219034211`, but the artifact had no `evidence-binding.json`; its nested `run.json` had null repository, head, commit, report-path, and report-digest fields. The report SHA-256 was `612ec1e9c3f7a4c58a5daf0109e22437bec72e51261ba9cfa6891b7829f8d6a2`. | Classify this as provider/content evidence only, not a clean exact-head security result or Merge approval. Preserve the report and digest for audit, require the trusted structured repository/head/run/job/report binding after central workflow integration, and rerun the current exact head; do not infer security, IRT, or Judge acceptance from the green job or report prose. | Observed 2026-08-14; unbound Strix provenance remains an active merge gate | + +| The structured-status consumer validated a `strix-reports` download by name and treated the provider's `run.json` identifier as the GitHub Actions run ID. That did not prove a unique non-expired artifact or bind the report to the status URL's outer workflow run. | Generate binding with the target repository, exact artifact name, and outer `$GITHUB_RUN_ID`; before download, require exactly one non-expired artifact of that name, then validate repository/name/head/run/scan/report/digest. Treat provider-internal IDs as descriptive only. Keep missing, duplicate, expired, or mismatched artifact metadata fail-closed and add a regression for each case. | Observed 2026-08-14; Goal/ADR expanded, central PR #965 exact-head re-review and protected Merge remain required | +| A fresh central review-request comment was initially written with a hand-transcribed head that did not equal the local `git rev-parse HEAD` or the PR head. The mismatch was caught before review use, and the stored comment was API-corrected and re-fetched with the exact full head. | Derive every review SHA programmatically immediately before posting, compare local/remote/PR refs, fetch the stored comment body, and patch or supersede any mismatch. A malformed request is request evidence only and never approval or Merge authority. | Reproduced and corrected 2026-08-14; Goal/ADR expanded, exact-head review identity remains a hard gate | + +| The current local MLX integrated smoke used contextual-orchestrator `ccfa292aafadc37b6a008ffa1fb3b1d4bc2e346e` and fast-mlsirm `d1114e5e20c9aeb4c1cd7c8c8b46053db314ae4a`: two criteria, four K=`3` binary-threshold calls, `12.569 s`, `1,956` total tokens, and a valid row `[2,2]`. | Preserve this as transport/contract/trace and multi-item shape evidence only. Do not promote one accepted row to semantic quality, unbiased polytomous measurement, positive-option-count confirmation, IRT readiness, approval, or Merge evidence; retain calibration, occupancy, exact-head checks, and protected review gates. | Verified 2026-08-14; semantic calibration and protected Merge remain open | +| The current authenticated gateway sweep used Gemma 4 e4b with MLX prompt/decode concurrency `4`, gateway admission `4`, and four requests per parallelism level. Parallelism `4` completed `4/4` in `0.731 s` (`5.471 req/s`), all IDs were unique/non-empty, while the serial sweep retained a `15.220 s` warm-up outlier and a warm repeat took `2.573 s`. | Keep this as route throughput and response-integrity evidence only. Use `4` as a measured candidate for this exact workload, not a universal setting or semantic/Judge/IRT result; retain warm-up, overload, token, and failure denominators and remeasure after model/server/prompt changes. | Verified 2026-08-15; quality calibration and protected Merge remain open | +| Central `.github` PR #965 was found to mix the Strix evidence fix with scheduler, model-pool policy, lock-materializer, architecture, and release-document changes. A protected-base successor PR #1009 was created at exact head `2833d8a1c2f2cbb02387a2af752db51298cc64c4` with only the direct Strix provenance/fail-closed/process-group scope; its 90 focused tests, `test_strix_quick_gate.sh`, and full local suite (`978 passed, 16 subtests`) passed, but hosted checks and independent approval are still pending. | Keep #965 closed/superseded and use only #1009 as the current central dependency. Require exact-head terminal checks, structured same-head artifact binding, independent last-push review, resolved threads, and final rules/refetch before treating the dependency as integrated; no local green result or predecessor evidence transfers. | Observed and corrected 2026-08-15; scope repaired, normal protected integration remains open | +| One transient `gh pr view` status snapshot reported contextual/fast heads and merge states that did not match the local, branch, or REST pull refs; immediate REST and branch refetches restored the verified exact heads and no acceptance action used the snapshot. | Treat any provider snapshot as non-authoritative until local `git rev-parse`, remote branch ref, REST pull `head.sha`/`base.sha`, and stored review-comment identity agree. On mismatch, discard check/review/merge evidence, refetch with bounded backoff, and never push, approve, or merge to reconcile an observation. | Observed and contained 2026-08-14; exact identity gate strengthened, protected Merge remains open | +| A check-run audit wrapper first left a query-bearing GitHub API path unquoted (`zsh nomatch`) and then relied on unquoted zsh word splitting for a repository/PR loop; the same omission recurred in a later parallel status command. Both produced no check evidence and were caught before acceptance. | Quote every query-bearing API path, pass repository and PR as separate explicit arguments without shell word-splitting assumptions, fail on command/API errors, assert returned check-run head SHAs against the exact PR ref, and treat absent/empty output as unknown rather than passing. Add a shell-level regression or use a quoting-safe helper so the audit cannot regress to raw zsh URLs. | Reproduced and corrected 2026-08-15; Goal/ADR expanded, audit transport remains a release gate | +| After the current exact-head pushes, contextual Tests run `31812903425`, fast CI run `31812903984`, and central materializer run `31812474225`/Strix run `31812472502` were queued; representative jobs had `runner_id=0` and empty `runner_name`, while predecessor runs were cancelled by the normal concurrency policy. | Keep all queued contexts non-passing and preserve run IDs as capacity evidence only. Do not infer a public outage, LibreSSL error, MLX/Judge failure, or clean security result; wait for terminal checks, then refetch exact heads, structured Strix binding, independent approval, threads, rules, and auto-merge state before normal Merge. | Observed 2026-08-15; hosted capacity and protected Merge remain open | +| A strict anchored Gemma 4 e4b calibration through contextual-orchestrator exact head `1c05e209e9d11359bfc8cbc39bcbf71b694e1363` and fast-mlsirm exact head `ed62e1d1723d1274c1c0483dca4f46bb4eb81665` evaluated 12 paired cases at K=`3,5,7` (48 binary boundary calls). All cases parsed and passed the ordinal gate, but manual gold `[1,1]` agreement was `0/12`; K=`3`/`5` stayed at `[1,0]`/score `0.25`, while one K=`7` shuffled-options control changed to `[1,2]`/score `0.75`, a paired `+0.5` outlier. | Treat this as descriptive option-count/order and semantic-calibration evidence, not a causal universal positive-K law. Keep the binary judge fail-closed; retain every gold mismatch and perturbation, and require replicated held-out human/gold recall, non-ceiling occupancy, and perturbation stability before semantic or IRT promotion. No keyword matching, position inference, category repair, retry, or silent drop is permitted. | Observed 2026-08-15; Goal expanded, semantic calibration and protected Merge remain open | +| The follow-up strict general prompt at fast-mlsirm `112b1956d9f19cdab20bbada6b596d65e8f5c827` improved explicit direct polytomous scoring to `11/12` pass with `11/11` gold exact among scored rows, while cumulative-threshold scoring passed only `5/12` and produced seven `JudgeFormatError` outcomes; the direct scored rows were stable `[1,1]`/`0.5`, but one K=`7` shuffled cumulative row was `[2,2]`/`1.0`. | Preserve method-specific denominators. Keep direct and cumulative explicit calibration-only, retain cumulative malformed/non-monotone/truncated outcomes, and use binary-threshold as the implicit production polytomous path until replicated held-out gold, occupancy, perturbation, and output-budget evidence supports a change. Do not repair arrays, infer categories, keyword-match, or silently drop failures. | Observed 2026-08-15; Goal expanded, semantic calibration and protected Merge remain open | +| A fresh current-source recheck used the authenticated gateway and direct Gemma 4 e4b worker at widths `1,2,4,5`; gateway width `5` returned `4/5` HTTP 200 plus an explicit `concurrency_limit_exceeded` HTTP 503, while direct MLX queued and returned `5/5` HTTP 200. The same pair ran three two-criterion K=`3` Judge probes through the required contextual route: implicit binary returned safe `[2,2]`, partial `[0,1]`, unsafe `[0,0]`; direct returned `[2,2]`, `[0,0]`, `[0,0]`; cumulative safe failed closed as non-monotone and the other two returned `[0,0]`. | Preserve explicit gateway admission and every Judge semantic failure. Do not raise concurrency, queue hidden work, repair non-monotone arrays, or promote direct's lower latency/small-sample behavior to quality or unbiasedness evidence; retain implicit binary as production default and keep direct/cumulative calibration-only until balanced human/gold, occupancy, perturbation, and failure-rate evidence is met. | Observed 2026-08-15 at contextual `719d9cc83393c616f0a552adad0b41ae55d5b346` + fast `e55a6c3e742e2688efe618267870e2007902857b`; exact-head review/check/Strix/Merge gates remain open | +| After the strict calibration code/docs pushes, contextual PR #109 exact head `bd78ca1a6f433b7d83d5a3f1b451cfa02d09cdc8` has `7 completed/skipped` and `15 queued` check-runs; fast PR #816 exact head `ed62e1d1723d1274c1c0483dca4f46bb4eb81665` has `7 completed/skipped` and `11 queued`. Both remain Draft, `REVIEW_REQUIRED`, and `BLOCKED`; latest request-only comments are `5295211638` and `5295211561`, each verified against the full head/base/central dependency identities. | Treat the new queued runs and absent formal approval as non-passing. The next docs push invalidates these exact-head checks again; re-fetch local/remote/PR refs, request-only comment bodies, terminal checks, structured same-head Strix, independent last-push approval, threads, rules, and auto-merge immediately before any normal Merge. Never self-approve, Admin-bypass, cancel/retry to hide the queue, or interpret skipped checks as success. | Observed 2026-08-15; Goal expanded, hosted capacity/review/provenance and protected Merge remain open | + +| Zotero Local API recheck found the relevant LLM-judge/option-order records already have original OA PDFs (`SHLVYKJC` -> `TVZMTEB8`, `UFZQ8WN6` -> `S5KQCN97`), while the response-category records `CWY355RP` and `MYPNHHWJ` have no child attachments. Local `PUT`/`DELETE` returned HTTP 501 and Connector `saveItems` did not create a parent note, so no metadata or reconstructed/mirror PDF was claimed as an update. Two accidental audit notes (`TSE2GUDB`, `N7DAW7JG`) were exact-scope marked deleted after a database backup; Zotero restart cleared their pending delete log and the collection listing no longer includes them. | Keep research provenance read-only unless an authorized Zotero write route is available. Attach only source-authorized original PDFs and verify key, parent, child, checksum, rights, and sync state; never use keyword matching, mirrors, or regenerated bytes as evidence. Preserve the explicit blocked-write state in ADRs and retry through desktop/Web API/Connector only when its write capability is verified. | Observed 2026-08-15; Zotero PDF provenance and authorized-write follow-up remain open | + +| After the latest ADR-only follow-up, verified local/remote/PR identities are contextual-orchestrator `aeae379f63a2544fc5269bc47b3f4aff094f2915` and linked fast-mlsirm `b4121d2e2071a02b1f497b7228b0ecde061fbb45`; contextual has `7` skipped plus `15` queued check-runs and fast has `7` skipped plus `11` queued. Both PRs remain Draft, `REVIEW_REQUIRED`, `BLOCKED`, and without an independent formal approval; CodeRabbit/security comments are comments, not approvals. | Keep both PRs open and fail-closed. Re-fetch exact branch/pull refs after every push, wait for terminal successful required checks and structured same-head Strix evidence, obtain an authorized independent current-head approval, resolve threads, and perform the final rules/auto-merge/refetch audit. Never self-approve, Admin-bypass, cancel/retry to hide queueing, or merge on local tests, bot comments, or skipped-only results. | Observed 2026-08-15; hosted capacity, independent review, Strix provenance, and protected Merge remain open | + +| A 2026-08-15 re-poll before the next documentation push verified contextual-orchestrator `efc4b5f11e77bd3c5061da4c3faa29b83f55110e` and fast-mlsirm `26e5404d7da00f4c652957a5e41ba586d0b5925e`; contextual had `7` skipped plus `15` queued check-runs and fast had `7` skipped plus `17` queued. Both remained Draft, `REVIEW_REQUIRED`, `BLOCKED`, and without an independent formal approval. | Treat the count change as hosted check materialization, not a pass or code-quality signal. Re-fetch after every push and require all exact-head checks to terminate successfully, structured same-head Strix evidence, an independent current-head approval, zero unresolved threads, and final rules/auto-merge/refetch before normal Merge. | Observed 2026-08-15; recorded as historical queue/review evidence | + +| Contextual PR #109 exact head `1003cea38df0ba789ae6ecf94329ae9998573ab0` produced Strix run `31819952638`/job `94830607001` with a CRITICAL authentication-bypass finding in `SecurityConfig.authorize`: the common `auth_token` was selected before scope-specific tokens, while direct `SecurityConfig` construction permitted mixed single/split modes. | Fix the shared security boundary, add a regression test, preserve the Strix report/digest as the triggering evidence, and rerun Strix against the new exact head. Do not classify the old failure as stale or bypass it; keep normal Merge closed until the new terminal security result, independent approval, resolved threads, and final refetch are present. | Remediated locally; new exact-head hosted checks and protected Merge remain required | +| Review of the first remediation found that the mutable `SecurityConfig` dataclass could be changed after initialization, so initialization-only validation was not a complete authorization invariant. | Re-check scope-specific precedence at every request, reject unknown scopes, add the mutation regression, and restart the exact-head review/check/Strix cycle after the follow-up commit. | Remediated locally; every predecessor check/review is stale and protected Merge remains closed | +| Central `.github` PR #1009 exact head `2833d8a1c2f2cbb02387a2af752db51298cc64c4` ran trusted-base workflow SHA `6eb06cdd08c79a06f7b390069d4ffa49e2eb7dba` in Strix run `31813452739`/job `94809347457`; NVIDIA NIM returned `agents.exceptions.ModelBehaviorError: Tool execute not found in agent strix`, no vulnerability report was produced, and the head's new gate classifier was not the code executed by `pull_request_target`. | Classify this as provider/model-tool-contract plus trusted-base evidence, never as a clean security result or a source finding. Keep the required check fail-closed, do not weaken fallback or publish a status-only success, and validate trusted workflow/gate changes only through the protected default-branch dispatch after integration; re-fetch exact head, structured binding, independent approval, resolved threads, and rules before normal Merge. | Observed 2026-08-15; Goal expanded, central PR #1009 remains blocked | +| During the same macOS recheck, `/usr/bin/curl` reported `SecureTransport` with `LibreSSL/3.3.6`, while the GitHub and GitHub API HTTPS probes returned HTTP/2 `200`; with the VPN route active, `route -n get` placed both destinations on `utun10`. Prior VPN-on/off comparison showed the failure only on the full-tunnel path, with MTU changes not resolving it. | Treat `LibreSSL` as the TLS backend reporting a broken/aborted network handshake, not as the root defect. Preserve certificate verification and do not reinstall or downgrade TLS; compare VPN-on/off routes, DNS, destination IP, peer/endpoint reachability, NAT, and MTU, then repair the WireGuard/Passepartout full-tunnel egress or endpoint-loop configuration with the VPN administrator. | Observed/rechecked 2026-08-15; Goal expanded, VPN server/path remediation remains external | +| After contextual ADR commit `491566e277c8dfd4ff28e34537b3a58253c33037`, contextual PR #109 had exact head `491566e277c8dfd4ff28e34537b3a58253c33037` with `13` completed and `13` queued check-runs; fast PR #816 was `f3249f613ad6a87e261dc2938ee1af2552bb67ca` with `34` completed and `1` queued; central PR #1009 was `2833d8a1c2f2cbb02387a2af752db51298cc64c4` with `84` completed, `1` queued, and the known Strix failure. All three remained open/blocked with `0` formal approvals. | Treat this snapshot as exact-head merge-gate evidence only. The next push invalidates predecessor checks and review state; require fresh terminal checks, structured same-head Strix evidence, resolved threads, an independent current-head approval, and final rules/refetch before normal protected Merge. Never infer approval from local passes, comments, queued checks, or the provider-failure diagnosis. | Observed 2026-08-15; Goal expanded, protected Merge remains open | +| A live Gemma 4 e4b transport sweep through the current MLX worker and authenticated contextual gateway measured direct/gateway throughput at widths `1,2,4`; width `4` was the local candidate peak (`5.552`/`5.538` req/s) with all four responses successful and non-empty. The gateway width-`8` sample returned `4/4` HTTP `429` responses because the fixed `60 requests / 60 seconds` limiter was already exhausted; it did not exercise concurrency admission and cannot be interpreted as model saturation or a width-`8` quality result. | Keep rate-limit, concurrency, warm-up, transport, and model-quality denominators separate. Pace or reset the measurement window through an authorized test configuration before comparing widths above `4`; retain `429` outcomes in the report, do not raise the production limiter or hide them, and do not change the measured concurrency bound without a larger workload. | Observed 2026-08-15; Goal/ADR expanded, gateway benchmark and protected Merge remain open | +| A review-request comment for contextual PR #109 was initially posted with an invalid 40-character SHA (`fcb82a1c7f43ea680a944eaf86c2fe8d4e6a52c9`) that did not exist; the verified local/remote/PR head was `fcb82a18e1597549f7f3bdfb37e81202c603d937`. The mismatch was detected by the exact-head check query, and the stored comment was API-patched and re-fetched before any review or merge use. | Never hand-transcribe or predict a full SHA. Derive it immediately from `git rev-parse HEAD`, compare local/remote/PR refs, post a request-only comment, and re-fetch the stored body; treat a `No commit found`/mismatch as invalid evidence and correct it before continuing the review loop. | Reproduced and corrected 2026-08-15; Goal/ADR expanded, exact-head identity remains a hard gate | +| After the latest exact contextual push, PR #109 head `3a59b67b2b232de298b9a0a8f0c5816e4ce8f219` had `7` completed/skipped and `15` queued check-runs after repeated polls; its PR REST state remained Draft/open with no current-head formal approval. The linked fast-mlsirm head `f3249f613ad6a87e261dc2938ee1af2552bb67ca` was green on its own older head, while central #1009 still had the known Strix failure. | Treat queued and skipped-only contextual checks, older linked results, request-only bot comments, and the central provider failure as non-passing merge evidence. Preserve the exact denominator, wait for terminal same-head checks and structured Strix provenance, obtain an independent approval, resolve threads, and re-fetch refs/rules before normal protected Merge. | Observed 2026-08-15; Goal/ADR expanded, hosted capacity and protected Merge remain open | +| A live benchmark harness first sent gateway model alias `mlx_e4b` to the direct MLX endpoint, producing `404` responses and invalid Hugging Face lookup errors; it also used four requests at width `8`, so its direct/gateway comparison and overload conclusion were invalid. A corrected sweep used the worker-advertised id `mlx-community/gemma-4-e4b-it-4bit` and eight requests at width `8`: direct `8/8` success at `7.066` req/s, gateway width `4` `4/4` success at `7.088` req/s, and gateway width `8` `4/8` success plus `4/8` explicit `503 concurrency_limit_exceeded` (`6.269` successful req/s). | Treat endpoint model ids, request count, concurrency width, HTTP status, successful throughput, and rate-limit state as separate benchmark fields. Reject any run with endpoint/model mismatch or an insufficient request count; retain every `404`/`503` denominator, keep gateway admission at four for this workload, and do not change production concurrency based on attempted-request rate. | Reproduced, corrected, and documented 2026-08-15; Goal/ADR expanded, semantic calibration and protected Merge remain open | + +| An authenticated `local://` gateway request returned `401` because the client discarded all loopback credentials; fixing it by reusing the remote `credential_key` would risk forwarding OpenAI/remote secrets to local workers. | Use an explicit `ModelAgent.local_credential_key` only for authenticated `local://`, keep direct `mlx://` keyless, fail closed on a missing local key, and require focused tests plus a live bearer-authenticated route. | Fixed in current working tree; local/KV tests and live structured Judge route passed, exact-head checks/review remain required | +| The authenticated gateway rejected direct-worker `chat_template_kwargs` with HTTP `400 unknown_fields`; forwarding provider-specific fields through a strict gateway is a contract violation. | Send template kwargs only to direct `mlx://`; configure them on the MLX worker behind `local://`, and preserve the explicit route distinction in tests and ADRs. | Fixed in current working tree; focused tests and live gateway smoke passed, exact-head checks/review remain required | +| Free-form Gemma Judge output was prose/Markdown in all four completed boundary calls, so strict parsing failed closed even though the gateway and model were healthy. | Add the method-specific strict JSON Schema transport in fast-mlsirm and route it through contextual-orchestrator's existing proxy adapter. Keep failures in the denominator and prohibit keyword matching, positional inference, category repair, retries, or silent drop. | Implemented in current working trees; contextual `73 passed`, fast Judge `48 passed`, live two-criterion row `[2,2]`; semantic calibration remains open | +| A later query-bearing GitHub job API audit again triggered zsh glob expansion before the same URL was rerun safely quoted; the first invocation produced no check evidence. | Treat any unquoted query URL or empty/error audit output as unknown, quote every query-bearing path, assert the returned head SHA, and add/use a quoting-safe audit helper before relying on checks for review or Merge. | Reproduced again 2026-08-15; ADR/Goal expanded, exact-head audit remains a hard gate | +| Contextual PR #109 exact head `60d9cfc9be2ce0426ed37746eb9a2768b8f3455d` produced Strix run `31831835133`/job `94869100782` with a successful zero-finding report and artifact `9231362799`, but no `evidence-binding.json`; `run.json` contained only a local temporary target and no repository, PR head, job, report path, or digest binding. The report digest was `33a47fa5855600b393d92c3a1a77e0ac15adb99a55406f99e5b2efba93c17c18`, and the same-head publish step was skipped. | Keep the success as provider/content evidence only, not a clean exact-head security or Merge result. Require the central trusted workflow to publish a structured binding for repository, full PR head, run/job, report path, and digest; reject unbound success and re-run the exact head after protected-main workflow integration. | Reproduced 2026-08-15; central provenance dependency and protected Merge remain open | +| The contextual exact-head code-scanning checks `Trivy` and `Scorecard` both became `neutral` because the PR's `security.yml` configuration was absent from protected `main` (`trivy-filesystem` and `supply-chain/branch-protection`), while the separate required `trivy-fs`/`scorecard` jobs were still pending or successful. | Keep the organization's CodeQL-only code-scanning rule unchanged; distinguish code-scanning alert comparison from required Security job results, record the missing configuration as a warning, and never treat neutral, skipped, or absent results as a pass. Re-audit after trusted workflow/ruleset integration. | Observed 2026-08-15; no local source bypass, governance/central workflow follow-up required | +| Fast PR #816 exact head `355f93b27ba4a0cb141e86b0fbc9127681edb750` produced Strix run `31833030282`/job `94872991843`; NVIDIA NIM failed with `agents.exceptions.ModelBehaviorError: Tool execute not found in agent strix`, no penetration-test report was produced, and the required check failed closed after 599 seconds. | Keep this as provider/model-tool-contract evidence, not a source vulnerability or clean security result. Preserve the failure denominator, do not publish a status-only success or retry to hide it, and require central trusted workflow/provider repair plus a structured same-head artifact before normal Merge. | Observed 2026-08-15; central Strix dependency and protected Merge remain open | + +## Risks and Mitigations + +| risk | likelihood | impact | mitigation | owner | +| --- | --- | --- | --- | --- | +| CI passes before a later force-push. | low | high | Re-check exact head immediately before merge. | maintainer | +| Review loop churns without convergence. | medium | medium | Keep each iteration scoped to evidence, add a regression test, and stop only at a concrete acceptance gate. | maintainer | +| Merge permission is unavailable. | medium | medium | Continue local/remote checks, preserve the PR, and report the exact permission state; do not bypass protection. | repository admin | +| Security-provider outage delays a required scan. | medium | high | Preserve the failed evidence, do not weaken the gate, and rerun the exact HEAD when an authorized provider is available. | CI owner | +| Gate-version drift creates inconsistent no-report semantics across linked PRs. | medium | high | Pin/upgrade the shared trusted gate together, require structured-report evidence for both repositories, and keep the discrepancy visible in the PR/ADR. | CI owner | +| Central scheduler policy is weaker than this ADR when branch protection requires zero approvals. | medium | critical | Enforce the complete exact merge-gate contract in both branch protection and the scheduler; reject direct and auto merge whenever any required protection, approval, thread, check, Strix, identity, or final-refetch control is absent or non-passing. | repository/CI owner | + +## Rollback / Exit Strategy + +If a merged change regresses, revert the merge commit through a new reviewed PR and keep the original ADR lineage. If merge is blocked, leave the verified PR open and record the external permission/check condition; do not delete work or weaken the gate. + +## Affected Components + +* contextual-orchestrator branch, PR, CI, and merge state +* fast-mlsirm branch, PR, CI, and merge state +* .github/workflows/ +* docs/planning/adrs/ + +## More Information + +This ADR is the operational extension of the active Goal: implementation, evidence, review remediation, and merge are all part of completion. The Goal is dynamically expanded whenever a new review or test finding changes the acceptance boundary. diff --git a/docs/planning/adrs/0005-irt-response-matrix-contract.md b/docs/planning/adrs/0005-irt-response-matrix-contract.md new file mode 100644 index 000000000..59a379020 --- /dev/null +++ b/docs/planning/adrs/0005-irt-response-matrix-contract.md @@ -0,0 +1,173 @@ +--- +id: "0005" +title: "Require multi-item response matrices at the IRT integration boundary" +status: accepted +proposed_date: "2026-08-11" +accepted_date: "2026-08-11" +deciders: + - "repository maintainer" +consulted: + - "fast-mlsirm IRT response validators" + - "fast-mlsirm LLM judge adapter" +informed: + - "contributors" +affected_components: + - "fast-mlsirm/python/fast_mlsirm/irt_contract.py" + - "fast-mlsirm/python/fast_mlsirm/llm_judge.py" + - "fast-mlsirm/tests/test_irt_contract.py" + - "fast-mlsirm/tests/test_llm_judge.py" +effort: S +supersedes: null +superseded-by: null +related: + - path: "docs/planning/adrs/0001-fail-closed-model-judgment.md" + relation: informational + - path: "docs/planning/adrs/0002-explicit-local-mlx-evaluation.md" + relation: informational +asr_triggers: + - kind: maintainability + evidence: "A scalar judge score has no item dimension and cannot identify multiple IRT item responses." + note: "The public integration validator rejects one-item response matrices." + - kind: maintainability + evidence: "Low-level fast-mlsirm numerical primitives intentionally accept some one-item diagnostic inputs." + note: "The cross-component contract is isolated instead of silently changing every numerical primitive." +success_criteria: + - metric: "IRT item columns" + target: "every cross-component dichotomous or polytomous matrix has at least two item columns" + measurement_window: "every adapter-to-IRT conversion" + source: "validate_irt_response_matrix and regression tests" + - metric: "response-domain validation" + target: "invalid shape, non-binary values, invalid category indices, infinities, and implicit polytomous category counts are rejected" + measurement_window: "every validation call" + source: "fast-mlsirm/tests/test_irt_contract.py" +--- + +# Require multi-item response matrices at the IRT integration boundary + +## Context + +fast-mlsirm exposes dichotomous and polytomous numerical primitives, but an +LLM-as-a-Judge normally produces one scalar decision or one scalar per rubric. +That scalar is not an IRT response matrix. The user clarified that an IRT +result must contain multiple dichotomous items or multiple polytomous items. + +> Existing fast-mlsirm response APIs describe inputs as persons by items, while several low-level validators allow one item for numerical tests. +> +> LLMJudgeResult previously exposed criterion scores but had no explicit conversion boundary for IRT item rows. +> +> A missing item dimension, an inferred category count, or a continuous score silently coerced into one item can produce an apparently valid but scientifically invalid IRT run. + +## Decision Drivers + +* Make the user’s multi-item IRT requirement executable at the integration boundary. +* Keep missing responses, binary domains, and ordered category domains explicit. +* Avoid breaking low-level one-item diagnostics that are useful for numerical and security tests. +* Prevent a single LLM verdict from being represented as a fake item bank. + +## Considered Options + +* Let each IRT model decide whether a single item is acceptable. +* Globally change every fast-mlsirm numerical primitive to require two items. +* Add one public cross-component validator and require LLM judge projections to expose multiple criterion items. + +## Decision Outcome + +Chosen option: "Validate multi-item response matrices at the cross-component boundary". + +| Driver | Model-local checks | Global breaking check | Shared integration validator | +| --- | --- | --- | --- | +| User contract | inconsistent | enforced but broad | enforced where results cross systems | +| Compatibility | high | low | high for existing primitives | +| Error observability | scattered | mixed | one actionable error boundary | +| LLM scalar misuse | possible | partly prevented | rejected explicitly | + +The public validate_irt_response_matrix function accepts a 2-D persons by +items matrix with at least two item columns. Dichotomous observed values are +0/1; polytomous values are integer indices from 0 through K-1 and require an +explicit K. NaN is the only missing-value marker. LLMJudgeResult.to_irt_row +requires at least two criteria and produces a deterministic row for an +explicitly requested dichotomous or polytomous projection. + +The projection is a shape and domain bridge, not a claim of unbiased +measurement. Category-count and prompt-perturbation calibration remain +mandatory under ADR 0006. + +### Consequences + +* Good, because one-item and scalar outputs fail before reaching an IRT model. +* Good, because category semantics and missingness are explicit and testable. +* Good, because existing numerical primitives retain their current narrow + diagnostic behavior. +* Bad, because callers must collect multiple rubric criteria and multiple + persons before fitting a meaningful model. +* Bad, because equal-width projection from continuous judge scores can retain + judge bias; it is intentionally not a calibration substitute. + +## Pros and Cons of the Options + +### Model-local checks + +* Good, because there is no new public helper. +* Bad, because a scalar emitted by an external judge can still be misrouted. +* Bad, because each model family can drift in shape and missing-value behavior. + +### Global breaking check + +* Good, because every numerical entry point would enforce the same minimum. +* Bad, because low-level diagnostics and existing compatibility tests use + one-item inputs intentionally. +* Bad, because a broad breaking change does not explain whether the source + result had multiple rubric items. + +### Shared integration validator (chosen) + +* Good, because it enforces the requirement exactly where external results + become IRT data. +* Good, because it keeps the numerical core stable and makes the contract + reusable by future adapters. +* Bad, because callers can bypass it if they deliberately call low-level + functions; documentation and review must keep the boundary visible. + +## Problem Register and Remediation Directions + +| Finding | Direction | State | +| --- | --- | --- | +| A scalar LLM verdict is not an IRT item matrix. | Require at least two criterion items in LLMJudgeResult.to_irt_row. | Implemented | +| A one-item persons-by-items matrix can look structurally valid. | Reject fewer than two item columns in the public integration validator. | Implemented | +| Polytomous category count can be inferred from a partial sample. | Require explicit n_categories at the integration boundary. | Implemented | +| Continuous criterion scores are not inherently ordinal observations. | Keep the projection explicit and calibrate category-count effects before fitting. | Ongoing | +| IRT estimation quality also depends on persons, item information, and factor coverage. | Add sample-size, item-information, and factor-anchor gates to the benchmark before interpreting fit. | Required next | +| Public numerical fitters could still receive one-item matrices when a caller bypassed the cross-component helper. | Enforce the same multi-item validator at public IRT fitter boundaries while leaving explicitly diagnostic low-level primitives compatible. | Implemented on fast-mlsirm follow-up branch; exact-head integration pending | +| contextual-orchestrator previously discarded fast-mlsirm criterion scores after deriving accepted/rejected, so downstream IRT consumers could not see the multi-item output contract. | Preserve only validated criterion scores and the fast-mlsirm dichotomous multi-item projection in verification metadata; reject an invalid projection rather than padding, collapsing, or repairing it. | Implemented in current local head; exact-head CI/review follow-up required | +| Low-level APIs and integration APIs have different compatibility goals. | Keep this contract documented and do not silently apply it to every existing primitive. | Implemented | + +## Risks and Mitigations + +| risk | likelihood | impact | mitigation | owner | +| --- | --- | --- | --- | --- | +| Callers bypass the validator. | medium | high | Export one public helper, test the LLM projection, and review IRT call sites for direct coercion. | maintainer | +| Equal-width bins create artificial category thresholds. | high | high | Run category-count perturbation and calibration experiments; do not report uncalibrated IRT estimates as ground truth. | evaluation owner | +| Requiring multiple criteria reduces one-criterion convenience. | medium | medium | Preserve ordinary scalar judge use; enforce the requirement only in to_irt_row. | maintainer | + +## Rollback / Exit Strategy + +If compatibility evidence requires one-item numerical primitives, retain the +integration validator and revert only an overly broad caller-level adoption. +Do not remove the multi-item contract or silently convert a scalar judge result +into an IRT item. + +## Affected Components + +* fast-mlsirm/python/fast_mlsirm/irt_contract.py +* fast-mlsirm/python/fast_mlsirm/llm_judge.py +* fast-mlsirm/tests/test_irt_contract.py +* fast-mlsirm/tests/test_llm_judge.py +* downstream IRT adapters and benchmark data preparation + +## More Information + +The local Zotero library now contains the IRT and response-category references +used for this decision. Relevant local item keys are MYPNHHWJ (ordered response +categories), CWY355RP (response categories), and DXADSGKY (IRT introduction). +The implementation and review gate are intentionally independent of any single +publisher or LLM provider. diff --git a/docs/planning/adrs/0006-polytomous-llm-judge-bias-calibration.md b/docs/planning/adrs/0006-polytomous-llm-judge-bias-calibration.md new file mode 100644 index 000000000..a6afbd9ca --- /dev/null +++ b/docs/planning/adrs/0006-polytomous-llm-judge-bias-calibration.md @@ -0,0 +1,387 @@ +--- +id: "0006" +title: "Calibrate polytomous LLM judgment against category and prompt bias" +status: accepted +proposed_date: "2026-08-11" +accepted_date: "2026-08-11" +deciders: + - "repository maintainer" +consulted: + - "fast-mlsirm LLM judge adapter" + - "fast-mlsirm IRT response research" + - "local Zotero literature collection" +informed: + - "contributors" +affected_components: + - "fast-mlsirm/python/fast_mlsirm/llm_judge.py" + - "fast-mlsirm/tests/test_llm_judge.py" + - "contextual_orchestrator/orchestrator.py" + - "docs/benchmarks/" + - "docs/planning/adrs/" +effort: M +supersedes: null +superseded-by: null +related: + - path: "docs/planning/adrs/0001-fail-closed-model-judgment.md" + relation: influences + - path: "docs/planning/adrs/0005-irt-response-matrix-contract.md" + relation: influences +asr_triggers: + - kind: security + evidence: "Absolute LLM scoring can shift under rubric order, score identifiers, reference scores, user framing, and answer-option order." + note: "Perturbation deltas are recorded instead of assuming one prompt is unbiased." + - kind: maintainability + evidence: "The user reports a positive tendency as the number of choices grows, but current literature does not prove a monotone LLM-specific effect." + note: "Treat the claim as a measured hypothesis and fail the benchmark gate on material positive drift." + - kind: performance + evidence: "A calibration design that calls a judge must preserve contextual-orchestrator routing, tracing, and local-model controls." + note: "The fast adapter accepts an injected orchestrator and adds no provider-specific transport." +success_criteria: + - metric: "category-count score drift" + target: "the same answer and rubric are evaluated at K in {2,3,5,7}; mean score and acceptance deltas are reported with uncertainty" + measurement_window: "every polytomous calibration benchmark" + source: "contextual-orchestrator traces and fast-mlsirm result records" + - metric: "prompt perturbation invariance" + target: "rubric order, score-ID labels, reference presence, answer-option order, and positive/negative framing are paired and compared" + measurement_window: "every judge calibration release" + source: "benchmark artifacts and regression reports" + - metric: "IRT-safe output" + target: "only multi-item dichotomous or explicitly categorized polytomous rows reach fast-mlsirm" + measurement_window: "every IRT conversion" + source: "ADR 0005 validator and tests" + - metric: "local judge service reliability" + target: "every compared model reports bounded latency, timeout, structured-parse success, token usage, and score drift; a timeout or malformed response is a failed comparison, not an omitted datum" + measurement_window: "every local-model calibration benchmark" + source: "contextual-orchestrator traces and benchmark records" +--- + +# Calibrate polytomous LLM judgment against category and prompt bias + +## Context + +Polytomous judgment introduces more than an IRT data-shape problem. A judge +must map evidence to ordered categories, and the category labels, rubric +order, reference examples, response-option order, and user framing can all +become unintended cues. The user’s hypothesis that more choices can make an +LLM more positive is therefore a high-value risk, but it must be tested rather +than encoded as an unverified universal law. + +> The local copy of Evaluating Scoring Bias in LLM-as-a-Judge reports scoring shifts caused by rubric order, score identifiers, and reference-answer scores. +> +> The local copies of the ICLR and NAACL studies report LLM selection or position bias under option changes, including experiments with different option counts and reordered choices. +> +> The local sycophancy study reports more positive feedback when user framing signals that a passage is liked, so agreement and politeness cannot be treated as evidence of quality. + +## Decision Drivers + +* Separate direct evidence of bias from the specific monotonic-positive hypothesis. +* Avoid making a multi-category choice list the sole source of an ordinal score. +* Keep judgment semantic and evidence-based; keyword matching remains forbidden. +* Preserve contextual-orchestrator routing for every fast-mlsirm judge call. +* Produce a repeatable benchmark that can distinguish category information from + category-induced positivity. + +## Considered Options + +* Use a single K-way score prompt and assume more categories add useful information. +* Collapse every judgment to a binary keyword or lexical match. +* Use structured criterion scoring with fixed anchors, derive acceptance in the + runtime, and run paired category/prompt perturbation calibration. +* Replace the local judge with a hosted evaluator without perturbation checks. + +## Decision Outcome + +Chosen option: "Use structured, criterion-level scoring plus perturbation +calibration; never assume that more categories are better or neutral". + +| Driver | Single K-way score | Keyword fallback | Structured calibrated judge | +| --- | --- | --- | --- | +| Semantic validity | medium | poor | high | +| Category-count bias visibility | low | none | explicit deltas | +| Language/negation robustness | medium | poor | model evidence with strict schema | +| IRT compatibility | ambiguous | binary-only and invalid as a default | explicit multi-item projection | +| Local runtime integration | simple | bypasses judge semantics | contextual-orchestrator trace | + +The default judge prompt keeps criterion-level scores, explicitly says not to +reward answer length, politeness, agreement, or a larger number of response +options, and derives accepted from the numeric score in the adapter rather +than trusting a redundant model boolean. A result can be projected into an +IRT row only through the multi-item contract in ADR 0005. + +The calibration benchmark must evaluate the same semantic case under K values +2, 3, 5, and 7, balanced category labels and rubric orders, with and without +reference examples, and with positive/negative/neutral framing controls. It +must report score mean, category occupancy, acceptance rate, pairwise +agreement, and the signed shift from a human or deterministic gold label when +available. A positive shift with more categories is a failure signal, not a +normalization target. + +The first end-to-end local MLX sweep is recorded in +`docs/benchmarks/2026-08-11-polytomous-llm-judge.md`. With the same worker +answer and rubric, the observed derived scores were 1.00, 0.75, 0.50, and +0.9167 for K=2,3,5,7, with acceptance changing across the sweep. This is not +evidence of a monotone positive effect; it is evidence that category-count +sensitivity is real enough to block an uncalibrated IRT interpretation. + +The same run also exposed provider-format failure modes: a local model emitted +numeric criterion keys, copied an instruction phrase into a JSON key, and +returned decimal values for integer categories. The fast adapter now uses an +exact literal schema containing the validated criterion IDs and explicit +ordered anchors, accepts only mathematically integral category values, and +rejects the rest without keyword or positional repair. + +After the prompt was made explicit about JSON-only output, no markdown fences, +integer category values, and a numeric top-level score, the same Gemma 4B case +was repeated twice at each K. All eight calls parsed successfully through the +contextual-orchestrator route. Mean scores were 0.50, 0.50, 0.00, and 0.75 for +K=2, 3, 5, and 7; acceptance counts were 0/2, 0/2, 0/2, and 2/2. The result +reproduces category-count sensitivity but is non-monotonic, so it is not +evidence for a universal positive-with-more-options law. + +A separate three-criterion case compared the cached Gemma 31B judge at the +same K values. Its derived score stayed at 0.3333 and acceptance stayed 0/2 at +every K, while the criterion categories moved with K; the eight responses all +parsed successfully. The cached 32B DeepSeek judge did not produce a +structured response within the 180-second request bound on its first K=2 call, +so the comparison was stopped and no quality conclusion was drawn for it. +This adds a reliability gate: the largest or most capable-looking local model +is not a performance win if it cannot return a bounded, parseable judge result. + +For high-stakes polytomous use, the fast adapter now provides an explicit +`category_method="cumulative_threshold"` mode and a bounded +`category_method="binary_threshold"` mode with an explicit `category_count`. +The binary method asks the model whether each criterion clears each ordered +boundary, validates the Boolean responses as monotone, and derives the +category from the number of cleared thresholds. When callers provide +`category_count` without a method, binary thresholds are now the default; +direct K-way output remains an explicit calibration-only choice. This reduces +dependence on one K-way score-ID choice, but it does not make the judge +unbiased: the same answer must still be calibrated across K, prompt +perturbations, models, and gold labels. + +The first direct-versus-threshold extension run on 2026-08-12 used the same +Gemma 4 e4b MLX judge, worker answer, two criteria, disabled thinking, +temperature 0, two repeats per K, and the same contextual-orchestrator route. +All 16 responses parsed and produced two-item polytomous rows accepted by the +ADR 0005 validator. Direct K-way scores were 1.0000 at K=2,3,5,7; cumulative +threshold scores were 1.0000, 1.0000, 0.5000, and 0.3333 respectively, with +acceptance changing from yes to no at K=5. This is a useful replication of +category-method sensitivity, not evidence of a universal directional bias. + +A second 2026-08-12 replication used the cached local Llama 3B judge under +neutral, liked, and disliked framing for one good and one unsafe release plan. +The good plan parsed in 11/18 calls and was accepted in 5/11 parsed calls; the +unsafe plan parsed in all 18 calls and was rejected in every case. Direct K-way +judging parsed 8/9 good-plan calls, while cumulative thresholds parsed 3/9. +The good-plan direct scores were framing-sensitive at K=7 (neutral `0.5833`, +liked/disliked `0.8333`) but equal at K=5 (`0.7500`), and the K=2/K=7 path was +not monotone. Seven good-plan failures were retained as failures: five invalid +JSON responses, one out-of-range category, and one non-monotone threshold +vector. This is evidence of local-model format and framing sensitivity, not a +universal positive-choice-count law. + +A same-route retry probe then evaluated a separate good release plan with the +same 3B model, two criteria, K in `{2,5,7}`, and neutral/liked/disliked +framing. All nine direct K-way responses parsed, but scores were respectively +`(0.0000, 1.0000, 0.8333)`, `(0.0000, 1.0000, 0.9167)`, and +`(0.0000, 0.7500, 1.0000)`. Four cumulative-threshold calls at K in +`{2,3,5,7}` each failed strict parsing; one identical second +contextual-orchestrator completion per failure recovered none. K=2 failed the +boundary-array shape contract, while K=3/5/7 failed monotonicity. This probe +does not justify a blind retry: any future recovery must be independently +specified, remain on the contextual-orchestrator path, retain first/final +parse status and cost, and accept only a final strict schema result. + +The binary-threshold calibration method was subsequently optimized at +fast-mlsirm exact commit `61e6be9`: when the injected contextual-orchestrator +exposes its already-bounded `client.local_concurrency`, independent boundary +calls reuse that limit. Generic injected orchestrators remain sequential by +default. This is a transport/latency optimization only; it does not reorder +criteria in the retained evidence, repair malformed output, infer a threshold, +or change the fail-closed monotonicity rule. The live MLX follow-up retained +both valid and failed cases, including unsafe K=5/K=7 results at `5.756/7.719 s` +and `2,422/3,620` tokens, while safe K=5/K=7 remained non-monotone failures. + +### Consequences + +* Good, because the suspected positive drift becomes measurable and + reproducible instead of being hidden in an aggregate score. +* Good, because score-ID and rubric-order perturbations are treated as + first-class regression cases. +* Good, because the fast-mlsirm adapter remains provider-neutral and every + call is visible in contextual-orchestrator traces. +* Bad, because calibration costs additional local model calls and requires + gold or human comparisons for strong conclusions. +* Bad, because a small local model can fail to emit valid structured output; + malformed output remains rejected rather than repaired lexically. + +## Pros and Cons of the Options + +### Single K-way score + +* Good, because it is easy to prompt and cheap to implement. +* Bad, because score-ID and category-count effects can be mistaken for quality. +* Bad, because it does not distinguish an ordinal response from a model’s + arbitrary numeric preference. + +### Keyword fallback + +* Good, because it is cheap. +* Bad, because it violates the explicit user requirement and fails on + negation, multilingual evidence, and mixed reports. +* Bad, because it cannot measure an ordered latent trait. + +### Structured calibrated judge (chosen) + +* Good, because it retains semantic evaluation while exposing perturbation + sensitivity. +* Good, because criterion scores become explicit multi-item candidates for + ADR 0005 rather than a hidden scalar. +* Bad, because calibration is more expensive than one unchallenged call. + +### Hosted evaluator + +* Good, because it may produce stronger raw judgments. +* Bad, because it violates the local-model performance objective and does not + remove category or prompt bias automatically. + +## Problem Register and Remediation Directions + +| Finding | Direction | State | +| --- | --- | --- | +| A local judge returned inconsistent score and accepted fields during a real MLX run. | Derive acceptance from the validated numeric score; reject non-JSON or non-boolean fields, never keyword-repair. | Implemented | +| A local category judge emitted numeric keys, instruction text as a key, and decimal category values. | Prompt an exact literal schema with criterion IDs and anchors; accept integral JSON numbers only and reject malformed output. | Implemented | +| A local Llama 3B judge emitted a non-numeric top-level `score` alongside integer category values; ignoring the redundant field would weaken the strict schema. | Validate the redundant top-level score's finite numeric shape, but derive the effective score and acceptance only from validated criterion categories; reject malformed fields without repair. | Implemented | +| The positive-with-more-options hypothesis lacks a direct monotonic LLM proof. | Add K=2/3/5/7 paired calibration and treat positive drift as a gate failure. | Required next | +| A real K=2/3/5/7 sweep changed the same case’s derived score and acceptance. | Keep category projection experimental, report the complete sweep, and block production IRT claims until replicated calibration is stable. | Implemented as benchmark; gate ongoing | +| A prompt-hardened two-repeat K=2/3/5/7 sweep parsed reliably but remained non-monotonic (K=5 below K=2/3; K=7 above them). | Keep strict parsing and report replication variance; expand balanced cases, criterion/rubric order, answer-option count/order, framing, and model variants before drawing a directional bias conclusion. | Required next | +| Score IDs and rubric order can change absolute judgments. | Randomize or balance labels/order and record signed perturbation deltas. | Required next | +| User framing can induce positive or negative sycophantic feedback. | Add neutral, liked, disliked, and authored framing controls; compare to content-only gold. | Required next | +| Equal-width score bins can create artificial polytomous thresholds. | Implement cumulative threshold judging or calibrated category mapping before production IRT use. | Ongoing | +| A K-way prompt can make the model choose among many score identifiers even when the underlying evidence is unchanged. | Expose opt-in cumulative-threshold judging with explicit K, exact Boolean arrays, monotonicity validation, derived categories, and the same multi-item IRT validator; keep direct K-way output experimental until paired calibration supports it. | Implemented on fast-mlsirm follow-up PR; calibration ongoing | +| A fresh same-route 3B MLX direct probe scored the unsafe case `0.0`, `0.5`, `0.8333` and the partial-evidence case `0.0`, `1.0`, `0.0` at K=`2,5,7`; acceptance therefore changed with the number of score identifiers | Resolve an omitted method to bounded binary thresholds whenever `category_count` is present; retain direct K-way only for explicit calibration and record every semantic miss as failed evidence rather than repairing it | Implemented in fast-mlsirm follow-up; exact-head review required | +| The paired binary probe returned score `0.0` for both safe and unsafe answers at K=`5,7`, avoiding the direct positive drift in this sample but under-recognizing the safe answer | Keep the default fail-closed and require held-out human/gold recall, category occupancy, and parse/provider denominators before treating any local model/prompt as IRT-ready; do not claim bias removal from this probe | Observed 2026-08-14; calibration gate remains open | +| The real `_FastMLSIJudgeAdapter` path with fast-mlsirm `9d18f53` and contextual-orchestrator `a0a354a` selected binary thresholds when K=5 was supplied without a method: the unsafe case produced a valid rejected `(0,0)` result in 8 calls, while the safe case failed monotonicity after 8 calls | Treat the integrated unsafe result as contract evidence only and retain the safe failure as a calibration datum; keep malformed/non-monotone output fail-closed, record trace/usage/latency, and require held-out human/gold recall before IRT production | Observed 2026-08-14; integrated default verified, semantic calibration remains open | +| The real MLX safe-case failure was initially only the text `criterion thresholds must be monotone`, but fast-mlsirm `d1eca0c2fed89991e647802f0b27a91f0f6fe2bd` captured `semantic_status=non_monotone`, `parse_status=passed`, `8/8` completed calls, `8` trace steps, and `2,639` provider tokens in bounded exception evidence | Keep complete semantic failures in the denominator and separate non-monotonicity from provider/parse failures; retain ordered records and usage without retry, keyword matching, positional repair, or IRT coercion | New evidence 2026-08-14; failure-evidence contract implemented, calibration remains open | +| A fresh anchored K=5 probe through the real adapter showed Gemma 4 e4b returning strict `(4,4)`/score `1.0` in `3,031` tokens and `11.96 s`; the same rubric left Llama 3B with repeated safe false negatives and Llama 1B with eight malformed boundary responses | Compare judge models on balanced gold cases using semantic recall, false-positive/false-negative rates, parse/provider denominators, category occupancy, latency, and usage; use the stronger result as a candidate only, never as a universal bias correction | New evidence 2026-08-14; fast-mlsirm `dd44a95`, model calibration remains open | +| The binary prompt previously allowed a K-only ordinal interpretation with no criterion-specific definitions, so the model could not reliably distinguish intermediate categories even when the answer contained operational controls | Allow complete per-criterion `category_anchors` of length K, bind each boundary to its matching anchor as untrusted rubric data, record anchor presence, and keep omitted-anchor runs exploratory rather than treating them as calibrated IRT evidence | Implemented in fast-mlsirm `dd44a95`; held-out anchor/gold calibration required | +| A cumulative-threshold prompt can still produce inconsistent or non-monotone boundary judgments, and its score can differ from direct K-way output. | Reject non-Boolean, wrong-length, or false-then-true vectors; record category method, K, trace, usage, parse success, score, acceptance, and perturbation deltas in every benchmark. | Implemented in adapter/tests; calibration ongoing | +| Multiple criteria can still be correlated or cover one latent dimension poorly. | Require item coverage review, factor anchors, and sample-size checks before interpreting IRT fit. | Required next | +| A single judge call can hide model drift. | Preserve contextual-orchestrator trace, model identity, prompt variant, category count, and usage in benchmark records. | Implemented in adapter trace and the 2026-08-11 benchmark artifact | +| A one-person, two-item matrix can pass a shape check while remaining insufficient for IRT estimation. | Require multiple persons, item-information, and factor-coverage checks before fitting or interpreting an IRT model. | Required next | +| A larger local judge can preserve the aggregate score while moving criterion categories, and a cached 32B judge timed out before producing structured output. | Gate model comparisons on bounded latency, timeout rate, strict-parse success, token usage, category occupancy, and score/acceptance drift; never treat a timeout as a missing or positive result. | Implemented in the 2026-08-12 benchmark; reliability calibration ongoing | +| A cached local 3B judge failed to emit valid or ordinally coherent structured output in 7/18 good-plan calls, and framing changed some K=7 scores. | Keep every malformed/monotonicity failure in the denominator; compare a separately measured bounded retry or stronger local judge only through contextual-orchestrator, and never add keyword, positional, or silent-drop repair. | Recorded in the 2026-08-12 benchmark; reliability/framing calibration required | +| An identical second contextual-orchestrator completion recovered none of four cumulative-threshold failures in a follow-up 3B probe, while direct K-way scores shifted across K and framing. | Keep blind retry out of the production contract. If recovery is pursued, compare a bounded independent binary-threshold decomposition or stronger local judge on held-out paired cases, record added latency/tokens and first/final parse status, and preserve strict fail-closed parsing. | Measured 2026-08-12; Goal expanded and calibration required | +| A live bearer-authenticated gateway probe on the same Llama 3B answer and two criteria at `K=5` produced direct `1.0000` (`4/4`, accepted) versus cumulative-threshold `0.0000` (`0/0`, rejected), with both strict parses valid and one contextual-orchestrator trace step each. | Treat this as paired method sensitivity, not as a positive-bias conclusion. Keep method/K/trace/usage in the denominator, compare balanced held-out cases, and retain the multi-item polytomous validator; never repair the disagreement lexically or positionally. | New evidence 2026-08-12; calibration required | +| A fresh two-case 3B MLX probe through contextual-orchestrator produced direct scores of `0.5 -> 1.0 -> 1.0` for a safe release plan and `0.0 -> 0.0 -> 0.3333` for an unsafe plan at K `2,5,7`; cumulative thresholds parsed only at K=5 and failed JSON/monotonicity at K=2/7. | Preserve all 12 comparisons, including four strict-parse failures, in the denominator. Do not promote direct or cumulative to an unbiased default; add an opt-in bounded binary-threshold decomposition and compare its latency, calls, tokens, semantic recall, and human/gold agreement on held-out paired cases. | New evidence 2026-08-14; Goal/ADR expanded, calibration remains required | +| The binary-threshold follow-up reduced each boundary to a Boolean contextual-orchestrator call, but the safe case still failed monotonicity at K=5/7 while the unsafe case parsed at score `0.0` using 8/12 calls and `2,606/3,940` tokens. | Keep binary decomposition experimental and fail-closed. Record its call budget and semantic under-recognition; do not short-circuit, synthesize, or repair higher categories without an explicit ordinal measurement design and held-out gold evidence. | New evidence 2026-08-14; method implemented, calibration required | +| OA metadata, local Zotero attachment state, and network retrievability are separate: Zotero `9.0.6` exposes read-only Local API reads; the official Jones--Loe SAGE PDF returned anti-bot `403`; the official Iannario De Gruyter PDF returned a WAF `202` response with zero bytes; and Zotero items `CWY355RP`/`MYPNHHWJ` have no child attachment. | Record rights, canonical landing/PDF URLs, and retrieval evidence separately. Attach only byte-verified original PDFs through an authorized Zotero/API route, and retry from an authorized route or a write-capable Zotero version. Never regenerate, OCR-rebuild, or substitute a PDF while claiming it is the original. | Required follow-up | +| OpenAlex/Unpaywall/Crossref revalidation identifies Jones--Loe as gold OA with CC BY metadata and Iannario as a CC BY 4.0 published version; local Zotero rights fields agree (`Open access` and `Creative Commons Attribution 4.0 International`), but neither record has a child attachment. The Cao et al. AAAI-26 study also shows substantial option-only answer bias when the question is removed, with contamination more explanatory than position or answer popularity. | Keep Jones--Loe and Iannario citation-only until an authorized route yields byte-verified original parent attachments; never count 403/202 HTML, reconstructed files, or unauthorized mirrors. Add option-only/no-question, shuffled-option, replaced-distractor, and contamination-aware controls so a positive score at larger `K` is not misattributed to option count. Keep Cao citation-only because its PDF is all-rights-reserved. | Revalidated and expanded 2026-08-14; rights are corrected, while PDF retrieval and calibration controls remain required | +| Follow-up OA retrieval found a rights/access conflict for Iannario: the [De Gruyter article](https://www.degruyterbrill.com/document/doi/10.1515/ijb-2021-0013/html) and Crossref/OpenAlex metadata state CC BY 4.0, while the [IRIS record](https://www.iris.unina.it/handle/11588/877609) labels its editorial PDF authorized-users-only and [RePEc](https://ideas.repec.org/a/bpj/ijbist/v18y2022i2p593-611n2.html) reports full-text restriction. The [SAGE Jones--Loe PDF](https://journals.sagepub.com/doi/pdf/10.1177/2158244013489691) remains publisher-original but returned anti-bot 403 to the local downloader. | Treat the license and retrieval state as separate fields; do not attach either binary until an authorized route produces a byte-verified original and the rights are reconciled. Do not use ResearchGate, a WAF/HTML response, OCR, or a reconstructed PDF as the claimed original. | Rechecked 2026-08-14; PDF attachment remains an explicit Goal item | +| Before this follow-up, fast-mlsirm had no bounded reusable control that ran baseline, option-only/no-question, shuffled-option, and distractor-replacement variants through the existing contextual-orchestrator judge while retaining provider/parse/IRT failures and gold agreement. | Implement `JudgeCalibrationCase`/`JudgeCalibrationReport` and run every variant through an injected `ContextualOrchestratorJudge`; preserve contamination status, caller-supplied gold categories, multi-criterion polytomous rows, trace/usage, and every failure. Do not retry, repair, keyword-match, infer category positions, or interpret score deltas as causal option-count bias. | Implemented in fast-mlsirm at exact head `5a072705c840ea70d87a73bf737d5b193ef428cb` 2026-08-14; exact-head review/check follow-up required | +| A Gemma 4 e4b loopback smoke through the new paired controls completed four variants for both 3-option and 5-option cases at K=`3`; all 8 rows were `[2,2]`, all 8 matched the held-out gold categories, and every paired score delta was `0.0` (`20.442 s/7,868` and `20.277 s/8,110` provider tokens). | Record this as a bounded integration and negative-observation smoke only. Expand persons/items, correct-option positions, option counts, models, framing, contamination controls, and human/gold anchors before estimating or rejecting a general positive option-count effect or IRT readiness. | Observed 2026-08-14; semantic bias calibration remains open | +| A same-case K=`3` model comparison through the real MLX route gave 1B Llama `0/4` passed with four bounded JSON/format failures; 3B Llama `4/4`, gold `4/4`, all deltas `0.0`, `24.139 s`, `7,960` tokens; Gemma 4 e4b `4/4`, gold `4/4`, all deltas `0.0`, `39.731 s`, `7,860` tokens. | Treat 1B as a structured-output reliability failure for this prompt, and 3B/Gemma as candidate models only. Expand balanced persons/items, option positions/counts, framing, contamination controls, and human/gold anchors before model promotion or IRT interpretation; latency/token differences are workload evidence, not quality proof. | Observed 2026-08-14; semantic calibration remains open | +| `ContextualOrchestratorJudge` previously validated only that an injected object had `complete()`, leaving a direct-provider or unrelated transport possible even though every Judge call is required to use contextual-orchestrator. | Require the exact `contextual-orchestrator-contract-v1` provenance marker before construction, expose it from `_FastMLSIJudgeAdapter`, and reject unmarked transports before any call. Marked test doubles are contract tests only and do not loosen production routing. | Fixed in the current cross-repository follow-up; focused tests and exact-head review/check follow-up required | +| A fresh held-out Gemma 4 e4b paired calibration through the exact contextual-orchestrator route returned the highest category `[2, 2]` for baseline, option-only, shuffled-option, and replaced-distractor variants; all four passed and matched gold, but every criterion was saturated and every paired score delta was `0.0` (4 rows, 7,626 provider tokens). | Treat this as ceiling saturation, not evidence of no positive-choice-count bias or judge neutrality. Add per-criterion category-occupancy reporting, expand difficult/partial/unsupported gold items and option-count/position/model strata, and block IRT interpretation until held-out recall, false-positive rates, and non-ceiling occupancy are demonstrated. Never repair, keyword-match, or collapse saturated rows into a favorable result. | Observed 2026-08-14 on contextual `7f47665f0d837debc9db82060347ff3502469239` + fast `830d3c0c159a52a5131859dd549b0d89f8b9d02d`; Goal/ADR expanded, occupancy implementation and calibration follow-up required | +| A fresh live K=`3` anchored comparison through contextual-orchestrator found e4b produced valid safe `[2,2]` and unsafe `[1,1]` rows, while 3B failed the unsafe case as non-monotone, 31B failed the safe boundary after `96.93 s`, and DeepSeek completed zero of eight boundary groups within about `100 s` each. | Use evidence-based role eligibility: exclude 31B, DeepSeek, and the previously failing 1B from `verifier`; select e4b as the current primary and keep 3B as an explicit lower-priority candidate. Preserve all semantic, parse, timeout, and latency failures; this is not a bias correction or IRT-readiness claim, and larger balanced gold/perturbation calibration remains mandatory. | Observed 2026-08-14; Goal/ADR expanded, contextual routing updated, calibration remains open | +| A balanced held-out K=`3`/K=`7` edge-position run through contextual-orchestrator `d3480cc` and fast-mlsirm `dbbd41d` covered first/last correct-option positions and four presentation variants. It produced 11 valid `[2,2]` rows and 5 strict `JudgeFormatError` failures across 16 paired outcomes/64 boundary calls; every valid criterion was at ceiling category `2`, only one group had a complete baseline/control comparison, and elapsed time was `1,044.7 s`. | Treat the result as ceiling-saturated, incomplete reliability evidence: preserve all five failures, do not infer neutrality or positive option-count bias from `11/11` conditional gold agreement, and do not interpret the rows as IRT-ready. Add harder partial/unsupported gold items, non-ceiling anchors, more persons/items/models, balanced option counts/positions, and an explicit completion-time budget before the next calibration gate. | New evidence 2026-08-14; Goal/ADR expanded, bias calibration and completion-path reliability remain open | + +| A dedicated-port Gemma 4 e4b run through contextual-orchestrator `63451a0` and fast-mlsirm `3c2fecf` evaluated partial and unsupported held-out answers at K=`3`/K=`7` with edge correct-option positions and four controls. It produced 15 passed and 1 strict non-monotone failure across 16 outcomes/128 boundary calls in `202.781 s`; conditional gold exact agreement was `5/15`. Evidence-quality occupancy covered categories 0/1/2 evenly, but risk-awareness had occupancy `{0:7,1:0,2:8}`. Option-only controls raised unsupported evidence quality from 0 to 1 and partial baselines were mostly over-scored `[2,2]` against gold `[1,1]`. | Keep all outcomes, semantic failures, and control deltas in the denominator; treat this as evidence of semantic miscalibration and control sensitivity, not a positive-K causal estimate or IRT-ready dataset. Add harder intermediate risk anchors, more human/gold items and persons, model/position/count/framing strata, and a completion-time budget before verifier promotion. No keyword matching, retry, repair, positional inference, or silent drop. | New evidence 2026-08-14; Goal/ADR expanded, semantic calibration remains open | +| A dedicated-port Llama 3B rerun through contextual-orchestrator `62100d3` and fast-mlsirm `57795b1` evaluated partial K=`3`/correct-first and unsupported K=`7`/correct-last groups with the same four controls. It produced 5 passed and 3 strict non-monotone failures across 8 outcomes/64 boundary calls in `67.454 s`; all valid rows saturated at `[2,2]`, category occupancy was maximum-only for both criteria, and conditional gold exact agreement was `0/5`. | Keep Llama 3B out of the verifier role and treat the result as ceiling-saturated semantic/reliability evidence. Preserve all failures and valid over-scores in the denominator; require non-ceiling held-out gold recall, false-positive/false-negative rates, and replicated option-count/position/framing strata before any promotion or IRT interpretation. No keyword matching, retry, positional inference, category repair, or silent drop. | New evidence 2026-08-14; Goal/ADR expanded, 3B verifier eligibility remains closed | +| The K-stratified report follow-up used contextual-orchestrator `b30697d06d1160b6a892fbdd26112316fb53a202` and fast-mlsirm `22596ab714e20e9b4d1aa7f50f621deec010f622` with Gemma 4 e4b at K=`3`/`5`, partial/unsupported gold anchors, and four controls. All 16 outcomes/64 boundary calls were structurally valid in `221.505 s`, but exact gold agreement was only `7/16`; all partial rows were `[2,2]`, and unsupported K=`5` shuffled became `[0,1]` (`+0.25` score delta). | Use the new option-count/variant summary as descriptive measurement infrastructure, not a causal K effect or correction. Treat the K=`5` shuffled shift as control sensitivity, preserve the full denominator, and expand balanced positions/orders, non-ceiling anchors, persons/items/models, and human/gold strata before verifier promotion or polytomous IRT interpretation. No keyword matching, retry, positional inference, repair, or silent drop. | New live evidence 2026-08-14; Goal/ADR expanded, semantic calibration remains open | +| The current exact-head Gemma 4 e4b `direct` sweep through contextual-orchestrator completed 12/12 anchored two-criterion outcomes at K=`2,3,5,7`: safe remained score `1.0`, unsafe remained `0.0`, and partial moved from `0.0` at K=`2` to `0.5` at K=`3,5,7`. | Treat the partial step change as category-count sensitivity, not a monotone positive-bias result. Keep direct K-way output calibration-only, retain all multi-item rows and costs, and require replicated non-ceiling human/gold agreement before any IRT or verifier claim. | Verified 2026-08-14; contextual `bc882c0`, fast `a536292`, calibration remains open | +| The same exact-head binary-threshold run produced valid K=`3` rows for safe `[2,2]`, unsafe `[0,0]`, and partial `[0,0]`; at K=`5`, unsafe remained valid `[0,0]`, while safe and partial had all 8 boundary calls parse but failed the monotonicity contract. | Preserve complete non-monotone outcomes as semantic failures, not transport omissions; keep binary threshold fail-closed with no repair or category synthesis, report its call/token/latency cost, and require balanced gold recall and category occupancy before IRT use. | Verified 2026-08-14; contextual `bc882c0`, fast `a536292`, semantic calibration remains open | + +| The current exact-head safe-case integrated recheck through the contextual adapter completed all four K=`3` binary boundaries with two criteria, score `1.0`, categories `[2,2]`, row `[2,2]`, 4.949 s, and 1,923 provider tokens. The paired gateway probe also showed a cold/warm-up width-1 latency of 2,170.42 ms, a width-4 wave of 591.34 ms, unique successful IDs, and a bounded fifth-request `503`. | Preserve this as route/contract and cost evidence only. Do not treat a single safe case or throughput plateau as semantic calibration, neutrality, or IRT readiness; retain the existing K-stratified non-monotone failures and require balanced non-ceiling gold, occupancy, false-positive/negative, and perturbation strata before promotion. | Verified 2026-08-14; context `8f922d8`, fast `47c5fbd`, calibration remains open | + +## Risks and Mitigations + +| risk | likelihood | impact | mitigation | owner | +| --- | --- | --- | --- | --- | +| Calibration cost is too high on local hardware. | high | medium | Use a bounded paired suite and reuse fixed cases; do not omit the perturbation axes that define the risk. | evaluation owner | +| A prompt instruction suppresses but does not measure bias. | high | high | Keep perturbation experiments and report deltas; prompt wording alone is not evidence of neutrality. | evaluation owner | +| Positive framing is confused with answer quality. | medium | high | Use content-only gold labels and neutral controls; keep sycophancy as a separate metric. | evaluation owner | +| Category projection is used as a validated IRT instrument too early. | medium | high | ADR 0005 rejects scalar/one-item input and this ADR blocks uncalibrated category claims. | maintainer | +| A large local model consumes device time or stalls before strict output. | medium | high | Use a bounded request timeout, record every timeout/parse failure, compare quality only on completed structured results, and keep a smaller verified fallback for exploratory work. | evaluation owner | + +## Rollback / Exit Strategy + +If a mitigation prompt reduces agreement with gold labels, roll back only that +prompt revision and keep the perturbation benchmark. Do not roll back to +keyword matching or to an unobserved single K-way score. If cumulative +threshold judging is not implemented, keep equal-width projection explicitly +experimental and do not use it for a production IRT claim. + +## Affected Components + +* fast-mlsirm/python/fast_mlsirm/llm_judge.py +* fast-mlsirm/tests/test_llm_judge.py +* fast-mlsirm/python/fast_mlsirm/irt_contract.py +* contextual-orchestrator orchestration, trace, and local-MLX benchmark paths +* docs/benchmarks/ and future category-bias reports + +## More Information + +The local Zotero collection was searched through the running Zotero Local +Connector API. It contains four accessible PDF attachments: Li et al. item +SHLVYKJC with attachment TVZMTEB8; Zheng et al. item GSZ4D83U with attachment +J44YVR37; Pezeshkpour and Hruschka item UFZQ8WN6 with attachment S5KQCN97; and Sharma et al. item +YDM7VXSG with attachment 47VH4PC7. The first, third, and fourth records expose +permissive OA terms used by the repository manifest; Zheng's PDF is retained +in Zotero for local research but its redistribution terms are not asserted. +Response-category records MYPNHHWJ and CWY355RP were also added for the +psychometric comparison. Jones--Loe is an OA SAGE Open record whose official +PDF endpoint returned anti-bot `403` locally. Iannario's publisher, Crossref, +OpenAlex, and local Zotero rights metadata identify the record as CC BY 4.0, +but its official PDF endpoint returned a WAF response and the Zotero item has +no child attachment. Cao et al. was added through the local Connector API as +item `393S5NXZ`; its record is all-rights-reserved, so its PDF was not copied. +None of these records is represented by a fabricated or regenerated PDF. + +The Jones--Loe publisher page and PDF are marked open access and the original +is readable through the web research path, but the local download path still +returned 403. Iannario is licensed CC BY 4.0, but the local official download +returned a WAF response; license metadata alone does not prove that a fetched +byte stream is the publisher original or a valid Zotero parent attachment. +The four attached PDFs above remain the only attachments counted as verified +originals until an additional OA file passes a PDF magic header, size, +checksum, provenance, and Zotero attachment-parent check. + +Revalidation on 2026-08-12 confirmed the four attachment records and their +local files: `TVZMTEB8`, `J44YVR37`, `S5KQCN97`, and `47VH4PC7` each have a +matching PDF file size and MD5 recorded by the Zotero Local API. The running +client reports `X-Zotero-Version: 9.0.6`, so its Local API is read-only and +cannot perform the write/file-upload phase needed for the Jones--Loe +attachment. Official Zotero documentation now describes local writes and file +uploads for Zotero 10+ with an authorized local API key; this installation is +not that write-capable path; `/api/local/authorize` is also absent and item +PATCH is unsupported. Direct retries against the official Iannario PDF URL +returned HTTP 202 with zero bytes, while the official Jones--Loe PDF URL +returned HTTP 403. The OA landing pages and canonical PDF URLs remain +recorded below; web-crawler text, an HTML error page, a regenerated PDF, or an +unauthorised mirror must not be counted as the original. The Goal therefore +remains open for both publisher-original attachments until a byte-verified +parent attachment or a documented, authorised retrieval route is available. + +An Internet Archive capture of the canonical De Gruyter PDF was also +revalidated on 2026-08-12 as a 19-page PDF (1,074,249 bytes, MD5 +`263d2effa1d7cc5bdc2748878e7f32d4`) captured from the publisher URL on +2024-04-13. It is historical retrieval evidence only: it is not a current +publisher endpoint, not a Zotero parent attachment, and its byte identity as +the requested publisher original was not independently established. It must +not be copied into the repository or counted as the requested OA original +until provenance and authorization are verified. + +Primary sources: + +* https://arxiv.org/abs/2506.22316 +* https://proceedings.iclr.cc/paper_files/paper/2024/hash/54dd9e0cff6d9214e20d97eb2a3bae49-Abstract-Conference.html +* https://aclanthology.org/2024.findings-naacl.130/ +* https://www.anthropic.com/research/towards-understanding-sycophancy-in-language-models +* https://www.degruyterbrill.com/document/doi/10.1515/ijb-2021-0013/html +* https://web.archive.org/web/20240413073305id_/https://www.degruyter.com/document/doi/10.1515/ijb-2021-0013/pdf +* https://journals.sagepub.com/doi/10.1177/2158244013489691 +* https://journals.sagepub.com/doi/pdf/10.1177/2158244013489691 +* https://www.iris.unina.it/handle/11588/877609 +* https://www.zotero.org/support/dev/web_api/v3/local_api +* https://www.zotero.org/support/dev/web_api/v3/file_upload diff --git a/docs/planning/adrs/0007-sast-transport-and-sql-hardening.md b/docs/planning/adrs/0007-sast-transport-and-sql-hardening.md new file mode 100644 index 000000000..a76e03297 --- /dev/null +++ b/docs/planning/adrs/0007-sast-transport-and-sql-hardening.md @@ -0,0 +1,158 @@ +--- +id: "0007" +title: "Harden provider transport and SQL ledger against scanner findings" +status: accepted +proposed_date: "2026-08-11" +accepted_date: "2026-08-11" +deciders: + - "repository maintainer" +consulted: + - "Semgrep SAST workflow" + - "provider transport and cost ledger callers" +informed: + - "contributors" +affected_components: + - "contextual_orchestrator/orchestrator.py" + - "contextual_orchestrator/cost_ledger.py" + - "contextual_orchestrator/__main__.py" + - "tests/test_provider_tls.py" + - "tests/test_local_mlx.py" + - "tests/test_cost_ledger.py" +effort: M +supersedes: null +superseded-by: null +related: + - path: "docs/planning/adrs/0002-explicit-local-mlx-evaluation.md" + relation: influences + - path: "docs/planning/adrs/0003-keyverse-authentication-boundary.md" + relation: informational +asr_triggers: + - kind: security + evidence: "The first PR Semgrep run reported unverified TLS construction, dynamic urllib URL use, and three SQL string-construction findings." + note: "Resolve the trust-boundary findings in code; scanner suppression is not the default remediation." + - kind: maintainability + evidence: "The provider transport and DB-API ledger support multiple protocol/driver shapes." + note: "Keep explicit protocol variants and fixed SQL templates instead of adding a broad dependency or dynamic query builder." +success_criteria: + - metric: "provider transport safety" + target: "remote HTTPS uses a verifying SSL context, local HTTP is loopback-only, and non-HTTP request URLs are rejected before I/O" + measurement_window: "every provider transport test and PR SAST run" + source: "ModelClient transport tests and Semgrep" + - metric: "ledger query safety" + target: "all DB-API SQL statements use fixed templates with bound values and an explicit qmark/pyformat parameter-style allow-list" + measurement_window: "every SQL ledger operation" + source: "SqlLedgerStore tests and Semgrep" +--- + +# Harden provider transport and SQL ledger against scanner findings + +## Context + +The first remote PR security run found five blocking findings. Three were +reported in the SQL ledger, one in the explicit TLS opt-out, and one in the +provider request transport. The ledger already used fixed column names and +bound values, but its f-strings made that safety difficult for the scanner and +left the parameter-style boundary implicit. A follow-up scan showed that the +first fixed-template change still interpolated the fixed column list, so Ruff +S608 remained reproducible even though runtime values were bound. + +> Semgrep reported raw-query construction at the three fixed SQL execution sites in cost_ledger.py. +> +> Semgrep reported ssl._create_unverified_context and a dynamic urllib URL in the provider transport. +> +> The local MLX path needs plain HTTP only for a validated loopback endpoint; remote providers must retain HTTPS verification. + +## Decision Drivers + +* Remove real insecure TLS behavior from the public API. +* Keep local mlx-lm usable without sending a credential or requiring TLS. +* Make the URL trust boundary visible to both code review and static analysis. +* Preserve sqlite3 and psycopg compatibility without adding SQLAlchemy for one ledger. +* Keep SQL values bound and SQL identifiers fixed. + +## Considered Options + +* Suppress the five Semgrep rules with comments and keep the implementation. +* Add SQLAlchemy and retain urllib with a disabled-TLS development flag. +* Use verifying TLS only, a small stdlib http.client transport after strict URL validation, and fixed SQL templates for each supported DB-API parameter style. + +## Decision Outcome + +Chosen option: "Fix the trust boundaries and make safe variants explicit". + +| Driver | Suppress findings | Add broad dependency / keep bypass | Fixed stdlib transport and SQL templates | +| --- | --- | --- | --- | +| Remote TLS verification | Fails | Fails | Satisfies | +| Local loopback MLX support | Preserves | Preserves | Preserves | +| SQL value binding | Obscures review | Delegates to dependency | Explicitly preserves | +| Dependency and maintenance cost | Low now, high risk | High | Low | +| Scanner and review evidence | Weak | Mixed | Strong | + +Remote transport always uses a verifying SSL context; custom CA bundles remain +supported, but TLS verification cannot be disabled. The CLI option that offered +an insecure TLS bypass is removed. Provider I/O uses `http.client` after +validating the request URL as HTTP(S), with the existing agent-level HTTPS, +loopback, DNS, and credential checks remaining in force. The validated sockaddr +is carried into the connection so the socket does not resolve the hostname a +second time; the original hostname remains the TLS SNI and HTTP Host identity. +The local `mlx://` scheme is translated to loopback HTTP only by the validated +provider URL path. + +`SqlLedgerStore` accepts only `qmark` and `pyformat`. Select/insert statements +use fixed SQL templates for each parameter style and fixed column lists; start +and end windows select one of four fixed query templates. Values remain DB-API +parameters and never become SQL text. + +The final Semgrep run identified the stdlib `HTTPSConnection` call itself even +though the code passes the already reviewed verifying SSL context. A +rule-specific `nosemgrep` annotation is retained at that one call site, with +the transport and TLS tests remaining the source-of-truth checks. This is a +documented false-positive boundary, not a suppression of certificate +verification or URL validation. + +### Consequences + +* Good, because the remote SAST gate checks the same invariant the runtime uses. +* Good, because a caller cannot accidentally turn off certificate verification. +* Good, because local MLX remains a deliberate loopback exception rather than a general HTTP exception. +* Good, because the cost ledger remains dependency-free and portable. +* Bad, because callers relying on `verify_tls=False` must use a trusted CA bundle or local HTTP loopback instead. +* Bad, because adding another DB-API parameter style requires one reviewed set of fixed templates. + +## Problem Register and Remediation Directions + +| Finding | Direction | State | +| --- | --- | --- | +| Semgrep/Ruff flagged three f-string SQL statements that still interpolated a fixed column list. | Declare complete qmark/pyformat INSERT and SELECT templates as literals, keep all values bound, and add a pyformat regression covering seed, append, and all four query windows. | Implemented locally 2026-08-12; exact-head CI/SAST revalidation required | +| `paramstyle` accepted arbitrary values and silently selected pyformat. | Reject styles other than qmark and pyformat at construction. | Implemented | +| `ssl._create_unverified_context` made an insecure remote mode executable. | Remove the bypass and keep `ssl.create_default_context` or a validated custom CA bundle. | Implemented | +| `urllib.request.urlopen` accepted a dynamically assembled request URL. | Use `http.client` with scheme/host/userinfo validation at the final I/O boundary. | Implemented | +| Semgrep flagged the reviewed `HTTPSConnection` API despite its explicit verifying context. | Keep the verifying context and add only the exact rule-specific suppression at that call site; retain transport regression tests. | Implemented | +| DNS could return a safe address during validation and a different address during connection. | Return the validated sockaddr and pin every HTTP, HTTPS, streaming, passthrough, and batch connection to it while retaining hostname SNI/Host semantics. | Implemented | +| A scanner-clean result could regress without a transport test. | Add a non-HTTP rejection regression and rerun the full SAST/CI gate. | Implemented / ongoing CI confirmation | +| `actionlint`/ShellCheck flagged unquoted `FUZZ_SECONDS` expansions in every fuzz target, leaving the workflow vulnerable to word-splitting/globbing if the time-budget value changed. | Quote the shell expansion at every fuzz invocation and require the workflow lint gate to remain clean. | Fixed locally 2026-08-13; exact-head CI/SAST revalidation required | + +## Risks and Mitigations + +| risk | likelihood | impact | mitigation | owner | +| --- | --- | --- | --- | --- | +| A future caller bypasses `_validate_provider` and calls private transport directly. | low | high | Final transport validation rejects non-HTTP URLs and userinfo; keep private methods covered. | maintainer | +| A custom CA bundle is untrusted. | low | high | Treat the path as deployment configuration; require file existence and normal SSL context loading. | deployment owner | +| SQL template additions reintroduce dynamic identifiers. | medium | high | Keep column names in fixed constants and values in parameter tuples; rerun Semgrep. | maintainer | + +## Rollback / Exit Strategy + +If an external DB-API driver requires another parameter style, add a new fixed +template set and tests through a follow-up ADR. Do not restore disabled TLS or +dynamic SQL concatenation as a compatibility shortcut. If the local transport +needs a proxy later, add an explicit, reviewed proxy boundary rather than +reintroducing general urllib URL handling. + +## Affected Components + +* contextual_orchestrator/orchestrator.py +* contextual_orchestrator/cost_ledger.py +* contextual_orchestrator/__main__.py +* tests/test_provider_tls.py +* tests/test_local_mlx.py +* tests/test_cost_ledger.py diff --git a/docs/planning/adrs/0008-fast-judge-review-hardening.md b/docs/planning/adrs/0008-fast-judge-review-hardening.md new file mode 100644 index 000000000..3d3adbd63 --- /dev/null +++ b/docs/planning/adrs/0008-fast-judge-review-hardening.md @@ -0,0 +1,184 @@ +--- +id: "0008" +title: "Harden the fast-mlsirm contextual judge after review" +status: accepted +proposed_date: "2026-08-11" +accepted_date: "2026-08-11" +deciders: + - "repository maintainer" +consulted: + - "fast-mlsirm CodeRabbit review" + - "fast-mlsirm judge and IRT callers" +informed: + - "contributors" +affected_components: + - "fast-mlsirm/python/fast_mlsirm/llm_judge.py" + - "fast-mlsirm/python/fast_mlsirm/irt_contract.py" + - "fast-mlsirm/tests/test_llm_judge.py" + - "fast-mlsirm/tests/test_irt_contract.py" + - "fast-mlsirm/README.md" +effort: S +supersedes: null +superseded-by: null +related: + - path: "docs/planning/adrs/0005-irt-response-matrix-contract.md" + relation: influenced-by + - path: "docs/planning/adrs/0006-polytomous-llm-judge-bias-calibration.md" + relation: influenced-by + - path: "docs/planning/adrs/0007-sast-transport-and-sql-hardening.md" + relation: influenced-by +asr_triggers: + - kind: security + evidence: "Review found predictable prompt delimiters around untrusted judge input, substring extraction that accepted wrapped model output, malformed public result mappings that could break IRT projection, and an unbounded criteria iterable." + note: "Keep model-controlled content data-only, require one complete JSON value, validate mapping boundaries before sorting or set comparison, and bound iterable consumption." + - kind: maintainability + evidence: "Review found inconsistent exception types, coercive mapping normalization, mutable ADR links, and missing malformed-output tests." + note: "Make the public contract explicit, use documented ValueError validation for malformed criterion inputs, reject conversion-hook numeric subclasses, and pin documentation to immutable evidence." +success_criteria: + - metric: "judge trust-boundary validation" + target: "untrusted task/answer/reference data is serialized as JSON, malformed model fields raise JudgeFormatError, and criterion inputs reject invalid runtime types with documented ValueError failures" + measurement_window: "every fast-mlsirm judge test and PR review" + source: "tests/test_llm_judge.py and CodeRabbit review" + - metric: "IRT-safe projection" + target: "dichotomous and polytomous rows contain only validated categories and retain the multi-item contract" + measurement_window: "every judge-to-IRT conversion" + source: "tests/test_llm_judge.py and tests/test_irt_contract.py" + - metric: "documentation reproducibility" + target: "ADR links resolve through an immutable contextual-orchestrator commit" + measurement_window: "every README review" + source: "fast-mlsirm README" +--- + +# Harden the fast-mlsirm contextual judge after review + +## Context + +The first fast-mlsirm PR added a provider-neutral judge routed through +contextual-orchestrator and a multi-item IRT projection. Its automated review +then identified several small but real weaknesses: model-controlled missing +fields escaped as generic `ValueError`, direct `LLMJudgeResult` construction +could project a negative category, malformed public result mappings could fail +before the intended `JudgeFormatError`, criteria iterables were not bounded +during consumption, mapping inputs were silently coerced, and predictable +XML-like prompt tags could be closed by answer text. The same review also found +unpinned ADR links and missing failure-path coverage. + +> The review found two actionable comments and additional lint, test, prompt-boundary, and documentation findings. +> +> The user requires every plausible problem to become an explicit remediation direction, not a keyword or positional fallback. +> +> The fast-mlsirm PR must continue through review, remediation, re-test, and exact-head merge rather than stopping at a green local run. + +## Decision Drivers + +* Keep the contextual-orchestrator-only LLM-as-a-Judge path strict and fail closed. +* Prevent untrusted evaluation text from changing prompt structure. +* Preserve the dichotomous-or-polytomous multi-item IRT contract. +* Provide an ordinal polytomous path that does not rely only on one K-way score-ID choice. +* Make type, exception, test, and documentation behavior reproducible. + +## Considered Options + +* Treat review comments as optional style suggestions and keep the implementation. +* Add broad schema and prompt dependencies for the judge boundary. +* Apply small stdlib-only validation, JSON serialization, explicit error translation, and focused tests. + +## Decision Outcome + +Chosen option: "Harden the existing provider-neutral judge with small explicit boundary checks". + +| Driver | Defer review findings | Add broad dependency | Explicit validation and focused tests | +| --- | --- | --- | --- | +| Model-output fail-closed behavior | Inconsistent | Depends on schema runtime | Preserved with `JudgeFormatError` | +| Prompt data boundary | Predictable tags remain | More operational surface | JSON payload with system-level data instruction | +| IRT category safety | Negative direct scores can leak | Hidden in dependency | `_score` plus bounded category projection | +| Reproducibility and maintenance | Mutable links and weak tests | Higher dependency cost | Immutable links and targeted regression tests | + +`JudgeCriterion` now rejects non-string identifiers/descriptions and non-numeric +weights without coercion or incidental exception leakage; malformed criterion +inputs use documented `ValueError` failures. IRT projection validates direct +criterion-score and category mapping boundaries before sorting or set +comparison, then keeps category indices within bounds while retaining the +requirement for at least two criteria. Criteria are bounded while the iterable +is consumed. Model-controlled answer and rationale failures are translated to +`JudgeFormatError`; the caller's task and answer validation remains ordinary +input validation. + +The user prompt carries task, answer, reference, and rubric as one JSON data +object rather than predictable open/close tags. The system instruction still +requires the model to ignore instructions inside those values. Failure-path +tests cover missing answer, missing rationale, and non-mapping completions; +category-bound tests cover invalid `n_categories`; and import ordering plus +the test regex are kept lint-clean. The response parser now passes the complete +bounded answer to `json.loads` and rejects prefixes, suffixes, and Markdown +fences instead of extracting the first and last braces. README ADR links use +the immutable contextual commit that contains ADR 0005 and ADR 0006. + +The follow-up polytomous path adds explicit +`category_method="cumulative_threshold"` and bounded +`category_method="binary_threshold"` modes. With an explicit category count, +the binary mode asks one Boolean question for each ordered boundary of each +criterion. The adapter rejects malformed or false-then-true responses, derives +the category and weighted score itself, and retains the existing exact-schema, +contextual-orchestrator, and multi-item IRT requirements. Direct K-way +categories remain available only for explicit calibration. Omitting the method +now selects binary thresholds for polytomous output because a live 3B probe +changed an unsafe answer from `0.0` at K=2 to `0.8333` at K=7 and changed a +partial answer from `0.0` to `1.0` at K=5; binary output remains a fail-closed +guard, not a claim of unbiased or high-recall judgment. + +## Problem Register and Remediation Directions + +| Finding | Direction | State | +| --- | --- | --- | +| Criterion fields could raise incidental `TypeError` or accept coercive mapping values. | Validate runtime types explicitly and stop string/float coercion in `_criteria`. | Implemented | +| Invalid public criterion field types exposed inconsistent `TypeError` failures. | Normalize malformed criterion field validation to documented `ValueError` failures and test both direct and mapping inputs. | Implemented | +| Numeric weight subclasses could execute a custom `__float__` hook during validation. | Accept only exact built-in `int`/`float` weights before conversion and test a hooked subclass remains uncalled. | Implemented | +| Direct criterion scores could produce a negative or non-integral IRT category. | Validate scores with the same bounded score contract and clamp the projection to the legal category range. | Implemented | +| Public result mappings could be non-mappings or have non-string keys, causing incidental `TypeError` during IRT projection. | Validate `criterion_scores` and `criterion_categories` mapping/key boundaries before sorting or set comparison and fail with `JudgeFormatError`. | Implemented | +| A caller-controlled criteria iterable could exceed the configured maximum before validation completed. | Enforce `MAX_JUDGE_CRITERIA` during iteration, before normalizing an additional value. | Implemented | +| Missing model answer/rationale used generic `ValueError`. | Translate model-controlled bounded-text failures to `JudgeFormatError`. | Implemented | +| Predictable XML tags could be closed by untrusted answer text. | Serialize evaluation inputs as one JSON data payload. | Implemented | +| Response parsing extracted a brace-delimited substring and accepted wrappers/fences around model JSON. | Parse the complete bounded answer as exactly one JSON object and reject any surrounding text or Markdown fence. | Implemented | +| The fast adapter's `json.loads` accepted duplicate object members with last-value-wins semantics and ignored unknown top-level fields, weakening the strict judge contract. | Parse with a duplicate-rejecting `object_pairs_hook`, require the exact mode-specific top-level field set including the advisory boolean, and add top-level plus nested duplicate/unknown-field regressions. | Implemented on fast-mlsirm follow-up branch; retain exact-schema tests | +| Parsed advisory `accepted` name was overwritten by derived acceptance. | Rename the advisory field and derive acceptance only from the validated score. | Implemented | +| Public export order and a regex assertion were lint-fragile. | Reorder `__all__` and escape the literal test pattern. | Implemented | +| Invalid `n_categories` and malformed completion paths lacked tests. | Add focused `pytest.raises` coverage and preserve the multi-item checks. | Implemented | +| README ADR links targeted mutable/nonexistent `main` paths. | Pin links to the immutable contextual-orchestrator commit containing the referenced ADRs. | Implemented | +| A polytomous K-way choice can expose the judge to score-ID and category-count effects. | Add an opt-in cumulative-threshold mode with explicit K, exact criterion IDs, Boolean boundary vectors, monotonicity validation, derived categories, and focused IRT-row tests. | Implemented on fast-mlsirm follow-up branch; exact-head review pending | +| Threshold output can be syntactically valid but ordinally incoherent, or can disagree with direct K-way output. | Fail closed on non-monotone thresholds and record category method, K, score, acceptance, parse status, trace, and token usage in paired MLX calibration runs; do not claim bias removal. | Implemented in adapter/tests and 2026-08-12 exploratory run; calibration ongoing | +| The 2026-08-14 paired 3B probe showed direct scores rising with K for a safe case (`0.5 -> 1.0 -> 1.0`) and for an unsafe case at K=7 (`0.0 -> 0.0 -> 0.3333`); cumulative parsing/monotonicity failures remained 4/6. | Keep direct and cumulative methods experimental; add the bounded `binary_threshold` method as a fail-closed calibration probe, record its extra calls/tokens/latency and semantic misses, and require held-out human/gold agreement before changing a default. Never use keyword, positional, or silent repair. | Goal expanded 2026-08-14; binary method implemented on fast-mlsirm exact follow-up, calibration ongoing | +| A fresh same-route 3B probe returned the unsafe answer at direct scores `0.0`, `0.5`, `0.8333` and the partial answer at `0.0`, `1.0`, `0.0` for K=`2,5,7`; a binary probe returned `0.0` for both safe and unsafe answers at K=`5,7`, including a safe semantic false negative. | Make omitted polytomous method selection fail closed into bounded binary thresholds; keep direct K-way explicit and calibration-only, preserve the false negative in the denominator, and require held-out human/gold recall before any IRT production claim. Never use keyword, positional, or silent repair. | Implemented on fast-mlsirm `608cfbd`; exact-head review/check follow-up required | +| The real contextual adapter smoke at K=5 confirmed the omitted-method default: unsafe evidence returned a valid rejected `(0,0)` row after 8 calls, while safe evidence produced a non-monotone threshold failure after 8 calls. | Keep the default integrated and fail-closed, retain both valid and failed comparisons with trace/usage/latency, and separate transport/default-selection proof from semantic recall; no category coercion or keyword repair is allowed. | Observed 2026-08-14; exact fast head `9d18f53`, semantic calibration remains open | +| The integrated safe-case exception previously discarded boundary-level evidence; fast-mlsirm `d1eca0c2fed89991e647802f0b27a91f0f6fe2bd` now retains bounded `.evidence` showing `parse_status=passed`, `semantic_status=non_monotone`, `8/8` completed calls, `8` trace steps, and `2,639` tokens. | Make failure evidence part of the judge contract, preserve it in the calibration denominator, and keep semantic non-monotonicity distinct from transport/parse failure; do not retry blindly, keyword-match, repair, or emit an IRT row. | Implemented 2026-08-14; exact-head review/check follow-up required | +| The anchored model comparison through contextual-orchestrator found Gemma 4 e4b strict and semantically positive on the safe case, Llama 3B repeatedly false-negative, and Llama 1B malformed on every boundary. | Record model identity, anchor presence, category row, parse/semantic status, calls, trace, tokens, and latency; prefer a measured quality/latency candidate only after balanced held-out gold calibration and never silently fall back to a weaker judge. | Observed 2026-08-14; fast-mlsirm `dd44a95`, calibration remains open | +| K-only intermediate category numbers were under-specified for a small local judge, so a valid Boolean response could still have no stable ordinal meaning. | Support complete per-criterion `category_anchors` and carry only the matching anchor as rubric data to each boundary; reject mixed/incomplete anchor sets and label omitted-anchor runs exploratory. | Implemented 2026-08-14; exact-head review/check follow-up required | +| Binary-threshold boundary calls were independent but serial, making larger K calibration expensive even though contextual-orchestrator already exposes bounded local concurrency. | Reuse the injected gateway's `client.local_concurrency` only for independent boundary calls; keep generic injected transports sequential, preserve deterministic evidence order, aggregate trace/usage, and validate all thresholds before returning. | Implemented in fast-mlsirm `61e6be9`; exact-source targeted `58 passed`, full `3630 passed`, live MLX evidence retained | +| The cached local Llama 3B judge failed strict structured parsing in 7/18 good-plan calls, including invalid JSON, an out-of-range category, and a non-monotone threshold vector; framing also shifted some K=7 scores. | Keep failures in the reliability denominator and test any bounded retry or stronger local-judge selection as a separate contextual-orchestrator experiment. Never repair by keyword/position or silently omit a failed call. | Recorded in 2026-08-12 benchmark; required calibration follow-up | +| contextual-orchestrator's exact fast-mlsirm preflight imported the judge symbols successfully but reported the integration unavailable because the fast package root omitted `CONTEXTUAL_ORCHESTRATOR_CONTRACT_V1`. | Make the versioned cross-repository contract a public package-root export, compare it during preflight, and cover the export with a regression test. Preserve the distinction between an integration/import failure and a judge semantic failure; do not bypass preflight or infer availability from a direct class smoke. | Fixed in fast-mlsirm `a536292`; contextual `a07c11f`; exact-head CI/review follow-up required | +| Fresh exact-head review found that binary-threshold failure evidence retained raw provider output and arbitrary exception text, which could contain task, answer, reference, rubric, or provider diagnostics. | Remove both fields before evidence leaves the judge, retain only allowlisted status/identity/usage fields plus a stable package-owned failure code, and add provider-output and exception-text sentinel regressions. | Fixed in fast-mlsirm `a536292`; full suite `3714 passed` with 2 warnings; exact-head CI/review follow-up required | + +## Risks and Mitigations + +| risk | likelihood | impact | mitigation | owner | +| --- | --- | --- | --- | --- | +| A model treats JSON string content as instructions despite the data boundary. | medium | high | Keep the system instruction explicit, never use model output as executable content, and review perturbation results. | maintainer | +| Clamping masks a caller-created invalid score. | low | medium | `_score` rejects non-finite/out-of-range values before the bounded projection; production results still originate from strict judge parsing. | maintainer | +| Immutable documentation ref becomes hard to update. | low | low | Add a new pinned link when the contextual ADR set changes; do not return to mutable `main` links. | documentation owner | + +## Rollback / Exit Strategy + +If a downstream caller depends on coercive criterion values, migrate that caller +to explicit `JudgeCriterion` construction rather than restoring silent coercion. +If a future structured message API is introduced, retain the JSON data contract +or an equivalent typed payload and keep `JudgeFormatError` as the fail-closed +boundary. Do not restore keyword matching, positional repair, or one-item IRT +conversion. + +## Affected Components + +* fast-mlsirm/python/fast_mlsirm/llm_judge.py +* fast-mlsirm/python/fast_mlsirm/irt_contract.py +* fast-mlsirm/tests/test_llm_judge.py +* fast-mlsirm/tests/test_irt_contract.py +* fast-mlsirm/README.md diff --git a/docs/planning/adrs/0009-supply-chain-dependency-cooldown.md b/docs/planning/adrs/0009-supply-chain-dependency-cooldown.md new file mode 100644 index 000000000..06e799b97 --- /dev/null +++ b/docs/planning/adrs/0009-supply-chain-dependency-cooldown.md @@ -0,0 +1,108 @@ +--- +id: "0009" +title: "Add an explicit Dependabot dependency cooldown" +status: accepted +proposed_date: "2026-08-11" +accepted_date: "2026-08-11" +deciders: + - "repository maintainer" +consulted: + - "Strix security scan" + - "GitHub Dependabot configuration" +informed: + - "contributors" +affected_components: + - ".github/dependabot.yml" + - "tests/test_repository_security_metadata.py" + - "docs/planning/adrs/" +effort: S +supersedes: null +superseded-by: null +related: + - path: "docs/planning/adrs/0004-pr-review-merge-loop.md" + relation: informational + - path: "docs/planning/adrs/0007-sast-transport-and-sql-hardening.md" + relation: informational +asr_triggers: + - kind: security + evidence: "The current-head Strix scan reported a critical Dependabot cooldown finding in .github/dependabot.yml, alongside provider failures that required fail-closed handling." + note: "Make dependency-update timing an explicit repository policy and retain the security scan as the validation gate." + - kind: maintainability + evidence: "A scanner finding exposed an implicit dependency-update policy that was not protected by repository metadata tests." + note: "Keep the setting adjacent to each ecosystem and assert both entries in the repository security metadata test." +success_criteria: + - metric: "dependency update cooldown" + target: "GitHub Actions and pip Dependabot update entries each declare cooldown.default-days: 7" + measurement_window: "every Dependabot configuration review and security scan" + source: ".github/dependabot.yml and tests/test_repository_security_metadata.py" +--- + +# Add an explicit Dependabot dependency cooldown + +## Context + +The current-head Strix scan for PR #109 produced a structured critical finding +against the repository's dependency-update configuration. The finding targeted +the absence of an explicit cooldown for newly published package versions. + +> Strix reported `package_managers.dependabot.dependabot-missing-cooldown.dependabot-missing-cooldown` with high confidence. +> The finding recommended a seven-day cooldown for each update ecosystem. +> GitHub's current Dependabot options document `cooldown.default-days` as the explicit setting for supported package managers and distinguish version-update cooldown from security updates. + +## Decision Drivers + +* Reduce exposure to newly published malicious or unstable dependency versions. +* Keep the policy explicit and reviewable for both configured ecosystems. +* Preserve timely Dependabot security updates and avoid adding a dependency-management tool. + +## Considered Options + +* Leave the implicit platform default and accept scanner noise. +* Add an explicit seven-day cooldown to every configured ecosystem. +* Disable Dependabot version updates and manage all updates manually. + +## Decision Outcome + +Chosen option: "Declare a seven-day cooldown for GitHub Actions and pip version updates". + +| Driver | Implicit default | Explicit seven-day cooldown | Disable automated updates | +| --- | --- | --- | --- | +| Supply-chain exposure window | Unclear and scanner-visible | Bounded and reviewable | Manual process can drift | +| Security updates | Platform behavior remains implicit | Security updates remain outside version cooldown | Delayed by human workflow | +| Maintenance cost | Low now, weak evidence | One small config and metadata assertion | High | + +Each `github-actions` and `pip` update entry in `.github/dependabot.yml` now has +`cooldown.default-days: 7`. This applies to version-update proposals; it does +not disable or intentionally delay Dependabot security updates. The repository +metadata test asserts that both ecosystems retain the explicit setting. + +## Problem Register and Remediation Directions + +| Finding | Direction | State | +| --- | --- | --- | +| Dependabot entries had no explicit cooldown and Strix reported a critical supply-chain finding. | Add `cooldown.default-days: 7` to every configured ecosystem and protect both entries with a repository metadata regression test. | Implemented in current head | +| An external scanner finding could be mistaken for provider noise. | Keep the Strix gate fail-closed and inspect structured artifacts separately from provider 429/410 evidence. | Implemented in current review loop | + +## Risks and Mitigations + +| risk | likelihood | impact | mitigation | owner | +| --- | --- | --- | --- | --- | +| A seven-day delay postpones a non-security version update. | low | medium | Keep Dependabot security updates enabled and review urgent version updates explicitly. | maintainer | +| A future ecosystem entry omits the cooldown. | medium | high | Require one `default-days: 7` entry per configured ecosystem in the metadata test and Strix review. | maintainer | + +## Rollback / Exit Strategy + +If a documented dependency requires a shorter version-update window, change the +specific ecosystem cooldown only through a reviewed ADR update and retain the +security-update path. Do not remove the explicit setting to silence a scanner. + +## Affected Components + +* .github/dependabot.yml +* tests/test_repository_security_metadata.py +* docs/planning/adrs/0009-supply-chain-dependency-cooldown.md + +## More Information + +* [GitHub Dependabot options reference](https://docs.github.com/en/code-security/reference/supply-chain-security/dependabot-options-reference) +* [GitHub guidance on dependency update cooldown](https://docs.github.com/en/code-security/tutorials/secure-your-dependencies/optimizing-pr-creation-version-updates) diff --git a/docs/rest_api_design.md b/docs/rest_api_design.md index 9378e5a37..b000dbe53 100644 --- a/docs/rest_api_design.md +++ b/docs/rest_api_design.md @@ -19,6 +19,7 @@ | `GET` | `/v1/batch/embeddings/{batch_id}` | Poll an embeddings batch; returns reduced vectors + recorded cost once completed | | `GET` | `/api/v1/agent_pools` | List model agents | | `GET` | `/api/v1/orchestration_policies/default_policy` | Read active policy | +| `GET` | `/api/v1/provider_readiness/latest` | Read or explicitly refresh bounded provider chat readiness | | `GET` | `/api/v1/analytics_snapshots/latest` | Read local runtime KPI and guardrail snapshot | | `GET` | `/api/v1/sales_readiness/latest` | Read local enterprise-pilot readiness criteria and evidence | | `GET` | `/api/v1/commercial_readiness/latest` | Read KRW 2,000,000,000 commercial due-diligence readiness criteria and evidence | diff --git a/examples/agents.local.json b/examples/agents.local.json new file mode 100644 index 000000000..25043c5e0 --- /dev/null +++ b/examples/agents.local.json @@ -0,0 +1,80 @@ +{ + "agents": [ + { + "id": "contextual_orchestrator", + "model": "contextual-orchestrator", + "base_url": "local://127.0.0.1:18000/v1", + "provider_name": "contextual-orchestrator", + "tags": ["orchestration", "planning", "reasoning", "verification", "writing"], + "priority": 5, + "provider_exclusions": ["thinker", "worker", "verifier", "synthesizer"] + }, + { + "id": "mlx_gemma_4_31b_it", + "model": "mlx-community/gemma-4-31b-it-4bit", + "base_url": "mlx://127.0.0.1:8080/v1", + "provider_name": "mlx-lm", + "tags": ["reasoning", "research", "coding", "writing", "verification"], + "priority": 4, + "provider_exclusions": ["verifier"] + }, + { + "id": "mlx_deepseek_r1_qwen_32b", + "model": "outlier-ai/deepseek-r1-distill-qwen-32b-mlx-4bit", + "base_url": "mlx://127.0.0.1:8080/v1", + "provider_name": "mlx-lm", + "tags": ["reasoning", "research", "coding", "verification"], + "priority": 4, + "provider_exclusions": ["verifier"] + }, + { + "id": "mlx_gemma_4_e4b_it", + "model": "mlx-community/gemma-4-e4b-it-4bit", + "base_url": "mlx://127.0.0.1:8080/v1", + "provider_name": "mlx-lm", + "tags": ["reasoning", "research", "coding", "writing", "verification"], + "priority": 3 + }, + { + "id": "mlx_llama_3_2_3b_instruct", + "model": "mlx-community/llama-3.2-3b-instruct-4bit", + "base_url": "mlx://127.0.0.1:8080/v1", + "provider_name": "mlx-lm", + "tags": ["fast", "reasoning", "coding", "writing", "verification"], + "priority": 2 + }, + { + "id": "mlx_llama_3_2_1b_instruct", + "model": "mlx-community/llama-3.2-1b-instruct-4bit", + "base_url": "mlx://127.0.0.1:8080/v1", + "provider_name": "mlx-lm", + "tags": ["fast", "writing", "coding"], + "priority": 1, + "provider_exclusions": ["verifier"] + }, + { + "id": "llama_cpp_embeddinggemma", + "model": "embeddinggemma", + "base_url": "local://127.0.0.1:8082/v1", + "provider_name": "llama.cpp", + "tags": ["embedding"], + "priority": 0 + }, + { + "id": "lmstudio_gemma_4_e4b_it", + "model": "lmstudio-community/gemma-4-E4B-it-MLX-4bit", + "base_url": "local://127.0.0.1:1234/v1", + "provider_name": "lm-studio", + "tags": ["reasoning", "coding", "writing"], + "priority": 0 + }, + { + "id": "lmstudio_embeddinggemma", + "model": "mlx-community/embeddinggemma-300m-8bit", + "base_url": "local://127.0.0.1:1234/v1", + "provider_name": "lm-studio", + "tags": ["embedding"], + "priority": 0 + } + ] +} diff --git a/examples/agents.mlx.json b/examples/agents.mlx.json new file mode 100644 index 000000000..103a31b38 --- /dev/null +++ b/examples/agents.mlx.json @@ -0,0 +1,12 @@ +{ + "agents": [ + { + "id": "local_fast_agent", + "model": "mlx-community/llama-3.2-3b-instruct-4bit", + "base_url": "mlx://127.0.0.1:8080/v1", + "provider_name": "mlx-lm", + "tags": ["reasoning", "writing", "coding", "verification"], + "priority": 1 + } + ] +} diff --git a/fuzz/corpus/judge/valid.json b/fuzz/corpus/judge/valid.json new file mode 100644 index 000000000..49122bedd --- /dev/null +++ b/fuzz/corpus/judge/valid.json @@ -0,0 +1 @@ +{"decision":"ACCEPT","reason":"The evidence satisfies the criterion."} diff --git a/fuzz/corpus/judge/wrapped.txt b/fuzz/corpus/judge/wrapped.txt new file mode 100644 index 000000000..b200f5057 --- /dev/null +++ b/fuzz/corpus/judge/wrapped.txt @@ -0,0 +1 @@ +prefix {"decision":"ACCEPT","reason":"The wrapper must be rejected."} diff --git a/fuzz/fuzz_model_judge.py b/fuzz/fuzz_model_judge.py new file mode 100755 index 000000000..9058f8de1 --- /dev/null +++ b/fuzz/fuzz_model_judge.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""Atheris coverage-guided harness for strict model-judge response parsing.""" + +import sys +from pathlib import Path + +import atheris + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +with atheris.instrument_imports(): + from fuzz.targets import exercise_model_judge_reply + + +def one_input(data: bytes) -> None: + fdp = atheris.FuzzedDataProvider(data) + exercise_model_judge_reply(fdp.ConsumeUnicodeNoSurrogates(fdp.remaining_bytes())) + + +def main() -> None: + atheris.Setup(sys.argv, one_input) + atheris.Fuzz() + + +if __name__ == "__main__": + main() diff --git a/fuzz/requirements-atheris.in b/fuzz/requirements-atheris.in index b930f7524..5fa77c8bc 100644 --- a/fuzz/requirements-atheris.in +++ b/fuzz/requirements-atheris.in @@ -1,3 +1,5 @@ -# Atheris coverage-guided job deps (Python 3.11). Compile: uv pip compile fuzz/requirements-atheris.in --generate-hashes --python-version 3.11 --universal -o fuzz/requirements-atheris.txt +# Atheris coverage-guided job deps. Compile: uv pip compile fuzz/requirements-atheris.in --generate-hashes --python-version 3.12 --universal -o fuzz/requirements-atheris.txt +# Atheris 3.1.0 has wheels for the Python 3.12 fuzz runner and the central +# coverage-evidence image; 3.0.0 is no longer available to that image. pip -atheris==3.0.0 +atheris==3.1.0 diff --git a/fuzz/requirements-atheris.txt b/fuzz/requirements-atheris.txt index b3e913ba6..616be93d6 100644 --- a/fuzz/requirements-atheris.txt +++ b/fuzz/requirements-atheris.txt @@ -1,10 +1,9 @@ # This file was autogenerated by uv via the following command: -# uv pip compile fuzz/requirements-atheris.in --generate-hashes --python-version 3.11 --universal -o fuzz/requirements-atheris.txt -atheris==3.0.0 \ - --hash=sha256:1f0929c7bc3040f3fe4102e557718734190cf2d7718bbb8e3ce6d3eb56ef5bb3 \ - --hash=sha256:510e502c57b6dc615fb174066407af620d4c7f73cf08a782c86e7761bf12c4eb \ - --hash=sha256:8a5c8a781467c187da40fd29139784193e2647058831f837f675d0bb8cbd8746 \ - --hash=sha256:a402cdca8a650d1371050b1f9552eb4cdc488d2db64950d603c4560318365eac +# uv pip compile fuzz/requirements-atheris.in --generate-hashes --python-version 3.12 --universal -o fuzz/requirements-atheris.txt +atheris==3.1.0 \ + --hash=sha256:315a0b5c819852b1ffe1ca72efc389c7724881f2c33e4aacb8c6bcec49bd5011 \ + --hash=sha256:ec5e11f21a4c197fe91f7aea2b2de88e623c73a21fc07b105ac6329a1588457b \ + --hash=sha256:f8a9f51ce8369026e8eb7b7174835e8c4c85a1a6db5d9add36c15100779d2a39 # via -r fuzz/requirements-atheris.in pip==26.1.2 \ --hash=sha256:382ff9f685ee3bc25864f820aa50505825f10f5458ffff07e30a6d96e5715cab \ diff --git a/fuzz/targets.py b/fuzz/targets.py index d0c344462..c82ceba0b 100644 --- a/fuzz/targets.py +++ b/fuzz/targets.py @@ -8,7 +8,7 @@ ``AttributeError``, ``RecursionError``, ``SystemError`` or a hang; and * structural invariants on any successful result (shape, types, idempotence). -CodeGraph (``codegraph explore``) surfaced these four surfaces as the ones that +CodeGraph (``codegraph explore``) surfaced these five surfaces as the ones that consume untrusted bytes/JSON: 1. ``server._coerce_json`` / ``_validate_mode`` / ``_validate_messages`` / @@ -18,6 +18,8 @@ over arbitrary trace payloads (regex + recursion). 4. ``orchestrator.TaskOrchestrator.run`` (+ ``sse_stream_body``) -- end-to-end prompt processing on a mock (offline) provider. +5. ``orchestrator._parse_model_judge_reply`` -- strict parsing of untrusted + model-generated verdicts. No network, no secrets, no filesystem: every target runs fully offline. """ @@ -31,6 +33,7 @@ from contextual_orchestrator.orchestrator import ( ModelAgent, TaskOrchestrator, + _parse_model_judge_reply, chat_completion_chunks, redact_text, redact_value, @@ -188,3 +191,13 @@ def exercise_orchestration(prompt: str, mode: str) -> None: continue assert frame.startswith("data: ") json.loads(frame[len("data: "):]) + + +def exercise_model_judge_reply(reply: str) -> None: + """Drive strict model-judge parsing over arbitrary untrusted text.""" + try: + decision, reason = _parse_model_judge_reply(reply) + except ValueError: + return + assert decision in {"ACCEPT", "REJECT"} + assert isinstance(reason, str) and reason.strip() diff --git a/pyproject.toml b/pyproject.toml index 65bd69eac..355d3e4cf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ db = [ "psycopg[binary]>=3.2", ] fuzz = [ - "atheris==3.0.0; python_version < '3.13'", + "atheris==3.1.0; python_version >= '3.12'", ] [tool.contextual_orchestrator] diff --git a/tests/fuzz/test_fuzz_properties.py b/tests/fuzz/test_fuzz_properties.py index 7e7b3f347..8958370b3 100644 --- a/tests/fuzz/test_fuzz_properties.py +++ b/tests/fuzz/test_fuzz_properties.py @@ -18,6 +18,7 @@ from fuzz.targets import ( exercise_agent_config, + exercise_model_judge_reply, exercise_orchestration, exercise_redaction, exercise_request_body, @@ -108,3 +109,9 @@ def test_redaction_never_crashes_and_is_idempotent(text: str) -> None: ) def test_orchestration_on_arbitrary_prompt(prompt: str, mode: str) -> None: exercise_orchestration(prompt, mode) + + +@_SETTINGS +@given(st.text(max_size=4096)) +def test_model_judge_parser_rejects_or_validates_arbitrary_text(reply: str) -> None: + exercise_model_judge_reply(reply) diff --git a/tests/test_batch_optimizer.py b/tests/test_batch_optimizer.py index 0211fb19a..210ee2628 100644 --- a/tests/test_batch_optimizer.py +++ b/tests/test_batch_optimizer.py @@ -7,13 +7,17 @@ from __future__ import annotations -from pathlib import Path import sys +from pathlib import Path +from unittest.mock import patch + +import pytest sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 -from contextual_orchestrator.orchestrator import ModelClient, optimize_orchestration # noqa: E402 +import contextual_orchestrator.orchestrator as orchestrator_module +from contextual_orchestrator import ModelAgent, TaskOrchestrator +from contextual_orchestrator.orchestrator import ModelClient, optimize_orchestration class _CountingClient(ModelClient): @@ -38,6 +42,23 @@ def batch_chat(self, agent: ModelAgent, requests: dict, temperature: float = 0.2 } +class _InvalidBatchClient(_CountingClient): + """Produces one malformed batch result to verify fail-closed persistence.""" + + def __init__(self, kind: str) -> None: + super().__init__() + self.kind = kind + + def batch_chat(self, agent: ModelAgent, requests: dict, temperature: float = 0.2, # type: ignore[override] + poll_interval: float = 5.0, poll_timeout: float = 3600.0) -> dict: + results = super().batch_chat(agent, requests, temperature, poll_interval, poll_timeout) + if self.kind == "missing": + results.pop("task_1") + else: + results["task_1"]["content"] = None + return results + + def _orch(client: ModelClient | None = None) -> TaskOrchestrator: return TaskOrchestrator( [ModelAgent("general_agent", "model-x", tags=("reasoning", "writing"))], @@ -82,11 +103,12 @@ def test_optimizer_use_batch_routes_via_batch_and_matches_serial() -> None: def test_conduct_config_stays_serial_even_with_use_batch() -> None: client = _CountingClient() - optimize_orchestration( - [{"name": "conduct_cfg", "orchestrator": _orch(client), "mode": "conduct"}], - TASKS[:1], lambda task, answer: 1.0, use_batch=True) + with patch.object(orchestrator_module, "_resolve_fast_mlsirm_components", return_value=None): + optimize_orchestration( + [{"name": "conduct_cfg", "orchestrator": _orch(client), "mode": "conduct"}], + TASKS[:1], lambda task, answer: 1.0, use_batch=True) assert client.batch_calls == 0 # multi-step cannot batch - assert client.chat_calls == 4 # thinker/worker/verifier/synthesizer + assert client.chat_calls == 4 # thinker/worker/verifier/synthesizer; missing fast-mlsirm fails closed def test_mock_default_batch_route_works_without_usage() -> None: @@ -96,6 +118,28 @@ def test_mock_default_batch_route_works_without_usage() -> None: assert orchestrator.spend_analytics()["by_model"][0]["usage_source"] == "estimated" +@pytest.mark.parametrize("kind", ["missing", "content"]) +def test_batch_route_rejects_incomplete_or_empty_provider_results(kind: str) -> None: + orchestrator = _orch(_InvalidBatchClient(kind)) + + with pytest.raises(RuntimeError, match="batch provider"): + orchestrator.batch_route([t["prompt"] for t in TASKS]) + assert orchestrator._workflow_runs == {} + + +def test_batch_chat_rejects_incomplete_local_result_set() -> None: + client = ModelClient() + agent = ModelAgent("local_agent", "model-x", base_url="local://127.0.0.1:1") + requests = { + "task_0": [{"role": "user", "content": "one"}], + "task_1": [{"role": "user", "content": "two"}], + } + with patch.object(client, "_local_batch_chat", return_value={ + "task_0": {"content": "ok", "usage": None}, + }), pytest.raises(RuntimeError, match="incomplete or unexpected"): + client.batch_chat(agent, requests) + + if __name__ == "__main__": for name, fn in sorted(globals().items()): if name.startswith("test_") and callable(fn): diff --git a/tests/test_batch_routing.py b/tests/test_batch_routing.py index baa3a0ce5..1d74e43be 100644 --- a/tests/test_batch_routing.py +++ b/tests/test_batch_routing.py @@ -2,8 +2,9 @@ from __future__ import annotations -from pathlib import Path import sys +import threading +from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) @@ -19,7 +20,6 @@ from contextual_orchestrator.cost_ledger import PriceBook, PriceEntry # noqa: E402 from contextual_orchestrator.kv_config import InMemoryConfigStore # noqa: E402 - # --------------------------------------------------------------------------- # Sync-vs-batch decision # --------------------------------------------------------------------------- @@ -105,6 +105,23 @@ def runner(messages, mode): assert answers == {"a": "echo:one", "b": "echo:two"} +def test_local_backend_honors_bounded_concurrency() -> None: + barrier = threading.Barrier(2, timeout=1.0) + + def runner(messages, mode): + barrier.wait() + return {"answer": messages[-1]["content"], "mode": mode} + + backend = LocalBatchBackend(runner, max_concurrency=2) + requests = [ + BatchRequest(messages=[{"role": "user", "content": "one"}], custom_id="a"), + BatchRequest(messages=[{"role": "user", "content": "two"}], custom_id="b"), + ] + + job = backend.submit(requests) + assert [item.custom_id for item in backend.retrieve(job)] == ["a", "b"] + + # --------------------------------------------------------------------------- # pg-llm-batch backend (mocked async client mirroring BatchAPIClient) # --------------------------------------------------------------------------- diff --git a/tests/test_cli_auth.py b/tests/test_cli_auth.py new file mode 100644 index 000000000..b89d47748 --- /dev/null +++ b/tests/test_cli_auth.py @@ -0,0 +1,204 @@ +"""CLI server-auth resolution stays explicit or KV-backed.""" + +from __future__ import annotations + +import sys +import types +from io import StringIO +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator.__main__ import _resolve_auth_token, main +from contextual_orchestrator.credentials import ( + InMemoryCredentialBackend, + set_backend, +) + + +def test_auth_token_resolution_prefers_explicit_then_kv() -> None: + backend = InMemoryCredentialBackend() + backend.set("gateway_token", "from-kv") + set_backend(backend) + try: + assert _resolve_auth_token("explicit", "gateway_token") == "explicit" + assert _resolve_auth_token("", "gateway_token") == "from-kv" + try: + _resolve_auth_token("", "missing_token") + except ValueError as exc: + assert "not configured" in str(exc) + else: # pragma: no cover + raise AssertionError("missing auth credential was accepted") + finally: + set_backend(None) + + +def test_partial_split_tokens_fail_before_kv_lookup() -> None: + stderr = StringIO() + with ( + patch.object(sys, "argv", ["contextual-orchestrator", "--serve", "--admin-token", "admin"]), + patch.object(sys, "stderr", stderr), + patch( + "contextual_orchestrator.__main__.get_credential", + side_effect=AssertionError("KV lookup was premature"), + ), + ): + try: + main() + except SystemExit as exc: + assert exc.code == 2 + assert "--admin-token-key" in stderr.getvalue() + assert "--inference-token-key" in stderr.getvalue() + else: # pragma: no cover + raise AssertionError("partial split token mode must be rejected") + + +def test_key_only_split_tokens_select_split_mode() -> None: + backend = InMemoryCredentialBackend() + backend.set("admin_key", "admin-from-kv") + backend.set("inference_key", "inference-from-kv") + set_backend(backend) + try: + with patch.object( + sys, + "argv", + [ + "contextual-orchestrator", + "--serve", + "--admin-token-key", + "admin_key", + "--inference-token-key", + "inference_key", + ], + ), patch("contextual_orchestrator.__main__.serve") as serve: + main() + security = serve.call_args.kwargs["security"] + assert security.auth_token == "" + assert security.admin_token == "admin-from-kv" + assert security.inference_token == "inference-from-kv" + finally: + set_backend(None) + + +def test_invalid_local_provider_options_fail_at_parser_boundary() -> None: + invalid_options = ( + (["--local-concurrency", "0"], "positive integer"), + (["--local-concurrency", "-1"], "positive integer"), + (["--local-concurrency", "65"], "1..64"), + (["--max-concurrent-runs", "0"], "positive integer"), + (["--max-concurrent-runs", "65"], "1..64"), + (["--chat-template-args", "[]"], "JSON object"), + (["--chat-template-args", "null"], "JSON object"), + (["--chat-template-args", "{"], "valid JSON object"), + ) + + for options, expected_message in invalid_options: + stderr = StringIO() + with ( + patch.object(sys, "argv", ["contextual-orchestrator", *options]), + patch.object(sys, "stderr", stderr), + patch( + "contextual_orchestrator.__main__.ModelClient", + side_effect=AssertionError("invalid CLI input reached ModelClient"), + ), + ): + try: + main() + except SystemExit as exc: + assert exc.code == 2 + assert expected_message in stderr.getvalue() + else: # pragma: no cover + raise AssertionError("invalid local provider option was accepted") + + +def test_server_concurrency_is_explicit_and_bounded() -> None: + with ( + patch.object( + sys, + "argv", + [ + "contextual-orchestrator", + "--serve", + "--auth-token", + "token", + "--local-concurrency", + "16", + "--max-concurrent-runs", + "16", + ], + ), + patch("contextual_orchestrator.__main__.load_agents", return_value=[]), + patch("contextual_orchestrator.__main__.ModelClient"), + patch("contextual_orchestrator.__main__.TaskOrchestrator"), + patch("contextual_orchestrator.__main__.serve") as serve, + ): + main() + assert serve.call_args.kwargs["security"].max_concurrent_runs == 16 + + +def test_sampling_temperature_uses_descriptive_name_and_legacy_alias() -> None: + for option in ("--sampling-temperature", "--temperature"): + with ( + patch.object( + sys, + "argv", + ["contextual-orchestrator", "--serve", "--auth-token", "token", option, "0.7"], + ), + patch("contextual_orchestrator.__main__.load_agents", return_value=[]), + patch("contextual_orchestrator.__main__.ModelClient") as model_client, + patch("contextual_orchestrator.__main__.TaskOrchestrator"), + patch("contextual_orchestrator.__main__.serve"), + ): + main() + assert model_client.call_args.kwargs["temperature"] == 0.7 + + +def test_fast_mlsirm_preflight_reports_missing_transitive_dependency() -> None: + stderr = StringIO() + real_import = __import__ + + def import_without_fast(name: str, *args, **kwargs): + if name == "fast_mlsirm": + raise ModuleNotFoundError("No module named 'numpy'", name="numpy") + return real_import(name, *args, **kwargs) + + with ( + patch.object(sys, "argv", ["contextual-orchestrator", "check-fast-mlsirm"]), + patch.object(sys, "stdout", stderr), + patch("builtins.__import__", side_effect=import_without_fast), + ): + try: + main() + except SystemExit as exc: + assert exc.code == 1 + else: # pragma: no cover + raise AssertionError("missing judge dependency must fail closed") + assert '"missing_module": "numpy"' in stderr.getvalue() + + +def test_fast_mlsirm_preflight_accepts_the_versioned_contract() -> None: + fake_package = types.ModuleType("fast_mlsirm") + fake_package.__version__ = "test" + fake_package.CONTEXTUAL_ORCHESTRATOR_CONTRACT_V1 = "contextual-orchestrator-contract-v1" + fake_package.ContextualOrchestratorJudge = type("ContextualOrchestratorJudge", (), {}) + fake_package.JudgeCriterion = type("JudgeCriterion", (), {}) + fake_package.JudgeFormatError = type("JudgeFormatError", (Exception,), {}) + stdout = StringIO() + with ( + patch.dict(sys.modules, {"fast_mlsirm": fake_package}), + patch.object(sys, "argv", ["contextual-orchestrator", "check-fast-mlsirm"]), + patch.object(sys, "stdout", stdout), + ): + main() + assert '"available": true' in stdout.getvalue() + assert '"contextual_contract": true' in stdout.getvalue() + + +if __name__ == "__main__": + test_auth_token_resolution_prefers_explicit_then_kv() + test_partial_split_tokens_fail_before_kv_lookup() + test_key_only_split_tokens_select_split_mode() + test_invalid_local_provider_options_fail_at_parser_boundary() + test_sampling_temperature_uses_descriptive_name_and_legacy_alias() + print("ok") diff --git a/tests/test_cost_ledger.py b/tests/test_cost_ledger.py index 8051712a9..b1cf1d95e 100644 --- a/tests/test_cost_ledger.py +++ b/tests/test_cost_ledger.py @@ -38,6 +38,37 @@ def query(self, start=None, end=None): return [] +class _PyformatCursor: + def __init__(self, connection) -> None: + self._connection = connection + self._cursor = connection._sqlite.cursor() + + def execute(self, statement, params=()): + if statement.count("%s") != len(params): + raise AssertionError("pyformat placeholders and values diverged") + self._connection.executions.append((statement, tuple(params))) + self._cursor.execute(statement.replace("%s", "?"), tuple(params)) + return self + + def fetchone(self): + return self._cursor.fetchone() + + def fetchall(self): + return self._cursor.fetchall() + + +class _PyformatConnection: + def __init__(self) -> None: + self._sqlite = sqlite3.connect(":memory:") + self.executions = [] + + def cursor(self): + return _PyformatCursor(self) + + def commit(self) -> None: + self._sqlite.commit() + + def test_price_computation_uses_per_1k_rates() -> None: ledger = _priced_ledger() # 1000 prompt * $2/1k + 500 completion * $4/1k = 2.0 + 2.0 = 4.0 @@ -243,6 +274,36 @@ def test_sql_ledger_store_on_sqlite_creates_objects_and_rolls_up() -> None: assert by_company["acme"]["cost_amount"] == 15.0 # 6 + 9 +def test_sql_ledger_rejects_unknown_parameter_style() -> None: + try: + SqlLedgerStore(sqlite3.connect(":memory:"), paramstyle="named") + except ValueError as exc: + assert "paramstyle must be qmark or pyformat" in str(exc) + else: # pragma: no cover + raise AssertionError("unknown DB-API parameter style must be rejected") + + +def test_sql_ledger_store_pyformat_binds_all_query_windows() -> None: + connection = _PyformatConnection() + store = SqlLedgerStore(connection, paramstyle="pyformat") + ledger = _priced_ledger(store=store) + ledger.record_usage(provider="openai", model="gpt-x", prompt_tokens=1000, + completion_tokens=0, created_at=100) + ledger.record_usage(provider="openai", model="gpt-x", prompt_tokens=2000, + completion_tokens=0, created_at=200) + ledger.record_usage(provider="openai", model="gpt-x", prompt_tokens=3000, + completion_tokens=0, created_at=300) + + assert [row["created_at"] for row in store.query()] == [100, 200, 300] + assert [row["created_at"] for row in store.query(start=150)] == [200, 300] + assert [row["created_at"] for row in store.query(end=300)] == [100, 200] + assert [row["created_at"] for row in store.query(start=150, end=300)] == [200] + insert_calls = [call for call in connection.executions if "INSERT INTO llm_usage_records" in call[0]] + assert insert_calls[0][1][1] == 100 + query_calls = [call for call in connection.executions if "SELECT usage_record_id" in call[0]] + assert [call[1] for call in query_calls[-4:]] == [(), (150,), (300,), (150, 300)] + + def test_ledger_table_names_follow_two_word_snake_case() -> None: for name in ("llm_usage_records", "cost_attribution_dimensions", "llm_price_entries"): assert is_two_word_snake_case(name) diff --git a/tests/test_cost_router.py b/tests/test_cost_router.py index 19cf25d10..4364e667a 100644 --- a/tests/test_cost_router.py +++ b/tests/test_cost_router.py @@ -2,16 +2,16 @@ from __future__ import annotations -from pathlib import Path import sys +from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from contextual_orchestrator import ( # noqa: E402 CostLedger, CostRoutingCoordinator, - InMemoryUsageTelemetrySink, InMemoryConfigStore, + InMemoryUsageTelemetrySink, ModelAgent, NonBlockingLedgerStore, PriceBook, @@ -113,6 +113,21 @@ def test_batch_completion_records_on_retrieve() -> None: assert records[0]["team_name"] == "beta" +def test_default_local_batch_backend_reuses_orchestrator_concurrency() -> None: + class _Client: + local_concurrency = 3 + + class _Orchestrator: + client = _Client() + + def complete(self, messages, *, mode): + return {"answer": messages[-1]["content"], "mode": mode} + + coordinator = CostRoutingCoordinator(_Orchestrator()) + + assert coordinator.batch_backend.max_concurrency == 3 + + def test_cost_report_rolls_up_across_sync_and_batch() -> None: coordinator = _coordinator() coordinator.complete([{"role": "user", "content": "sync one"}], attribution={"company": "acme"}) diff --git a/tests/test_generated_workflow.py b/tests/test_generated_workflow.py index 8c65b759c..3409d538c 100644 --- a/tests/test_generated_workflow.py +++ b/tests/test_generated_workflow.py @@ -12,10 +12,12 @@ import json from pathlib import Path import sys +from unittest.mock import patch sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 +import contextual_orchestrator.orchestrator as orchestrator_module # noqa: E402 from contextual_orchestrator.orchestrator import ModelClient # noqa: E402 @@ -56,12 +58,13 @@ def _orch(plan_text: str) -> tuple[TaskOrchestrator, _PlannerClient]: def test_generated_plan_executes_with_natural_language_subtasks() -> None: orchestrator, client = _orch(json.dumps(PLAN)) - result = orchestrator.conduct([{"role": "user", "content": "solve the hard problem"}]) + with patch.object(orchestrator_module, "_resolve_fast_mlsirm_components", return_value=None): + result = orchestrator.conduct([{"role": "user", "content": "solve the hard problem"}]) assert result["plan_source"] == "generated" assert [row["subtask"] for row in result["trace"]] == [s["subtask"] for s in PLAN["steps"]] - assert result["answer"] == "step-output(4)" # the synthesizer (last step) answers - assert len(client.calls) == 5 # 1 planner call + 4 steps + assert result["answer"] == "step-output(2)" # fail-closed judge leaves the worker answer + assert len(client.calls) == 5 # 1 planner call + 4 steps; missing fast-mlsirm fails closed def test_access_lists_actually_isolate_context() -> None: diff --git a/tests/test_healthz.py b/tests/test_healthz.py index 90dfc03c0..71c1ddcae 100644 --- a/tests/test_healthz.py +++ b/tests/test_healthz.py @@ -5,6 +5,7 @@ import sys import threading import urllib.request +from urllib.error import HTTPError from pathlib import Path ROOT = Path(__file__).resolve().parents[1] @@ -16,7 +17,10 @@ def test_healthz_is_unauthenticated_liveness() -> None: - orchestrator = TaskOrchestrator([ModelAgent("probe_agent", "mock-agent", tags=("reasoning",))]) + orchestrator = TaskOrchestrator([ + ModelAgent("probe_agent", "mock-agent", tags=("reasoning",)), + ModelAgent("disabled_probe_agent", "disabled-mock-agent", disabled=True), + ]) server = build_server( orchestrator, port=0, @@ -38,11 +42,58 @@ def test_healthz_is_unauthenticated_liveness() -> None: assert body["status"] == "ok" assert body["service"] == "contextual-orchestrator" assert body["agent_count"] == 1 + assert body["enabled_agent_count"] == 1 + assert body["candidate_count"] == 2 + assert body["provider_readiness"] == "unprobed" assert body["batch_backend"] assert body["embedding_batch_backend"] assert body["usage_record_count"] == 0 +def test_provider_readiness_refresh_is_authenticated_and_explicit() -> None: + orchestrator = TaskOrchestrator([ + ModelAgent("probe_agent", "mock-agent", tags=("reasoning",)), + ModelAgent("disabled_probe_agent", "disabled-mock-agent", disabled=True), + ]) + server = build_server( + orchestrator, + port=0, + security=SecurityConfig(admin_token="admin_secret", inference_token="inference_secret"), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + port = server.server_address[1] + url = f"http://127.0.0.1:{port}/api/v1/provider_readiness/latest" + + def request(path: str, token: str | None = "admin_secret") -> tuple[int, dict]: + headers = {} if token is None else {"Authorization": f"Bearer {token}"} + req = urllib.request.Request(path, headers=headers) + with urllib.request.urlopen(req, timeout=5) as response: + return response.status, json.loads(response.read().decode("utf-8")) + + try: + status, unprobed = request(url) + assert status == 200 + assert unprobed["status"] == "unprobed" + + status, refreshed = request(f"{url}?refresh=true") + assert status == 200 + assert refreshed["status"] == "ready" + assert refreshed["ready_agent_count"] == 1 + assert refreshed["items"][1]["status"] == "disabled" + + try: + request(f"{url}?refresh=true", token=None) + except HTTPError as exc: + assert exc.code == 401 + else: # pragma: no cover + raise AssertionError("provider readiness must require admin authentication") + finally: + server.shutdown() + thread.join(timeout=5) + + if __name__ == "__main__": test_healthz_is_unauthenticated_liveness() + test_provider_readiness_refresh_is_authenticated_and_explicit() print("ok") diff --git a/tests/test_local_mlx.py b/tests/test_local_mlx.py new file mode 100644 index 000000000..8c2b3d815 --- /dev/null +++ b/tests/test_local_mlx.py @@ -0,0 +1,812 @@ +"""Explicit loopback mlx-lm transport without weakening remote egress rules.""" + +from __future__ import annotations + +import json +import socket +import sys +import urllib.request +from pathlib import Path +from unittest.mock import patch + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from contextual_orchestrator import ModelAgent, TaskOrchestrator, load_agents # noqa: E402 +from contextual_orchestrator.credentials import NotConfigured # noqa: E402 +from contextual_orchestrator.orchestrator import ( # noqa: E402 + ModelClient, + _chat_to_responses_payload, + _is_local_provider_url, + _responses_to_chat_payload, +) + + +def test_local_candidate_registry_keeps_all_discovered_entries() -> None: + agents = load_agents(str(Path(__file__).resolve().parents[1] / "examples/agents.local.json")) + orchestrator = TaskOrchestrator(agents) + + assert {agent.model for agent in orchestrator.candidates} >= { + "contextual-orchestrator", + "mlx-community/gemma-4-31b-it-4bit", + "mlx-community/llama-3.2-3b-instruct-4bit", + "outlier-ai/deepseek-r1-distill-qwen-32b-mlx-4bit", + "embeddinggemma", + } + assert all(not agent.disabled for agent in orchestrator.candidates) + assert len(orchestrator.candidates) == len(orchestrator.agents) + verifier_exclusions = { + agent.model: agent.provider_exclusions + for agent in orchestrator.candidates + if agent.model in { + "mlx-community/llama-3.2-1b-instruct-4bit", + "mlx-community/gemma-4-31b-it-4bit", + "outlier-ai/deepseek-r1-distill-qwen-32b-mlx-4bit", + } + } + assert verifier_exclusions == { + "mlx-community/llama-3.2-1b-instruct-4bit": ("verifier",), + "mlx-community/gemma-4-31b-it-4bit": ("verifier",), + "outlier-ai/deepseek-r1-distill-qwen-32b-mlx-4bit": ("verifier",), + } + assert orchestrator._select_agent( + "Evaluate this answer for evidence and risk.", "verifier" + ).model == "mlx-community/gemma-4-e4b-it-4bit" + assert any( + agent.model == "contextual-orchestrator" + and set(agent.provider_exclusions) == {"thinker", "worker", "verifier", "synthesizer"} + for agent in orchestrator.candidates + ) + + +class _Response: + def __init__(self, payload: dict) -> None: + self.payload = payload + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self) -> bytes: + import json + + return json.dumps(self.payload).encode("utf-8") + + +def test_mlx_loopback_uses_http_without_a_credential() -> None: + agent = ModelAgent("local_agent", "local-model", base_url="mlx://127.0.0.1:8080/v1") + client = ModelClient(max_retries=0, temperature=0.0, chat_template_args={"enable_thinking": False}) + seen = [] + + def open_provider(request, _destination=None): + seen.append(request) + return _Response({ + "choices": [{"message": {"content": "local-ok"}}], + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}, + }) + + with patch.object(client, "_open_provider", side_effect=open_provider): + assert client.chat(agent, [{"role": "user", "content": "ping"}]) == "local-ok" + assert seen[0].full_url == "http://127.0.0.1:8080/v1/chat/completions" + assert "Authorization" not in seen[0].headers + import json + + assert json.loads(seen[0].data)["chat_template_kwargs"] == {"enable_thinking": False} + assert client.take_usage()["total_tokens"] == 3 + + +def test_authenticated_local_gateway_uses_only_its_explicit_kv_credential() -> None: + agent = ModelAgent( + "gateway_agent", + "local-model", + base_url="local://127.0.0.1:8080/v1", + credential_key="OPENAI_API_KEY", + local_credential_key="LOCAL_GATEWAY_TOKEN", + ) + client = ModelClient( + max_retries=0, + temperature=0.0, + chat_template_args={"enable_thinking": False}, + ) + seen = [] + + def open_provider(request, _destination=None): + seen.append(request) + return _Response({"choices": [{"message": {"content": "gateway-ok"}}]}) + + def credential(name: str) -> str | None: + return {"LOCAL_GATEWAY_TOKEN": "gateway-secret"}.get(name) + + with patch("contextual_orchestrator.orchestrator.get_credential", side_effect=credential), patch.object( + client, "_open_provider", side_effect=open_provider + ): + assert client.chat(agent, [{"role": "user", "content": "ping"}]) == "gateway-ok" + + assert seen[0].full_url == "http://127.0.0.1:8080/v1/chat/completions" + assert seen[0].get_header("Authorization") == "Bearer gateway-secret" + assert "chat_template_kwargs" not in json.loads(seen[0].data) + + +def test_authenticated_local_gateway_requires_its_kv_credential() -> None: + agent = ModelAgent( + "gateway_agent", + "local-model", + base_url="local://127.0.0.1:8080/v1", + local_credential_key="LOCAL_GATEWAY_TOKEN", + ) + with patch("contextual_orchestrator.orchestrator.get_credential", return_value=None): + with pytest.raises(NotConfigured, match="LOCAL_GATEWAY_TOKEN"): + ModelClient(max_retries=0).chat(agent, [{"role": "user", "content": "ping"}]) + + +def test_local_gateway_credential_cannot_be_attached_to_mlx_worker() -> None: + with pytest.raises(ValueError, match="local:// gateway"): + ModelAgent( + "local_agent", + "local-model", + base_url="mlx://127.0.0.1:8080/v1", + local_credential_key="LOCAL_GATEWAY_TOKEN", + ) + + +def test_provider_probe_verifies_registry_then_uses_one_bounded_completion_without_retry() -> None: + agent = ModelAgent("local_agent", "local-model", base_url="mlx://127.0.0.1:8080/v1") + client = ModelClient(max_retries=2, local_max_retries=2, chat_template_args={"enable_thinking": False}) + seen: list[tuple[object, float | None]] = [] + + def open_provider(request, _destination=None, *, timeout=None): + seen.append((request, timeout)) + if request.get_method() == "GET": + return _Response({"object": "list", "data": [{"id": "local-model"}]}) + return _Response({ + "choices": [{"message": {"content": "OK"}}], + "usage": {"prompt_tokens": 6, "completion_tokens": 1, "total_tokens": 7}, + }) + + with patch.object(client, "_open_provider", side_effect=open_provider): + report = client.probe(agent, timeout=1.25) + + assert report["status"] == "ready" + assert report["usage"]["total_tokens"] == 7 + assert len(seen) == 2 + assert seen[0][0].get_method() == "GET" + assert seen[0][0].full_url == "http://127.0.0.1:8080/v1/models" + assert seen[1][0].get_method() == "POST" + assert seen[1][1] == 1.25 + import json + + payload = json.loads(seen[1][0].data) + assert payload["max_tokens"] == 1 + assert payload["temperature"] == 0.0 + assert payload["chat_template_kwargs"] == {"enable_thinking": False} + + +def test_provider_probe_rejects_a_local_model_registry_mismatch() -> None: + agent = ModelAgent("local_agent", "requested-model", base_url="mlx://127.0.0.1:8080/v1") + client = ModelClient(max_retries=0) + with patch.object( + client, + "_open_provider", + return_value=_Response({"object": "list", "data": [{"id": "other-model"}]}), + ) as open_provider: + report = client.probe(agent, timeout=0.5) + + assert report["status"] == "not_ready" + assert report["error_type"] == "RuntimeError" + assert report["failure_code"] == "provider_model_not_registered" + assert "error" not in report + assert open_provider.call_count == 1 + + +def test_provider_probe_reports_timeout_without_retry() -> None: + agent = ModelAgent("local_agent", "local-model", base_url="mlx://127.0.0.1:8080/v1") + client = ModelClient(max_retries=2, local_max_retries=2) + with patch.object(client, "_open_provider", side_effect=TimeoutError("probe timeout")) as open_provider: + report = client.probe(agent, timeout=0.5) + + assert report["status"] == "not_ready" + assert report["error_type"] == "TimeoutError" + assert report["failure_code"] == "provider_probe_failed" + assert "error" not in report + assert open_provider.call_count == 1 + + +def test_provider_probe_does_not_serialize_provider_exception_text() -> None: + agent = ModelAgent("local_agent", "local-model", base_url="mlx://127.0.0.1:8080/v1") + client = ModelClient(max_retries=0) + with patch.object(client, "_open_provider", side_effect=RuntimeError("provider-output-secret")): + report = client.probe(agent, timeout=0.5) + + serialized = json.dumps(report) + assert "provider-output-secret" not in serialized + assert report["error_type"] == "RuntimeError" + assert report["failure_code"] == "provider_probe_failed" + + +def test_provider_readiness_report_keeps_liveness_unprobed_until_refresh() -> None: + client = ModelClient() + orchestrator = TaskOrchestrator([ + ModelAgent("ready_agent", "ready-model"), + ModelAgent("disabled_agent", "disabled-model", disabled=True), + ], client=client) + with patch.object(client, "probe", return_value={"status": "ready", "agent_id": "ready_agent", "model": "ready-model"}) as probe: + unprobed = orchestrator.provider_readiness_report() + refreshed = orchestrator.provider_readiness_report(refresh=True, timeout=2.0) + + assert unprobed["status"] == "unprobed" + assert unprobed["items"][0]["status"] == "unprobed" + assert refreshed["status"] == "ready" + assert refreshed["ready_agent_count"] == 1 + assert refreshed["items"][1]["status"] == "disabled" + probe.assert_called_once_with(orchestrator.agents[0], timeout=2.0) + + +def test_provider_readiness_refresh_serializes_concurrent_probes() -> None: + import threading + + client = ModelClient() + orchestrator = TaskOrchestrator([ModelAgent("ready_agent", "ready-model")], client=client) + entered = threading.Event() + release = threading.Event() + counters = {"active": 0, "max_active": 0} + counter_lock = threading.Lock() + + def probe(_agent, *, timeout): + del timeout + with counter_lock: + counters["active"] += 1 + counters["max_active"] = max(counters["max_active"], counters["active"]) + entered.set() + release.wait(timeout=2) + with counter_lock: + counters["active"] -= 1 + return {"status": "ready", "agent_id": "ready_agent", "model": "ready-model"} + + with patch.object(client, "probe", side_effect=probe): + first = threading.Thread(target=lambda: orchestrator.provider_readiness_report(refresh=True)) + second = threading.Thread(target=lambda: orchestrator.provider_readiness_report(refresh=True)) + first.start() + assert entered.wait(timeout=2) + second.start() + release.set() + first.join(timeout=2) + second.join(timeout=2) + + assert counters["max_active"] == 1 + + +def test_local_provider_serializes_model_switches_and_bounds_waiters() -> None: + import threading + + first_agent = ModelAgent("first_agent", "model-a", base_url="mlx://127.0.0.1:8080/v1") + second_agent = ModelAgent("second_agent", "model-b", base_url="mlx://127.0.0.1:8080/v1") + first_client = ModelClient(max_retries=0, timeout=1.0) + second_client = ModelClient(max_retries=0, timeout=0.05) + entered = threading.Event() + release = threading.Event() + errors: list[BaseException] = [] + active = 0 + max_active = 0 + counter_lock = threading.Lock() + + def slow_open(_request, _destination=None): + nonlocal active, max_active + with counter_lock: + active += 1 + max_active = max(max_active, active) + entered.set() + release.wait(timeout=2) + with counter_lock: + active -= 1 + return _Response({"choices": [{"message": {"content": "OK"}}]}) + + def call(client, agent): + try: + client.chat(agent, [{"role": "user", "content": "ping"}]) + except BaseException as exc: # noqa: BLE001 - thread result is asserted below + errors.append(exc) + + with patch.object(first_client, "_open_provider", side_effect=slow_open), patch.object( + second_client, "_open_provider", side_effect=slow_open + ): + first = threading.Thread(target=call, args=(first_client, first_agent)) + second = threading.Thread(target=call, args=(second_client, second_agent)) + first.start() + assert entered.wait(timeout=1) + second.start() + second.join(timeout=1) + release.set() + first.join(timeout=1) + + assert not first.is_alive() + assert not second.is_alive() + assert max_active == 1 + assert len(errors) == 1 + assert isinstance(errors[0], TimeoutError) + + +def test_reasoning_only_response_explains_local_template_fix() -> None: + agent = ModelAgent("local_agent", "local-model", base_url="mlx://127.0.0.1:8080/v1") + client = ModelClient(max_retries=0) + with patch.object( + client, + "_open_provider", + return_value=_Response({"choices": [{"message": {"reasoning": "still thinking"}}]}), + ): + try: + client.chat(agent, [{"role": "user", "content": "ping"}]) + except RuntimeError as exc: + assert "enable_thinking" in str(exc) + else: # pragma: no cover + raise AssertionError("reasoning-only provider response must fail clearly") + + +def test_response_without_content_or_reasoning_fails_clearly() -> None: + agent = ModelAgent("local_agent", "local-model", base_url="mlx://127.0.0.1:8080/v1") + with pytest.raises(RuntimeError, match="assistant content"): + ModelClient()._response_content(agent, {"choices": [{"message": {}}]}) + + +def test_local_responses_passthrough_adapts_to_chat_transport() -> None: + agent = ModelAgent("local_agent", "local-model", base_url="mlx://127.0.0.1:8080/v1") + client = ModelClient(max_retries=0, chat_template_args={"enable_thinking": False}) + with patch.object(client, "_validate_provider", return_value=None), patch.object( + client, + "_send_raw_with_retry", + return_value={ + "id": "chatcmpl-local", + "model": "local-model", + "created": 123, + "choices": [{ + "message": {"role": "assistant", "content": "OK"}, + "finish_reason": "stop", + }], + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}, + }, + ) as send: + response = client.proxy_send( + agent, + "responses", + { + "model": "local-model", + "instructions": "Be concise.", + "input": [{ + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "ping"}], + }], + "stream": True, + "tools": [{ + "type": "function", + "name": "lookup", + "parameters": {"type": "object"}, + }], + }, + ) + assert response["object"] == "response" + assert response["output_text"] == "OK" + forwarded = send.call_args.args[2] + assert forwarded["messages"] == [ + {"role": "system", "content": "Be concise."}, + {"role": "user", "content": "ping"}, + ] + assert forwarded["tools"] == [{ + "type": "function", + "function": {"name": "lookup", "parameters": {"type": "object"}}, + }] + assert forwarded["chat_template_kwargs"] == {"enable_thinking": False} + + +def test_local_responses_passthrough_omits_empty_template_arguments() -> None: + agent = ModelAgent("local_agent", "local-model", base_url="mlx://127.0.0.1:8080/v1") + client = ModelClient() + with patch.object(client, "_validate_provider", return_value=None), patch.object( + client, + "_send_raw_with_retry", + return_value={"choices": [{"message": {"content": "OK"}}]}, + ) as send: + client.proxy_send(agent, "responses", {"input": "ping"}) + + assert "chat_template_kwargs" not in send.call_args.args[2] + + +def test_local_responses_adapter_preserves_supported_items_and_controls() -> None: + payload = _responses_to_chat_payload( + { + "model": "local-model", + "instructions": [ + {"type": "input_text", "text": "system"}, + {"type": "ignored", "text": 17}, + " rules", + ], + "input": [ + "plain input", + {"type": "message", "role": "developer", "content": "developer note"}, + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "prior answer"}], + }, + {"type": "message", "role": "user", "content": []}, + { + "type": "function_call_output", + "call_id": "call_1", + "output": [{"type": "output_text", "text": "tool result"}], + }, + { + "type": "function_call", + "call_id": "call_2", + "name": "lookup", + "arguments": '{"q":"x"}', + }, + {"type": "reasoning", "id": "reasoning_1"}, + {"type": "item_reference", "id": "item_1"}, + ], + "max_output_tokens": 99, + "temperature": 0.1, + "tools": [ + {"type": "file_search"}, + {"type": "function", "name": "lookup", "parameters": {"type": "object"}}, + ], + "tool_choice": {"type": "function", "name": "lookup", "extra": "ignored"}, + } + ) + + assert payload["messages"] == [ + {"role": "system", "content": "system rules"}, + {"role": "user", "content": "plain input"}, + {"role": "system", "content": "developer note"}, + {"role": "assistant", "content": "prior answer"}, + {"role": "tool", "tool_call_id": "call_1", "content": "tool result"}, + { + "role": "assistant", + "content": "", + "tool_calls": [{ + "id": "call_2", + "type": "function", + "function": {"name": "lookup", "arguments": '{"q":"x"}'}, + }], + }, + ] + assert payload["max_tokens"] == 99 + assert payload["temperature"] == 0.1 + assert payload["tools"] == [{ + "type": "function", + "function": {"name": "lookup", "parameters": {"type": "object"}}, + }] + assert payload["tool_choice"] == {"type": "function", "function": {"name": "lookup"}} + + +def test_local_responses_adapter_rejects_non_string_input() -> None: + with pytest.raises(ValueError, match="string or item list"): + _responses_to_chat_payload({"input": {"unexpected": "mapping"}}) + + +@pytest.mark.parametrize( + "item", + [ + 17, + {"type": "message", "role": "moderator", "content": "unsupported role"}, + {"type": "unknown"}, + ], +) +def test_local_responses_adapter_rejects_unsupported_items(item: object) -> None: + with pytest.raises(ValueError): + _responses_to_chat_payload({"input": [item]}) + + +def test_local_responses_adapter_accepts_string_input() -> None: + payload = _responses_to_chat_payload({"input": "plain text"}) + assert payload["messages"] == [{"role": "user", "content": "plain text"}] + + +def test_local_responses_response_maps_reasoning_and_tool_calls() -> None: + response = _chat_to_responses_payload( + { + "id": "chatcmpl-1", + "model": "local-model", + "created": 123, + "choices": [{ + "message": { + "reasoning": "internal reasoning", + "tool_calls": [ + "malformed", + {"id": "call_1", "function": {"name": "lookup", "arguments": "{}"}}, + ], + }, + "finish_reason": "length", + }], + "usage": {"prompt_tokens": 2, "completion_tokens": 3, "total_tokens": 5}, + }, + {"model": "local-model", "metadata": {"trace": "test"}}, + ) + + assert response["status"] == "incomplete" + assert response["output_text"] == "internal reasoning" + assert response["usage"] == {"input_tokens": 2, "output_tokens": 3, "total_tokens": 5} + assert response["metadata"] == {"trace": "test"} + assert response["output"][-1] == { + "id": "fc_call_1", + "type": "function_call", + "status": "completed", + "call_id": "call_1", + "name": "lookup", + "arguments": "{}", + } + + tool_only = _chat_to_responses_payload( + {"choices": [{"message": {"content": "", "tool_calls": [{"id": "call_2", "function": {}}]}}]}, + {}, + ) + assert tool_only["output"][0]["type"] == "function_call" + + +def test_local_provider_scheme_validation_rejects_remote_and_malformed_ports() -> None: + assert _is_local_provider_url("mlx://127.0.0.1:8080/v1") + assert not _is_local_provider_url("mlx://example.com:8080/v1") + assert not _is_local_provider_url("mlx://127.0.0.1:not-a-port/v1") + + +@pytest.mark.parametrize( + "kwargs", + [ + {"max_retries": -1}, + {"max_retries": True}, + {"local_max_retries": -1}, + {"local_max_retries": True}, + {"local_concurrency": 0}, + {"local_concurrency": False}, + {"local_concurrency": 1.5}, + {"local_concurrency": 65}, + ], +) +def test_local_transport_limits_reject_invalid_values(kwargs: dict[str, object]) -> None: + with pytest.raises(ValueError): + ModelClient(**kwargs) + + +def test_provider_transport_rejects_invalid_port_and_resolution_failures() -> None: + client = ModelClient() + with pytest.raises(RuntimeError, match="invalid port"): + client._open_provider(urllib.request.Request("http://127.0.0.1:not-a-port/v1")) + + with patch("contextual_orchestrator.orchestrator.socket.getaddrinfo", side_effect=socket.gaierror): + with pytest.raises(RuntimeError, match="could not be resolved"): + client._resolve_addresses("unresolvable.example", 443) + with patch("contextual_orchestrator.orchestrator.socket.getaddrinfo", return_value=[]): + with pytest.raises(RuntimeError, match="no stream address"): + client._resolve_addresses("empty.example", 443) + + +def test_https_provider_uses_verifying_connection_and_resolved_destination() -> None: + class FakeResponse: + status = 200 + + class FakeConnection: + def __init__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + self.request_args = None + + def request(self, *args, **kwargs): + self.request_args = (args, kwargs) + + def getresponse(self): + return FakeResponse() + + def close(self): + return None + + client = ModelClient() + connection = FakeConnection() + request = urllib.request.Request("https://provider.example/v1/chat/completions", method="POST") + with patch( + "contextual_orchestrator.orchestrator.http.client.HTTPSConnection", + return_value=connection, + ) as https_connection: + response = client._open_provider( + request, + (socket.AF_INET, ("127.0.0.1", 443)), + ) + + assert response.status == 200 + https_connection.assert_called_once_with( + "provider.example", 443, timeout=client.timeout, context=client._ssl_context + ) + assert connection.request_args[0][0] == "POST" + + +def test_validated_connect_binds_source_address() -> None: + class FakeSocket: + def __init__(self): + self.bound = None + self.connected = None + self.timeout = None + self.closed = False + + def settimeout(self, value): + self.timeout = value + + def bind(self, address): + self.bound = address + + def connect(self, address): + self.connected = address + + def close(self): + self.closed = True + + fake = FakeSocket() + with patch("contextual_orchestrator.orchestrator.socket.socket", return_value=fake) as socket_factory: + result = ModelClient._connect_validated( + (socket.AF_INET, ("127.0.0.1", 443)), + 2.0, + ("127.0.0.1", 0), + ) + + assert result is fake + socket_factory.assert_called_once_with(socket.AF_INET, socket.SOCK_STREAM) + assert fake.timeout == 2.0 + assert fake.bound == ("127.0.0.1", 0) + assert fake.connected == ("127.0.0.1", 443) + + +def test_local_provider_url_rejects_query_data_at_transport_boundary() -> None: + agent = ModelAgent("local_agent", "local-model", base_url="mlx://127.0.0.1:8080/v1?unsafe=1") + with pytest.raises(RuntimeError, match="query data"): + ModelClient()._provider_url(agent, "/chat/completions") + with pytest.raises(RuntimeError, match="query data"): + ModelClient()._validate_provider(agent) + + +def test_provider_url_rejects_non_http_scheme_at_builder_boundary() -> None: + agent = ModelAgent("bad_agent", "bad-model", base_url="file:///tmp/provider") + with pytest.raises(RuntimeError, match=r"http\(s\) provider URL"): + ModelClient()._provider_url(agent, "/chat/completions") + + +def test_provider_validation_rejects_non_loopback_and_remote_query_data() -> None: + client = ModelClient() + local = ModelAgent("local_agent", "local-model", base_url="mlx://127.0.0.1:8080/v1") + with patch.object(client, "_resolve_addresses", return_value=[(socket.AF_INET, ("192.0.2.1", 8080))]): + with pytest.raises(RuntimeError, match="non-loopback"): + client._validate_provider(local) + + remote_query = ModelAgent( + "remote_agent", + "remote-model", + base_url="https://provider.example/v1?unsafe=1", + credential_key="remote-key", + ) + remote = ModelAgent( + "remote_agent", + "remote-model", + base_url="https://provider.example/v1", + credential_key="remote-key", + ) + with patch("contextual_orchestrator.orchestrator.get_credential", return_value="secret"): + with pytest.raises(RuntimeError, match="query data"): + client._validate_provider(remote_query) + with patch.object( + client, + "_resolve_addresses", + return_value=[(socket.AF_INET, ("93.184.216.34", 443))], + ): + assert client._validate_provider(remote) == (socket.AF_INET, ("93.184.216.34", 443)) + + +def test_remote_transport_uses_kv_credential_for_chat_and_stream() -> None: + agent = ModelAgent( + "remote_agent", + "remote-model", + base_url="https://provider.example/v1", + credential_key="remote-key", + ) + + class StreamingResponse: + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def __iter__(self): + return iter([b'data: {"choices":[{"delta":{"content":"delta"}}]}\n', b"data: [DONE]\n"]) + + client = ModelClient() + with patch("contextual_orchestrator.orchestrator.get_credential", return_value="remote-secret"), patch.object( + client, + "_open_provider", + side_effect=[ + _Response({"choices": [{"message": {"content": "chat"}}]}), + StreamingResponse(), + ], + ) as open_provider: + assert client._send(agent, {"model": agent.model}) == "chat" + assert list(client._stream_send(agent, {"model": agent.model, "stream": True})) == ["delta"] + + for call in open_provider.call_args_list: + assert call.args[0].get_header("Authorization") == "Bearer remote-secret" + + +def test_remote_http_is_still_rejected() -> None: + agent = ModelAgent("remote_agent", "remote-model", base_url="http://127.0.0.1:8080/v1") + try: + with patch("contextual_orchestrator.orchestrator.get_credential", return_value="local"): + ModelClient(max_retries=0).chat(agent, [{"role": "user", "content": "ping"}]) + except RuntimeError as exc: + assert "https" in str(exc) + else: # pragma: no cover + raise AssertionError("plain http provider must remain rejected") + + +def test_provider_transport_rejects_non_http_url_before_io() -> None: + client = ModelClient(max_retries=0) + request = urllib.request.Request("file:///tmp/not-a-provider", method="GET") + try: + client._open_provider(request) + except RuntimeError as exc: + assert "HTTP(S) URL" in str(exc) + else: # pragma: no cover + raise AssertionError("non-HTTP provider URL must be rejected") + + +def test_local_batch_preserves_ids_and_usage() -> None: + agent = ModelAgent("local_agent", "local-model", base_url="mlx://127.0.0.1:8080/v1") + client = ModelClient(max_retries=0, local_concurrency=2) + calls = [] + + def fake_chat(_agent, messages, temperature=None): + calls.append((messages[0]["content"], temperature)) + client._local.usage = {"completion_tokens": 1} + return messages[0]["content"] + + with patch.object(client, "chat", side_effect=fake_chat): + result = client.batch_chat( + agent, + {"one": [{"role": "user", "content": "1"}], "two": [{"role": "user", "content": "2"}]}, + temperature=0.0, + ) + assert {key: value["content"] for key, value in result.items()} == {"one": "1", "two": "2"} + assert all(value["usage"] == {"completion_tokens": 1} for value in result.values()) + assert sorted(calls) == [("1", 0.0), ("2", 0.0)] + + +def test_local_batch_default_uses_sequential_path() -> None: + agent = ModelAgent("local_agent", "local-model", base_url="mlx://127.0.0.1:8080/v1") + client = ModelClient() + with patch.object(client, "chat", side_effect=lambda _agent, messages, temperature=None: messages[0]["content"]): + result = client.batch_chat( + agent, + {"one": [{"role": "user", "content": "1"}], "two": [{"role": "user", "content": "2"}]}, + ) + + assert {key: value["content"] for key, value in result.items()} == {"one": "1", "two": "2"} + + +def test_patch_agent_rejects_disabling_last_enabled_agent() -> None: + orchestrator = TaskOrchestrator([ModelAgent("only_agent", "mock-only")]) + with pytest.raises(ValueError, match="last enabled"): + orchestrator.patch_agent("default", "only_agent", {"status": "disabled"}) + + +def test_stream_chat_forwards_local_template_arguments() -> None: + agent = ModelAgent("local_agent", "local-model", base_url="mlx://127.0.0.1:8080/v1") + client = ModelClient(chat_template_args={"enable_thinking": False}) + with patch.object(client, "_validate_provider", return_value=None), patch.object( + client, "_stream_send", return_value=iter(("delta",)) + ) as stream_send: + assert list(client.stream_chat(agent, [{"role": "user", "content": "ping"}])) == ["delta"] + + assert stream_send.call_args.args[1]["chat_template_kwargs"] == {"enable_thinking": False} + + +if __name__ == "__main__": + for name, fn in sorted(globals().items()): + if name.startswith("test_") and callable(fn): + fn() + print(f"ok {name}") + print("ok") diff --git a/tests/test_model_judge.py b/tests/test_model_judge.py index 6db8c5e9c..b06ee0acb 100644 --- a/tests/test_model_judge.py +++ b/tests/test_model_judge.py @@ -1,21 +1,25 @@ -"""Model-based verifier judge — the recorded fix for term-matching false negatives. +"""Structured model-based verifier judging. -A verifier report that *discusses* risks tripped the term matcher (observed on the -real-OpenAI generated-workflow run). With verifier_judge="model", a verifier-selected -model replies ACCEPT/REJECT; ambiguous replies or judge failures keep the term verdict. -Default "terms" is unchanged (no extra model call). +Keyword matching is deliberately rejected: verifier reports can quote risks, +use negation, or be written in another language. The judge must return an +explicit structured verdict and uncertainty must fail closed. """ from __future__ import annotations from dataclasses import replace from pathlib import Path +import contextual_orchestrator.orchestrator as orchestrator_module import sys +from types import SimpleNamespace +from unittest.mock import patch + +import pytest sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 -from contextual_orchestrator.orchestrator import ModelClient # noqa: E402 +from contextual_orchestrator.orchestrator import ModelClient, _parse_model_judge_reply # noqa: E402 RISKY_VERIFIER_REPORT = "The plan is sound overall but discusses downtime risks and error handling." @@ -29,63 +33,140 @@ def __init__(self, judge_reply: str) -> None: self.judge_reply = judge_reply self.calls = 0 - def chat(self, agent: ModelAgent, messages: list, temperature: float = 0.2) -> str: # type: ignore[override] + def chat(self, agent: ModelAgent, messages: list, temperature: float | None = None) -> str: # type: ignore[override] self.calls += 1 if self.calls == 3: - return RISKY_VERIFIER_REPORT # term matcher sees "risk"/"error" -> would reject + return RISKY_VERIFIER_REPORT if self.calls == 5: return self.judge_reply return f"step-output({self.calls})" -def _orch(judge_reply: str, judge_mode: str = "model") -> tuple[TaskOrchestrator, _ScriptedClient]: +def _orch(judge_reply: str) -> tuple[TaskOrchestrator, _ScriptedClient]: client = _ScriptedClient(judge_reply) orchestrator = TaskOrchestrator( [ModelAgent("general_agent", "model-x", tags=("reasoning", "writing", "planning", "research"))], client=client, ) - orchestrator.policy = replace(orchestrator.policy, verifier_judge=judge_mode) return orchestrator, client +class _ScriptedCriterion: + def __init__(self, criterion_id: str, description: str, weight: float) -> None: + self.criterion_id = criterion_id + self.description = description + self.weight = weight + + +class _ScriptedFastJudge: + def __init__(self, adapter, *, mode: str, accept_threshold: float) -> None: + self.adapter = adapter + self.mode = mode + self.accept_threshold = accept_threshold + + def judge(self, *, task: str, answer: str, criteria: tuple) -> object: + del task, answer, criteria + completion = self.adapter.complete([{"role": "user", "content": "judge"}], mode=self.mode) + decision, reason = _parse_model_judge_reply(completion["answer"]) + accepted = decision == "ACCEPT" + return SimpleNamespace( + accepted=accepted, + rationale=reason, + criterion_scores={"evidence_quality": 1.0, "risk_signal": 1.0}, + usage=completion.get("usage"), + orchestration_mode=self.mode, + to_irt_row=lambda *, item_type: (int(accepted), int(accepted)), + ) + + +def _scripted_fast_components() -> orchestrator_module.FastMLSIRMJudgeComponents: + return orchestrator_module.FastMLSIRMJudgeComponents( + judge_cls=_ScriptedFastJudge, + criterion_cls=_ScriptedCriterion, + format_error=ValueError, + ) + + MESSAGES = [{"role": "user", "content": "design and verify the migration plan"}] -def test_terms_judge_false_negatives_on_risk_vocabulary() -> None: - # Baseline showing the problem the model judge fixes. - orchestrator, client = _orch("unused", judge_mode="terms") - result = orchestrator.conduct(MESSAGES) - assert result["verification"]["accepted"] is False # term matcher trips on "risks"/"error" - assert client.calls == 4 # no judge call in terms mode +def test_keyword_matching_never_decides() -> None: + orchestrator, _ = _orch("unused") + result = orchestrator._judge_verifier_output("verified and good", "planner", "worker") + assert result["accepted"] is False + assert "keyword matching" in result["reason"] + +def test_legacy_keyword_policy_is_rejected() -> None: + orchestrator, _ = _orch("unused") + try: + replace(orchestrator.policy, verifier_judge="terms") + except ValueError as exc: + assert "keyword-based" in str(exc) + else: # pragma: no cover + raise AssertionError("keyword-based verifier policy was accepted") -def test_model_judge_accept_overrides_term_false_negative() -> None: - orchestrator, client = _orch("ACCEPT") - result = orchestrator.conduct(MESSAGES) + +def test_structured_model_judge_accepts() -> None: + orchestrator, client = _orch('{"decision":"ACCEPT","reason":"The report supports the answer."}') + with patch.object( + orchestrator_module, + "_resolve_fast_mlsirm_components", + return_value=_scripted_fast_components(), + ): + result = orchestrator.conduct(MESSAGES) assert result["verification"]["accepted"] is True assert result["verification"]["judge"] == "model" - assert client.calls == 5 # exactly one extra judge call - assert result["answer"] == "step-output(4)" # synthesizer answers when accepted + assert client.calls == 5 + assert result["answer"] == "step-output(4)" -def test_model_judge_reject_is_respected() -> None: - orchestrator, _ = _orch("REJECT — the migration plan loses writes.") - result = orchestrator.conduct(MESSAGES) +def test_structured_model_judge_rejects() -> None: + orchestrator, _ = _orch('{"decision":"REJECT","reason":"The migration plan loses writes."}') + with patch.object( + orchestrator_module, + "_resolve_fast_mlsirm_components", + return_value=_scripted_fast_components(), + ): + result = orchestrator.conduct(MESSAGES) assert result["verification"]["accepted"] is False assert result["verification"]["judge"] == "model" - assert result["answer"] == "step-output(2)" # falls back to the worker output + assert result["answer"] == "step-output(2)" + + +def test_plain_keyword_reply_is_rejected() -> None: + orchestrator, _ = _orch("ACCEPT because the report looks fine") + with patch.object( + orchestrator_module, + "_resolve_fast_mlsirm_components", + return_value=_scripted_fast_components(), + ): + result = orchestrator.conduct(MESSAGES) + assert result["verification"]["accepted"] is False + assert "invalid structured verdict" in result["verification"]["reason"] + assert result["answer"] == "step-output(2)" -def test_ambiguous_judge_reply_keeps_term_verdict() -> None: - orchestrator, _ = _orch("well, it depends on many factors") - result = orchestrator.conduct(MESSAGES) - assert result["verification"]["accepted"] is False # term verdict retained - assert "judge" not in result["verification"] +def test_judge_rejects_wrapped_extra_and_duplicate_json() -> None: + for reply in ( + 'prefix {"decision":"ACCEPT","reason":"valid"}', + '{"decision":"ACCEPT","reason":"valid","extra":true}', + '{"decision":"ACCEPT","decision":"REJECT","reason":"ambiguous"}', + ): + orchestrator, _ = _orch(reply) + with patch.object( + orchestrator_module, + "_resolve_fast_mlsirm_components", + return_value=_scripted_fast_components(), + ): + result = orchestrator.conduct(MESSAGES) + assert result["verification"]["accepted"] is False + assert "invalid structured verdict" in result["verification"]["reason"] -def test_judge_failure_keeps_term_verdict() -> None: +def test_judge_failure_fails_closed() -> None: class _FailingJudge(_ScriptedClient): - def chat(self, agent: ModelAgent, messages: list, temperature: float = 0.2) -> str: # type: ignore[override] + def chat(self, agent: ModelAgent, messages: list, temperature: float | None = None) -> str: # type: ignore[override] self.calls += 1 if self.calls == 3: return RISKY_VERIFIER_REPORT @@ -98,10 +179,297 @@ def chat(self, agent: ModelAgent, messages: list, temperature: float = 0.2) -> s [ModelAgent("general_agent", "model-x", tags=("reasoning", "writing", "planning", "research"))], client=client, ) - orchestrator.policy = replace(orchestrator.policy, verifier_judge="model") - result = orchestrator.conduct(MESSAGES) - assert result["verification"]["accepted"] is False # fallback, request not broken - assert "judge" not in result["verification"] + with patch.object( + orchestrator_module, + "_resolve_fast_mlsirm_components", + return_value=_scripted_fast_components(), + ): + result = orchestrator.conduct(MESSAGES) + assert result["verification"]["accepted"] is False + assert result["verification"]["judge"] == "model" + assert "failed closed" in result["verification"]["reason"] + assert result["answer"] == "step-output(2)" + + +def test_fast_mlsirm_path_is_used_when_available() -> None: + class _FakeJudge: + def __init__(self, orchestrator, mode: str = "route", accept_threshold: float = 0.7) -> None: + self.adapter = orchestrator + self.mode = mode + self.accept_threshold = accept_threshold + + def judge(self, **_) -> object: + self.adapter.complete([{"role": "user", "content": "ping"}]) + return type("Result", (), { + "accepted": True, + "rationale": "structured score exceeded threshold", + "criterion_scores": {"evidence_quality": 0.8, "risk_signal": 0.9}, + "usage": {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7}, + "orchestration_mode": self.mode, + "to_irt_row": lambda *, item_type: (1, 1), + }) + + class _FormatError(Exception): + pass + + class _Criterion: + def __init__(self, criterion_id: str, description: str, weight: float) -> None: + self.criterion_id = criterion_id + self.description = description + self.weight = weight + + orchestrator, _ = _orch("unused") + orchestrator.policy = replace(orchestrator.policy, workflow_planning="conduct") + with patch.object( + orchestrator_module, + "_resolve_fast_mlsirm_components", + return_value=orchestrator_module.FastMLSIRMJudgeComponents( + judge_cls=_FakeJudge, + criterion_cls=_Criterion, + format_error=_FormatError, + ), + ): + with patch.object( + orchestrator, + "_invoke", + return_value=("judge completion", "backup_judge", {"total_tokens": 7}), + ): + result = orchestrator._model_judge_verification( + "task", + {"verifier_output": "report"}, + ) + + assert result["accepted"] is True + assert result["judge"] == "model" + assert result["judge_agent_id"] == "backup_judge" + assert result["judge_orchestration_mode"] == "route" + assert result["judge_usage"] == {"prompt_tokens": 4, "completion_tokens": 3, "total_tokens": 7} + assert result["judge_criterion_scores"] == {"evidence_quality": 0.8, "risk_signal": 0.9} + assert result["judge_irt_item_type"] == "dichotomous" + assert result["judge_irt_row"] == [1, 1] + assert result["reason"] == "structured score exceeded threshold" + + +def test_fast_mlsirm_adapter_accepts_contextual_judge_mode_keyword() -> None: + orchestrator, _ = _orch("unused") + adapter = orchestrator_module._FastMLSIJudgeAdapter( + orchestrator, + "task", + "general_agent", + mode="route", + ) + assert adapter.client is orchestrator.client + assert ( + adapter.contextual_orchestrator_contract + == "contextual-orchestrator-contract-v1" + ) + with patch.object( + orchestrator, + "_invoke", + return_value=("judge completion", "general_agent", None), + ): + completion = adapter.complete( + [{"role": "user", "content": "ping"}], + mode="conduct", + ) + assert completion["answer"] == "judge completion" + assert completion["mode"] == "conduct" + + +def test_fast_mlsirm_adapter_routes_structured_completion_through_gateway() -> None: + orchestrator, _ = _orch("unused") + adapter = orchestrator_module._FastMLSIJudgeAdapter( + orchestrator, + "task", + "general_agent", + mode="route", + ) + response_format = { + "type": "json_schema", + "json_schema": {"name": "judge", "strict": True, "schema": {"type": "object"}}, + } + with patch.object( + orchestrator, + "proxy_completion", + return_value={ + "choices": [{"message": {"content": '{"meets_threshold":true,"rationale":"ok"}'}}], + "usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}, + }, + ) as proxy: + completion = adapter.complete_structured( + [{"role": "user", "content": "judge"}], + mode="conduct", + response_format=response_format, + ) + + proxy.assert_called_once_with( + { + "model": "model-x", + "messages": [{"role": "user", "content": "judge"}], + "temperature": orchestrator.client.temperature, + "max_tokens": orchestrator.client.max_output_tokens, + "response_format": response_format, + } + ) + assert completion["answer"] == '{"meets_threshold":true,"rationale":"ok"}' + assert completion["mode"] == "conduct" + assert completion["trace"][0]["usage"]["total_tokens"] == 5 + + +def test_fast_mlsirm_judge_contract_does_not_pass_threshold_to_judge_call() -> None: + class _Judge: + def __init__(self, _orchestrator, *, mode: str, accept_threshold: float) -> None: + assert mode == "route" + assert accept_threshold == 0.7 + + def judge(self, *, task: str, answer: str, criteria: tuple) -> object: + assert task == "task" + assert answer == "report" + assert len(criteria) == 2 + return type("Result", (), { + "accepted": True, + "rationale": "valid", + "usage": {}, + "orchestration_mode": "route", + }) + + class _Criterion: + def __init__(self, criterion_id: str, description: str, weight: float) -> None: + self.criterion_id = criterion_id + self.description = description + self.weight = weight + + orchestrator, _ = _orch("unused") + with patch.object( + orchestrator_module, + "_resolve_fast_mlsirm_components", + return_value=orchestrator_module.FastMLSIRMJudgeComponents( + judge_cls=_Judge, + criterion_cls=_Criterion, + format_error=ValueError, + ), + ): + result = orchestrator._model_judge_verification("task", {"verifier_output": "report"}) + + assert result["accepted"] is True + assert result["judge_orchestration_mode"] == "route" + + +def test_fast_mlsirm_invalid_irt_projection_fails_closed() -> None: + class _Judge: + def __init__(self, _orchestrator, *, mode: str, accept_threshold: float) -> None: + del mode, accept_threshold + + def judge(self, **_) -> object: + return type("Result", (), { + "accepted": True, + "rationale": "valid score but invalid item vector", + "criterion_scores": {"only_item": 0.8}, + "usage": {}, + "orchestration_mode": "route", + "to_irt_row": lambda *, item_type: (1,), + }) + + class _Criterion: + def __init__(self, criterion_id: str, description: str, weight: float) -> None: + self.criterion_id = criterion_id + self.description = description + self.weight = weight + + orchestrator, _ = _orch("unused") + with patch.object( + orchestrator_module, + "_resolve_fast_mlsirm_components", + return_value=orchestrator_module.FastMLSIRMJudgeComponents( + judge_cls=_Judge, + criterion_cls=_Criterion, + format_error=ValueError, + ), + ): + result = orchestrator._model_judge_verification("task", {"verifier_output": "report"}) + + assert result["accepted"] is False + assert "multi-item IRT projection" in result["reason"] + + +def test_fast_mlsirm_format_error_fails_closed() -> None: + class _FormatError(Exception): + pass + + class _FlakyJudge: + def __init__(self, _orchestrator, mode: str = "route", accept_threshold: float = 0.7) -> None: + del mode, accept_threshold + + def judge(self, **_) -> None: + raise _FormatError("invalid structured verdict") + + class _Criterion: + def __init__(self, criterion_id: str, description: str, weight: float) -> None: + self.criterion_id = criterion_id + self.description = description + self.weight = weight + + orchestrator, _ = _orch("unused") + with patch.object( + orchestrator_module, + "_resolve_fast_mlsirm_components", + return_value=orchestrator_module.FastMLSIRMJudgeComponents( + judge_cls=_FlakyJudge, + criterion_cls=_Criterion, + format_error=_FormatError, + ), + ): + result = orchestrator._model_judge_verification("task", {"verifier_output": "report"}) + + assert result["accepted"] is False + assert result["judge"] == "model" + assert result["reason"] == "model judge returned an invalid structured verdict; verification failed closed" + + +def test_broken_fast_mlsirm_import_does_not_bypass_required_judge_path() -> None: + orchestrator, _ = _orch("unused") + with patch.object( + orchestrator_module, + "_resolve_fast_mlsirm_components", + side_effect=RuntimeError("broken fast-mlsirm import"), + ): + result = orchestrator._model_judge_verification("task", {"verifier_output": "report"}) + + assert result["accepted"] is False + assert result["reason"] == "fast-mlsirm judge could not be loaded; verification failed closed" + + +@pytest.mark.parametrize( + ("reply", "message"), + [ + ('{"decision":"MAYBE","reason":"uncertain"}', "allowed enum"), + ('{"decision":"ACCEPT","reason":""}', "reason is missing"), + ('{"decision":"ACCEPT","reason":17}', "reason is missing"), + ], +) +def test_model_judge_parser_rejects_invalid_structured_values(reply: str, message: str) -> None: + with pytest.raises(ValueError, match=message): + _parse_model_judge_reply(reply) + + +def test_model_judge_parser_rejects_oversized_reply() -> None: + with pytest.raises(ValueError, match="maximum size"): + _parse_model_judge_reply("x" * 32_001) + + +def test_missing_fast_mlsirm_does_not_use_a_direct_judge_fallback() -> None: + orchestrator, _ = _orch("unused") + with patch.object(orchestrator_module, "_resolve_fast_mlsirm_components", return_value=None), patch.object( + orchestrator, "_invoke" + ) as invoke: + result = orchestrator._model_judge_verification( + "task", + {"verifier_output": "report"}, + ) + + assert result["accepted"] is False + assert result["reason"] == "fast-mlsirm judge is unavailable; verification failed closed" + invoke.assert_not_called() if __name__ == "__main__": diff --git a/tests/test_openai_passthrough.py b/tests/test_openai_passthrough.py index d50342289..c58e5d56a 100644 --- a/tests/test_openai_passthrough.py +++ b/tests/test_openai_passthrough.py @@ -14,18 +14,22 @@ import urllib.request from pathlib import Path +import pytest + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 -from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 +from contextual_orchestrator.server import SecurityConfig, build_server, responses_sse_body # noqa: E402 def _build() -> TaskOrchestrator: return TaskOrchestrator( agents=[ ModelAgent("planner_agent", "mock-planner", tags=("planning", "reasoning")), + ModelAgent("disabled_builder_duplicate", "mock-builder", disabled=True), ModelAgent("builder_agent", "mock-builder", tags=("coding", "implementation")), ModelAgent("reviewer_agent", "mock-reviewer", tags=("verification", "review")), + ModelAgent("disabled_candidate", "disabled-model", disabled=True), ] ) @@ -61,6 +65,44 @@ def test_proxy_completion_forwards_tools() -> None: assert result["echo"]["tools"] == tools +def test_proxy_completion_honors_an_enabled_requested_worker_model() -> None: + result = _build().proxy_completion({ + "model": "mock-builder", + "messages": [{"role": "user", "content": "call a tool"}], + "tools": [], + }) + + assert result["model"] == "mock-builder" + + +def test_proxy_completion_rejects_an_unknown_requested_model() -> None: + try: + _build().proxy_completion({ + "model": "not-configured", + "messages": [{"role": "user", "content": "call a tool"}], + "tools": [], + }) + except ValueError as exc: + assert "not configured" in str(exc) + else: # pragma: no cover + raise AssertionError("unknown explicit model must not silently fall back") + + +def test_proxy_completion_rejects_disabled_and_malformed_requested_models() -> None: + with pytest.raises(RuntimeError, match="disabled"): + _build().proxy_completion({ + "model": "disabled-model", + "messages": [{"role": "user", "content": "call a tool"}], + }) + + for requested_model in (17, ""): + with pytest.raises(ValueError, match="non-empty string"): + _build().proxy_completion({ + "model": requested_model, + "messages": [{"role": "user", "content": "call a tool"}], + }) + + def test_proxy_completion_responses_endpoint_returns_response_object() -> None: orch = _build() result = orch.proxy_completion( @@ -125,6 +167,50 @@ def test_http_responses_endpoint_passes_through() -> None: assert body["object"] == "response" +def test_http_models_endpoint_lists_configured_models() -> None: + server, port, token = _serve() + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/models", + headers={"authorization": f"Bearer {token}", "connection": "close"}, + method="GET", + ) + try: + with urllib.request.urlopen(request, timeout=5) as response: + status = response.status + body = json.loads(response.read().decode("utf-8")) + finally: + server.shutdown() + assert status == 200 + assert body["object"] == "list" + assert {item["id"] for item in body["data"]} == { + "contextual-orchestrator", "mock-planner", "mock-builder", "mock-reviewer", "disabled-model" + } + assert body["data"][0]["kind"] == "orchestrator" + assert all(item["readiness"] == "unprobed" for item in body["data"]) + assert next(item for item in body["data"] if item["id"] == "mock-builder")["status"] == "active" + assert next(item for item in body["data"] if item["id"] == "disabled-model")["status"] == "disabled" + + +def test_responses_stream_has_completion_event() -> None: + body = { + "id": "resp_test", + "object": "response", + "status": "completed", + "output": [{ + "id": "msg_test", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "OK", "annotations": []}], + }], + } + stream = responses_sse_body(body) + assert "event: response.output_text.delta" in stream + assert '"delta": "OK"' in stream + assert "event: response.completed" in stream + assert stream.endswith("data: [DONE]\n\n") + + def test_http_plain_prompt_still_uses_orchestration_path() -> None: server, port, token = _serve() url = f"http://127.0.0.1:{port}/v1/chat/completions" diff --git a/tests/test_provider_integration.py b/tests/test_provider_integration.py index 95794480b..49238f17c 100644 --- a/tests/test_provider_integration.py +++ b/tests/test_provider_integration.py @@ -11,8 +11,10 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import json from pathlib import Path +import socket import sys import threading +import urllib.request sys.path.insert(0, str(Path(__file__).resolve().parents[1])) @@ -80,6 +82,20 @@ def test_send_real_http_round_trip_and_usage_capture() -> None: assert client._local.usage == usage # provider-reported usage captured from a real response +def test_open_provider_uses_validated_destination_without_dns_relookup() -> None: + with _FakeProvider([(200, _completion("pinned"))]) as provider: + port = provider._server.server_address[1] + request = urllib.request.Request( + f"http://provider.example:{port}/chat/completions", + data=b"{}", + headers={"content-type": "application/json"}, + method="POST", + ) + with ModelClient()._open_provider(request, (socket.AF_INET, ("127.0.0.1", port))) as response: + assert json.loads(response.read())["choices"][0]["message"]["content"] == "pinned" + assert provider.request_count == 1 + + def test_transient_5xx_retries_then_succeeds_over_http() -> None: with _FakeProvider([(503, {}), (503, {}), (200, _completion("recovered"))]) as provider: client = ModelClient(max_retries=3, retry_backoff=0.0) diff --git a/tests/test_provider_reliability.py b/tests/test_provider_reliability.py index dd5ead98c..a8a36a8ba 100644 --- a/tests/test_provider_reliability.py +++ b/tests/test_provider_reliability.py @@ -6,17 +6,20 @@ from __future__ import annotations -from pathlib import Path import socket +import ssl import sys +import threading import urllib.error +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 from contextual_orchestrator.orchestrator import ( # noqa: E402 - ModelClient, TRANSIENT_HTTP_STATUS, + ModelClient, is_transient_error, ) @@ -34,6 +37,9 @@ def test_transient_classification_matches_status_and_network_errors() -> None: assert is_transient_error(urllib.error.URLError("dns")) assert is_transient_error(TimeoutError("read timeout")) assert is_transient_error(socket.timeout("slow")) + assert is_transient_error(ssl.SSLEOFError("peer closed TLS stream")) + assert is_transient_error(ssl.SSLSyscallError("SSL_ERROR_SYSCALL")) + assert not is_transient_error(ssl.SSLCertVerificationError("certificate verify failed")) assert not is_transient_error(ValueError("bad json")) @@ -46,7 +52,7 @@ def __init__(self) -> None: self._sleep = delays.append # capture backoff instead of sleeping self.attempts = 0 - def _send(self, agent: ModelAgent, payload: dict) -> str: # type: ignore[override] + def _send(self, agent: ModelAgent, payload: dict, destination=None) -> str: # type: ignore[override] self.attempts += 1 if self.attempts < 3: raise _http_error(503) @@ -61,13 +67,88 @@ def _send(self, agent: ModelAgent, payload: dict) -> str: # type: ignore[overri assert all(0.0 <= d <= client.retry_backoff_cap for d in delays) +def test_local_retry_budget_is_zero_by_default_to_avoid_queue_multiplication() -> None: + class LocalDownClient(ModelClient): + def __init__(self) -> None: + super().__init__(max_retries=5, retry_backoff=0.0) + self.attempts = 0 + + def _send(self, agent: ModelAgent, payload: dict, destination=None) -> str: # type: ignore[override] + self.attempts += 1 + raise urllib.error.URLError("local server is busy") + + client = LocalDownClient() + agent = ModelAgent("local_worker", "local-model", base_url="mlx://127.0.0.1:8080/v1") + try: + client._send_with_retry(agent, {"model": agent.model}) + except RuntimeError: + pass + else: # pragma: no cover + raise AssertionError("a failed local request must not succeed") + assert client.attempts == 1 + + +def test_local_retry_budget_can_be_explicitly_opted_into() -> None: + class LocalFlakyClient(ModelClient): + def __init__(self) -> None: + super().__init__(max_retries=5, local_max_retries=1, retry_backoff=0.0) + self.attempts = 0 + + def _send(self, agent: ModelAgent, payload: dict, destination=None) -> str: # type: ignore[override] + self.attempts += 1 + if self.attempts == 1: + raise urllib.error.URLError("local server restarted") + return "recovered" + + client = LocalFlakyClient() + agent = ModelAgent("local_worker", "local-model", base_url="local://127.0.0.1:8080/v1") + assert client._send_with_retry(agent, {"model": agent.model}) == "recovered" + assert client.attempts == 2 + + +def test_local_retry_budget_is_not_capped_by_remote_retry_default() -> None: + class LocalFlakyClient(ModelClient): + def __init__(self) -> None: + super().__init__(max_retries=0, local_max_retries=2, retry_backoff=0.0) + self.attempts = 0 + + def _send(self, agent: ModelAgent, payload: dict, destination=None) -> str: # type: ignore[override] + self.attempts += 1 + if self.attempts < 3: + raise urllib.error.URLError("local server is restarting") + return "recovered" + + client = LocalFlakyClient() + agent = ModelAgent("local_worker", "local-model", base_url="mlx://127.0.0.1:8080/v1") + assert client._send_with_retry(agent, {"model": agent.model}) == "recovered" + assert client.attempts == 3 + + +def test_local_passthrough_retry_budget_is_not_capped_by_remote_retry_default() -> None: + class LocalRawFlakyClient(ModelClient): + def __init__(self) -> None: + super().__init__(max_retries=0, local_max_retries=2, retry_backoff=0.0) + self.attempts = 0 + + def _send_raw(self, agent: ModelAgent, endpoint: str, payload: dict, destination=None) -> dict: # type: ignore[override] + self.attempts += 1 + if self.attempts < 3: + raise urllib.error.URLError("local server is restarting") + return {"ok": True} + + client = LocalRawFlakyClient() + agent = ModelAgent("local_worker", "local-model", base_url="local://127.0.0.1:8080/v1") + assert client._send_raw_with_retry(agent, "chat/completions", {}) == {"ok": True} + assert client.attempts == 3 + + def test_permanent_error_is_not_retried() -> None: class BadRequestClient(ModelClient): def __init__(self) -> None: super().__init__(max_retries=5, retry_backoff=0.0) self.attempts = 0 - def _send(self, agent: ModelAgent, payload: dict) -> str: # type: ignore[override] + def _send(self, agent: ModelAgent, payload: dict, destination=None) -> str: # type: ignore[override] self.attempts += 1 raise _http_error(400) @@ -171,6 +252,22 @@ def test_success_clears_prior_failures() -> None: assert "primary_worker" not in orchestrator._circuit +def test_circuit_breaker_counts_concurrent_failures() -> None: + orchestrator, _ = _two_worker_orchestrator(down_id="primary_worker") + calls = orchestrator.circuit_failure_threshold * 4 + barrier = threading.Barrier(calls, timeout=2.0) + + def record_failure(_index: int) -> None: + barrier.wait() + orchestrator._record_failure("primary_worker") + + with ThreadPoolExecutor(max_workers=calls) as pool: + list(pool.map(record_failure, range(calls))) + + assert orchestrator._circuit["primary_worker"]["failures"] == float(calls) + assert orchestrator._circuit_open("primary_worker") is True + + def test_mock_path_is_unchanged_no_failover_no_circuit_state() -> None: # Regression guard: the default mock path must behave exactly as before — # single attempt, no failover metadata, no circuit state recorded. diff --git a/tests/test_provider_tls.py b/tests/test_provider_tls.py index e78e90f42..becd18cba 100644 --- a/tests/test_provider_tls.py +++ b/tests/test_provider_tls.py @@ -1,9 +1,8 @@ """Provider TLS trust configuration for ModelClient. A live run against a corporate OpenAI-compatible gateway failed because urllib could -not verify its certificate chain (custom CA not in Python's trust store), and there -was no way to supply a CA bundle short of disabling verification globally. This adds -a per-client CA bundle / verify toggle. Default stays verified against the system store. +not verify its certificate chain (custom CA not in Python's trust store). A per-client +CA bundle now solves that without an insecure verification bypass. """ from __future__ import annotations @@ -24,10 +23,13 @@ def test_default_verifies_against_system_store() -> None: assert context.check_hostname is True -def test_insecure_skip_verify_disables_checks() -> None: - context = ModelClient(verify_tls=False)._ssl_context - assert context.verify_mode == ssl.CERT_NONE - assert context.check_hostname is False +def test_insecure_skip_verify_is_rejected() -> None: + try: + ModelClient(verify_tls=False) + except ValueError as exc: + assert "cannot be disabled" in str(exc) + else: # pragma: no cover + raise AssertionError("provider TLS verification bypass must be rejected") def test_ca_bundle_is_loaded() -> None: diff --git a/tests/test_repository_security_metadata.py b/tests/test_repository_security_metadata.py index b1c9ee785..cbc0b12d2 100644 --- a/tests/test_repository_security_metadata.py +++ b/tests/test_repository_security_metadata.py @@ -1,6 +1,5 @@ -from pathlib import Path import re - +from pathlib import Path ROOT_DIR = Path(__file__).resolve().parents[1] @@ -71,9 +70,44 @@ def test_security_workflow_covers_core_repository_security_process(): def test_dependabot_tracks_actions_and_python_dependencies(): dependabot_text = read_text(".github/dependabot.yml") - assert "package-ecosystem: github-actions" in dependabot_text - assert "package-ecosystem: pip" in dependabot_text - assert "timezone: Asia/Seoul" in dependabot_text + entries = { + match.group(1): match.group(2) + for match in re.finditer( + r"(?ms)^ - package-ecosystem:\s+([^\n]+)\n(.*?)(?=^ - package-ecosystem:|\Z)", + dependabot_text, + ) + } + + assert set(entries) == {"github-actions", "pip"} + for entry in entries.values(): + assert "timezone: Asia/Seoul" in entry + assert re.search(r"(?m)^ cooldown:\n default-days: 7$", entry) + + +def test_review_adr_requires_enforced_exact_head_merge_controls(): + adr_text = read_text("docs/planning/adrs/0004-pr-review-merge-loop.md") + normalized_adr_text = " ".join(adr_text.split()) + + required_controls = [ + "`requiredApprovals >= 1`", + "`enforce_admins=true`", + "`reviewDecision=APPROVED`", + "independent current-head approval", + "zero active unresolved threads", + "terminal successful required checks", + "structured same-head Strix evidence", + "final re-fetch immediately before", + "one recorded `verified_head_sha`", + "the PR head, any check SHA, or the reviewed diff changes", + "the merge stops and the complete gate is re-evaluated on the new head", + ] + for required_control in required_controls: + assert required_control in normalized_adr_text + + assert ( + "Branch protection and the central scheduler must each reject direct and " + "auto merge when any control is absent or non-passing." + ) in normalized_adr_text def test_codeowners_requires_repository_owner_review(): diff --git a/tests/test_routing_eval.py b/tests/test_routing_eval.py index f6f9bbc76..7531e6ccc 100644 --- a/tests/test_routing_eval.py +++ b/tests/test_routing_eval.py @@ -10,6 +10,7 @@ from pathlib import Path import sys +from dataclasses import replace sys.path.insert(0, str(Path(__file__).resolve().parents[1])) @@ -65,6 +66,25 @@ def test_eval_does_not_persist_runs() -> None: assert orchestrator._workflow_runs == {} +def test_auto_route_uses_length_threshold() -> None: + orchestrator = _orch() + orchestrator.policy = replace( + orchestrator.policy, + route_text_length_threshold=10_000, + conduct_hint_threshold=99, + ) + + prompt = "x" * 2000 + assert orchestrator.would_route([{"role": "user", "content": prompt}], mode="auto") + + orchestrator.policy = replace( + orchestrator.policy, + route_text_length_threshold=1, + conduct_hint_threshold=99, + ) + assert not orchestrator.would_route([{"role": "user", "content": prompt}], mode="auto") + + if __name__ == "__main__": for name, fn in sorted(globals().items()): if name.startswith("test_") and callable(fn): diff --git a/tests/test_sales_readiness.py b/tests/test_sales_readiness.py index d690d5573..9eea75fca 100644 --- a/tests/test_sales_readiness.py +++ b/tests/test_sales_readiness.py @@ -111,6 +111,31 @@ def test_sales_readiness_report_marks_enterprise_pilot_ready() -> None: assert {"criterion_name", "status", "label", "evidence", "remediation"}.issubset(row) +def test_sales_readiness_accepts_external_bearer_verifier() -> None: + orchestrator = build() + exercise_runtime(orchestrator) + report = orchestrator.sales_readiness_report( + locale_bundles=ADMIN_TRANSLATIONS, + security_profile={ + "auth_mode": "external_bearer_verifier", + "allow_public_bind": False, + "expose_trace_by_default": False, + "rate_limit_requests": 60, + "max_concurrent_runs": 8, + }, + ) + rows = criteria_by_name(report) + assert rows["security_posture"]["status"] == "pass" + assert report["readiness_summary"]["fail"] == 0 + + +def test_sales_readiness_rejects_unknown_auth_mode() -> None: + result = build()._security_posture_criterion({"auth_mode": "unknown"}) + + assert result["status"] == "fail" + assert "no bearer token" in result["evidence"] + + def test_sales_readiness_warns_for_single_token_local_deployment() -> None: orchestrator = build() exercise_runtime(orchestrator) @@ -134,6 +159,25 @@ def test_sales_readiness_warns_for_single_token_local_deployment() -> None: assert "split admin and inference tokens" in rows["security_posture"]["remediation"] +def test_provider_egress_report_skips_local_and_checks_remote_agents() -> None: + orchestrator = TaskOrchestrator( + [ + ModelAgent("local_agent", "local-model", base_url="mlx://127.0.0.1:8080/v1"), + ModelAgent( + "remote_agent", + "remote-model", + base_url="https://provider.example/v1", + credential_key="remote-key", + ), + ] + ) + + result = orchestrator._provider_egress_criterion() + + assert result["status"] == "pass" + assert "1 remote providers" in result["evidence"] + + def test_sales_readiness_endpoint_openapi_and_admin_surface() -> None: assert "/api/v1/sales_readiness/latest" in OPENAPI_SPEC["paths"] assert OPENAPI_SPEC["paths"]["/api/v1/sales_readiness/latest"]["get"]["operationId"] == ( diff --git a/tests/test_security_hardening.py b/tests/test_security_hardening.py index 67134ea6b..3eb5c2bf0 100644 --- a/tests/test_security_hardening.py +++ b/tests/test_security_hardening.py @@ -2,11 +2,13 @@ import json import os +import socket import threading import urllib.error import urllib.request from pathlib import Path import sys +from unittest.mock import patch sys.path.insert(0, str(Path(__file__).resolve().parents[1])) @@ -20,6 +22,25 @@ def build() -> TaskOrchestrator: return TaskOrchestrator([ModelAgent("general_agent", "mock-generalist", tags=("reasoning", "writing"))]) +def test_external_bearer_verifier_is_fail_closed_and_scoped() -> None: + seen: list[tuple[str, str]] = [] + + def verify(token: str, scope: str) -> bool: + seen.append((token, scope)) + return token == "keyverse-token" and scope == "inference" + + security = SecurityConfig(bearer_verifier=verify) + security.authorize({"authorization": "Bearer keyverse-token"}, "inference", "127.0.0.1") + try: + security.authorize({"authorization": "Bearer keyverse-token"}, "admin", "127.0.0.1") + except Exception as exc: + assert "invalid" in str(exc) + else: # pragma: no cover + raise AssertionError("external verifier accepted the wrong scope") + assert seen == [("keyverse-token", "inference"), ("keyverse-token", "admin")] + assert security.readiness_profile()["auth_mode"] == "external_bearer_verifier" + + def post_json(url: str, payload: dict[str, object], token: str | None = None) -> tuple[int, dict[str, object]]: headers = {"content-type": "application/json", "connection": "close"} if token: @@ -95,6 +116,29 @@ def test_admin_and_inference_tokens_are_separate() -> None: assert "trace" not in inference_body["orchestration"] +def test_single_and_split_token_modes_cannot_be_combined() -> None: + try: + SecurityConfig(auth_token="shared_secret", admin_token="admin_secret", inference_token="inference_secret") + except ValueError as exc: + assert str(exc) == "single auth_token cannot be combined with split tokens" + else: # pragma: no cover + raise AssertionError("mixed single and split token modes must be rejected") + + +def test_scope_token_precedes_mutated_shared_token() -> None: + security = SecurityConfig(admin_token="admin_secret", inference_token="inference_secret") + security.auth_token = "mutated_shared_secret" + + try: + security.authorize({"authorization": "Bearer mutated_shared_secret"}, "admin", "127.0.0.1") + except Exception as exc: + assert "invalid" in str(exc) + else: # pragma: no cover + raise AssertionError("scope token must remain authoritative after field mutation") + + security.authorize({"authorization": "Bearer admin_secret"}, "admin", "127.0.0.1") + + def test_loopback_without_configured_token_is_rejected() -> None: server = build_server(build(), port=0, security=SecurityConfig()) thread = threading.Thread(target=server.serve_forever, daemon=True) @@ -204,6 +248,16 @@ def test_concurrency_limit_rejects_when_slots_are_full() -> None: security.release_run_slot() +def test_concurrency_limit_rejects_unbounded_or_non_integer_configuration() -> None: + for value in (0, 65, False, 1.5): + try: + SecurityConfig(auth_token="secret_token", max_concurrent_runs=value) # type: ignore[arg-type] + except ValueError as exc: + assert "max_concurrent_runs" in str(exc) + else: # pragma: no cover + raise AssertionError("invalid max_concurrent_runs configuration was accepted") + + def test_chat_completion_response_requires_explicit_trace() -> None: result = { "mode": "route", @@ -255,11 +309,9 @@ def test_external_provider_requires_resolvable_credential_and_public_https() -> def test_external_provider_rejects_insecure_or_unlisted_hosts() -> None: - client = ModelClient() + client = ModelClient(allowed_provider_hosts={"example.com"}) insecure_agent = ModelAgent("insecure_agent", "gpt-example", "http://api.openai.com/v1", "MODEL_KEY") unlisted_agent = ModelAgent("unlisted_agent", "gpt-example", "https://api.openai.com/v1", "MODEL_KEY") - previous = os.environ.get("CONTEXTUAL_ORCHESTRATOR_ALLOWED_PROVIDER_HOSTS") - os.environ["CONTEXTUAL_ORCHESTRATOR_ALLOWED_PROVIDER_HOSTS"] = "example.com" # Register the credential so validation proceeds to the host-safety checks. backend = InMemoryCredentialBackend() backend.set("MODEL_KEY", "sk-host-check") @@ -281,10 +333,28 @@ def test_external_provider_rejects_insecure_or_unlisted_hosts() -> None: raise AssertionError("unlisted provider should fail") finally: set_backend(None) - if previous is None: - os.environ.pop("CONTEXTUAL_ORCHESTRATOR_ALLOWED_PROVIDER_HOSTS", None) - else: - os.environ["CONTEXTUAL_ORCHESTRATOR_ALLOWED_PROVIDER_HOSTS"] = previous + + +def test_provider_allowlist_ignores_request_time_environment_changes() -> None: + client = ModelClient(allowed_provider_hosts={"provider.example"}) + agent = ModelAgent( + "remote_agent", + "remote-model", + base_url="https://provider.example/v1", + credential_key="remote-key", + ) + with patch.dict( + "contextual_orchestrator.orchestrator.os.environ", + {"CONTEXTUAL_ORCHESTRATOR_ALLOWED_PROVIDER_HOSTS": "other.example"}, + ), patch( + "contextual_orchestrator.orchestrator.get_credential", + return_value="secret", + ), patch.object( + client, + "_resolve_addresses", + return_value=[(socket.AF_INET, ("93.184.216.34", 443))], + ): + assert client._validate_provider(agent) == (socket.AF_INET, ("93.184.216.34", 443)) def test_provider_transport_rejects_local_url_schemes_before_urllib() -> None: diff --git a/tests/test_streaming.py b/tests/test_streaming.py index 789fb7084..f111199b4 100644 --- a/tests/test_streaming.py +++ b/tests/test_streaming.py @@ -13,11 +13,12 @@ import threading import urllib.error import urllib.request +from unittest.mock import patch sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 -from contextual_orchestrator.orchestrator import chat_completion_chunks, sse_stream_body # noqa: E402 +from contextual_orchestrator.orchestrator import chat_completion_chunks, chat_completion_response, sse_stream_body # noqa: E402 from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 @@ -54,6 +55,16 @@ def test_empty_answer_produces_role_and_stop_only() -> None: assert chunks[1]["choices"][0]["finish_reason"] == "stop" +def test_completion_ids_remain_unique_when_created_in_one_millisecond() -> None: + result = {"answer": "OK", "mode": "route"} + with patch("contextual_orchestrator.orchestrator.time.time", return_value=1_786_698_100.0): + response_ids = {chat_completion_response(result)["id"] for _ in range(128)} + chunk_ids = {chat_completion_chunks(result)[0]["id"] for _ in range(128)} + + assert len(response_ids) == 128 + assert len(chunk_ids) == 128 + + def test_sse_body_frames_and_done_terminator() -> None: body = sse_stream_body(chat_completion_chunks({"answer": "abc", "mode": "route"})) assert body.endswith("data: [DONE]\n\n")