diff --git a/.github/workflows/opencode-hourly-loop.yml b/.github/workflows/opencode-hourly-loop.yml index ca1fa4cd4..49a296e84 100644 --- a/.github/workflows/opencode-hourly-loop.yml +++ b/.github/workflows/opencode-hourly-loop.yml @@ -88,29 +88,35 @@ jobs: NVIDIA_NIM_API_KEY_SUB: ${{ secrets.NVIDIA_NIM_API_KEY_SUB }} steps: - name: Checkout repository + timeout-minutes: 5 uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 with: persist-credentials: true # the agent needs to push fixes/branches - name: Set up Python + timeout-minutes: 5 uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 with: python-version: "3.12" - name: Set up Node for the OpenCode CLI + timeout-minutes: 5 uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # actions/setup-node@v7.0.0 with: node-version: "22" - name: Install pinned runtime dependencies + timeout-minutes: 10 run: python -m pip install --require-hashes -r requirements.lock - name: Install OpenCode CLI + timeout-minutes: 10 run: | npm ci --ignore-scripts node scripts/ci/install_locked_opencode.mjs - name: Generate ephemeral loopback gateway token + timeout-minutes: 1 run: | gateway_token=$(python -c 'import secrets; print(secrets.token_urlsafe(32))') echo "::add-mask::$gateway_token" @@ -126,17 +132,24 @@ jobs: --auth-token-key CONTEXTUAL_ORCHESTRATOR_TOKEN \ --host 127.0.0.1 --port 8000 \ > /tmp/gateway.log 2>&1 & - for _attempt in $(seq 1 30); do + gateway_pid=$! + while :; do if curl -fsS -H "Authorization: Bearer ${CONTEXTUAL_ORCHESTRATOR_TOKEN}" \ http://127.0.0.1:8000/healthz >/dev/null 2>&1; then echo "gateway ready" exit 0 fi + if ! kill -0 "$gateway_pid" 2>/dev/null; then + wait "$gateway_pid" || true + echo "::error::gateway exited before becoming healthy" + tail -50 /tmp/gateway.log + exit 1 + fi sleep 2 done - echo "::error::gateway did not become healthy"; tail -50 /tmp/gateway.log; exit 1 - name: Point OpenCode at the local gateway + timeout-minutes: 1 run: | set -euo pipefail umask 077 @@ -162,6 +175,7 @@ jobs: JSON - name: Build trusted same-repository PR allowlist + timeout-minutes: 5 env: GH_TOKEN: ${{ github.token }} run: | @@ -177,5 +191,6 @@ jobs: "$(cat .github/opencode/hourly-loop-prompt.md)" - name: Dump gateway log on failure + timeout-minutes: 5 if: failure() run: tail -100 /tmp/gateway.log diff --git a/CHANGELOG.d/bootstrap-ambiguous-admission.md b/CHANGELOG.d/bootstrap-ambiguous-admission.md new file mode 100644 index 000000000..06e49a51d --- /dev/null +++ b/CHANGELOG.d/bootstrap-ambiguous-admission.md @@ -0,0 +1,10 @@ +# Fail closed on ambiguous bootstrap admission + +- Reject provider/model-group bootstrap capacity cutoffs when selected and + excluded candidates have equal or incomplete price evidence, and reject any + diversity pass that would change the price-evidenced candidate sequence + without an explicit decision model. +- Preserve raw availability evidence without adding weights, quotas, fuzzy + identity, or lexical provider/model admission. +- Document the provider-diversity migration boundary and keep ADR 0032 Proposed + until protected delivery and exact-head verification. diff --git a/CHANGELOG.d/bootstrap-report-persisted-identity.md b/CHANGELOG.d/bootstrap-report-persisted-identity.md new file mode 100644 index 000000000..b58f30c5f --- /dev/null +++ b/CHANGELOG.d/bootstrap-report-persisted-identity.md @@ -0,0 +1,3 @@ +### Fixed + +- Durable provider and provider-catalog bootstrap reports now expose the resolved persisted agent identity in both `selected_agent_ids` and `enabled_agent_ids`. Legacy discovered-agent migration therefore no longer reports two identifier generations for the same selected endpoint; ephemeral bootstrap reports continue to use generated identities because no persisted identity exists. diff --git a/CHANGELOG.d/bootstrap-report-selection-order.md b/CHANGELOG.d/bootstrap-report-selection-order.md new file mode 100644 index 000000000..d63342bc7 --- /dev/null +++ b/CHANGELOG.d/bootstrap-report-selection-order.md @@ -0,0 +1,3 @@ +### Fixed + +- Durable provider bootstrap reports now preserve the selector's cost and model-group ordering after persisted identity resolution instead of alphabetically sorting enabled agent IDs. Pool membership and operator-identity collision checks remain fail closed. diff --git a/CHANGELOG.d/endpoint-race-callback-settlement.md b/CHANGELOG.d/endpoint-race-callback-settlement.md new file mode 100644 index 000000000..0321784be --- /dev/null +++ b/CHANGELOG.d/endpoint-race-callback-settlement.md @@ -0,0 +1,3 @@ +### Fixed + +- Endpoint race observer failures now settle the manually managed `Future` exactly once instead of leaving an unbounded race in a permanent `RUNNING` state. This preserves the prior executor-backed exception semantics while retaining daemon-owned race workers that cannot block interpreter shutdown. Regression coverage exercises callback failures after both successful and failed provider attempts with `deadline_seconds=None`. diff --git a/CHANGELOG.d/openrouter-not-evidence-only.md b/CHANGELOG.d/openrouter-not-evidence-only.md new file mode 100644 index 000000000..e687c148d --- /dev/null +++ b/CHANGELOG.d/openrouter-not-evidence-only.md @@ -0,0 +1 @@ +Stopped treating OpenRouter as a whole-account, ZDR-motivated serving exclusion. `PROVIDER_MODEL_SOURCES`'s `openrouter` entry no longer sets `evidence_only=True`: ZDR eligibility is a route/model-level property, never grounds to block an entire provider account from serving, and OpenRouter was the one provider source with genuinely reliable native pricing/`is_free` evidence, so excluding it directly caused `orchestrator/free`'s previously-documented structural emptiness (ADR 0041). Also fixed a backwards side effect of the old flag: `_apply_discovered_model_evidence` could never mark OpenRouter's own rows `zdr_capable=True` even when they exactly matched OpenRouter's own declared ZDR feed. The former cross-provider evidence application from PR #901 is corrected: OpenRouter's feed attests only OpenRouter rows. Matching model identifiers at another provider do not establish that endpoint's retention policy; independent provider attestations remain unchanged. Since OpenRouter can multiplex one model id across several backing providers, `ModelClient` now pins every OpenRouter request made under an active `zdr_only` scope with OpenRouter's own documented `"provider": {"zdr": true}` request-time enforcement, applied at the shared `_send`/`_stream_send`/`_send_raw` transport chokepoints, and at the async Batch API path (`_batch_run`) via the same helper on each uploaded JSONL request body. diff --git a/CHANGELOG.d/provider-embedding-daemon-worker-pool.md b/CHANGELOG.d/provider-embedding-daemon-worker-pool.md new file mode 100644 index 000000000..1423a961e --- /dev/null +++ b/CHANGELOG.d/provider-embedding-daemon-worker-pool.md @@ -0,0 +1,5 @@ +### Fixed + +- `ProviderEmbeddingBatchBackend` (`batch_routing.py`) no longer drives its durable job queue through a `concurrent.futures.ThreadPoolExecutor`. That executor's worker threads register with `concurrent.futures.thread`'s own interpreter-exit hook, which unconditionally joins every still-running worker at shutdown regardless of daemon status; combined with this org's default no-deadline `ModelClient.timeout=None` policy, a provider embedding runner that never returned would block process shutdown forever even after `close()`. A private `_DaemonWorkerPool` — a fixed-size pool of `threading.Thread(daemon=True)` workers pulling from a queue, exposing the same `submit()`/`shutdown()` surface — replaces it at every construction site while every durability/claim/recovery/publish guarantee is unchanged. Regression coverage runs a hung provider embedding runner in a separate interpreter and confirms normal process exit after `close()`. +- `_DaemonWorkerPool.submit()` now fails closed with `RuntimeError` once `shutdown()` has run, matching `ThreadPoolExecutor.submit`'s closed-pool contract, instead of silently queuing a task behind the shutdown sentinels where no worker would ever pick it up. +- `_DaemonWorkerPool.shutdown()` now closes admission (`self._shutdown = True`) atomically under `_workers_lock` before draining the queue or queuing stop sentinels, instead of draining first. Previously a `submit()` that landed between the drain and the flag being set could enqueue work that survived `cancel_futures=True` and later ran on a worker despite the cancellation. `shutdown()` remains idempotent (a repeated call skips re-draining/re-queuing and only joins the already-captured workers). diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f0b268c9..25266c8cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,35 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ### Fixed +- `endpoint_race.race_first_valid` now drives each equivalent-endpoint + attempt from a raw `threading.Thread(daemon=True)` worker built on a bare + `concurrent.futures.Future`, instead of a `concurrent.futures.ThreadPoolExecutor`. + `ThreadPoolExecutor` registers every worker it starts with + `concurrent.futures.thread`'s own interpreter-exit hook, which + unconditionally joins each still-running worker at shutdown regardless of + that worker thread's own daemon flag; combined with this org's default + no-deadline `ModelClient.timeout=None`, a losing race participant blocked + in a provider call that never returns could hang process shutdown forever + even though the winner already answered the caller (Devin Review finding + on #971, "Endpoint races block process shutdown"). `set_running_or_notify_cancel()` + on the bare `Future` preserves the existing "cancelled before it started + never calls the provider" duplicate-cost guarantee, and every other + coordination primitive (`wait()`, `future.cancel()`, `future.result()`, + `future.exception()`) behaves identically to the prior executor-backed + futures — "first valid response wins" semantics, unbounded wait for the + active call, and cancellation/drain provenance are unchanged. +- OpenRouter free-model endpoint discovery (`_openrouter_free_model_endpoints`) + now fans its per-model fetch out across a fixed pool of at most 8 daemon + worker threads pulling model IDs from a queue, instead of allocating one + `threading.Thread` object per free model and gating only the *work* (not + thread creation itself) behind an 8-slot semaphore. A catalog of hundreds + or thousands of free models previously still allocated and started that + many native OS threads up front — real kernel/stack overhead each — before + the semaphore ever limited anything, risking memory exhaustion or stalling + discovery before a single fetch could begin. Live thread count now stays + bounded regardless of catalog size; daemon-only workers and the abandon- + on-deadline behavior for the enclosing bounded discovery call are + unchanged. - Workflow workers now preserve the caller message array exactly once, while the added envelope carries only the subtask and Conductor-style prior-step access list instead of duplicating the task or source attachments. @@ -408,6 +437,19 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) retried as if it were a network blip. Fixes the shared classifier itself (not just the discovery retry call site), so every current and future caller of `is_transient_error` benefits. +- (Devin review on #953) `_pin_openrouter_zdr` no longer raises a bare + `TypeError` from `dict()` when a caller-supplied `provider` field is + present but not an object (an int, bool, list, or string) under an active + `zdr_only` scope. It now validates the field and raises a named `ValueError` + ("provider must be an object with optional OpenRouter routing keys") + instead, matching this codebase's existing convention for malformed + caller-input fields. Every call site sharing this one choke point (chat, + streaming, tools/binary-media passthrough, and the batch JSONL path) + benefits; a valid `provider` object or an absent/`None` one keep their + existing behavior unchanged. +- OpenRouter embedding Batch JSONL now carries the same request-scoped + `provider.zdr=true` enforcement when `zdr_only` selects an attested + OpenRouter embedding agent. - `batch_route` no longer fabricates a hardcoded, ungated `{"accepted": True, "verifier_output": ""}` verification verdict for every batched answer regardless of `policy.realtime_judge`. It now calls the same @@ -854,6 +896,200 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) otherwise. `_write_sse` relies on `_begin_sse`'s already-set marker rather than touching it itself, since it is only ever called after a prior successful header flush. +- (2026-09-02, PR #971) `ModelClient`'s constructor no longer defaults + `max_retries` to a hand-picked `2`: no cited standard, paper, or the org's + own research (Fugu, Conductor, TRINITY) establishes that number, and a + fresh audit found it was never justified. RFC 9110 §9.2.2 constrains *when* + replay can be safe for idempotent semantics, and NIST SP 800-204 discusses + retry/circuit-breaker resilience as a pattern, but neither identifies a + specific numeric retry allocation for this library. The default is now + `0`: a default `ModelClient` allocates zero automatic provider transport + retries, independent of provider, model, or reasoning identity + (`tests/test_no_heuristic_default_transport_retry.py`). Explicit nonzero + retry budgets remain caller-owned configuration, never a library-authored + default (ADR 0001 amendment, 2026-09-02). +- (2026-09-02, PR #971) A default (no configured deadline) synchronous + `/v1/embeddings` request against a provider-backed member returned 503: + `embedding_deadline` collapsed to `+inf`, and `ProviderEmbeddingBatchBackend.wait` + passed that straight into `threading.Event.wait(timeout=...)`, which raises + `OverflowError` for a non-finite timeout on CPython -- directly contradicting + #971's own no-implicit-deadline policy. `wait()` now translates a non-finite + timeout to `None` (block indefinitely) instead + (`tests/test_provider_embedding_batch_backend.py::test_provider_batch_wait_survives_infinite_deadline`). +- (2026-09-02, PR #971) A failed configured-gateway structured-chat probe + recorded the failing model only under its new fingerprinted id + (`agent_id_for`); a persisted agent kept under its pre-fingerprint legacy + id (`legacy_agent_id_for`) was silently dropped from `runtime_models` and + never reached the disable path, leaving a failed legacy endpoint enabled + indefinitely. `_auto_discover_runtime_agents` now accepts either id form + when checking for an existing persisted agent + (`tests/test_auto_discovery_server.py::test_failed_gateway_probe_disables_legacy_id_persisted_agent`). +- (2026-09-02, PR #971) A privacy-scoped (`zdr_only=True`) embedding batch's + pinned `agent_id` was replayed verbatim by `ProviderEmbeddingBatchBackend` + after a process restart recovers a durably queued job, with no + re-validation that the agent still carried the `privacy:zdr` tag -- an + operator could remove ZDR support or repoint the agent between submission + and a resumed execution and the batch would still run. `_run_provider_embeddings` + now re-checks the request's own recorded `zdr_only` against the resolved + agent's current tags at execution time and fails closed if they no longer + match + (`tests/test_provider_embedding_batch_backend.py::test_recovered_privacy_scoped_embedding_batch_revalidates_current_agent_tags`). +- (2026-09-02, PR #971) A synchronous `/v1/embeddings` member result that + came back without raising but was not `completed` (or had no embeddings) + bypassed `orchestrator._record_embedding_failure`, so the circuit breaker + never opened and no `embedding_endpoint_failed` analytics event was + recorded for that failure mode -- a repeatedly incomplete member was + retried forever instead of being quarantined like a raised exception. It + now routes through the same failure recorder + (`tests/test_embeddings_model_pool_http_honesty.py::test_http_embeddings_quarantines_repeated_incomplete_document_with_failure_evidence`). +- (2026-09-02, PR #971) `CostRoutingCoordinator` derived the durable + provider-embedding claim lease from `ModelClient.timeout`, so a durable + (Valkey-backed) job registry raised `ValueError: durable provider backend + claim lease must be positive` at startup whenever the client had no + configured deadline (the default since #971) -- a crash directly caused by + conflating an internal locking heartbeat with the caller's request + deadline. The claim lease now falls back to a fixed, positive default + independent of `ModelClient.timeout`. Separately, `execution_timeout_seconds=None` + previously fell back to the job registry's storage retention window (7 + days), silently expiring an intentionally unbounded embedding job; it now + stays genuinely unbounded (`+inf` deadline) + (`tests/test_provider_embedding_batch_backend.py::test_durable_provider_embedding_backend_survives_unbounded_client_timeout`, + `::test_unbounded_execution_timeout_never_substitutes_registry_retention`). +- (2026-09-02, PR #971) `OpenRouterUptimeCollector._fetch_uptime` adopted + #971's inference no-fixed-deadline policy (`timeout=None`) even though it + is unrelated background telemetry polled sequentially on one dedicated + sweep thread that `stop()` cannot interrupt mid-request: one unresponsive + OpenRouter endpoint would hang that thread forever, leaking it and + indefinitely starving every later member of an uptime update. The fetch + now keeps its own fixed, independent bound + (`tests/test_openrouter_uptime.py::test_uptime_fetch_does_not_hang_forever_on_an_unresponsive_endpoint`). +- (2026-09-02, PR #971) `discover_all_models`'s per-provider loop + (`model_discovery.py`) called `discover_provider_models` directly and + in-line, with no separate bound or cancellation mechanism -- a stalled + provider catalog request (a connection accepted but never answered, or + in tests a mock that never returns) blocked discovery of every later, + healthy provider forever, regardless of `DISCOVERY_TIMEOUT_SECONDS` + (which only bounds one socket read at a time and stays `None` by + default). Each provider's discovery attempt now runs on its own daemon + thread bounded by a new, separately configured + `PROVIDER_DISCOVERY_DEADLINE_SECONDS` (default 30.0s, independent of + both `DISCOVERY_TIMEOUT_SECONDS` and `ModelClient.timeout` -- neither is + reused or repurposed): once the deadline elapses the caller stops + waiting, records a `ProviderDiscoveryError(error_code="discovery_timeout")` + for that provider, and moves on; the abandoned thread is daemonized so + it cannot block interpreter shutdown + (`tests/test_model_discovery.py::test_discover_all_models_bounds_a_stalled_provider_so_later_providers_still_complete`). +- (2026-09-02, PR #971) `select_bootstrap_discovered_agents`'s first pass + admitted at most one endpoint per model group (the provider-declared + exact model identity) but never checked provider identity, so several + cheap, distinctly-named models from one provider could fill most or all + of a bootstrap pool before a genuinely independent alternative provider + was ever tried -- an apparently diverse pool (distinct model names) that + was actually one provider's outage away from total failure, contrary to + this repo's own documented "provider-diverse selection" claim for this + function. The first pass now admits at most one endpoint per provider + *and* per model group; once every viable provider has contributed once + (or capacity runs out), a second pass fills remaining slots from + still-untried model groups regardless of provider, and a final pass + falls back to duplicate model-group endpoints only once real diversity + is exhausted. `contextual_orchestrator/provider_bootstrap.py`'s separate, + honestly-named `select_model_group_diverse_models` (which never claimed + provider diversity, and whose own tests deliberately rank a known + same-provider price ahead of diversity) was left unchanged + (`tests/test_discovery_bootstrap_selection.py::test_bootstrap_selector_spans_multiple_providers_before_repeating_one`, + `::test_bootstrap_selector_prefers_model_group_diversity`, + `::test_bootstrap_selector_falls_back_to_duplicate_model_group_when_capacity_remains`, + `tests/test_review_gateway.py::test_build_review_orchestrator_uses_model_group_diversity`). +- (2026-09-02, PR #971) The privacy re-validation added for a recovered + `zdr_only` embedding batch (above) only checked the resolved agent's + current tags; it never restored the ambient `request_policy` scope before + executing the batch, and the batch's homogeneity check compared only + `model`/`agent_id`, not `zdr_only`. Concretely: (1) `_pin_openrouter_zdr` + (the code that adds OpenRouter's enforcing `provider.zdr: true` request + field) branches on the `request_policy` contextvar, not on `first.zdr_only` + directly, and that contextvar is never in effect on the background worker + thread that replays a recovered job -- so a recovered ZDR batch's actual + HTTP request to OpenRouter silently omitted the ZDR pin even though the tag + check above believed it had already re-validated privacy safety; and (2) a + batch that somehow mixed `zdr_only=True` and `zdr_only=False` requests + under the same `agent_id` executed entirely under `first`'s policy instead + of being rejected. `_run_provider_embeddings` now wraps its embedding-shard + execution in `self.orchestrator.request_policy(first.zdr_only)` (the same + pattern used at every other client call site in `cost_router.py`) and + extends the existing route-homogeneity check to also require every request + in the batch share `first.zdr_only`, failing closed with the same + `RuntimeError` style otherwise (Devin Review on #971) + (`tests/test_pr971_review_quality_regressions.py::test_recovered_zdr_batch_reenters_request_privacy_scope`, + `::test_provider_embedding_batch_rejects_mixed_privacy_identity`). +- (2026-09-02, PR #971) The per-provider `discovery_deadline` bound added + above covered only `discover_provider_models` inside `discover_all_models`'s + per-provider loop. Three *shared* metadata fetches the same function makes + outside that loop stayed unbounded: `_fetch_models_dev_metadata` (runs once, + before the loop, whenever any registered source declares + `models_dev_provider_id`), and `_openrouter_zdr_model_ids` / + `openrouter_paid_inference_available` (run after the loop; the latter only + once an OpenRouter credential is registered) -- each received only + `timeout` (the per-socket-read timeout, `None`/unbounded by default), so a + stalled Models.dev, OpenRouter ZDR, or OpenRouter credits endpoint could + block `discover_all_models` -- and therefore first-boot pool bootstrapping + -- indefinitely (Devin Review, bug id + `BUG_pr-review-job-93783e6ce7a2440ab487ebce4076fe6f_0002`). The + per-provider bound is now a shared primitive, `_run_bounded_by_deadline` + (same mechanism as before: a daemon thread plus + `worker.join(timeout=discovery_deadline)`; `_discover_provider_models_bounded` + is a thin instantiation of it), and all three shared fetches run through + it under the same `discovery_deadline`. On timeout each returns the exact + fallback value it already returns for an ordinary fetch failure, so the + fail-closed posture is unchanged, never weakened: `_fetch_models_dev_metadata` + returns `None` (`_merge_models_dev_metadata` already treats `None` as "no + evidence" and passes provider rows through unenriched, so the affected + provider's own catalog listing still succeeds); `_openrouter_zdr_model_ids` + returns an empty `set()` (`_apply_discovered_model_evidence` already + short-circuits on an empty set and never marks a model ZDR-capable without + positive evidence); and `openrouter_paid_inference_available` returns + `None` (`apply_openrouter_spend_admission`'s existing fail-closed rule + already denies `spend_admitted` for a paid, non-free OpenRouter row unless + `paid_available is True`). `discovery_deadline=None` continues to opt + every one of these calls back into the pre-#971 unbounded wait + (`tests/test_model_discovery.py::test_discover_all_models_bounds_a_stalled_models_dev_metadata_fetch`, + `::test_discover_all_models_bounds_a_stalled_openrouter_zdr_fetch`, + `::test_discover_all_models_bounds_a_stalled_openrouter_paid_inference_fetch`). +- (2026-09-02, PR #971) `_openrouter_free_model_endpoints` fanned its + per-model endpoint fetch out across a + `concurrent.futures.ThreadPoolExecutor`. That executor's worker threads + register with an interpreter-exit hook (`concurrent.futures.thread`'s own + `atexit` handler) that unconditionally joins every still-running worker at + shutdown, regardless of whether the thread that *created* the executor is + itself `daemon=True` -- verified with a local repro: a hung fetch blocked + process shutdown even from inside this module's already-daemonized, + already-bounded per-provider discovery thread (CodeRabbit, re-confirming + the shared-metadata-deadline finding above from a different angle). The + fetch fan-out now uses plain `threading.Thread(daemon=True)` workers + (concurrency-capped at 8 in flight via a semaphore, matching the prior + `max_workers`), which carry no such registration, so a hung fetch is + abandoned like every other stalled discovery-time network call in this + module and the process can still exit + (`tests/test_model_discovery.py::test_openrouter_free_model_endpoints_hang_does_not_block_process_exit`). +- (2026-09-02, PR #971) The `/v1/batch/embeddings` handler's member-failover + loop called `observe_success`/`_record_success` for the just-tried agent + unconditionally, before ever inspecting the returned document's `status` + -- unlike the sibling `/v1/embeddings` fix above (same root cause), this + loop's own terminal-status check (`is_complete = document.get("status") + == "completed"`) only ran afterward, purely to pick the response's HTTP + status code, not to gate success recording or failover. So a + `complete_embeddings_batch` call that returned normally with a terminal + failure document (`status` of `failed`, `cancelled`, or `rejected` -- + `CostRoutingCoordinator.embeddings_batch_document`'s own terminal + vocabulary) still marked that endpoint healthy and cleared its circuit, + `break`ing out of the loop before any other candidate was tried; a + genuinely broken endpoint kept being selected by every later request + instead of failing over (Devin Review on #971). The loop now checks the + document's status before recording success: a terminal-failure document is + routed through the same shared `_record_embedding_failure` recorder an + exception would use and the loop `continue`s to the next candidate; only a + non-terminal-failure document (`completed`, or an in-flight status such as + `queued`/`validating`/`running`) records success and breaks + (`tests/test_pr971_review_quality_regressions.py::test_terminal_embedding_batch_document_fails_over_before_marking_health`). ### Added diff --git a/README.md b/README.md index 6eedf5755..50947ff10 100644 --- a/README.md +++ b/README.md @@ -290,8 +290,9 @@ is read from a **KV config store**, never `os.getenv`. - **Health.** `GET /healthz` is an unauthenticated liveness probe that returns only service identity and process status; it never discloses worker topology, backend names, usage volume, or upstream readiness. Admins can use - `GET /api/v1/provider_readiness/latest?refresh=true` for one bounded, - non-retrying chat probe per enabled worker. + `GET /api/v1/provider_readiness/latest?refresh=true` for one explicitly + cancellable, non-retrying chat probe per enabled worker. Concurrent refreshes + return `refresh_in_progress` instead of blocking behind a slow provider. - **Standalone + optional pg-llm-batch integration.** The hub runs standalone with the in-memory config store and local batch backend; wiring a Postgres DSN and an installed/deployed `pg_llm_batch` client activates the KV/secret stores, @@ -401,8 +402,10 @@ python tests/test_discovery_bootstrap_selection.py python tests/test_chat_capability.py python tests/test_review_gateway.py python tests/test_provider_bootstrap.py +python tests/test_provider_bootstrap_report_identity.py python tests/test_provider_bootstrap_secret_normalization.py python tests/test_provider_catalog_bootstrap.py +python tests/test_provider_catalog_bootstrap_report_identity.py python tests/test_provider_catalog_credential_promotion.py python tests/test_provider_catalog_store.py python tests/test_tool_execution_fallback.py diff --git a/contextual_orchestrator/__main__.py b/contextual_orchestrator/__main__.py index 9c6150bb0..ed1e4bb51 100644 --- a/contextual_orchestrator/__main__.py +++ b/contextual_orchestrator/__main__.py @@ -28,6 +28,7 @@ ProviderModelSource, agent_from_discovered, agent_id_for, + legacy_agent_id_for, configured_gateway_source, discover_all_models, free_discovered_models, @@ -569,8 +570,8 @@ def _discover_models_command(argv: list[str]) -> None: type=_non_negative_int, default=0, metavar="N", - help="Enable a price-honest, provider-diverse discovered agent pool in --agents-db (auto-optimization bootstrap; " - "requires --agents-db; 0 disables, the default, leaving every discovered agent inert).", + help="Enable a price-evidenced discovered agent pool in --agents-db; unmodeled diversity reordering fails closed " + "(requires --agents-db; 0 disables, the default, leaving every discovered agent inert).", ) parser.add_argument( "--free-only", @@ -626,7 +627,7 @@ def _discover_models_command(argv: list[str]) -> None: if not model.evidence_only ] bootstrap = TaskOrchestrator( - discovered_agents, + [], agents_db=args.agents_db, allow_empty_agents=True, ) @@ -634,7 +635,21 @@ def _discover_models_command(argv: list[str]) -> None: bootstrap.sync_discovered_agents(discovered_agents) if args.enable_cheapest: for model in select_bootstrap_discovered_agents(reported, price_book, args.enable_cheapest): - agent_id = agent_id_for(model) + incoming = agent_from_discovered(model) + matches = [ + candidate + for candidate in bootstrap.candidates + if "discovered" in candidate.tags + and candidate.provider_name == incoming.provider_name + and candidate.credential_name == incoming.credential_name + and candidate.model == incoming.model + ] + if not matches: + continue + agent_id = next( + (candidate.id for candidate in matches if candidate.id == incoming.id), + matches[-1].id, + ) bootstrap.patch_agent("default", agent_id, {"status": "active"}) enabled_agent_ids.append(agent_id) finally: @@ -759,14 +774,32 @@ def _auto_discover_runtime_agents(orchestrator: TaskOrchestrator) -> dict[str, l or "embedding" in model.capabilities or ( agent_id_for(model) in failed_configured_gateway_probe_ids - and agent_id_for(model) in existing_by_id + # A failed probe is only ever recorded under the new + # fingerprinted id, but a persisted agent from before + # model-group fingerprinting may still be keyed by its + # legacy id (see ``existing`` below); accept either so a + # failed legacy-id endpoint reaches the disable path + # instead of being silently dropped and left enabled. + and ( + agent_id_for(model) in existing_by_id + or legacy_agent_id_for(model) in existing_by_id + ) ) ) ] - discovered_chat_agent_ids = {agent_id_for(model) for model in chat_models} + # Include the legacy id form too: an already-persisted agent matched via + # the legacy_agent_id_for fallback below keeps its existing (pre-model- + # group) id rather than adopting the new hash-suffixed one, so a candidate + # that is genuinely one of the freshly-discovered chat models can still + # be persisted under either id. + discovered_chat_agent_ids = {agent_id_for(model) for model in chat_models} | { + legacy_agent_id_for(model) for model in chat_models + } agents = [] for model in runtime_models: - existing = existing_by_id.get(agent_id_for(model)) + existing = existing_by_id.get(agent_id_for(model)) or existing_by_id.get( + legacy_agent_id_for(model) + ) embedding_routable = "embedding" in model.capabilities and model.spend_admitted spend_routable = is_routable_discovered_model(model) or embedding_routable structured_routable = agent_id_for(model) not in failed_configured_gateway_probe_ids @@ -822,6 +855,7 @@ def _auto_discover_runtime_agents(orchestrator: TaskOrchestrator) -> dict[str, l existing, disabled=not routable or preserve_disabled, tags=tuple(dict.fromkeys(tags)), + group_name=existing.group_name or agent_from_discovered(model).group_name, ) ) elif existing is not None and any(tag in existing.tags for tag in ("spend:blocked", "structured:blocked")): @@ -846,6 +880,14 @@ def _auto_discover_runtime_agents(orchestrator: TaskOrchestrator) -> dict[str, l "structured:blocked:preserve-disabled", } ), + group_name=existing.group_name or agent_from_discovered(model).group_name, + ) + ) + elif existing is not None and not existing.group_name: + agents.append( + replace( + existing, + group_name=agent_from_discovered(model).group_name, ) ) elif existing is not None and limits_changed: diff --git a/contextual_orchestrator/api_contract.py b/contextual_orchestrator/api_contract.py index 03f89ba3a..c8ab29ac9 100644 --- a/contextual_orchestrator/api_contract.py +++ b/contextual_orchestrator/api_contract.py @@ -606,7 +606,7 @@ "/api/v1/provider_readiness/latest": { "get": { "operationId": "get_latest_provider_readiness", - "summary": "Read or explicitly refresh bounded provider chat readiness", + "summary": "Read or explicitly refresh provider chat readiness", "security": [{"admin_bearer_auth": []}], "parameters": [{ "name": "refresh", diff --git a/contextual_orchestrator/batch_routing.py b/contextual_orchestrator/batch_routing.py index 5650e441d..6fc87773f 100644 --- a/contextual_orchestrator/batch_routing.py +++ b/contextual_orchestrator/batch_routing.py @@ -25,6 +25,8 @@ import hashlib import json import logging +import math +import queue import threading import time import uuid @@ -623,6 +625,7 @@ class EmbeddingBatchRequest: routing_agent_id: str | None = None zdr_only: bool = False agent_id: Optional[str] = None + provider_routing: Optional[Dict[str, Any]] = None def wire_custom_id(self) -> str: """Return a provider-safe id while retaining the internal request mapping. @@ -639,14 +642,16 @@ def wire_custom_id(self) -> str: def to_jsonl_line(self, endpoint: str = "/v1/embeddings") -> Dict[str, Any]: """Render this request as an OpenAI Batch API embeddings JSONL line.""" + body: Dict[str, Any] = {"model": self.model, "input": self.input_text} + if self.provider_routing is not None: + body["provider"] = dict(self.provider_routing) return { # The provider body stays OpenAI-compatible; the backend's tracked # request map carries the immutable route identity separately. "custom_id": self.wire_custom_id(), "method": "POST", "url": endpoint, - # ``zdr_only`` is enforced before this provider JSONL is built. - "body": {"model": self.model, "input": self.input_text}, + "body": body, } @@ -767,6 +772,148 @@ def retrieve(self, job: BatchJob) -> List[EmbeddingBatchResultItem]: return self._results.get(job.job_id, []) +class _DaemonWorkerPool: + """A fixed-size daemon worker pool with a `ThreadPoolExecutor`-shaped surface. + + ``ProviderEmbeddingBatchBackend`` used to hand its long-lived, durable + job queue to a real :class:`concurrent.futures.ThreadPoolExecutor`. That + executor registers every worker thread it starts with + ``concurrent.futures.thread``'s own interpreter-exit ``atexit`` hook, + which unconditionally *joins* each still-running worker at shutdown + regardless of that worker's own daemon flag (the same verified failure + mode as ``endpoint_race.race_first_valid`` and + ``model_discovery._openrouter_free_model_endpoints`` elsewhere in this + PR). Combined with this org's default no-deadline + ``ModelClient.timeout=None`` policy -- and the fact that + ``execution_timeout_seconds`` here is only a *cooperative* deadline + checked after ``runner()`` returns, never a preemptive cancellation of + the in-flight call -- a provider embedding runner that never returns + would make that join, and therefore process shutdown, hang forever even + though ``close()`` had already been called. + + This pool keeps the concurrency bound (a *fixed* number of persistent + daemon workers, not one raw thread per queued job -- durable recovery + can replay many pending jobs at once, and one native thread per job + would reintroduce the unbounded-thread-count problem the fixed-pool + fetch in ``model_discovery`` was written to avoid) while carrying no + exit-hook registration at all: an abandoned worker is silently dropped + at interpreter exit, exactly like every other raw ``daemon=True`` + thread in this codebase's timeout-safe call sites. + + ``submit()``/``shutdown()`` intentionally mirror + :class:`concurrent.futures.ThreadPoolExecutor`'s method names and + ``shutdown(wait=, cancel_futures=)`` signature so every existing + ``ProviderEmbeddingBatchBackend`` call site -- and the ``_executor`` + attribute callers substitute test doubles into -- keeps working + unchanged. Workers are also spawned lazily, one per ``submit()`` up to + ``max_workers``, exactly like ``ThreadPoolExecutor._adjust_thread_count``: + each new worker's first ``queue.get()`` finds its triggering item + already queued rather than racing a ``Condition`` wakeup against an + already-idle worker, so callers that (like several existing tests) + observe a durable job's state immediately after ``submit()`` returns + keep the same effectively-synchronous-for-fast-runners timing the + ``ThreadPoolExecutor``-backed implementation had. + + ``submit()`` also mirrors ``ThreadPoolExecutor``'s closed-pool contract: + once ``shutdown()`` has run, a later ``submit()`` raises ``RuntimeError`` + instead of silently queuing work behind the shutdown sentinels, where it + would never be picked up by any worker (every worker already exits on + its own sentinel and none are spawned after shutdown, since + ``submit()`` is the only place that spawns them). ``ProviderEmbeddingBatchBackend`` + itself never races ``submit()`` against ``shutdown()`` this way -- + ``start()`` and ``close()`` both serialize through the backend's + ``_executor_lock`` and ``start()`` checks ``self._closed`` first -- but + this pool is a standalone primitive and must not depend on every caller + reproducing that locking to avoid stranding a job. + """ + + def __init__(self, max_workers: int, *, thread_name_prefix: str = "provider_embedding_worker") -> None: + self._max_workers = max_workers + self._thread_name_prefix = thread_name_prefix + self._queue: queue.Queue[tuple[Callable[..., Any], tuple[Any, ...]] | None] = queue.Queue() + self._workers: list[threading.Thread] = [] + self._workers_lock = threading.Lock() + self._shutdown = False + + def _drain(self) -> None: + while True: + item = self._queue.get() + if item is None: # shutdown sentinel + return + fn, args = item + try: + fn(*args) + except BaseException: # noqa: BLE001 - a worker task must never kill its worker + _LOGGER.exception("provider embedding worker task raised") + + def submit(self, fn: Callable[..., Any], *args: Any) -> None: + """Queue ``fn(*args)`` for a pool worker. Fire-and-forget: no Future. + + Raises ``RuntimeError`` if ``shutdown()`` has already run, matching + ``ThreadPoolExecutor.submit``'s closed-pool contract, instead of + enqueuing work behind the shutdown sentinels where no worker would + ever pick it up. + """ + with self._workers_lock: + if self._shutdown: + raise RuntimeError("cannot schedule new work after shutdown") + self._queue.put((fn, args)) + if len(self._workers) < self._max_workers: + worker = threading.Thread( + target=self._drain, + name=f"{self._thread_name_prefix}_{len(self._workers)}", + daemon=True, + ) + self._workers.append(worker) + worker.start() + + def shutdown(self, *, wait: bool = True, cancel_futures: bool = False) -> None: + """Stop accepting new work; mirrors ``ThreadPoolExecutor.shutdown``. + + ``cancel_futures=True`` drops anything still sitting in the queue + that no worker has picked up yet -- it never runs, exactly like + ``ThreadPoolExecutor``'s own ``cancel_futures``. It cannot, and does + not need to, interrupt a worker already inside ``fn(*args)``: that + worker is a daemon thread and is simply abandoned, not joined, when + ``wait=False``. + + Admission is closed *first*, atomically, under ``_workers_lock`` -- + before anything else (draining the queue or queuing stop sentinels) + happens. ``submit()`` checks the same flag under the same lock, so + the two methods can never interleave: a ``submit()`` that acquires + the lock after this closes admission observes the closed pool and + raises immediately (see ``submit``'s docstring) instead of racing + the cancellation drain below, and a ``submit()`` already holding the + lock is guaranteed to finish enqueuing before this method can + proceed -- so its item is still present in the queue when the drain + below runs and is correctly cancelled. Either way no submission can + land *between* the drain and admission closing, which is what let a + racing ``submit()`` slip past ``cancel_futures=True`` before this + fix (ContextualWisdomLab/contextual-orchestrator#971). + + Idempotent: a repeated call observes ``self._shutdown`` already set + and skips re-draining the queue and re-queuing sentinels (both of + which are only correct to do once), while still joining the + already-captured worker list when ``wait=True``. + """ + with self._workers_lock: + first_shutdown = not self._shutdown + self._shutdown = True + workers = list(self._workers) + if first_shutdown: + if cancel_futures: + while True: + try: + self._queue.get_nowait() + except queue.Empty: + break + for _ in workers: + self._queue.put(None) + if wait: + for worker in workers: + worker.join() + + class ProviderEmbeddingBatchBackend: """Queue provider embedding work and expose a durable polling lifecycle.""" @@ -785,7 +932,7 @@ def __init__( raise ValueError("max_concurrency must be a positive integer") self._runner = runner self._max_concurrency = max_concurrency - self._executor: ThreadPoolExecutor | None = None + self._executor: _DaemonWorkerPool | None = None self._executor_lock = threading.Lock() self._closed = threading.Event() self._registry = job_registry or JobRegistryFactory() @@ -804,11 +951,13 @@ def __init__( ) if execution_timeout_seconds is not None and execution_timeout_seconds <= 0: raise ValueError("provider embedding execution timeout must be positive") - self._execution_timeout_seconds = ( - execution_timeout_seconds - if execution_timeout_seconds is not None - else self._registry.retention_seconds - ) + # ``None`` is a genuine "no execution deadline" state (the caller's + # ``ModelClient`` has no configured wall-clock timeout) and must stay + # that way -- it previously fell back to the registry's storage + # retention window, silently expiring an intentionally unbounded + # embedding job once that window elapsed. ``_execution_deadline`` + # below treats ``None`` as an unbounded (``+inf``) deadline instead. + self._execution_timeout_seconds = execution_timeout_seconds self._terminal_events: Dict[str, threading.Event] = {} self._results: Dict[str, List[EmbeddingBatchResultItem]] = ( job_registry.mapping( @@ -856,7 +1005,7 @@ def __init__( if self._states.get(job_id) in {"queued", "running"} ] if pending_job_ids: - self._executor = ThreadPoolExecutor(max_workers=self._max_concurrency) + self._executor = _DaemonWorkerPool(self._max_concurrency) for job_id in pending_job_ids: self._terminal_events[job_id] = threading.Event() self._executor.submit(copy_context().run, self._run_job, job_id) @@ -915,7 +1064,7 @@ def start(self, job: BatchJob) -> None: self._states[job.job_id] = "queued" self._terminal_events[job.job_id] = threading.Event() if self._executor is None: - self._executor = ThreadPoolExecutor(max_workers=self._max_concurrency) + self._executor = _DaemonWorkerPool(self._max_concurrency) self._executor.submit(copy_context().run, self._run_job, job.job_id) def _run_job(self, job_id: str) -> None: @@ -956,7 +1105,12 @@ def _run_job(self, job_id: str) -> None: event.set() def _execution_deadline(self, job_id: str) -> float: - """Persist a bounded lifetime beginning with the first execution claim.""" + """Persist this job's lifetime beginning with the first execution claim. + + ``+inf`` when the backend was built with no execution timeout (a + genuinely unbounded job) -- every deadline comparison below already + treats an infinite epoch as "never expires" without further changes. + """ existing = self._deadlines.get(job_id) if existing is not None: return float(existing) @@ -967,8 +1121,14 @@ def _execution_deadline(self, job_id: str) -> float: ): existing = self._deadlines.get(job_id) if existing is None: - request_count = len(self._requests[job_id]) - deadline = time.time() + self._execution_timeout_seconds * max(1, request_count) + if self._execution_timeout_seconds is None: + deadline = float("inf") + else: + request_count = len(self._requests[job_id]) + deadline = ( + time.time() + + self._execution_timeout_seconds * max(1, request_count) + ) set_if_absent = getattr(self._deadlines, "set_if_absent", None) if callable(set_if_absent): set_if_absent(job_id, deadline) @@ -1112,10 +1272,17 @@ def _publish_terminal( self._states[job_id] = status def wait(self, job: BatchJob, *, timeout: float) -> Dict[str, Any]: - """Wait within the caller's explicit deadline for a terminal state.""" + """Wait within the caller's explicit deadline for a terminal state. + + ``timeout`` may be ``float("inf")`` when the caller has no wall-clock + deadline (contextual-orchestrator's no-implicit-deadline default); + ``threading.Event.wait`` raises ``OverflowError`` for a non-finite + timeout on CPython, so a non-finite value is translated to ``None`` + (block indefinitely) rather than passed through. + """ event = self._terminal_events.get(job.job_id) if event is not None: - event.wait(timeout=timeout) + event.wait(timeout=timeout if math.isfinite(timeout) else None) return self.poll(job) def poll(self, job: BatchJob) -> Dict[str, Any]: diff --git a/contextual_orchestrator/conventions.py b/contextual_orchestrator/conventions.py index 5aa24d056..906212d27 100644 --- a/contextual_orchestrator/conventions.py +++ b/contextual_orchestrator/conventions.py @@ -4,8 +4,8 @@ import re - TWO_WORD_SNAKE_CASE = re.compile(r"^[a-z][a-z0-9]*_[a-z0-9]+(?:_[a-z0-9]+)*$") +_LEGACY_DISCOVERED_MODEL_SLUG = re.compile(r"[^a-z0-9]+") def is_two_word_snake_case(value: str) -> bool: @@ -17,3 +17,9 @@ def require_object_name(value: str, field_name: str) -> None: """Raise a validation error when an object name is not compliant.""" if not is_two_word_snake_case(value): # pragma: no cover raise ValueError(f"{field_name} must be two or more words in snake_case: {value!r}") + + +def legacy_discovered_agent_id(provider_name: str, model_id: str) -> str: + """Reproduce the exact pre-fingerprint discovered-agent identifier.""" + model_slug = _LEGACY_DISCOVERED_MODEL_SLUG.sub("_", model_id.lower()).strip("_") + return f"{provider_name}_{model_slug or 'model'}" diff --git a/contextual_orchestrator/cost_router.py b/contextual_orchestrator/cost_router.py index 751c9e366..617bec91f 100644 --- a/contextual_orchestrator/cost_router.py +++ b/contextual_orchestrator/cost_router.py @@ -61,6 +61,13 @@ _DEFAULT_EMBEDDING_MAX_INPUTS_PER_REQUEST = 1 _BATCH_LEDGER_SETTLEMENT_TIMEOUT_SECONDS = 1.0 _EMBEDDING_UNIT_RE = re.compile(r"\S+\s*|\s+", re.UNICODE) +# The durable provider-embedding claim lease is an internal locking/heartbeat +# interval (how long one worker holds a job claim before it must renew), not +# a caller-facing request deadline. It must stay a fixed, positive default +# independent of ``ModelClient.timeout`` -- deriving it from that (optional, +# now ``None``-by-default) client timeout meant a durable job registry raised +# at coordinator construction whenever the caller opted into "no deadline". +_DEFAULT_EMBEDDING_CLAIM_LEASE_SECONDS = 30.0 class BatchModelSelectionError(RuntimeError): @@ -232,16 +239,26 @@ def _run_embedding_shard( def _provider_embedding_backend(self) -> ProviderEmbeddingBatchBackend: client = getattr(self.orchestrator, "client", None) - client_timeout = float(getattr(client, "timeout", 0)) + # ``client.timeout`` is ``None`` when the client has no fixed + # wall-clock deadline (the default since #971's removal of fixed + # inference timeouts); treat that the same as an absent/zero + # attribute rather than raising out of ``float(None)``. + client_timeout = float(getattr(client, "timeout", None) or 0) + # The claim lease is an internal durability heartbeat, independent of + # the caller's request deadline: a durable registry always needs a + # positive lease, falling back to a fixed default rather than the + # (possibly absent) client timeout. Execution stays genuinely + # unbounded (``None``) when the caller configured no deadline -- + # ``ProviderEmbeddingBatchBackend`` no longer substitutes the + # registry's storage retention window for that. + claim_lease_seconds = ( + client_timeout if client_timeout > 0 else _DEFAULT_EMBEDDING_CLAIM_LEASE_SECONDS + ) if self.job_registry.durable else None return ProviderEmbeddingBatchBackend( self._run_provider_embeddings, job_registry=self.job_registry, max_concurrency=getattr(client, "local_concurrency", 1), - claim_lease_seconds=( - client_timeout - if self.job_registry.durable and client_timeout > 0 - else None - ), + claim_lease_seconds=claim_lease_seconds, execution_timeout_seconds=client_timeout if client_timeout > 0 else None, ) @@ -256,32 +273,60 @@ def _run_provider_embeddings( if first.agent_id is not None else self.orchestrator.select_capability_agent("embedding", first.model) ) + # ``first.agent_id`` may be a route pinned at submission time under + # an active ``zdr_only`` policy (see ``_resolve_embedding_target``) + # and later replayed by ``ProviderEmbeddingBatchBackend`` after a + # process restart recovers a durably queued job -- an arbitrarily + # long gap during which an operator could have removed the agent's + # ZDR tag or repointed it to a non-ZDR route. Re-validate the + # request's own recorded privacy scope against the agent's *current* + # tags here, at the point of execution, rather than trusting the + # pinned id blindly; the ambient ``request_policy`` contextvar used + # at submission time is not (and cannot be) in effect on this + # worker thread. + if first.zdr_only and "privacy:zdr" not in agent.tags: + raise RuntimeError( + f"embedding agent {agent.id!r} no longer satisfies zdr_only; " + "refusing to execute a recovered privacy-scoped batch" + ) if any( - request.model != first.model or request.agent_id != first.agent_id + request.model != first.model + or request.agent_id != first.agent_id + or request.zdr_only != first.zdr_only for request in requests ): - raise RuntimeError("provider embedding batch must retain one selected route") + raise RuntimeError( + "provider embedding batch must retain one selected route and privacy policy" + ) max_tokens, _max_chars, max_inputs = self._embedding_request_limits() vectors: List[List[float]] = [] prompt_tokens = 0 shard: List[EmbeddingBatchRequest] = [] shard_tokens = 0 - for request in requests: - request_tokens = request.token_count or len(request.input_text.encode("utf-8")) - if shard and ( - len(shard) >= max_inputs or shard_tokens + request_tokens > max_tokens - ): + # Re-establish the ambient ``request_policy`` scope for the actual + # client call(s) below. It is not in effect on this worker thread + # (see the recovery comment above) but the client's OpenRouter ZDR + # pin (``_pin_openrouter_zdr``) reads it, not ``first.zdr_only`` + # directly -- without this, a recovered ``zdr_only`` batch's request + # would silently omit ``provider.zdr`` even though the tag check + # above already re-validated the route. + with self.orchestrator.request_policy(first.zdr_only): + for request in requests: + request_tokens = request.token_count or len(request.input_text.encode("utf-8")) + if shard and ( + len(shard) >= max_inputs or shard_tokens + request_tokens > max_tokens + ): + shard_vectors, shard_usage = self._run_embedding_shard(agent, shard) + vectors.extend(shard_vectors) + prompt_tokens += shard_usage + shard = [] + shard_tokens = 0 + shard.append(request) + shard_tokens += request_tokens + if shard: shard_vectors, shard_usage = self._run_embedding_shard(agent, shard) vectors.extend(shard_vectors) prompt_tokens += shard_usage - shard = [] - shard_tokens = 0 - shard.append(request) - shard_tokens += request_tokens - if shard: - shard_vectors, shard_usage = self._run_embedding_shard(agent, shard) - vectors.extend(shard_vectors) - prompt_tokens += shard_usage return vectors, prompt_tokens def _refresh_embedding_backend(self) -> None: @@ -1317,7 +1362,12 @@ def submit_embeddings_batch( if agent_id is not None and (not isinstance(agent_id, str) or not agent_id): raise TypeError("agent_id must be a non-empty string when provided") self._refresh_embedding_backend() - resolved_model, resolved_agent_id = self._resolve_embedding_target(model, zdr_only, agent_id) + resolved_model, resolved_agent_id, resolved_provider = self._resolve_embedding_target( + model, zdr_only, agent_id + ) + provider_routing = ( + {"zdr": True} if zdr_only and resolved_provider == "openrouter" else None + ) backend = self._embedding_backend_for_route(resolved_model, resolved_agent_id) shared_attribution = dict(attribution or {}) requests, part_counts, part_limits = self._build_embedding_requests( @@ -1326,6 +1376,7 @@ def submit_embeddings_batch( attribution=shared_attribution, zdr_only=zdr_only, agent_id=resolved_agent_id, + provider_routing=provider_routing, ) reserve = getattr(backend, "reserve", None) start = getattr(backend, "start", None) @@ -1346,7 +1397,7 @@ def submit_embeddings_batch( def _resolve_embedding_target( self, model: str, zdr_only: bool, agent_id: Optional[str] - ) -> tuple[str, Optional[str]]: + ) -> tuple[str, Optional[str], Optional[str]]: """Resolve one embedding member without losing a caller's member choice. An explicit caller-supplied ``agent_id`` always wins, and an explicit @@ -1373,16 +1424,16 @@ def _resolve_embedding_target( and not zdr_only and not unspecified_model ): - return model, None + return model, None, None selection_model = None if unspecified_model else model with self.orchestrator.request_policy(zdr_only): candidates = self.orchestrator._capability_agents("embedding", selection_model) if agent_id is None: chosen = self._cheapest_capability_candidate(candidates) - return chosen.model, chosen.id + return chosen.model, chosen.id, _resolved_provider_name(chosen) for candidate in candidates: if candidate.id == agent_id: - return candidate.model, candidate.id + return candidate.model, candidate.id, _resolved_provider_name(candidate) raise RuntimeError(f"embedding agent {agent_id!r} is not eligible for this request") def _build_embedding_requests( @@ -1393,6 +1444,7 @@ def _build_embedding_requests( attribution: Dict[str, Any], zdr_only: bool, agent_id: Optional[str], + provider_routing: Optional[Dict[str, Any]], ) -> tuple[List[EmbeddingBatchRequest], List[int], Dict[str, int]]: """Map original embedding inputs into token-budgeted provider parts.""" max_tokens, max_chars, max_inputs = self._embedding_request_limits() @@ -1417,6 +1469,7 @@ def _build_embedding_requests( token_count=token_count, zdr_only=zdr_only, agent_id=agent_id, + provider_routing=provider_routing, ) ) return requests, part_counts, { @@ -1895,6 +1948,31 @@ def _provider_from_base_url(base_url: str) -> str: return host +def _resolved_provider_name(agent: Any) -> str: + """Return a canonical provider name for one selected agent snapshot. + + ``base_url`` is what actually decides an outbound HTTP destination; + ``provider_name`` is a free-text label unvalidated at ``ModelAgent`` + construction, so it can be empty *or* nonempty-but-wrong (a typo, a + stale copy-paste). Trusting a nonempty ``provider_name`` unconditionally + — the previous ``agent.provider_name or ...`` short-circuit — let an + agent whose ``base_url`` is OpenRouter's own endpoint report a different + provider identity, which made ``submit_embeddings_batch``'s ZDR pin + (``provider_routing = {"zdr": True} if resolved_provider == "openrouter" + ...``) silently skip OpenRouter requests under an active ``zdr_only`` + scope. The exact destination hostname is checked first and is + authoritative whenever it is OpenRouter's, mirroring + ``orchestrator._resolved_openrouter_provider`` so both ZDR-pin choke + points (the embedding-batch path here and the chat/streaming/raw/batch + JSONL path there) share one normalization rule (CodeRabbit review on + #953, discussion_r3898471887 / discussion_r3898659143). + """ + host = _provider_from_base_url(agent.base_url) + if host == "openrouter.ai": + return "openrouter" + return agent.provider_name or host + + def _positive_int(value: Any, default: int) -> int: """Return ``value`` as a positive int, or ``default`` when invalid.""" try: diff --git a/contextual_orchestrator/endpoint_race.py b/contextual_orchestrator/endpoint_race.py index 7d6b5c682..91ab2cabf 100644 --- a/contextual_orchestrator/endpoint_race.py +++ b/contextual_orchestrator/endpoint_race.py @@ -2,9 +2,10 @@ from __future__ import annotations -from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait +from concurrent.futures import FIRST_COMPLETED, Future, wait from contextvars import copy_context from dataclasses import dataclass +import threading import time from typing import Callable, Generic, TypeVar @@ -62,6 +63,7 @@ class EndpointAttempt(Generic[T]): contract: EndpointEquivalenceContract call: Callable[[], T] cancellation_supported: bool = False + cancel: Callable[[], None] | None = None @dataclass(frozen=True) @@ -79,7 +81,7 @@ def race_first_valid( attempts: list[EndpointAttempt[T]], *, validate: Callable[[T], bool], - deadline_seconds: float, + deadline_seconds: float | None, max_concurrency: int, on_attempt_complete: Callable[[str, T | None, BaseException | None], None] | None = None, ) -> RaceOutcome[T]: @@ -94,7 +96,7 @@ def race_first_valid( raise ValueError("immediate_race requires concurrency capacity of at least two") if max_concurrency < len(attempts): raise ValueError("immediate_race capacity must cover every declared endpoint") - if deadline_seconds <= 0: + if deadline_seconds is not None and deadline_seconds <= 0: raise ValueError("deadline_seconds must be positive") contract = attempts[0].contract if any(attempt.contract != contract for attempt in attempts[1:]): @@ -103,33 +105,105 @@ def race_first_valid( raise ValueError("endpoint identifiers must be unique") started = time.monotonic() - pool = ThreadPoolExecutor( - max_workers=min(max_concurrency, len(attempts)), - thread_name_prefix="equivalent_endpoint_race", - ) - def execute(attempt: EndpointAttempt[T]) -> T: + + def execute(attempt: EndpointAttempt[T], future: Future[T]) -> None: + if not future.set_running_or_notify_cancel(): + # Cancelled before this worker got a chance to start: the + # provider was never called, preserving duplicate-cost honesty. + return try: value = attempt.call() if not validate(value): raise ValueError("endpoint returned an invalid completed response") except BaseException as exc: + reported_exc = exc if on_attempt_complete is not None: - on_attempt_complete(attempt.endpoint_id, None, exc) - raise + try: + on_attempt_complete(attempt.endpoint_id, None, exc) + except BaseException as callback_exc: + # ThreadPoolExecutor used to settle its Future with an + # exception raised by the worker callback. Preserve that + # contract explicitly now that raw daemon workers drive + # bare Futures: an observer failure must never strand a + # Future in RUNNING state under an unbounded race. + reported_exc = callback_exc + future.set_exception(reported_exc) + return if on_attempt_complete is not None: - on_attempt_complete(attempt.endpoint_id, value, None) - return value - - futures: dict[Future[T], tuple[int, EndpointAttempt[T]]] = { - pool.submit(copy_context().run, execute, attempt): (index, attempt) - for index, attempt in enumerate(attempts) - } + try: + on_attempt_complete(attempt.endpoint_id, value, None) + except BaseException as callback_exc: + future.set_exception(callback_exc) + return + future.set_result(value) + + # Bare `concurrent.futures.Future` objects driven by raw `daemon=True` + # threads -- never `ThreadPoolExecutor`. `ThreadPoolExecutor` registers + # every worker it starts with `concurrent.futures.thread`'s own + # interpreter-exit hook, which unconditionally *joins* each + # still-running worker at shutdown regardless of that worker thread's + # own daemon flag (mirrors the verified fix and rationale documented on + # `model_discovery._openrouter_free_model_endpoints`). Combined with + # this org's default no-deadline `ModelClient.timeout=None`, a losing + # race participant blocked in an unbounded provider call that never + # returns would make that join -- and therefore process shutdown -- + # hang forever, even though the winner already answered the caller. A + # raw daemon thread carries no such registration and is safely + # abandoned at interpreter exit if still running. Building on bare + # `Future` objects (the documented mechanism for custom executors) + # keeps every coordination primitive below -- `wait()`, + # `future.cancel()`, `future.result()`, `future.exception()`, and + # `set_running_or_notify_cancel()`'s "cancelled-before-start never + # calls the provider" guarantee -- byte-for-byte identical to the + # prior executor-backed futures. + futures: dict[Future[T], tuple[int, EndpointAttempt[T]]] = {} + for index, attempt in enumerate(attempts): + future: Future[T] = Future() + futures[future] = (index, attempt) + ctx = copy_context() + threading.Thread( + target=ctx.run, + args=(execute, attempt, future), + name=f"equivalent_endpoint_race_{index}", + daemon=True, + ).start() pending = set(futures) last_error: BaseException | None = None + cancellation_outcomes: dict[Future[T], str] = {} + + def cancel_loser(future: Future[T], attempt: EndpointAttempt[T]) -> str: + if future in cancellation_outcomes: + return cancellation_outcomes[future] + if future.cancelled(): + outcome = "queued_cancelled" + elif future.done(): + outcome = "failed" if future.exception() is not None else "completed" + elif future.cancel(): + outcome = "queued_cancelled" + elif ( + contract.cancellation_supported + and attempt.cancellation_supported + and attempt.cancel is not None + ): + try: + attempt.cancel() + except Exception: + outcome = "safe_drain" + else: + outcome = "cancellation_requested" + else: + outcome = "safe_drain" + cancellation_outcomes[future] = outcome + return outcome + try: while pending: - remaining = deadline_seconds - (time.monotonic() - started) - if remaining <= 0: + remaining = ( + None + if deadline_seconds is None + else deadline_seconds - (time.monotonic() - started) + ) + if remaining is not None and remaining <= 0: raise TimeoutError("equivalent endpoint race exceeded its deadline") done, pending = wait(pending, timeout=remaining, return_when=FIRST_COMPLETED) if not done: @@ -145,10 +219,8 @@ def execute(attempt: EndpointAttempt[T]) -> T: for loser_future, (_, loser) in futures.items(): if loser_future is future: continue - cancelled = loser_future.cancel() - outcome = "cancelled" if cancelled else "safe_drain" + outcome = cancel_loser(loser_future, loser) cancellations.append((loser.endpoint_id, outcome)) - pool.shutdown(wait=False, cancel_futures=True) return RaceOutcome( value=value, winner_endpoint_id=winner.endpoint_id, @@ -157,5 +229,6 @@ def execute(attempt: EndpointAttempt[T]) -> T: completion_ms=round((time.monotonic() - started) * 1000, 3), ) finally: - pool.shutdown(wait=False, cancel_futures=True) + for future in pending: + cancel_loser(future, futures[future][1]) raise RuntimeError("all equivalent endpoints failed validation") from last_error diff --git a/contextual_orchestrator/model_discovery.py b/contextual_orchestrator/model_discovery.py index b4ae2f135..c82864da6 100644 --- a/contextual_orchestrator/model_discovery.py +++ b/contextual_orchestrator/model_discovery.py @@ -14,18 +14,20 @@ from __future__ import annotations from decimal import Decimal -from concurrent.futures import ThreadPoolExecutor +import hashlib import json import logging import math +import queue import re import ssl +import threading import time import urllib.error import urllib.request import certifi from dataclasses import dataclass, replace -from typing import TYPE_CHECKING, Any, Literal, Mapping, Sequence +from typing import TYPE_CHECKING, Any, Callable, Literal, Mapping, NoReturn, Sequence, TypeVar from urllib.parse import quote, urlsplit, urlunsplit from .chat_capability import ( @@ -33,7 +35,8 @@ is_general_chat_candidate, requires_non_text_input, ) -from .credentials import get_credential +from .conventions import legacy_discovered_agent_id +from .credentials import NotConfigured, get_credential from .orchestrator import ( AUTH_SCHEME_RAW_TOKEN, ModelAgent, @@ -46,16 +49,36 @@ if TYPE_CHECKING: from .cost_ledger import PriceBook +DISCOVERY_TIMEOUT_SECONDS: float | None = None +# Bounded, cancellable wall-clock budget for one provider's *entire* catalog +# discovery attempt (`discover_provider_models`, including every fetch and +# retry it makes internally) inside `discover_all_models`'s per-provider +# loop -- AND for the shared Models.dev / OpenRouter ZDR / OpenRouter +# credits metadata fetches `discover_all_models` makes outside that loop +# (`_fetch_models_dev_metadata`, `_openrouter_zdr_model_ids`, +# `openrouter_paid_inference_available`). This is a wholly separate concern +# from `DISCOVERY_TIMEOUT_SECONDS` (the per-HTTP-call socket timeout passed +# *into* each fetch, which stays unbounded by default so a slow-but-live +# catalog response is not mistaken for an unavailable provider) and from +# `ModelClient.timeout` (the caller's model-*inference* deadline, which #971 +# deliberately defaults to no elapsed-time limit). Provider catalog listing +# at bootstrap/discovery time is a different concern from serving a +# completion: one stalled request -- even one whose hang the per-call socket +# timeout cannot bound, e.g. a connection accepted but never answered, or in +# tests a mock that blocks forever -- must never block the rest of discovery +# forever. Every one of these calls runs on its own daemon thread (see +# `_run_bounded_by_deadline`) and stops waiting once this deadline elapses; +# the abandoned thread cannot block interpreter shutdown (daemon) and its +# eventual result, if any, is simply discarded. Pass `discovery_deadline=None` +# explicitly to opt back into unbounded waiting for all of these calls. +PROVIDER_DISCOVERY_DEADLINE_SECONDS: float = 30.0 _LOGGER = logging.getLogger(__name__) -DISCOVERY_TIMEOUT_SECONDS = 15.0 -# One bounded retry for a provider's primary model-list fetch, reusing the same +# One retry for a provider's primary model-list fetch, reusing the same # transient-vs-terminal classification completion calls already trust -# (is_transient_error). A short, fixed delay and a shortened retry timeout keep -# the added worst case small and predictable for CI callers with their own -# overall time budget (see ContextualWisdomLab/.github's review sidecar). +# (is_transient_error). Discovery has no default wall-clock deadline: a slow +# provider catalog must not be mistaken for an unavailable provider. # Non-transient failures (auth/config errors, malformed responses) are never # retried — a retry cannot fix those and would only waste the time budget. -_DISCOVERY_RETRY_TIMEOUT_SECONDS = 5.0 _DISCOVERY_RETRY_DELAY_SECONDS = 0.5 # Some discovery endpoints (verified live: models.dev returns Cloudflare HTTP # 403 error 1010) reject urllib's default "Python-urllib/X.Y" user agent as a @@ -549,7 +572,7 @@ def _positive_int_metadata(value: object) -> int | None: return None -def _fetch_json(url: str, *, api_key: str = "", auth_scheme: str = "Bearer", timeout: float) -> Any: +def _fetch_json(url: str, *, api_key: str = "", auth_scheme: str = "Bearer", timeout: float | None) -> Any: """Fetch JSON, sending any credential only to the original trusted HTTPS host. Plain ``urllib`` follows a 3xx redirect by copying the original request's @@ -600,8 +623,8 @@ def _fetch_json(url: str, *, api_key: str = "", auth_scheme: str = "Bearer", tim return json.loads(raw.decode("utf-8")) -def _fetch_models_dev_metadata(*, timeout: float) -> Any | None: - """Fetch the shared Models.dev catalog with a small bounded retry. +def _fetch_models_dev_metadata(*, timeout: float | None) -> Any | None: + """Fetch the shared Models.dev catalog with a small retry count. Every ``models_dev_provider_id``-joined source (``opencode_zen``, ``nvidia_nim``, ``nvidia_nim_sub``, ``openai``) shares this one @@ -615,7 +638,7 @@ def _fetch_models_dev_metadata(*, timeout: float) -> Any | None: Returns ``None`` -- the existing "no evidence" fail-closed signal :func:`_merge_models_dev_metadata` already handles -- only once every - bounded attempt has failed; a successful attempt returns immediately + attempt has failed; a successful attempt returns immediately without spending the rest of the retry budget. """ for attempt in range(_MODELS_DEV_FETCH_ATTEMPTS): @@ -632,7 +655,7 @@ def _fetch_configured_gateway_json( *, api_key: str, auth_scheme: str, - timeout: float, + timeout: float | None, ca_bundle: str | None = None, ) -> Any: """Fetch an operator URL through the gateway's pinned hardened transport.""" @@ -648,7 +671,7 @@ def _fetch_configured_gateway_json( origin = urlunsplit(("https", parsed.netloc, "", "", "")) client = ModelClient( ca_bundle=ca_bundle, - timeout=max(1, math.ceil(timeout)), + timeout=None if timeout is None else max(1, math.ceil(timeout)), allowed_provider_hosts={parsed.hostname}, ) agent = ModelAgent( @@ -710,7 +733,7 @@ def _open_trusted_discovery_request( def _fetch_json_same_host_https( - url: str, *, api_key: str = "", auth_scheme: str = "Bearer", timeout: float + url: str, *, api_key: str = "", auth_scheme: str = "Bearer", timeout: float | None ) -> Any: """Fetch JSON while rejecting redirects outside the original trusted HTTPS host.""" if not url.startswith("https://"): @@ -1217,9 +1240,37 @@ def _merge_openrouter_provider_privacy( def _openrouter_free_model_endpoints( - payload: Any, *, api_key: str, timeout: float + payload: Any, *, api_key: str, timeout: float | None ) -> dict[str, Any]: - """Fetch endpoint/provider mappings only for explicitly zero-price models.""" + """Fetch endpoint/provider mappings only for explicitly zero-price models. + + Fans the per-model fetch out across raw ``daemon=True`` threads, never + :class:`concurrent.futures.ThreadPoolExecutor` -- verified (a local + repro, mirroring this PR's other timeout regressions): a + ``ThreadPoolExecutor``'s worker threads register with an + interpreter-exit hook (``concurrent.futures.thread``'s own + ``atexit`` handler) that unconditionally joins every still-running + worker at shutdown, *regardless of the daemon status of whatever thread + created the executor*. A single model whose endpoint fetch hangs + (``timeout=None``'s unbounded socket read, or any hang a finite + per-call ``timeout`` misses) would therefore block process shutdown + even though every enclosing caller here (:func:`discover_provider_models` + for this OpenRouter source, bounded in turn by + :func:`_discover_provider_models_bounded`) already runs on its own + ``daemon=True`` thread. Plain ``threading.Thread(daemon=True)`` workers + carry none of that registration, so a hung fetch is abandoned exactly + like every other stalled discovery-time network call in this module: + the thread is silently discarded at interpreter exit, and shutdown is + never blocked on it. Concurrency is bounded by a *fixed pool of at most + 8 daemon workers* pulling model IDs from a queue, rather than one + ``threading.Thread`` object per model gated only by a semaphore around + its work: the latter still allocates and starts one native OS thread + per model up front (real kernel/stack overhead each) before the + semaphore ever limits anything, so a catalog of hundreds or thousands + of free models could exhaust memory or stall discovery before a single + fetch even began. A fixed pool keeps the live thread count bounded + regardless of catalog size. + """ rows = payload.get("data") if isinstance(payload, dict) else None model_ids = [ row["id"] @@ -1229,22 +1280,56 @@ def _openrouter_free_model_endpoints( and isinstance(row.get("pricing"), dict) and _pricing_is_free(row.get("pricing")) ] + if not model_ids: + return {} - def fetch(model_id: str) -> tuple[str, Any]: + def fetch(model_id: str) -> Any: author, separator, slug = model_id.partition("/") if not separator or not author or not slug: - return model_id, None + return None try: - return model_id, _fetch_json( + return _fetch_json( f"https://openrouter.ai/api/v1/models/{quote(author, safe='')}/{quote(slug, safe=':')}/endpoints", api_key=api_key, timeout=timeout, ).get("data") except (AttributeError, urllib.error.URLError, TimeoutError, ValueError, OSError): - return model_id, None + return None - with ThreadPoolExecutor(max_workers=min(8, len(model_ids) or 1)) as executor: - return dict(executor.map(fetch, model_ids)) + results: dict[str, Any] = {} + results_lock = threading.Lock() + work_queue: queue.Queue[str] = queue.Queue() + for model_id in model_ids: + work_queue.put(model_id) + + def run() -> None: + while True: + try: + model_id = work_queue.get_nowait() + except queue.Empty: + return + value = fetch(model_id) + with results_lock: + results[model_id] = value + + worker_count = min(8, len(model_ids)) + workers = [ + threading.Thread(target=run, name="openrouter-endpoints", daemon=True) + for _ in range(worker_count) + ] + for worker in workers: + worker.start() + for worker in workers: + # No join timeout: this whole call already executes inside an + # already-bounded, already-daemonized caller (see the docstring + # above), so blocking this thread forever on an abandoned peer is + # the same accepted tradeoff already documented for + # `_run_bounded_by_deadline` -- the daemon property is what matters + # for shutdown, not how long this particular thread blocks. A + # worker stuck on one hung fetch simply never drains the rest of + # the queue; the other workers keep making progress independently. + worker.join() + return results def _privacy_policy_urls( @@ -1479,7 +1564,7 @@ def _parse_bytez(payload: Any, source: ProviderModelSource) -> list[DiscoveredMo return _deduplicate_discovered_models(discovered) -def _openrouter_zdr_model_ids(*, timeout: float) -> set[str]: +def _openrouter_zdr_model_ids(*, timeout: float | None) -> set[str]: """Read public OpenRouter ZDR evidence for discovered provider models.""" api_key = get_credential("OPENROUTER_API_KEY") or "" try: @@ -1505,12 +1590,10 @@ def _openrouter_zdr_model_ids(*, timeout: float) -> set[str]: def _apply_discovered_model_evidence( discovered: list[DiscoveredModel], zdr_model_ids: set[str] ) -> list[DiscoveredModel]: - """Apply model-level ZDR evidence to matching rows from every provider. + """Apply OpenRouter ZDR evidence only to its own provider rows. - Providers may expose the same canonical model id as OpenRouter while using - a different upstream endpoint. Exact canonical ids are the only portable - identity; suffix matching would transfer privacy evidence to an unrelated - model that merely shares a display name. + Model identity does not establish another endpoint's retention policy. + Other providers retain their independently supplied privacy evidence. """ if not zdr_model_ids: return discovered @@ -1525,6 +1608,8 @@ def matches(model_id: str) -> bool: model, zdr_capable=not model.evidence_only and matches(model.model_id), ) + if model.provider_name == "openrouter" + else model for model in discovered ] @@ -1532,7 +1617,7 @@ def matches(model_id: str) -> bool: def discover_provider_models( source: ProviderModelSource, *, - timeout: float = DISCOVERY_TIMEOUT_SECONDS, + timeout: float | None = DISCOVERY_TIMEOUT_SECONDS, ca_bundle: str | None = None, models_dev_metadata: Any = _NOT_FETCHED, ) -> list[DiscoveredModel]: @@ -1545,8 +1630,8 @@ def discover_provider_models( preserves this function's existing lazy, per-call fetch-on-demand behavior for every other caller, tests included. - The primary model-list fetch gets one bounded retry (short fixed delay, - shortened timeout) when the failure is transient (5xx/timeout/connection + The primary model-list fetch gets one count-bounded retry (short fixed + delay) when the failure is transient (5xx/timeout/connection reset, per :func:`~contextual_orchestrator.orchestrator.is_transient_error`) — a single provider's momentary blip no longer has to zero out that provider's entire contribution for this discovery pass. A non-transient @@ -1583,7 +1668,7 @@ def discover_provider_models( "auth_scheme": source.auth_scheme, **({"ca_bundle": ca_bundle} if source.provider_name == "configured_gateway" else {}), } - attempt_timeouts = (timeout, min(timeout, _DISCOVERY_RETRY_TIMEOUT_SECONDS)) + attempt_timeouts = (timeout, timeout) payload: Any = None last_exc: Exception | None = None for attempt_index, attempt_timeout in enumerate(attempt_timeouts): @@ -1677,21 +1762,145 @@ def discover_provider_models( return result +_BoundedT = TypeVar("_BoundedT") + + +def _run_bounded_by_deadline( + fn: Callable[[], _BoundedT], + *, + discovery_deadline: float | None, + on_timeout: Callable[[], _BoundedT], + thread_name: str, +) -> _BoundedT: + """Run ``fn`` on its own daemon thread, abandoning it at ``discovery_deadline``. + + The shared bounded/cancellable primitive behind every network call this + module makes at discovery time: the whole-attempt bound around one + provider's :func:`discover_provider_models` call + (:func:`_discover_provider_models_bounded`), and the shared Models.dev / + OpenRouter ZDR / OpenRouter credits metadata fetches in + :func:`discover_all_models` that used to run outside any bound at all + (#971 review finding: "shared metadata fetches bypass discovery + deadline" -- Models.dev ran before the per-provider loop, the OpenRouter + ZDR and credits calls ran after it, none of them under + ``discovery_deadline``). + + Runs ``fn`` on its own daemon thread and stops waiting once + ``discovery_deadline`` elapses instead of blocking forever -- catching a + hang the per-request socket ``timeout`` cannot, e.g. a connection + accepted but never answered, a redirect loop, or (in tests) a mock that + never returns. The thread is daemonized specifically so an abandoned, + still-hung attempt cannot block interpreter shutdown; its result, if it + ever arrives, is simply discarded -- Python threads cannot be forcibly + killed, so "cancellable" here means "the caller stops waiting on it". + ``on_timeout`` is called instead of returning that discarded result, and + lets each caller keep its own already fail-closed "could not get an + answer" outcome (raising :class:`ProviderDiscoveryError` for the + per-provider loop; returning the wrapped function's own no-evidence + fallback -- ``None``, ``set()`` -- for the shared metadata fetches, the + exact value each already returns for an ordinary fetch failure) rather + than this helper inventing a new one. ``discovery_deadline=None`` opts + back into the unbounded wait every one of these calls used before #971. + + Known, accepted tradeoff (same as the pre-existing OpenRouter uptime + sweep thread this pattern was copied from): "abandon" only ever means + the *caller* stops waiting, not that the daemon thread or its underlying + socket actually stops running. Repeated discovery refreshes against a + dependency that stalls every time can accumulate abandoned threads and + open connections until each one's underlying call eventually returns, + errors, or the interpreter exits; daemon threads keep this from blocking + shutdown, but do not reclaim resources any sooner. + """ + if discovery_deadline is None: + return fn() + results: list[_BoundedT] = [] + failures: list[BaseException] = [] + + def run() -> None: + try: + results.append(fn()) + except BaseException as exc: # noqa: BLE001 -- re-raised on the caller's thread below + failures.append(exc) + + worker = threading.Thread(target=run, name=thread_name, daemon=True) + worker.start() + worker.join(timeout=discovery_deadline) + if worker.is_alive(): + # Still running past the deadline; abandon it rather than block the + # rest of discovery (or first-boot bootstrap) forever. + return on_timeout() + if failures: + raise failures[0] + return results[0] if results else on_timeout() + + +def _discover_provider_models_bounded( + source: ProviderModelSource, + *, + timeout: float | None, + ca_bundle: str | None, + models_dev_metadata: Any, + discovery_deadline: float | None, +) -> list[DiscoveredModel]: + """Run one provider's :func:`discover_provider_models` under a wall-clock bound. + + A thin, provider-specific instantiation of :func:`_run_bounded_by_deadline`: + on timeout, raises :class:`ProviderDiscoveryError` with error code + ``"discovery_timeout"`` (rather than returning a fallback value) so + :func:`discover_all_models`'s per-provider loop records it as a normal + per-provider discovery failure and moves on to the next source. + """ + + def _on_timeout() -> NoReturn: + raise ProviderDiscoveryError(source.provider_name, "discovery_timeout") + + return _run_bounded_by_deadline( + lambda: discover_provider_models( + source, + timeout=timeout, + ca_bundle=ca_bundle, + models_dev_metadata=models_dev_metadata, + ), + discovery_deadline=discovery_deadline, + on_timeout=_on_timeout, + thread_name=f"discover-provider-{source.provider_name}", + ) + + def discover_all_models( sources: tuple[ProviderModelSource, ...] = PROVIDER_MODEL_SOURCES, *, - timeout: float = DISCOVERY_TIMEOUT_SECONDS, + timeout: float | None = DISCOVERY_TIMEOUT_SECONDS, ca_bundle: str | None = None, + discovery_deadline: float | None = PROVIDER_DISCOVERY_DEADLINE_SECONDS, ) -> tuple[list[DiscoveredModel], list[ProviderDiscoveryError]]: """Discover models across every provider with a registered credential. One provider's failure never blocks the others: errors are collected and - returned alongside whatever models were successfully discovered. + returned alongside whatever models were successfully discovered. This + now includes a provider whose catalog fetch simply never returns -- + ``discovery_deadline`` (default :data:`PROVIDER_DISCOVERY_DEADLINE_SECONDS`, + a bounded, cancellable budget wholly separate from any model-inference + deadline; see :func:`_run_bounded_by_deadline`) bounds each provider's + *entire* discovery attempt so one stalled source can no longer block + discovery of every later, healthy provider forever. + + The same bound also covers the three *shared* metadata fetches below + that run outside the per-provider loop -- ``_fetch_models_dev_metadata`` + (before the loop), ``_openrouter_zdr_model_ids`` and + ``openrouter_paid_inference_available`` (after it). Each of these + already has an established, fail-closed "no evidence" fallback for an + ordinary fetch failure (``None``, ``set()``, ``None`` respectively); on + a timeout this function abandons the stalled fetch and uses that exact + same fallback rather than waiting forever, so first-boot pool + bootstrapping can no longer hang on a stalled Models.dev, OpenRouter ZDR, + or OpenRouter credits endpoint (#971 review finding: "shared metadata + fetches bypass discovery deadline"). Up to four sources (``opencode_zen``, ``nvidia_nim``, ``nvidia_nim_sub``, ``openai``) each want the same Models.dev catalog. When any registered source declares ``models_dev_provider_id``, fetch it here exactly once - (:func:`_fetch_models_dev_metadata`, with its own small bounded retry) and + (:func:`_fetch_models_dev_metadata`, with its own small retry count) and hand every source the identical parsed payload, instead of each source independently repeating the fetch inside :func:`discover_provider_models`. """ @@ -1702,15 +1911,24 @@ def discover_all_models( source.models_dev_provider_id and get_credential(source.credential_name) for source in sources ): - models_dev_metadata = _fetch_models_dev_metadata(timeout=timeout) + # Timeout fallback mirrors _fetch_models_dev_metadata's own + # ordinary-failure return: None, which _merge_models_dev_metadata + # already treats as "no evidence" and passes rows through unchanged. + models_dev_metadata = _run_bounded_by_deadline( + lambda: _fetch_models_dev_metadata(timeout=timeout), + discovery_deadline=discovery_deadline, + on_timeout=lambda: None, + thread_name="discover-models-dev-metadata", + ) for source in sources: try: discovered.extend( - discover_provider_models( + _discover_provider_models_bounded( source, timeout=timeout, ca_bundle=ca_bundle, models_dev_metadata=models_dev_metadata, + discovery_deadline=discovery_deadline, ) ) except ProviderDiscoveryError as exc: @@ -1718,18 +1936,44 @@ def discover_all_models( # OpenRouter's authenticated catalog supplies routable account-model rows; # its public ZDR endpoint adds route-specific privacy evidence without # turning the whole provider account into either ZDR-only or non-serving. + # Request-time ZDR enforcement for OpenRouter specifically is a runtime + # concern (see ModelClient's `provider: {"zdr": true}` pin), not a + # discovery-time exclusion: OpenRouter can multiplex a model across several + # backing providers, so a stale discovery-time snapshot cannot by itself + # guarantee which provider serves a given request. + # + # Timeout fallback mirrors _openrouter_zdr_model_ids's own + # ordinary-failure return: an empty set, which _apply_discovered_model_evidence + # already treats as "no evidence" and leaves every row's zdr_capable + # unchanged -- never marks a model ZDR-capable on missing/timed-out + # evidence, preserving the fail-closed posture. routed = _apply_discovered_model_evidence( _deduplicate_discovered_models(discovered), - _openrouter_zdr_model_ids(timeout=timeout), + _run_bounded_by_deadline( + lambda: _openrouter_zdr_model_ids(timeout=timeout), + discovery_deadline=discovery_deadline, + on_timeout=set, + thread_name="discover-openrouter-zdr-model-ids", + ), ) if any( source.provider_name == "openrouter" and get_credential(source.credential_name) for source in sources ): + # Timeout fallback mirrors openrouter_paid_inference_available's own + # ordinary-failure return: None ("could not determine"), which + # apply_openrouter_spend_admission already treats as fail-closed -- + # a paid (non-free) OpenRouter row is not spend_admitted unless + # paid_available is True, never on missing/timed-out evidence. routed = apply_openrouter_spend_admission( routed, - openrouter_paid_inference_available(timeout=timeout), + _run_bounded_by_deadline( + lambda: openrouter_paid_inference_available(timeout=timeout), + discovery_deadline=discovery_deadline, + on_timeout=lambda: None, + thread_name="discover-openrouter-paid-inference", + ), ) if _LOGGER.isEnabledFor(logging.INFO): _LOGGER.info( @@ -1742,7 +1986,7 @@ def discover_all_models( def openrouter_paid_inference_available( - *, timeout: float = DISCOVERY_TIMEOUT_SECONDS + *, timeout: float | None = DISCOVERY_TIMEOUT_SECONDS ) -> bool | None: """Return whether OpenRouter attests a strictly positive credit balance.""" api_key = get_credential("OPENROUTER_API_KEY") @@ -1802,7 +2046,19 @@ def _slug(value: str) -> str: def agent_id_for(discovered: DiscoveredModel) -> str: """Two-or-more-word snake_case id, matching this repo's naming convention.""" - return f"{discovered.provider_name}_{_slug(discovered.model_id)}" + fingerprint = hashlib.sha256(discovered.model_id.encode("utf-8")).hexdigest()[:10] + return f"{discovered.provider_name}_{_slug(discovered.model_id)}_{fingerprint}" + + +def legacy_agent_id_for(discovered: DiscoveredModel) -> str: + """Return the pre-fingerprint identifier used by durable discovered agents.""" + return legacy_discovered_agent_id(discovered.provider_name, discovered.model_id) + + +def model_group_name_for(discovered: DiscoveredModel) -> str: + """Use the provider-declared exact model identity as the logical group.""" + fingerprint = hashlib.sha256(discovered.model_id.encode("utf-8")).hexdigest()[:10] + return f"model_{_slug(discovered.model_id)}_{fingerprint}" def privacy_tags_for_discovered(discovered: DiscoveredModel) -> tuple[str, ...]: @@ -1841,6 +2097,10 @@ def is_routable_discovered_model(discovered: DiscoveredModel) -> bool: return ( not discovered.evidence_only and discovered.spend_admitted + and not ( + discovered.provider_name == "openrouter" + and discovered.model_id.casefold() == "openrouter/free" + ) and is_discovered_chat_candidate(discovered) ) @@ -1863,6 +2123,7 @@ def agent_from_discovered(discovered: DiscoveredModel, *, priority: int = 0) -> credential_key=discovered.credential_name, auth_scheme=discovered.auth_scheme, provider_name=discovered.provider_name, + group_name=model_group_name_for(discovered), tags=( "discovered", *(("cost:free",) if discovered.is_free else ()), @@ -2169,20 +2430,74 @@ def select_top_n_cheapest_discovered_agents( )[:limit] +def _require_unambiguous_bootstrap_boundary( + ranked: list[DiscoveredModel], + selected: list[DiscoveredModel], + price_book: "PriceBook", +) -> None: + """Reject a cutoff decided by identity or unmodeled diversity preference.""" + selected_identities = [_serving_identity(model) for model in selected] + ranked_prefix = [_serving_identity(model) for model in ranked[: len(selected)]] + if selected_identities != ranked_prefix: + raise ValueError( + "bootstrap diversity would displace lower-cost evidence without an " + "explicit decision model" + ) + if len(selected) >= len(ranked): + return + selected_identity_set = set(selected_identities) + selected_evidence = { + _discovery_price_key(model, price_book)[:2] for model in selected + } + excluded_evidence = { + _discovery_price_key(model, price_book)[:2] + for model in ranked + if _serving_identity(model) not in selected_identity_set + } + if selected_evidence & excluded_evidence: + raise ValueError( + "bootstrap admission is ambiguous at the capacity boundary; " + "provide comparable price evidence or increase the limit to include " + "the tied candidates" + ) + + def select_bootstrap_discovered_agents( discovered: list[DiscoveredModel], price_book: "PriceBook", limit: int, ) -> list[DiscoveredModel]: - """Build a deterministic, price-honest, provider-diverse initial pool. + """Build a deterministic, price-honest, provider- and model-group-diverse pool. Candidates retain the known-price-first ordering of :func:`select_top_n_cheapest_discovered_agents` (queried here with no effective cap so it returns the full ranked, deduplicated, routable - field), but the first pass takes at most one model from each - independently discovered provider account. Remaining capacity is filled - in the same deterministic cost order. No vendor or endpoint name is used - to infer a shared family or collapse credential state. Duplicate serving + field). Three ordered passes propose provider/model-group diversity: + + 1. At most one endpoint per *provider* and per model group. This is the + pass that actually delivers "provider-diverse" (not only + "model-group-diverse"): admitting several cheap, distinctly-named + models from a single provider before any other viable provider gets + a turn would build a pool that looks diverse by model identity while + remaining one provider's outage away from total failure -- exactly + the gap this pass closes. It does not reorder by price on its own; + it only *defers* a candidate whose provider already has a selected + endpoint, so a same-provider model is still preferred the moment no + untried provider remains. + 2. Once every provider with a viable candidate has contributed (or + capacity ran out), fill remaining slots from the deferred candidates + that still introduce a new model group, still in cost order. + 3. Any capacity still open (more slots than distinct model groups) is + filled from the remaining deterministic cost order, duplicate + endpoints included, exactly as before this pass existed. + + A bounded proposal is accepted only when it is identical to the + price-evidenced prefix. Otherwise provider/model labels would become an + undocumented utility function, so admission fails closed until an explicit + decision model supplies that evidence. No vendor or endpoint name is used + to infer a shared family or collapse credential state -- provider identity + is `DiscoveredModel.provider_name` exactly as reported by discovery (e.g. + `nvidia_nim` and `nvidia_nim_sub` remain independent). Duplicate serving identities never consume capacity twice. """ if limit <= 0: @@ -2195,16 +2510,33 @@ def select_bootstrap_discovered_agents( selected: list[DiscoveredModel] = [] deferred: list[DiscoveredModel] = [] + model_groups: set[str] = set() providers: set[str] = set() for model in ranked: - if model.provider_name in providers: + model_group = model_group_name_for(model) + if model_group in model_groups or model.provider_name in providers: deferred.append(model) continue + model_groups.add(model_group) providers.add(model.provider_name) selected.append(model) if len(selected) == limit: + _require_unambiguous_bootstrap_boundary(ranked, selected, price_book) + return selected + + still_deferred: list[DiscoveredModel] = [] + for model in deferred: + model_group = model_group_name_for(model) + if model_group in model_groups: + still_deferred.append(model) + continue + model_groups.add(model_group) + selected.append(model) + if len(selected) == limit: + _require_unambiguous_bootstrap_boundary(ranked, selected, price_book) return selected - selected.extend(deferred[: limit - len(selected)]) + selected.extend(still_deferred[: limit - len(selected)]) + _require_unambiguous_bootstrap_boundary(ranked, selected, price_book) return selected diff --git a/contextual_orchestrator/openrouter_uptime.py b/contextual_orchestrator/openrouter_uptime.py index 79583aef8..5cafd7abf 100644 --- a/contextual_orchestrator/openrouter_uptime.py +++ b/contextual_orchestrator/openrouter_uptime.py @@ -33,6 +33,17 @@ # they are percent-encoded below before request assembly. _OPENROUTER_UPTIME_ORIGIN = "https://openrouter.ai/api/v1" +# #971 removes the *inference* client's fixed wall-clock deadline (a user is +# actively waiting on a model completion, which can legitimately run long). +# This collector's HTTP GET is unrelated background telemetry on a single +# dedicated sweep thread: it polls every openrouter member sequentially in +# one loop, and ``stop()`` cannot interrupt a call already blocked inside +# ``urlopen`` (Python threads are not forcibly cancellable). Leaving this +# fetch unbounded means one unresponsive endpoint hangs the sweep thread +# forever -- leaking it and indefinitely starving every later member of an +# uptime update -- so it keeps its own fixed, independent bound instead. +_UPTIME_FETCH_TIMEOUT_SECONDS = 10.0 + class OpenRouterUptimeCollector: """Periodically fold measured upstream availability into prior ledgers.""" @@ -145,7 +156,9 @@ def _fetch_uptime(self, model_id: str) -> float | None: request = urllib.request.Request(url, method="GET") try: # nosemgrep: python.lang.security.audit.dynamic-urllib-use-detected.dynamic-urllib-use-detected - scheme/host is the fixed constant origin; model_id is percent-encoded before interpolation and never reaches the scheme/authority. - with urllib.request.urlopen(request, timeout=10.0) as response: + with urllib.request.urlopen( + request, timeout=_UPTIME_FETCH_TIMEOUT_SECONDS + ) as response: payload = json.loads(response.read().decode("utf-8")) endpoints = payload.get("data", {}).get("endpoints", []) uptimes = [ diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index fa85ba533..05050d629 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -8,6 +8,7 @@ from contextvars import ContextVar, copy_context from concurrent.futures import ThreadPoolExecutor import copy +import errno import hashlib from dataclasses import dataclass, replace from decimal import Decimal @@ -22,6 +23,7 @@ from pathlib import Path import random import re +import select import socket import ssl import sqlite3 @@ -41,7 +43,7 @@ is_general_chat_candidate, requires_non_text_input, ) -from .conventions import require_object_name +from .conventions import legacy_discovered_agent_id, require_object_name from .credentials import NotConfigured, get_credential from .release_authorization import evaluate_release_authorization from .model_group import ModelGroupRouter, canonical_group_name @@ -172,7 +174,6 @@ def _request_endpoint_partition() -> str: MODEL_CAPABILITIES = frozenset( {"text", "image", "video", "speech", "transcription", "embedding", "rerank", "audio"} ) -MAX_PROVIDER_PROBE_TIMEOUT = 30.0 _SAFE_PROVIDER_PROBE_ERROR_TYPES = frozenset({ "ConnectionError", "HTTPError", @@ -187,24 +188,81 @@ def _request_endpoint_partition() -> str: }) +class _ProviderCancellation: + """Close stdlib HTTP connections owned by one cancellable provider call.""" + + def __init__(self) -> None: + self._connections: set[http.client.HTTPConnection] = set() + self._lock = threading.Lock() + self._cancelled = False + self._cancelled_event = threading.Event() + + def register(self, connection: http.client.HTTPConnection) -> None: + """Register a live connection or reject it after cancellation.""" + with self._lock: + if not self._cancelled: + self._connections.add(connection) + return + try: + connection.close() + except Exception: + pass + raise _ProviderRequestCancelled("provider request was cancelled") + + def cancel(self) -> None: + """Best-effort close every registered connection exactly once.""" + with self._lock: + self._cancelled = True + self._cancelled_event.set() + connections = tuple(self._connections) + self._connections.clear() + for connection in connections: + sock = connection.sock + if sock is not None: + try: + sock.shutdown(socket.SHUT_RDWR) + except Exception: + pass + try: + connection.close() + except Exception: + pass + + def raise_if_cancelled(self) -> None: + """Abort a cancellable DNS or connect operation after explicit cancellation.""" + if self._cancelled_event.is_set(): + raise _ProviderRequestCancelled("provider request was cancelled") + + def run(self, call: Callable[[], Any]) -> Any: + """Run one call and translate transport fallout from cancellation.""" + token = _PROVIDER_CANCELLATION.set(self) + try: + return call() + except BaseException as exc: + if self._cancelled: + raise _ProviderRequestCancelled("provider request was cancelled") from exc + raise + finally: + _PROVIDER_CANCELLATION.reset(token) + self.cancel() + + +class _ProviderRequestCancelled(RuntimeError): + """A provider attempt stopped because its equivalent race already completed.""" + + +_PROVIDER_CANCELLATION: ContextVar[_ProviderCancellation | None] = ContextVar( + "provider_cancellation", default=None +) +_PROVIDER_DNS_SLOTS = threading.BoundedSemaphore(4) + + def _safe_provider_probe_error_type(exc: Exception) -> str: """Keep provider diagnostics package-owned instead of echoing exception classes.""" name = type(exc).__name__ return name if name in _SAFE_PROVIDER_PROBE_ERROR_TYPES else "UnknownError" -def _validate_provider_probe_timeout(timeout: float) -> float: - """Validate the finite, bounded timeout used by explicit readiness probes.""" - if isinstance(timeout, bool) or not isinstance(timeout, (int, float)): - raise ValueError("provider probe timeout must be a finite number") - value = float(timeout) - if not math.isfinite(value) or not 0.1 <= value <= MAX_PROVIDER_PROBE_TIMEOUT: - raise ValueError( - f"provider probe timeout must be between 0.1 and {MAX_PROVIDER_PROBE_TIMEOUT:g} seconds" - ) - return value - - class BudgetExceededError(RuntimeError): """Raised when an operator-configured spend budget is already exhausted.""" @@ -326,6 +384,66 @@ def _cost_usd_decimal(output_tokens: int, price_per_million: float) -> Decimal: ) _REQUEST_ZDR_ONLY: ContextVar[bool] = ContextVar("request_zdr_only", default=False) + +def _resolved_openrouter_provider(agent: ModelAgent) -> str: + """Canonical provider identity for the ZDR-pin decision, base_url-first. + + ``ModelAgent.provider_name`` is free-text and unvalidated at construction + (hand-authored JSON, ``model_discovery.py`` auto-discovery, or KV-driven + config can all leave it empty or typo'd). Trusting it verbatim here would + let an agent whose ``base_url`` is OpenRouter's own endpoint silently skip + the ``provider.zdr=true`` enforcement pin under an explicit ``zdr_only`` + scope while still routing bytes to OpenRouter (base_url decides where the + request goes; this function only decides whether the pin is applied) — + a silent ZDR-policy bypass, not a crash (CodeRabbit review on #953, + discussion_r3898471887). Treating the exact OpenRouter hostname as + authoritative also covers a nonempty typo in that free-text field. Every + call site that funnels through this shared choke point (chat, streaming, + raw, binary media, and non-embedding batch JSONL) therefore gets the same + protection the embedding batch path already has. + """ + host = urlparse(agent.base_url).hostname or "" + if host == "openrouter.ai": + return "openrouter" + return agent.provider_name or host + + +def _pin_openrouter_zdr(agent: ModelAgent, payload: dict[str, Any]) -> dict[str, Any]: + """Force OpenRouter to enforce zero-data-retention at request time. + + OpenRouter can multiplex one model id across several backing providers; + a discovery-time ZDR feed snapshot proves a route was ZDR-attested when + it was fetched, not which provider actually serves a later request. Their + documented ``provider: {"zdr": true}`` request field is OpenRouter's own + server-side enforcement (https://openrouter.ai/docs/features/provider-routing) + and is authoritative for the request being sent right now, so it is + applied here rather than trusted to have been decided correctly upstream. + A caller-supplied ``provider`` object (e.g. explicit routing preferences) + is preserved and only gains the ``zdr`` key. + + ``provider`` is an optional caller passthrough field reaching this shared + choke point unvalidated from every call site (chat, streaming, tools and + binary-media passthrough, and the batch JSONL path). A malformed truthy + non-mapping value (an int, bool, list, or string) must fail with a named, + caller-actionable validation error here rather than an opaque ``TypeError`` + from ``dict()`` deep inside provider-transport code (Devin review on #953). + + The "is this agent OpenRouter" check itself goes through + ``_resolved_openrouter_provider`` rather than a bare ``agent.provider_name`` + comparison, so a misconfigured agent (empty/wrong ``provider_name`` but a + ``base_url`` that is actually OpenRouter's) still gets pinned instead of + silently bypassing ZDR enforcement (CodeRabbit review on #953). + """ + if not _REQUEST_ZDR_ONLY.get() or _resolved_openrouter_provider(agent) != "openrouter": + return payload + provider_routing = payload.get("provider") + if provider_routing is not None and not isinstance(provider_routing, dict): + raise ValueError("provider must be an object with optional OpenRouter routing keys") + provider_routing = dict(provider_routing or {}) + provider_routing["zdr"] = True + return {**payload, "provider": provider_routing} + + SECRET_PATTERNS = ( re.compile(r"(?i)(api[_-]?key|token|secret|password)(['\"]?\s*[:=]\s*['\"]?)[A-Za-z0-9._~+/=-]{12,}"), re.compile(r"(?i)(bearer\s+)[A-Za-z0-9._~+/=-]{12,}"), @@ -1030,7 +1148,7 @@ def _local_provider_state(base_url: str) -> _LocalProviderState: def _local_provider_slot( agent: ModelAgent, capacity: int, - timeout: float, + timeout: float | None, ): """Bound local requests and serialize model switches on a shared endpoint.""" if not _is_local_provider_url(agent.base_url): @@ -1038,7 +1156,7 @@ def _local_provider_slot( return state = _local_provider_state(agent.base_url) - deadline = time.monotonic() + max(float(timeout), 0.0) + deadline = None if timeout is None else time.monotonic() + max(float(timeout), 0.0) with state.condition: while True: if state.active == 0: @@ -1051,6 +1169,9 @@ def _local_provider_slot( state.active += 1 break + if deadline is None: + state.condition.wait() + continue remaining = deadline - time.monotonic() if remaining <= 0: raise TimeoutError("local provider endpoint is busy past its request deadline") @@ -1693,9 +1814,9 @@ class ModelClient: def __init__( self, - timeout: int = 90, + timeout: float | None = None, max_output_tokens: int = 2048, - max_retries: int = 2, + max_retries: int = 0, local_max_retries: int = 0, retry_backoff: float = 0.5, retry_backoff_cap: float = 8.0, @@ -1705,8 +1826,15 @@ def __init__( ca_bundle: str | None = None, verify_tls: bool = True, allowed_provider_hosts: Iterable[str] | None = None, + *, + connect_timeout: float | None = None, ) -> None: + # No deadline is selected by default. Explicit legacy caller limits remain + # compatible; review workflows and readiness paths never supply them. self.timeout = timeout + if connect_timeout is not None and connect_timeout <= 0: + raise ValueError("connect_timeout must be positive") + self.connect_timeout = None if connect_timeout is None else float(connect_timeout) self.max_output_tokens = max_output_tokens if isinstance(max_retries, bool) or max_retries < 0: raise ValueError("max_retries must be >= 0") @@ -1733,12 +1861,19 @@ def __init__( self._sleep = time.sleep # Per-thread usage from the most recent chat() (the server is threaded). self._local = threading.local() + if not verify_tls: raise ValueError("provider TLS verification cannot be disabled; configure a trusted ca_bundle") # TLS trust for provider egress. The system trust store is the default; # ca_bundle points at a custom CA for a reviewed corporate gateway. self._ssl_context = self._build_ssl_context(ca_bundle) + @staticmethod + def cancellable_call(call: Callable[[], Any]) -> tuple[Callable[[], Any], Callable[[], None]]: + """Wrap one provider call with socket-closing cooperative cancellation.""" + cancellation = _ProviderCancellation() + return lambda: cancellation.run(call), cancellation.cancel + @staticmethod def _build_ssl_context(ca_bundle: str | None) -> ssl.SSLContext: if ca_bundle: @@ -1981,15 +2116,16 @@ def apply_effort_profile( applied["reasoning"] = {"effort": applied.pop("reasoning_effort")} return applied - def probe(self, agent: ModelAgent, *, timeout: float = DEFAULT_PROVIDER_PROBE_TIMEOUT) -> dict[str, Any]: - """Verify a local model registry, then run one bounded completion probe. + def probe(self, agent: ModelAgent, *, timeout: float | None = None) -> dict[str, Any]: + """Verify a local model registry, then run one unbounded completion probe. ``/health`` and ``/v1/models`` only prove process/model-registry liveness; this verifies the configured local model and deliberately exercises the - chat path with one output token. It never retries, so a stuck local queue - cannot be multiplied by the readiness check. + chat path with one output token. Registry lookup and model inference are + allowed to complete regardless of wall-clock duration and are cancelled + only by an explicit caller action. """ - probe_timeout = _validate_provider_probe_timeout(timeout) + del timeout # compatibility-only; readiness has no wall-clock deadline started = time.monotonic() if not is_chat_compatible_model_id(agent.model): return { @@ -2013,9 +2149,7 @@ def probe(self, agent: ModelAgent, *, timeout: float = DEFAULT_PROVIDER_PROBE_TI self._provider_url(agent, "/models"), method="GET", ) - with self._open_provider( - registry_request, destination, timeout=probe_timeout - ) as registry_response: + with self._open_provider(registry_request, destination) as registry_response: registry = json.loads( registry_response.read().decode("utf-8") ) @@ -2038,8 +2172,8 @@ def probe(self, agent: ModelAgent, *, timeout: float = DEFAULT_PROVIDER_PROBE_TI } if _is_direct_mlx_provider_url(agent.base_url) and self.chat_template_args: payload["chat_template_kwargs"] = self.chat_template_args - with _local_provider_slot(agent, self.local_concurrency, probe_timeout): - content = self._send(agent, payload, destination, timeout=probe_timeout) + with _local_provider_slot(agent, self.local_concurrency, self.timeout): + content = self._send(agent, payload, destination) usage = self.take_usage() if not content.strip(): failure_code = "provider_empty_probe_response" @@ -2145,6 +2279,7 @@ def _send( timeout: float | None = None, ) -> str: """Perform one provider HTTP request (isolated so retry/backoff stays testable).""" + payload = _pin_openrouter_zdr(agent, payload) payload = self._clamp_agent_token_budget(agent, payload) api_key = _provider_credential(agent) headers = {"content-type": "application/json"} @@ -2209,11 +2344,38 @@ def _connect_validated( """Connect to one already-resolved address without performing another DNS lookup.""" family, sockaddr = destination connection = socket.socket(family, socket.SOCK_STREAM) + cancellation = _PROVIDER_CANCELLATION.get() try: - connection.settimeout(timeout) if source_address is not None: connection.bind(source_address) - connection.connect(sockaddr) + if cancellation is None: + connection.settimeout(timeout) + connection.connect(sockaddr) + return connection + connection.setblocking(False) + result = connection.connect_ex(sockaddr) + if result not in {0, errno.EISCONN}: + if result not in {errno.EINPROGRESS, errno.EALREADY, errno.EWOULDBLOCK}: + raise OSError(result, os.strerror(result)) + deadline = None if timeout is None else time.monotonic() + timeout + while True: + cancellation.raise_if_cancelled() + wait = 0.05 + if deadline is not None: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("provider connection exceeded its explicit deadline") + wait = min(wait, remaining) + _readable, writable, exceptional = select.select( + (), (connection,), (connection,), wait + ) + if writable or exceptional: + error = connection.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) + if error: + raise OSError(error, os.strerror(error)) + break + cancellation.raise_if_cancelled() + connection.settimeout(timeout) return connection except Exception: connection.close() @@ -2221,8 +2383,38 @@ def _connect_validated( @staticmethod def _resolve_addresses(hostname: str, port: int) -> list[ProviderDestination]: + cancellation = _PROVIDER_CANCELLATION.get() try: - addresses = socket.getaddrinfo(hostname, port, type=socket.SOCK_STREAM) + if cancellation is None: + addresses = socket.getaddrinfo(hostname, port, type=socket.SOCK_STREAM) + else: + result: list[Any] = [] + finished = threading.Event() + + while not _PROVIDER_DNS_SLOTS.acquire(timeout=0.05): + cancellation.raise_if_cancelled() + + def resolve() -> None: + try: + result.append(socket.getaddrinfo(hostname, port, type=socket.SOCK_STREAM)) + except BaseException as exc: # propagated on the requesting thread + result.append(exc) + finally: + finished.set() + _PROVIDER_DNS_SLOTS.release() + + worker = threading.Thread(target=resolve, daemon=True, name="provider-dns") + try: + worker.start() + except BaseException: + _PROVIDER_DNS_SLOTS.release() + raise + while not finished.wait(0.05): + cancellation.raise_if_cancelled() + cancellation.raise_if_cancelled() + if isinstance(result[0], BaseException): + raise result[0] + addresses = result[0] except socket.gaierror as exc: raise RuntimeError(f"provider host {hostname!r} could not be resolved") from exc resolved = [(family, sockaddr) for family, _type, _proto, _canonname, sockaddr in addresses] @@ -2253,7 +2445,14 @@ def _open_provider( raise RuntimeError("provider request URL has an invalid port") from exc if destination is None: destination = self._resolve_addresses(parsed.hostname, port)[0] - connection_timeout = self.timeout if timeout is None else timeout + generation_timeout = self.timeout if timeout is None else timeout + connection_timeout = generation_timeout + if self.connect_timeout is not None: + connection_timeout = ( + self.connect_timeout + if generation_timeout is None + else min(self.connect_timeout, generation_timeout) + ) connection: http.client.HTTPConnection if parsed.scheme == "https": # The explicit verifying context is the security control for this reviewed API. @@ -2264,14 +2463,22 @@ def _open_provider( context=self._ssl_context, ) else: - connection = http.client.HTTPConnection(parsed.hostname, port, timeout=connection_timeout) + connection = http.client.HTTPConnection( + parsed.hostname, port, timeout=connection_timeout + ) connection._create_connection = ( # type: ignore[attr-defined] lambda _address, timeout, source_address: self._connect_validated( destination, timeout, source_address ) ) target = urlunsplit(("", "", parsed.path or "/", parsed.query, "")) + cancellation = _PROVIDER_CANCELLATION.get() + if cancellation is not None: + cancellation.register(connection) try: + connection.connect() + if connection.sock is not None: + connection.sock.settimeout(generation_timeout) connection.request( request.get_method(), target, @@ -2361,6 +2568,7 @@ def _stream_send( ): """Stream content deltas from a provider SSE response (real transport, testable).""" self._local.usage = None + payload = _pin_openrouter_zdr(agent, payload) payload = self._clamp_agent_token_budget(agent, payload) api_key = _provider_credential(agent) headers = {"content-type": "application/json", "accept": "text/event-stream"} @@ -2563,6 +2771,7 @@ def proxy_send_bytes( self, agent: ModelAgent, endpoint: str, payload: dict[str, Any] ) -> tuple[bytes, str]: """Passthrough a provider response whose body is binary media.""" + payload = _pin_openrouter_zdr(agent, payload) if agent.base_url.startswith("mock://"): return b"mock audio", "audio/mpeg" api_key = _provider_credential(agent) # pragma: no cover @@ -2730,6 +2939,7 @@ def _send_raw( destination: ProviderDestination | None = None, ) -> dict[str, Any]: # pragma: no cover """One provider HTTP request returning the FULL provider JSON (for passthrough).""" + payload = _pin_openrouter_zdr(agent, payload) payload = self._clamp_agent_token_budget(agent, payload) api_key = _provider_credential(agent) headers = {"content-type": "application/json"} @@ -2971,12 +3181,12 @@ def _batch_run( "url": "/v1/chat/completions", "body": self._clamp_agent_token_budget( agent, - self.apply_effort_profile(agent, { + _pin_openrouter_zdr(agent, self.apply_effort_profile(agent, { "model": agent.model, "messages": messages, "temperature": settings["temperature"] if temperature is None else temperature, "max_tokens": settings["max_output_tokens"], - }, effort_profile), + }, effort_profile)), ), }, ensure_ascii=False) for custom_id, messages in requests.items() @@ -4077,14 +4287,24 @@ def provider_readiness_report( self, *, refresh: bool = False, - timeout: float = DEFAULT_PROVIDER_PROBE_TIMEOUT, + timeout: float | None = None, ) -> dict[str, Any]: """Report provider liveness separately from an explicit chat readiness probe.""" + del timeout # compatibility-only; readiness has no wall-clock deadline if type(refresh) is not bool: raise ValueError("refresh must be a boolean") - probe_timeout = _validate_provider_probe_timeout(timeout) items: list[dict[str, Any]] = [] - with self._provider_readiness_lock: + acquired = not refresh or self._provider_readiness_lock.acquire(blocking=False) + if not acquired: + return { + "status": "refresh_in_progress", + "probe": "refresh", + "checked_at": None, + "agent_count": len(self.agents), + "ready_agent_count": 0, + "items": [], + } + try: for agent in self.candidates: provider = agent.provider_name or self._infer_provider_name(agent.base_url) if agent.disabled: @@ -4096,7 +4316,7 @@ def provider_readiness_report( }) continue if refresh: - item = dict(self.client.probe(agent, timeout=probe_timeout)) + item = dict(self.client.probe(agent)) item["provider"] = provider items.append(redact_value(item)) else: @@ -4106,6 +4326,9 @@ def provider_readiness_report( "provider": provider, "status": "unprobed", }) + finally: + if refresh: + self._provider_readiness_lock.release() active = [item for item in items if item["status"] != "disabled"] status = "unprobed" if not refresh else ( "ready" if active and all(item["status"] == "ready" for item in active) else "not_ready" @@ -4113,7 +4336,6 @@ def provider_readiness_report( return { "status": status, "probe": "refresh" if refresh else "none", - "timeout_seconds": probe_timeout, "checked_at": int(time.time()) if refresh else None, "agent_count": len(active), "ready_agent_count": sum(item["status"] == "ready" for item in active), @@ -6206,12 +6428,39 @@ def sync_discovered_agents(self, discovered_agents: list[ModelAgent]) -> dict[st (or the cost router) opts it in via ``patch_agent``. """ existing_by_id = {agent.id: index for index, agent in enumerate(self.candidates)} + legacy_discovered = { + (agent.provider_name, agent.model, agent.id): index + for index, agent in enumerate(self.candidates) + } + discovered_by_identity = { + (agent.provider_name, agent.credential_name, agent.model): index + for index, agent in enumerate(self.candidates) + if "discovered" in agent.tags + } updated_candidates = list(self.candidates) effective_discovered_agents: list[ModelAgent] = [] added: list[str] = [] updated: list[str] = [] for agent in discovered_agents: index = existing_by_id.get(agent.id) + if index is not None and "discovered" not in updated_candidates[index].tags: + continue + if index is None: + index = legacy_discovered.get( + ( + agent.provider_name, + agent.model, + legacy_discovered_agent_id(agent.provider_name, agent.model), + ) + ) + if index is None: + index = discovered_by_identity.get( + (agent.provider_name, agent.credential_name, agent.model) + ) + if index is not None: + if "discovered" not in updated_candidates[index].tags: + continue + agent = replace(agent, id=updated_candidates[index].id) if index is None: existing_by_id[agent.id] = len(updated_candidates) updated_candidates.append(agent) @@ -6219,7 +6468,7 @@ def sync_discovered_agents(self, discovered_agents: list[ModelAgent]) -> dict[st else: agent = replace( agent, - group_name=updated_candidates[index].group_name, + group_name=updated_candidates[index].group_name or agent.group_name, ) updated_candidates[index] = agent updated.append(agent.id) @@ -6231,7 +6480,7 @@ def sync_discovered_agents(self, discovered_agents: list[ModelAgent]) -> dict[st self.candidates = updated_candidates self.agents = [candidate for candidate in self.candidates if not candidate.disabled] self._rebuild_budget_meter() - for agent in discovered_agents: + for agent in effective_discovered_agents: self._routers_register_member(agent.id) if added or updated: self._append_audit_event( @@ -7446,7 +7695,12 @@ def _capability_agents(self, capability: str, model_name: str | None = None) -> ] if not ranked: raise RuntimeError(f"no enabled agent available for capability={capability}") - return ranked + healthy = [agent for agent in ranked if not self._circuit_open(agent.id)] + if not healthy: + raise RuntimeError( + f"all enabled agents temporarily unavailable for capability={capability}" + ) + return healthy def select_capability_agent(self, capability: str, model_name: str | None = None) -> ModelAgent: """Select a measured member supporting a capability, optionally within one group.""" @@ -7510,7 +7764,11 @@ def _record_endpoint_attempt( { "capability": capability, "endpoint_id": endpoint_id, - "validation_outcome": "provider_error" if error is not None else "completed", + "validation_outcome": ( + "cancelled" + if isinstance(error, _ProviderRequestCancelled) + else "provider_error" if error is not None else "completed" + ), "usage": usage, "duplicate_cost_evidence": ( "provider_reported_usage" if usage is not None @@ -7529,7 +7787,11 @@ def _record_race_attempt( ) -> None: """Share race completion evidence with normal stability/circuit ledgers.""" self._record_endpoint_attempt(endpoint_id, value, error, capability=capability) - if error is not None and not _is_request_too_large_error(error): + if ( + error is not None + and not isinstance(error, _ProviderRequestCancelled) + and not _is_request_too_large_error(error) + ): self._group_router.observe_failure(endpoint_id) self._record_failure(endpoint_id) @@ -7621,17 +7883,16 @@ def call(agent: ModelAgent) -> dict[str, Any] | tuple[bytes, str]: contract = EndpointEquivalenceContract(**race_members[0].endpoint_equivalence) # type: ignore[arg-type] attempt_completed, finalize_attempts = self._race_attempt_collector(capability) + def attempt(agent: ModelAgent) -> EndpointAttempt[Any]: + provider_call, cancel = self.client.cancellable_call(lambda: call(agent)) + return EndpointAttempt( + agent.id, contract, provider_call, + cancellation_supported=contract.cancellation_supported, + cancel=cancel if contract.cancellation_supported else None, + ) try: outcome = race_first_valid( - [ - EndpointAttempt( - agent.id, - contract, - lambda agent=agent: call(agent), - cancellation_supported=False, - ) - for agent in race_members - ], + [attempt(agent) for agent in race_members], validate=( ( lambda value: isinstance(value, tuple) @@ -7771,16 +8032,16 @@ def call(agent: ModelAgent) -> tuple[str, str, str, dict[str, Any] | None]: contract = EndpointEquivalenceContract(**race_members[0].endpoint_equivalence) # type: ignore[arg-type] attempt_completed, finalize_attempts = self._race_attempt_collector("text") + def attempt(agent: ModelAgent) -> EndpointAttempt[Any]: + provider_call, cancel = self.client.cancellable_call(lambda: call(agent)) + return EndpointAttempt( + agent.id, contract, provider_call, + cancellation_supported=contract.cancellation_supported, + cancel=cancel if contract.cancellation_supported else None, + ) try: outcome = race_first_valid( - [ - EndpointAttempt( - agent.id, - contract, - lambda agent=agent: call(agent), - ) - for agent in race_members - ], + [attempt(agent) for agent in race_members], validate=lambda value: isinstance(value[0], str) and bool(value[0]), deadline_seconds=self.client.timeout, max_concurrency=len(race_members), @@ -8070,6 +8331,29 @@ def _record_failure(self, agent_id: str) -> None: self.circuit_reset_seconds, ) + def _record_embedding_failure( + self, agent: ModelAgent, endpoint_path: str, exc: BaseException + ) -> None: + """Quarantine one failing embedding endpoint and retain secret-free evidence.""" + provider_status = getattr(exc, "provider_status", None) + if provider_status is None and isinstance(exc, urllib.error.HTTPError): + provider_status = exc.code + if isinstance(provider_status, bool) or not isinstance(provider_status, int): + provider_status = None + if provider_status != 413: + self._group_router.observe_failure(agent.id) + self._record_failure(agent.id) + self.record_analytics_event( + "embedding_endpoint_failed", + { + "endpoint_path": endpoint_path, + "agent_id": agent.id, + "model": agent.model, + "error_type": type(exc).__name__, + "provider_status": provider_status, + }, + ) + def _record_success(self, agent_id: str) -> None: with self._circuit_lock: cleared = self._circuit.pop(agent_id, None) @@ -15893,3 +16177,4 @@ def sse_stream_body(chunks: list[dict[str, Any]]) -> str: frames = [f"data: {json.dumps(chunk, ensure_ascii=False)}\n\n" for chunk in chunks] frames.append("data: [DONE]\n\n") return "".join(frames) + diff --git a/contextual_orchestrator/privacy_policy_analysis.py b/contextual_orchestrator/privacy_policy_analysis.py index 5328e2654..69cb3d0f7 100644 --- a/contextual_orchestrator/privacy_policy_analysis.py +++ b/contextual_orchestrator/privacy_policy_analysis.py @@ -176,7 +176,7 @@ async def _render_policy_document_with_camoufox(url: str) -> str: def crawl_policy_document( url: str, *, - timeout: float = 15.0, + timeout: float | None = None, camoufox_renderer: Callable[[str], str] | None = None, ) -> str: """Fetch one policy through Wardnet's DNS-pinned outbound boundary.""" @@ -214,7 +214,7 @@ def crawl_policy_document( }, method="POST", ) - client = ModelClient(timeout=max(1, int(timeout)), allowed_provider_hosts={wardnet.hostname}) + client = ModelClient(timeout=timeout, allowed_provider_hosts={wardnet.hostname}) if wardnet.scheme == "http": origin = urlunsplit(("local", wardnet.netloc, "", "", "")) agent = ModelAgent( diff --git a/contextual_orchestrator/provider_bootstrap.py b/contextual_orchestrator/provider_bootstrap.py index 5b9f9af2a..8c19f3443 100644 --- a/contextual_orchestrator/provider_bootstrap.py +++ b/contextual_orchestrator/provider_bootstrap.py @@ -35,6 +35,8 @@ discover_all_models, privacy_tags_for_discovered, is_routable_discovered_model, + legacy_agent_id_for, + model_group_name_for, refresh_price_book, ) from .orchestrator import ModelAgent, TaskOrchestrator @@ -69,7 +71,14 @@ class ProviderBootstrapError(RuntimeError): @dataclass(frozen=True) class ProviderBootstrapReport: - """Secret-free evidence emitted after one provider bootstrap run.""" + """Secret-free evidence emitted after one provider bootstrap run. + + When a durable agent pool is requested, ``selected_agent_ids`` uses the + resolved persisted identities and therefore names the same agents as + ``enabled_agent_ids``. This keeps legacy-ID migration from exposing two + identifier generations for one selected endpoint. Ephemeral runs have no + persisted identities, so ``selected_agent_ids`` uses the generated IDs. + """ registered_credentials: tuple[str, ...] discovered_model_count: int @@ -219,10 +228,52 @@ def _known_cost_sort_key( return (1, float("inf"), model.provider_name, model.model_id) -def select_provider_diverse_models( +def _require_unambiguous_model_group_boundary( + ordered: Sequence[DiscoveredModel], + selected: Sequence[DiscoveredModel], +) -> None: + """Reject a cutoff decided by identity or unmodeled diversity preference.""" + selected_identities = [ + (model.provider_name, model.credential_name, model.model_id) + for model in selected + ] + ranked_prefix = [ + (model.provider_name, model.credential_name, model.model_id) + for model in ordered[: len(selected)] + ] + if selected_identities != ranked_prefix: + raise ProviderBootstrapError( + "provider bootstrap diversity would displace lower-cost evidence " + "without an explicit decision model" + ) + if len(selected) >= len(ordered): + return + selected_identity_set = set(selected_identities) + selected_evidence = {_known_cost_sort_key(model)[:2] for model in selected} + excluded_evidence = { + _known_cost_sort_key(model)[:2] + for model in ordered + if (model.provider_name, model.credential_name, model.model_id) + not in selected_identity_set + } + if selected_evidence & excluded_evidence: + raise ProviderBootstrapError( + "provider bootstrap admission is ambiguous at the capacity boundary; " + "provide comparable price evidence or increase the limit to include " + "the tied candidates" + ) + + +def select_model_group_diverse_models( discovered: Sequence[DiscoveredModel], *, limit: int ) -> list[DiscoveredModel]: - """Choose a bounded compatible pool while preserving provider diversity.""" + """Choose a bounded pool, rejecting diversity that changes priced admission. + + Consumer migration: selection favors exact ``model_group`` identity and + known cost over provider spread. Consumers relying on provider-level + diversity must review migration (price-evidenced admission is + authoritative; provider spread alone does not displace cheaper evidence). + """ if limit < 1: raise ValueError("provider bootstrap model limit must be positive") unique: dict[tuple[str, str, str], DiscoveredModel] = {} @@ -232,13 +283,15 @@ def select_provider_diverse_models( unique[(model.provider_name, model.credential_name, model.model_id)] = model ordered = sorted(unique.values(), key=_known_cost_sort_key) selected: list[DiscoveredModel] = [] - seen_providers: set[str] = set() + seen_model_groups: set[str] = set() for model in ordered: - if model.provider_name in seen_providers: + model_group = model_group_name_for(model) + if model_group in seen_model_groups: continue selected.append(model) - seen_providers.add(model.provider_name) + seen_model_groups.add(model_group) if len(selected) >= limit: + _require_unambiguous_model_group_boundary(ordered, selected) return selected selected_keys = { (item.provider_name, item.credential_name, item.model_id) @@ -251,6 +304,7 @@ def select_provider_diverse_models( selected.append(model) if len(selected) >= limit: break + _require_unambiguous_model_group_boundary(ordered, selected) return selected @@ -270,13 +324,47 @@ def _synchronize_durable_agent_pool( """Activate exactly the selected discovered models in one durable agent pool.""" agents = [_active_agent_from_discovered(model) for model in selected] bootstrap = TaskOrchestrator( - agents, + [], agents_db=agents_db, allow_empty_agents=True, ) try: - selected_ids = {agent.id for agent in agents} + protected_ids = { + candidate.id + for candidate in bootstrap.candidates + if "discovered" not in candidate.tags + } + if any( + {agent.id, legacy_agent_id_for(model)} & protected_ids + for agent, model in zip(agents, selected, strict=True) + ): + raise ProviderBootstrapError( + "selected discovered models conflict with operator-managed agent identities" + ) bootstrap.sync_discovered_agents(agents) + selected_ids: set[str] = set() + ordered_selected_ids: list[str] = [] + for agent in agents: + matches = [ + candidate + for candidate in bootstrap.candidates + if "discovered" in candidate.tags + if candidate.provider_name == agent.provider_name + and candidate.credential_name == agent.credential_name + and candidate.model == agent.model + ] + if not matches: + continue + selected_id = next( + (item.id for item in matches if item.id == agent.id), + matches[-1].id, + ) + selected_ids.add(selected_id) + ordered_selected_ids.append(selected_id) + if len(selected_ids) != len(agents): + raise ProviderBootstrapError( + "selected discovered models conflict with operator-managed agent identities" + ) for candidate in list(bootstrap.candidates): if candidate.id in selected_ids: @@ -285,14 +373,12 @@ def _synchronize_durable_agent_pool( if not candidate.disabled: bootstrap.remove_agent("default", candidate.id) - for agent in agents: - bootstrap.patch_agent("default", agent.id, {"status": "active"}) + for agent_id in ordered_selected_ids: + bootstrap.patch_agent("default", agent_id, {"status": "active"}) # The patch loop above raises KeyError if any selected agent is missing from - # the pool, so the enabled set equals selected_ids by construction here. - return tuple( - sorted(agent.id for agent in bootstrap.agents if agent.id in selected_ids) - ) + # the pool. Preserve the selector's cost/model-group order in the report. + return tuple(ordered_selected_ids) finally: bootstrap.close() @@ -323,16 +409,17 @@ def bootstrap_provider_runtime( price_book = PriceBook(InMemoryConfigStore()) priced_count = refresh_price_book(discovered, price_book) - # select_provider_diverse_models returns at least one model for a non-empty + # select_model_group_diverse_models returns at least one model for a non-empty # input with a positive limit and raises ValueError for a non-positive one, # so the selection here is never empty. - selected = select_provider_diverse_models(eligible, limit=model_limit) - selected_ids = tuple(agent_id_for(model) for model in selected) + selected = select_model_group_diverse_models(eligible, limit=model_limit) + generated_selected_ids = tuple(agent_id_for(model) for model in selected) enabled_ids = ( _synchronize_durable_agent_pool(agents_db, selected) if agents_db else () ) + selected_ids = enabled_ids if agents_db else generated_selected_ids return ProviderBootstrapReport( registered_credentials=registered, diff --git a/contextual_orchestrator/provider_catalog_bootstrap.py b/contextual_orchestrator/provider_catalog_bootstrap.py index 4e6d5e612..6eebdd7e1 100644 --- a/contextual_orchestrator/provider_catalog_bootstrap.py +++ b/contextual_orchestrator/provider_catalog_bootstrap.py @@ -45,7 +45,7 @@ collect_provider_credentials, is_chat_serving_candidate, register_provider_credentials_atomically, - select_provider_diverse_models, + select_model_group_diverse_models, serving_tags_for_discovered, ) from .provider_catalog_store import ( @@ -622,7 +622,7 @@ def bootstrap_provider_catalog_runtime( price_book = PriceBook(InMemoryConfigStore()) priced_count = refresh_price_book(list(usable_models), price_book) - selected = select_provider_diverse_models( + selected = select_model_group_diverse_models( usable_models, limit=model_limit, ) @@ -630,12 +630,13 @@ def bootstrap_provider_catalog_runtime( raise ProviderBootstrapError( "provider bootstrap selected no persisted chat-compatible model" ) - selected_ids = tuple(agent_id_for(model) for model in selected) + generated_selected_ids = tuple(agent_id_for(model) for model in selected) enabled_ids = ( _synchronize_durable_agent_pool(agents_db, selected) if agents_db else () ) + selected_ids = enabled_ids if agents_db else generated_selected_ids durable_registered_credentials = tuple( name for name in registered if get_credential(name) is not None ) diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index eb6a77519..871cefd39 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -2330,6 +2330,8 @@ def _validate_mode(mode: Any) -> str: def _validate_capability_request(path: str, body: dict[str, Any]) -> None: """Validate the required trust-boundary fields for media/rerank passthrough.""" + if "provider" in body and not isinstance(body["provider"], dict): + raise RequestError(400, "invalid_provider", "provider must be an object") if "model" in body: model = body["model"] if not isinstance(model, str): @@ -4973,6 +4975,28 @@ def _validate_embeddings_model(body: dict[str, Any], orchestrator: Any | None = return model +# Terminal-failure batch document statuses that must never be treated as a +# healthy completion for endpoint-health purposes. Mirrors the vocabulary +# ``CostRoutingCoordinator.embeddings_batch_document`` (cost_router.py) uses +# to stop polling a batch job: "failed"/"cancelled"/"rejected" are terminal +# outcomes with no embeddings and no further transitions, distinct from +# "completed" (success) and from in-flight statuses such as "queued", +# "validating", or "running" (still eligible to become "completed" later). +_TERMINAL_EMBEDDING_BATCH_FAILURE_STATUSES = frozenset({"failed", "cancelled", "rejected"}) + + +def _available_embedding_agents(orchestrator: Any, model_name: str) -> list[Any]: + """Map temporary embedding quarantine to the public availability contract.""" + try: + return orchestrator._capability_agents("embedding", model_name) + except RuntimeError as exc: + raise RequestError( + 503, + "embeddings_unavailable", + "all enabled embedding-capable model group members are temporarily unavailable", + ) from exc + + def _validate_embeddings_encoding_format(body: dict[str, Any]) -> str | None: """OpenAI ``encoding_format`` — omit/null/empty, ``float``, or ``base64``. @@ -7229,8 +7253,8 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di ) # Same pool honesty as chat/Completions: do not silently serve # a different embedding deployment than the client requested. - embedding_agents = orchestrator._capability_agents( - "embedding", + embedding_agents = _available_embedding_agents( + orchestrator, TaskOrchestrator.AUTO_MODEL if model_was_omitted else model_name, ) embedding_agents = coordinator._cost_ordered_capability_candidates( @@ -7286,8 +7310,14 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di if not attribution.get("service"): attribution["service"] = "embeddings_api" started_at = time.perf_counter() - embedding_deadline = time.monotonic() + float( - orchestrator.client.timeout + # A ``None`` client timeout (the default since #971's + # removal of fixed inference deadlines) means "no + # wall-clock deadline" rather than "zero" -- keep the + # failover loop below waiting on every candidate instead + # of raising out of float(None) or exiting immediately. + client_timeout = orchestrator.client.timeout + embedding_deadline = time.monotonic() + ( + float(client_timeout) if client_timeout else float("inf") ) document = None last_embedding_error: Exception | None = None @@ -7309,18 +7339,29 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di )) except Exception as exc: # noqa: BLE001 - measured member failover last_embedding_error = exc - orchestrator._group_router.observe_failure(embedding_agent.id) + orchestrator._record_embedding_failure( + embedding_agent, "/v1/embeddings", exc + ) continue - if document.get("status") == "completed": + if document.get("status") == "completed" and document.get("embeddings") is not None: orchestrator._group_router.observe_success( embedding_agent.id, time.perf_counter() - attempt_started_at, ) + orchestrator._record_success(embedding_agent.id) break last_embedding_error = RuntimeError( f"embedding member ended with {document.get('status', 'unknown')}" ) - orchestrator._group_router.observe_failure(embedding_agent.id) + # Route a non-completed/embedding-less sync result through the + # same failure recorder as a raised exception (above): a bare + # ``observe_failure`` skips the circuit breaker and the + # ``embedding_endpoint_failed`` analytics event, so a member + # that keeps returning an incomplete document would never be + # quarantined. + orchestrator._record_embedding_failure( + embedding_agent, "/v1/embeddings", last_embedding_error + ) document = None if document is None: raise RequestError( @@ -7371,8 +7412,8 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di _require_pool_model( orchestrator, model_name, required_capability="embedding" ) - embedding_agents = orchestrator._capability_agents( - "embedding", + embedding_agents = _available_embedding_agents( + orchestrator, TaskOrchestrator.AUTO_MODEL if model_was_omitted else model_name, ) embedding_agents = coordinator._cost_ordered_capability_candidates( @@ -7412,13 +7453,32 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di )) except Exception as exc: # noqa: BLE001 - measured member failover last_embedding_error = exc - orchestrator._group_router.observe_failure(embedding_agent.id) + orchestrator._record_embedding_failure( + embedding_agent, "/v1/batch/embeddings", exc + ) continue - if document.get("status") == "completed": - orchestrator._group_router.observe_success( - embedding_agent.id, - time.perf_counter() - attempt_started_at, + if document.get("status") in _TERMINAL_EMBEDDING_BATCH_FAILURE_STATUSES: + # complete_embeddings_batch returned normally, but the + # document itself is a terminal failure (no exception + # was raised). Treat it exactly like a raised exception + # for failover purposes: route it through the same + # shared failure recorder used above so the endpoint's + # circuit is not falsely cleared by observe_success + # below, then try the next candidate instead of + # breaking out with a failed document. + last_embedding_error = RuntimeError( + f"embedding batch member ended with {document.get('status')}" ) + orchestrator._record_embedding_failure( + embedding_agent, "/v1/batch/embeddings", last_embedding_error + ) + document = None + continue + orchestrator._group_router.observe_success( + embedding_agent.id, + time.perf_counter() - attempt_started_at, + ) + orchestrator._record_success(embedding_agent.id) break if document is None: raise RequestError( diff --git a/docs/adr/0001-tool-execution-fallback-policy.md b/docs/adr/0001-tool-execution-fallback-policy.md index 8fa113b69..9e982375a 100644 --- a/docs/adr/0001-tool-execution-fallback-policy.md +++ b/docs/adr/0001-tool-execution-fallback-policy.md @@ -46,6 +46,27 @@ The primary model call `TaskOrchestrator._invoke` makes on every route/Conduct s `contextual_orchestrator.tool_fallback.classify_provider_transport_failure(retryable: bool)` now classifies this specific call directly from the provider taxonomy's own already-computed `retryable` flag — never from message text — and never returns `fail_closed`: retryable failures (429/500/502/503/504/408/network) get one bounded same-agent retry then sequential failover; non-retryable failures (401/403/404/413 handled earlier/422/...) fail over immediately. This is the same "generic provider transport failures keep the previous agent-failover behavior" intent this ADR already stated; it is now an explicit, provider-status-driven contract instead of an implicit one that depended on a failure message never mentioning a tool-fallback keyword. `classify_tool_failure` itself is unchanged and still governs genuine `ToolExecutionError` adapters and the provider's own explicit `tool_execution_stopped` signal (`_provider_tool_execution_stopped`), both of which keep failing closed exactly as this ADR specifies. Motivated by the `orchestrator/free` review-sidecar reliability gap tracked in `ContextualWisdomLab/.github` PR #1433. +## Amendment (2026-09-02): no-heuristics default transport retry allocation + +`ModelClient` previously defaulted `max_retries` to `2` — a hand-picked provider transport retry +count with no cited standard, paper, or the org's own research (Fugu, Conductor, TRINITY) +establishing that number. RFC 9110 §9.2.2 (cited above) constrains *when* a client may safely +replay a request — only for idempotent semantics, or when the original request is known never to +have applied — but it does not name a specific attempt count. NIST SP 800-204 (also cited above) +discusses retry and circuit-breaker resilience as a pattern, not a numeric allocation. Neither +source, nor Fugu/Conductor/TRINITY, identifies a specific retry budget for this library to adopt +as a default. + +That unjustified numeric default is therefore removed rather than re-justified: +`ModelClient.__init__`'s `max_retries` default is now `0`. A default `ModelClient` allocates zero +automatic provider transport retries, independent of provider, model, or reasoning-capability +identity — `tests/test_no_heuristic_default_transport_retry.py` is the regression contract. +Explicit nonzero retry budgets remain a caller-owned configuration surface (an explicit +`max_retries=` argument at construction time), never a library-authored default. This amendment +does not change the fallback matrix, the safety invariants, or `local_max_retries` (already `0` +by default); it only removes an unproven default from the provider-transport retry path this ADR +governs. + ## Safety invariants 1. Missing-tool handling changes agents; it never guesses an alias for the missing tool. diff --git a/docs/doctoring/current-main-provider-bootstrap.md b/docs/doctoring/current-main-provider-bootstrap.md index 6957d8bc8..aaf59b407 100644 --- a/docs/doctoring/current-main-provider-bootstrap.md +++ b/docs/doctoring/current-main-provider-bootstrap.md @@ -56,9 +56,13 @@ retained as a structured-output capability tag. The structured auto route uses only a synthesizer carrying that evidence and fails closed when none exists; model names are never used to infer support. -The bootstrap pool is provider-diverse before it is cost-ordered. Missing price is -`unknown`, not zero. This avoids treating a provider such as Bytez, whose public -catalog may use a non-token billing unit, as a fabricated free route. +The bootstrap proposes provider/model-group spread while retaining cost order. +Every diversity proposal is admitted only when it preserves the +price-evidenced candidate sequence; otherwise it fails closed until an explicit +allocation model supplies the missing utility evidence. Missing price is `unknown`, not zero. +This avoids both treating a provider such as Bytez, whose public catalog may use a +non-token billing unit, as a fabricated free route and treating provider labels as +undocumented outage weights. Candidate selection and durable serving activation are separate claims: @@ -104,7 +108,8 @@ the emitted report. The automatic pool remains a routing input rather than an unsupported claim that a single cheapest model is universally best. Quality/performance selection remains in the orchestrator's paper-grounded routing and orchestration layer; this bootstrap -only establishes a compatible candidate set and failure isolation. +only establishes a compatible candidate set. It does not claim failure isolation +from provider labels alone. National Institute of Standards and Technology. (2020). *Security and privacy controls for information systems and organizations* (NIST Special Publication diff --git a/docs/doctoring/provider-diverse-discovery-routing.md b/docs/doctoring/provider-diverse-discovery-routing.md index 0ad35a439..eea7f08b1 100644 --- a/docs/doctoring/provider-diverse-discovery-routing.md +++ b/docs/doctoring/provider-diverse-discovery-routing.md @@ -1,6 +1,6 @@ --- title: "Provider-diverse discovery and cost-honest failover routing" -status: "implemented" +status: "proposed" date: "2026-08-21" scope: "PR #770" --- @@ -12,10 +12,16 @@ scope: "PR #770" PR #770 makes model discovery fail closed for invalid catalog rows (a price that is negative, non-finite, or a nonzero value that underflows to zero), retains eligible candidates that simply have no reported price as an -explicit unknown-cost fallback, and selects a provider-diverse bootstrap -pool before ordinary chat routing. The selector is deterministic eligibility -and cost accounting; it is not a learned answer-quality judge and does not -claim to reproduce the learning systems in the cited work. +explicit unknown-cost fallback, and proposes a provider/model-group-diverse +bootstrap pool before ordinary chat routing. Provider/model-group labels are +not outage probabilities or utility evidence. A proposal therefore fails +closed when it changes the price-evidenced candidate sequence, or when selected +and excluded candidates have equal comparable cost or incomplete price +evidence. Operators must supply a released decision model, comparable price +evidence, or capacity for the whole competing class. The selector is +deterministic eligibility and cost accounting; it is not a learned +answer-quality or availability judge and does not claim to reproduce the +learning systems in the cited work. Virtual-model passthrough requests use that same provider-diverse pool for tools, structured output, and Responses payloads. Each candidate receives one @@ -30,7 +36,8 @@ DNS failure can advance without changing an explicitly requested concrete model. | --- | --- | --- | | Reject malformed, negative, or non-finite price rows | A cost-aware router must not treat missing or invalid evidence as zero cost. | Discovery and persisted-price tests reject the row before selection. | | Keep unknown-price candidates only as an explicit fallback | Cost optimization must remain honest when price evidence is incomplete. | Selection tests never rank an unknown price above a valid priced candidate. | -| Prefer distinct providers in the bootstrap pool | A gateway needs an upstream failover set rather than several aliases for one provider. | Provider-diversity tests assert the configured pool spans available providers. | +| Propose distinct providers without treating labels as utility | A gateway may benefit from independently failing upstreams, but provider names alone do not quantify that benefit. | Discovery-selector tests reject a more-expensive diversity proposal until an explicit decision model supplies the missing evidence. | +| Reject an evidence-tied capacity cutoff | Lexical provider/model identity is deterministic ordering metadata, not price, quality, or availability evidence. | Both selectors raise when a selected and excluded candidate share the same comparable-cost/unknown state. | | Fail over virtual-model passthrough once per provider | Preserve raw provider features without retry amplification; concrete model selection remains a caller contract. | Passthrough tests cover 404, 410, 429, 503, wrapped failures, caller errors, and exhaustion. | | Leave quality judgment to evaluation/review policy | Routing signals and answer-quality judgment have different failure modes. | Existing model-judge and fail-closed routing tests remain the quality boundary. | @@ -41,6 +48,17 @@ stack base under `docs/papers/` (`routellm-routing-2406.18665.pdf`, to the exact discovery selector explicit instead of treating inherited files as incidental documentation. +## Consumer migration boundary + +`provider_bootstrap.select_model_group_diverse_models` and +`model_discovery.select_bootstrap_discovered_agents` may propose exact +model-group/provider spread, but neither may let that proposal change the +price-evidenced candidate sequence without an explicit decision model. Consumers +that require provider-level redundancy must supply that released allocation +contract at the approved owner boundary. Neither selector may break an +equal/incomplete evidence tie or displace lower-cost evidence by provider or +model name. + ## APA 7 references Chen, L., Zaharia, M., & Zou, J. (2023). *FrugalGPT: How to use large diff --git a/docs/kv-credentials.md b/docs/kv-credentials.md index 6e6b16176..c68b9ec4f 100644 --- a/docs/kv-credentials.md +++ b/docs/kv-credentials.md @@ -316,7 +316,7 @@ box, all resolved through `get_credential` (never fabricated, never read from | OpenRouter | `OPENROUTER_API_KEY` | `Bearer ` | | NVIDIA NIM (primary)| `NVIDIA_NIM_API_KEY` | `Bearer ` | | NVIDIA NIM (sub) | `NVIDIA_NIM_API_KEY_SUB` | `Bearer ` | -| Bytez | `BYTEZ_API_KEY` | `Key ` | +| Bytez | `BYTEZ_API_KEY` | `` (no prefix) | | Configured OpenAI-compatible gateway | `LLM_GATEWAY_API_KEY` | `Bearer ` | For a configured gateway, the one-shot discovery/bootstrap boundary accepts @@ -330,7 +330,7 @@ When serving the persisted agents, pass the same host with `--allowed-provider-host`; startup discovery reads this injected runtime policy and never re-reads or promotes gateway environment values. -Bytez's `Key ` scheme (rather than `Bearer`) is why `ModelAgent` has an +Bytez's prefix-free `Authorization: ` scheme (rather than `Bearer`) is why `ModelAgent` has an `auth_scheme` field (default `"Bearer"`) — set it per agent when a provider doesn't use the OpenAI-compatible default. @@ -367,3 +367,4 @@ This credential seam is the durable first step of growing per-tenant scoping can grow behind without touching the routing engine. The Rust/Python hybrid gateway is a later, separately-approved effort and is **not** started here. + diff --git a/docs/planning/adrs/0002-explicit-local-mlx-evaluation.md b/docs/planning/adrs/0002-explicit-local-mlx-evaluation.md index ceb6e2b51..620fccfc3 100644 --- a/docs/planning/adrs/0002-explicit-local-mlx-evaluation.md +++ b/docs/planning/adrs/0002-explicit-local-mlx-evaluation.md @@ -255,8 +255,8 @@ Run the local transport and passthrough tests, the real mlx route/conduct/judge | A live local run showed the mlx-lm `prompt-concurrency=1` queue continuing to process abandoned large prompts after the client timed out; the server logged `BrokenPipe` and was restarted, while a default same-agent retry would add another expensive queued request. | Keep remote retry policy unchanged, but default explicit `mlx://`/loopback local requests to zero same-agent retries; require an explicit local retry opt-in, bound prompt/output budgets, expose provider readiness separately from process liveness, and ensure the supervisor owns one process per port with stale-request cleanup. | Local retry isolation implemented; readiness, prompt-budget, and lifecycle evidence required | | The local retry opt-in was applied to the loop bound but the stop condition still compared attempts with the remote `max_retries` field, silently capping an explicit local retry budget whenever it exceeded the remote default. | Compute one provider-specific retry limit and use it for both iteration and termination in normal and raw passthrough transports; retain zero local retries by default and regression-test an explicit local budget above the remote budget. | Fixed locally; exact-head CI/review follow-up required | | An explicit unknown model ID silently fell back to a different worker, and duplicate IDs could report or select a disabled record before an enabled record. | Reject unknown/non-string explicit model IDs, resolve duplicate IDs to an enabled candidate when one exists, and aggregate `/v1/models` status across all records so discovery and execution agree. | Implemented with passthrough and model-list regressions | -| `/v1/models` returned governance-enabled candidates as `active` even when only one MLX model was loaded; a caller could mistake registry membership for provider readiness. | Preserve `status` as governance state for compatibility, add `readiness: "unprobed"` to model entries and `provider_readiness: "unprobed"` to liveness, never perform a blocking synchronous provider probe in discovery, and add a bounded readiness/refresh contract before claiming serving availability. | Explicit boundary implemented; readiness probe/refresh remains required | -| Liveness endpoints could not distinguish a responsive mlx-lm process from a chat path that was timing out, so operators had no safe way to refresh serving readiness; concurrent admin refreshes could also multiply a stuck local queue. | Keep `/healthz` and `/v1/models` non-blocking and `unprobed`; add an admin-authenticated `provider_readiness_report(refresh=true)` that performs one `max_tokens=1` chat probe per enabled worker with a 0.1–30 second bound, no retries, redacted bounded errors, and one narrow process-local lock around the refresh sequence. | Implemented and exercised against live Gemma 4 e4b on 2026-08-14; exact-head CI/review follow-up required | +| `/v1/models` returned governance-enabled candidates as `active` even when only one MLX model was loaded; a caller could mistake registry membership for provider readiness. | Preserve `status` as governance state for compatibility, add `readiness: "unprobed"` to model entries and `provider_readiness: "unprobed"` to liveness, never perform a blocking synchronous provider probe in discovery, and add an explicitly cancellable readiness/refresh contract before claiming serving availability. | Explicit boundary implemented; readiness probe/refresh remains required | +| Liveness endpoints could not distinguish a responsive mlx-lm process from a chat path that was timing out, so operators had no safe way to refresh serving readiness; concurrent admin refreshes could also multiply a stuck local queue. | Keep `/healthz` and `/v1/models` non-blocking and `unprobed`; add an admin-authenticated `provider_readiness_report(refresh=true)` that performs one `max_tokens=1` chat probe per enabled worker with no fixed wall-clock deadline or retries, redacted errors, explicit cancellation, and one narrow process-local lock around the refresh sequence. | Implemented and exercised against live Gemma 4 e4b on 2026-08-14; timeout policy superseded by ADR 0032; exact-head CI/review follow-up required | | Three anchored K=5 Gemma 4 e4b calibration reruns after fast-mlsirm `17e19ec90643a8dfcc464cd7dde0b63949539a32` prompt hardening produced complete/failed-closed counts `2/6`, `5/6`, and `4/6`, with complete-row cell accuracy `25.0%`, `40.0%`, and `37.5%`; partial/unsupported answers were still over-scored and unsafe/irrelevant cases could remain non-monotone. | Treat the prompt change as ordinal-contract hardening only; preserve all semantic failures and over-scores, do not promote Gemma or infer positive-K bias removal, and require larger held-out human/gold recall, randomized order/framing perturbations, and category occupancy before IRT use. | Observed 2026-08-14 through contextual head `c2bb2b2f85b3eae1c0c0138dff7f4a39cd744cd0`; calibration remains open | | The fast branch ref and GitHub PR #816 pull ref temporarily diverged after the calibration push (`17e19ec` versus predecessor `7605c154`), leaving no checks on the new commit until a normal subsequent named-branch push reconciled exact head `2cd12090f6f4ef8188da15fc6a5704a6ad7063c7`. | Treat branch refs, pull refs, checks, reviews, and rulesets as separate exact-head evidence; record the drift, do not force-push/cancel predecessor runs, and bind all future review/check claims to the reconciled fast head and the linked contextual head. | Resolved by normal push on 2026-08-14; governance follow-up remains required | | fast-mlsirm PR #816 then advanced from `2cd12090f6f4ef8188da15fc6a5704a6ad7063c7` to `ebd76b4664147c18a3e1cfcc3d689e916a2fff08` after a real failure-evidence gap was found: parsed non-monotone Boolean boundaries were not retained in bounded evidence. | Keep only the validated `meets_threshold` Boolean per ordered boundary, never retain full model output or repair the ordinal result, and invalidate predecessor review/check evidence after the normal push; require fresh exact-head review/check evidence for the linked judge implementation. | Implemented in fast-mlsirm `ebd76b4`; focused `62 passed`, full Python `3654 passed`; exact-head review/check follow-up required | @@ -271,8 +271,8 @@ Run the local transport and passthrough tests, the real mlx route/conduct/judge | A fresh same-route model comparison used the real anchored binary-threshold judge with two criteria and baseline/option-only controls. Llama 1B failed both cases (`0/2` passed; all four boundary calls per case completed but strict parsing failed), while Llama 3B and Gemma 4 e4b each passed both cases with polytomous rows `[2,2]`, gold agreement `2/2`, and no provider failures. | Keep the 1B candidate available for fast non-verifier work but exclude it from the `verifier` role in the local registry; allow stronger eligible local agents to handle judge calls through the same gateway. Treat 3B/Gemma as candidates only, retain saturation and gold/perturbation evidence, and never use keyword or silent-repair fallback. | Observed 2026-08-14 through contextual-orchestrator `1614c7f40e5629c07dcfaa97d62b048a7bb459bf`; verifier exclusion implemented in the local registry, broader calibration remains required | | A fresh same-route two-case comparison at K=`3` found Gemma 4 e4b passed both structured boundary comparisons (`[2,2]` safe, `[1,1]` unsafe) in `7.73 s`/`4.87 s`; Llama 3B passed the safe case but failed the unsafe case as non-monotone; Gemma 4 31B failed the safe boundary after `96.93 s`; DeepSeek R1 Qwen 32B completed no boundary in either `100 s` case. | Keep all models discoverable for non-verifier work, but exclude 31B and DeepSeek from `verifier` alongside the previously excluded 1B; select e4b as the current verifier primary and retain 3B as a lower-priority candidate. Treat this as workload-specific reliability evidence only, preserve every failure in the calibration denominator, and require larger balanced gold/perturbation calibration before promotion or IRT claims. | Observed 2026-08-14 through the contextual-orchestrator adapter; routing exclusions implemented, calibration remains required | -| A local `mlx://127.0.0.1:8080/v1` endpoint shared the machine with an unrelated wildcard listener; `/health` remained HTTP 200 while chat completions returned zero bytes and timed out. Candidate port 18080 was also occupied by a Colima SSH forward. | Treat one-process-per-port ownership as part of local serving readiness: reserve a dedicated loopback port, verify the actual MLX model registry and one bounded completion before calibration, and let the local supervisor report a port-owner/configuration mismatch without terminating unrelated processes. Keep the explicit `mlx://` scheme, bounded concurrency, zero default local retries, and TLS verification rules. | Observed 2026-08-14; dedicated MLX port 18083 restored `ModelClient.probe()` readiness, lifecycle hardening remains required | -| `ModelClient.probe()` previously exercised only a completion after endpoint validation, so an incompatible local listener could produce an opaque timeout even when `/health` was 200. | For explicit local providers, verify `/v1/models` contains the configured model before the one-token completion probe; retain bounded timeouts, zero default local retries, and fail-closed error reporting. | Implemented 2026-08-14; focused `41 passed`, full `387 passed`, live registry-plus-completion probe ready on dedicated port 18083 | +| A local `mlx://127.0.0.1:8080/v1` endpoint shared the machine with an unrelated wildcard listener; `/health` remained HTTP 200 while chat completions returned zero bytes and timed out. Candidate port 18080 was also occupied by a Colima SSH forward. | Treat one-process-per-port ownership as part of local serving readiness: reserve a dedicated loopback port, verify the actual MLX model registry and one explicitly cancellable, unbounded completion before calibration, and let the local supervisor report a port-owner/configuration mismatch without terminating unrelated processes. Keep the explicit `mlx://` scheme, bounded concurrency, zero default local retries, and TLS verification rules. | Observed 2026-08-14; timeout policy superseded by ADR 0032; dedicated MLX port 18083 restored `ModelClient.probe()` readiness, lifecycle hardening remains required | +| `ModelClient.probe()` previously exercised only a completion after endpoint validation, so an incompatible local listener could produce an opaque timeout even when `/health` was 200. | For explicit local providers, verify `/v1/models` contains the configured model before the one-token completion probe; retain explicit cancellation, zero default local retries, and fail-closed error reporting without an elapsed-time cutoff. | Implemented 2026-08-14; timeout policy superseded by ADR 0032; focused `41 passed`, full `387 passed`, live registry-plus-completion probe ready on dedicated port 18083 | | A warm authenticated gateway recheck at 09:52 UTC measured `1/1`, `2/2`, and `4/4` successful short completions at wave times `307.89 ms`, `327.23 ms`, and `579.67 ms`; a fifth simultaneous request was explicitly rejected as `503 concurrency_limit_exceeded`, and all successful completion IDs were unique. | Keep the measured `max_concurrent_runs=4` admission bound for this server/model configuration, preserve explicit overload failure and UUID IDs, and re-measure after model, prompt, token, server, or device changes. Do not promote one short workload to a global concurrency default or a quality claim. | Observed 2026-08-14; no new code change justified by this sample | | The current exact-head integrated judge smoke used contextual-orchestrator `474b667b576f8a019db51d892db41a605e3a0a85` and fast-mlsirm `a536292cc05bd16287dab16431bc0c3fef74ba81` with Gemma 4 e4b, two criteria, K=`3`, and four `binary_threshold` calls; it completed in `19.354 s`, used `2,163` provider tokens, and produced the required two-column polytomous row `[2,2]`. | Retain this as current-head route/contract evidence only. Keep multiple criteria mandatory, preserve the model/category result and measured cost, and require balanced held-out gold, perturbation stability, and category occupancy before semantic or IRT claims. Continue to reject keyword matching, positional inference, category repair, scalar synthesis, and silent drop. | Verified 2026-08-14; no new code change justified, semantic calibration and protected PR gates remain required | | The gateway classified DNS/connection/timeout failures as transient but did not classify `ssl.SSLEOFError` or `ssl.SSLSyscallError`, so a VPN-path `SSL_ERROR_SYSCALL` could fail without the existing bounded retry policy; certificate verification errors are a different trust-boundary failure. | Treat TLS socket EOF/SYSCALL errors as transient network failures, keep `SSLCertVerificationError` non-transient, retain certificate verification, and cover both branches. Never use `GIT_SSL_NO_VERIFY`, a global proxy, or certificate replacement as a retry fix. | Fixed in contextual-orchestrator `1b22ff0`; targeted `14 passed`, full `388 passed`, exact-head CI/review follow-up required | diff --git a/docs/planning/adrs/0032-model-group-cost-aware-discovery.md b/docs/planning/adrs/0032-model-group-cost-aware-discovery.md index 201d141f4..f59772d9a 100644 --- a/docs/planning/adrs/0032-model-group-cost-aware-discovery.md +++ b/docs/planning/adrs/0032-model-group-cost-aware-discovery.md @@ -1,15 +1,15 @@ # ADR 0032: Measured model groups and cost-aware discovery -- Status: Accepted on PR #834; protected-main delivery pending +- Status: Proposed on PR #971; protected-main delivery and exact-head verification pending - Date: 2026-08-25 - Figma file ID: `vsZMd8WAv42HDRgcZuNcWk`; this change reuses the existing Agent Pool table rather than introducing a new visual pattern. - Product/technical specification: [`docs/model-group-product-technical-spec.md`](../../model-group-product-technical-spec.md) ## Product requirement -Operators need one logical model name when several providers expose the same underlying model under unrelated identifiers. Groups are entirely operator-defined: discovery never infers equivalence from provider or model names, and no model family is built in. Model discovery remains provider-specific and retains the complete catalog; zero-cost entries are additionally classified so cost policy can distinguish free, priced, and unknown-price models. +Operators need one logical model name when several provider accounts expose the same underlying model. Discovery assigns rows with the same provider-declared exact model identifier to one `model_group`; operators may explicitly group differently named deployments. It never infers equivalence from provider names or fuzzy model-name similarity. Model discovery remains account-specific and retains the complete catalog; zero-cost entries are additionally classified so cost policy can distinguish free, priced, and unknown-price models. -Every configured KV credential is a separate provider-account and catalog boundary. Discovery queries every registered credential independently and retains rows by `(provider_name, credential_name, model_id)`; it never assumes that two keys for the same vendor expose the same models, entitlements, price, privacy policy, availability, or failure state. NVIDIA's primary/sub keys are one example, not a special case. Provider-family inference is absent. Only an explicit operator-defined `model_group` may assert that deployments represent one logical model and may therefore share measured routing decisions. +Every configured KV credential is a separate provider-account and catalog boundary. Discovery queries every registered credential independently and retains rows by `(provider_name, credential_name, model_id)`; it never assumes that two keys for the same vendor expose the same models, entitlements, price, privacy policy, availability, or failure state. NVIDIA's primary/sub keys are one example, not a special case. Provider-family grouping is absent. Only exact provider-declared model identity or an explicit operator-defined `model_group` may assert that deployments represent one logical model and may therefore share measured routing decisions. ## Decision and technical contract @@ -21,7 +21,13 @@ Stability uses the posterior mean of a Bernoulli success probability under a uni Each group member also reports the maximum completed requests and provider-reported total tokens observed in any trailing 60-second window as `max_observed_rpm` and `max_observed_tpm`. These are achieved lower bounds from real gateway traffic, not inferred provider quotas or promises of sustainable capacity. Requests with absent total-token usage still count toward RPM but add nothing to TPM; the gateway never estimates missing tokens for this evidence. The counters reset with the existing process-local routing ledger and never cause probe traffic. -OpenRouter discovery reads its provider-reported per-token prices and recognizes explicit zero prices. OpenCode Zen discovery intersects its documented `/zen/v1/models` availability response with the `opencode` catalog in Models.dev, which OpenCode documents as a source for its own model catalog. Only structured cost records whose declared monetary components are all exactly zero are classified free. A missing, malformed, unmatched, or temporarily unavailable metadata record remains unknown; model-name suffixes are never treated as price evidence. All available models remain discoverable for later policy decisions. +OpenRouter discovery reads its provider-reported per-token prices and recognizes explicit zero prices. OpenCode Zen discovery intersects its documented `/zen/v1/models` availability response with the `opencode` catalog in Models.dev, which OpenCode documents as a source for its own model catalog. Only structured cost records whose declared monetary components are all exactly zero are classified free. A missing, malformed, unmatched, or temporarily unavailable metadata record remains unknown; model-name suffixes are never treated as price evidence. All available models remain discoverable for later policy decisions. A bounded +bootstrap selection never treats lexical provider/model ordering as decision +evidence: when the capacity boundary splits candidates with the same comparable +cost state, including unknown price, it fails closed until the operator supplies +evidence or includes the complete tied class. Exact model-group spread remains +the direct provider-bootstrap contract; provider spread is an additional +discovery-CLI availability constraint, documented for consumer migration. Privacy discovery is also model-specific rather than inferred from price. The OpenRouter ZDR endpoint inventory is joined to every paid and free catalog row; @@ -43,6 +49,29 @@ does not exclude non-ZDR routes from ordinary requests. Missing or failed ZDR evidence therefore fails closed only for `zdr_only` selection, not for general inference. +Every OpenRouter wire transport applies the provider's documented +`provider.zdr=true` request-time enforcement inside that policy scope. This +includes JSON chat, streaming, structured passthrough, chat and embedding +batch JSONL, and the binary-response speech transport; the response media +type does not weaken the privacy contract of its JSON request body. + +OpenRouter discovery retains the concrete free-model list returned by its model +catalog, including exact `vendor/model:free` identifiers and any row whose +complete structured monetary price is zero. The aggregate `openrouter/free` +Free Models Router is not a serving candidate: contextual-orchestrator is the +router and must select, measure, and report the concrete model group itself. +Unknown or incomplete prices remain unknown rather than free. + +Model inference has no fixed wall-clock timeout. This applies to ordinary +generation, the initial completion ping, warm-up, readiness probes, retries, +and repair attempts; a slow model such as DeepSeek is not declared unavailable +merely because it takes minutes or hours. Operators may cancel work, and a +superseded request may be cancelled. DNS resolution, TCP/TLS establishment, +health/readiness checks, provider catalogs, and ZDR-list retrieval likewise have +no fixed wall-clock deadline. Superseded races close registered +loser sockets only when their reviewed equivalence contract declares cancellation +support. + OpenAI catalog rows also retain OpenAI's official data-controls documentation as policy evidence. Because approval and enablement are organization/project settings that the Models API does not disclose, discovery leaves the actual ZDR @@ -98,6 +127,14 @@ sequenceDiagram ## Data, web, and operational boundaries +Provider model-list and OpenRouter ZDR/policy/credit metadata have no default +wall-clock timeout. DNS resolution, TCP/TLS establishment, local health/readiness +checks, and model inference have no default timeout. A slow catalog or reasoning +model is not evidence that the route is unavailable; cancellation is an +explicit operator or superseded-request action. Retry counts may remain finite +after an explicit transport failure, but elapsed time alone must not discard a +provider or model. + Group membership survives restart in normalized `model_group` and `model_group_member` relations; the agent JSON payload no longer duplicates `group_name`. Startup migrates legacy payload membership transactionally without dropping agent configuration. Authenticated REST resources provide `GET/POST /api/v1/model_groups` and `GET/PATCH/DELETE /api/v1/model_groups/{group_name}`; deleting a group retains its provider agents. The existing worker-agent create/PATCH API also accepts `group_name`. Admin provides a keyboard/native-form editor backed by those same resources and shows capability coverage, posterior success probability, and EWMA latency instead of fabricated capacity/success figures. The observation ledger intentionally resets on process restart: persisting it without a measurement horizon would let stale provider incidents dominate current routing. Add a normalized, time-windowed observation table when multi-instance aggregation and an explicit retention/decay policy are specified. ```mermaid diff --git a/docs/planning/adrs/0041-generalize-models-dev-cost-classification.md b/docs/planning/adrs/0041-generalize-models-dev-cost-classification.md index f4f2eaa7b..8e6326331 100644 --- a/docs/planning/adrs/0041-generalize-models-dev-cost-classification.md +++ b/docs/planning/adrs/0041-generalize-models-dev-cost-classification.md @@ -177,6 +177,29 @@ nothing about the retry can turn a paid model free. Motivated by the `orchestrator/free` review-sidecar reliability gap in `ContextualWisdomLab/.github` PR #1433. +## Amendment (2026-08-31): OpenRouter is no longer `evidence_only` + +This ADR's Context section characterized OpenRouter's `evidence_only=True` +(commit `952996ec`) as settled, deliberate ZDR-privacy hardening "that stays +untouched." That characterization was false; it was reversed on direct +review this pass: ZDR eligibility is a route/model-level property +(`is_zdr_model`, exact feed matching), never grounds to block an entire +provider account from serving. OpenRouter's `ProviderModelSource` entry no +longer sets `evidence_only=True`. + +This directly closes the gap this ADR's Context section itself identified +("only OpenRouter's own API ever reports real pricing... and OpenRouter... +never serves inference... `orchestrator/free` was therefore structurally +empty in practice"): OpenRouter can now serve like every other discovered +provider, independent of and in addition to this ADR's Models.dev join for +the other five sources. The provider-neutral ZDR-evidence-application +contract (OpenRouter's feed also crediting matching rows from *other* +providers, insisted on during PR #901's review) is unchanged; what changed +is that OpenRouter's own rows are no longer the one arbitrary exception to +it. See `docs/product-technical-gap-baseline.md`'s 2026-08-31 entry for the +full mechanism, the request-time ZDR-pinning enforcement this required, and +its stated scope limits. + ## References Models.dev. (2026). *Models.dev API*. https://models.dev/api.json diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index d145a0b1d..07a2d3904 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1,5 +1,1007 @@ # Contextual Orchestrator: Product & Technical Gap Baseline +## 2026-09-08 PR #971: unmodeled diversity displacement + full-pool ordering (fail-closed) + +Scope is the four-commit chain on branch +`fix/model-group-timeout-openrouter` at exact head `838cbcb7`: +`575148b9` fail-closed ambiguous bootstrap admission, +`50b0c869` equal-price admission boundary cover, +`7206c5f6` reject unmodeled diversity displacement, +`838cbcb7` reject unmodeled full-pool ordering. + +`575148b9` `fix(routing): fail closed on ambiguous bootstrap admission` +is the GREEN for the RED already recorded in the `2026-09-07 PR #971: +fail-closed ambiguous bootstrap admission` entry above +(`eeb9cc1bafe579032ab48778fa08c24e0b3f0aa1` RED, Security and Quality +run `34071330949`, job `101589111271`, `2 failed, 3490 passed, +2 skipped`). Referenced here, not re-argued: tied comparable-cost +candidates fail closed with an operator action instead of letting +lexical provider/model identity decide admission. + +`50b0c869` `test(routing): cover equal-price admission boundary` is +test-only, no production change. It pins the equal-price tie boundary +so the `575148b9` fail-closed rule has an executable contract on +identical comparable-cost admission. + +`7206c5f6` `fix(routing): reject unmodeled diversity displacement` is +the bounded-cutoff fail-closed GREEN. RED: a diversity proposal could +displace a price-evidenced candidate at the cutoff without a modeled +rule. GREEN: compare the diversity proposal against the price-evidenced +sequence and reject unmodeled displacement instead of silently +substituting. No new ranking, weight, quota, provider preference, or +learned-quality claim is added. + +`838cbcb7` `test(routing): reject unmodeled full-pool ordering` is +test-only at the exact head, no production change. It pins the +complete-pool reordering boundary fail-closed: `tests/test_model_discovery_boundaries.py` +`+28` and `tests/test_provider_bootstrap.py` `+15`. Unmodeled +full-pool reordering fails closed rather than returning a silently +re-ranked pool. + +Local verification on exact head `838cbcb7`: +`tests/test_pr971_review_quality_regressions.py` `4 passed`, +discovery/bootstrap selection `186 passed`, +`interrogate` `100.0%` (`679/679`). `CodeRabbit` `52.2%` is stale +(head moved since). Temporary source-fix workflows are already removed; +only `tests/test_pr971_review_quality_regressions.py` remains. ADR 0032 +stays Proposed while PR #971 remains open. + +New gaps recorded, not resolved: issue `#1110` (measure +accepted-request to durable route-decision latency, `OPEN 2026-09-09`) +and issue `#1114` (export bounded authorized request-outcome +associations, `OPEN 2026-09-09`) are absent from this baseline. + +Promotion contract still Draft: exact-head hosted `GREEN` is required +(`Hypothesis`, `Atheris`, `CodeQL`, supply chain, `dependency-review`, +`OSV`, `Trivy`, `Scorecard`, `coverage-evidence`, `opencode-review`, +`strix`, `scan-pr-queue`) plus a qualifying review. The `strix` +cancelled and `CodeQL`-compat failure are central-lane, not local. + +## 2026-09-07 PR #971: fail-closed ambiguous bootstrap admission + +External review thread `PRRT_kwDOTB3CTs6eh3BQ` identified that both +bootstrap selectors used provider/model-group passes and then let lexical +provider/model identity decide a capacity cutoff when price evidence was equal +or incomplete. Exact test-only commit +`eeb9cc1bafe579032ab48778fa08c24e0b3f0aa1` made those two cases executable; +Security and Quality run `34071330949`, job `101589111271`, failed exactly +`2 failed, 3490 passed, 2 skipped` because neither selector raised. + +The smallest GREEN retains the existing provider/model-group availability +constraints but removes lexical identity as admission evidence: if a selected +and excluded candidate share the same comparable-cost state, including the +all-unknown state, selection fails closed with an operator action to provide +comparable price evidence or raise the limit to include the entire tied class. +No weight, quota, fuzzy identity, provider preference, or learned-quality claim +is added. The direct provider bootstrap selector promises exact model-group +spread; the discovery CLI selector additionally promises provider spread, so +consumers that need provider-level redundancy must use the latter boundary. +ADR 0032 is Proposed while this PR remains open. Status remains Proposed until +the successor exact head passes focused/full quality, security, and required +review lanes. + + +## 2026-09-07 PR #971: durable bootstrap selection order + +Observation: exact predecessor `1e59d4fc9a628a898e404cd1bcacb412747c3b5b` +passed the unchanged full suite and coverage-guided fuzz lane, but external +review thread `PRRT_kwDOTB3CTs6elTYC` remained valid. The ephemeral bootstrap +report retained `select_model_group_diverse_models` order, while +`_synchronize_durable_agent_pool` converted the resolved identities to a set +and returned an alphabetically sorted tuple. A restart-backed report could +therefore discard the selector's cost/model-group order without changing pool +membership. + +RED commit `98aed1a811cb894881b4c9aeb20de4f0b00fb634` adds a two-model +durable-pool contract whose selected order is deliberately the reverse of +agent-ID lexical order. The smallest GREEN retains a set only for membership +and collision checks, records resolved persisted identities in selector order, +activates them in that order, and returns the ordered tuple. It introduces no +new ranking, quota, provider heuristic, timeout, or dependency. Status remains +Proposed until the successor head completes the focused regression, full suite, +security, and required review lanes; the other three open #971 architecture +findings remain separate. + +## 2026-09-02 PR #971: main-merge conflict resolution, remaining ThreadPoolExecutor shutdown-blocks, and a verified false positive + +Observation time: 2026-09-02 Asia/Seoul, later the same day as the entries +below. Closes out the rest of today's #971 session: the branch's actual +`main`-merge conflict (a real divergence, not the shallow-clone false alarm +noted in an earlier pass), the two remaining `ThreadPoolExecutor`-atexit-join +instances of the root cause already fixed twice below (this file's own +"unbounded per-model OS thread allocation" and "shared discovery metadata +fetches" entries), a raw-`Future` regression that root-cause fix itself +introduced, a shutdown-contract hardening on the new `_DaemonWorkerPool` +primitive, and an external self-fix proposal traced and confirmed non-causal +before being left unapplied. Every claim below is independently re-verified +against `git log`/`git show` on the actual branch history and current-head +source rather than restated from a hand-off. + +### Merge-conflict resolution: `main` -> branch, `mergeable_state` `dirty` -> `blocked` + +GitHub's `mergeable_state` for #971 was genuinely `dirty` -- confirmed by a +real trial merge, not the shallow-clone false alarm noted in an earlier pass +this same day. Root cause: `main`'s `0db4e5a7` ("fix(review): remove +heuristic candidate cap and ranking") dropped the `max_agents` parameter from +`review_gateway.py`'s `build_review_orchestrator`/CLI entirely as part of +moving admission to evidence-only (no cap, no ranking), while this branch's +own `tests/test_review_gateway.py` still carried tests written against the +pre-removal API and threaded `max_agents=` through several fixtures +(`test_build_review_orchestrator_routes_to_cheapest_selected_agent`, +`test_build_review_orchestrator_uses_model_group_diversity`, +`test_build_review_orchestrator_rejects_invalid_agent_limit`, +`test_main_rejects_invalid_agent_limit_without_traceback`) -- both sides had +independently rewritten the same test file since the PR's base diverged. + +Resolved via the standard recipe: `git fetch origin main`, `git merge +origin/main` (merge commit `d9320266`, merging `main` at `464da471` into the +branch at `41aaeff9`), with `git status` reporting exactly one conflicted +path, `tests/test_review_gateway.py`. Before resolving, verified the +diversity-selection logic the branch-only tests nominally exercised is +separately and currently covered elsewhere: +`tests/test_discovery_bootstrap_selection.py::test_bootstrap_selector_prefers_model_group_diversity` +and `::test_bootstrap_selector_spans_multiple_providers_before_repeating_one`, +plus `tests/test_model_discovery_boundaries.py`'s provider-name assertions on +`select_bootstrap_discovered_agents`, all still exist and pass at current +head and exercise the same selector the dropped +`test_build_review_orchestrator_uses_model_group_diversity` only wrapped in +an HTTP-gateway fixture around. With that confirmed, resolved the conflict by +taking `main`'s version of the file (its evidence-only-admission test suite, +no `max_agents` anywhere) rather than reintroducing a parameter `main` had +deliberately removed -- dropping the two `#971`-branch-only tests that had +gone stale against `main`'s evolved API instead of trying to keep both APIs +alive. Verified with the targeted suite (`tests/test_review_gateway.py`, +`tests/test_discovery_bootstrap_selection.py`, +`tests/test_model_discovery_boundaries.py`, +`tests/test_endpoint_race_callback_settlement.py`, +`tests/test_daemon_worker_pool_shutdown.py`: 53 passed, 0 failed), then +pushed. GitHub's `mergeable_state` for #971 now reads `blocked` (checks-only; +re-confirmed live at time of writing, head `558c8470`), not `dirty`. + +### Remaining `ThreadPoolExecutor`-atexit-join instances of the same root cause + +The `ThreadPoolExecutor`-registers-an-atexit-join gotcha already fixed twice +in the entries below (OpenRouter free-endpoint fetch; shared discovery +metadata fetches) recurred in two more call sites found by further Devin +Review passes on this same branch, both fixed with the identical +daemon-thread/bare-`Future` pattern: + +- **`endpoint_race.race_first_valid`** (`endpoint_race.py`, commit + `9b28bd22`) fanned its equivalent-endpoint race attempts out across a + `ThreadPoolExecutor`. Combined with this org's default no-deadline + `ModelClient.timeout=None`, a losing race participant stuck in an + uncancellable provider call that never returns could hang process shutdown + forever, even though the winner had already answered the caller and + `race_first_valid` had already returned. Fixed by driving each attempt from + a raw `threading.Thread(daemon=True)` built on a bare + `concurrent.futures.Future` (the documented low-level primitive + `ThreadPoolExecutor` itself is built on): `set_running_or_notify_cancel()` + preserves the existing "cancelled before it started never calls the + provider" duplicate-cost guarantee, and `wait()`/`future.cancel()`/ + `future.result()`/`future.exception()` behave identically to the prior + executor-backed futures. All 22 pre-existing tests in + `tests/test_endpoint_race.py` and + `tests/test_endpoint_race_terminal_provenance.py` pass unmodified. New + regression + `tests/test_endpoint_race_process_exit.py::test_uncancellable_loser_never_returning_does_not_block_process_exit` + (a subprocess test: one attempt blocks on an `Event` nothing ever sets, the + fast attempt wins, script falls through to a normal unforced exit) hung the + full 15s bound and was killed (RED) against the pre-fix code, and exits in + well under 5s (GREEN) with the fix. Full suite at that commit: 3430 passed, + 6 failed (same named pre-existing sandbox gaps as the entries below), 2 + skipped. +- **`ProviderEmbeddingBatchBackend`** (`batch_routing.py`, commit `59b2fc87`) + drove its durable, pollable job queue through a `ThreadPoolExecutor` at + every construction site (crash recovery in `__init__`, and `start()`); + `cost_router.py`'s `_provider_embedding_backend()` passes + `execution_timeout_seconds=None` by default, and even a finite value there + is only a cooperative check made after a runner returns, never a preemptive + cancellation of an in-flight call -- so a hung provider-embedding runner + could block that join, and therefore process shutdown, forever, even after + `close()` had already been called. Fixed with a new `_DaemonWorkerPool` + primitive: a fixed-size pool of `threading.Thread(daemon=True)` workers + pulling `(fn, args)` off a `queue.Queue`, exposing the exact + `submit()`/`shutdown(wait=, cancel_futures=)` surface `ThreadPoolExecutor` + has (including the surface a pre-existing test double reaches into + directly). The pool stays fixed-size (matching `max_concurrency`), workers + spawn lazily on `submit()`, and every durability/claim/recovery/publish + code path (`_run_job`, `_run_claimed_job`, `_execution_deadline`, + `_fail_expired_job`, `_publish_terminal`, reserve/start/cancel/poll/wait) + is unchanged. New regression + `tests/test_provider_embedding_batch_backend_process_exit.py::test_hung_provider_embedding_runner_does_not_block_process_exit_after_close` + (subprocess test, same shape as above) is RED against the pre-fix code and + GREEN with the fix. `tests/test_provider_embedding_batch_backend.py` (24 + tests) passes unmodified; the broader affected 18-file suite: 294 passed, 4 + pre-existing/out-of-scope failures. Full suite: 3433 passed, 6 failed, 2 + skipped -- matches the branch's known baseline exactly, no regressions. + +### Regression from the `endpoint_race.py` raw-`Future` refactor: unsettled `Future` on a raising callback + +`race_first_valid`'s raw-`Future` rewrite above (`9b28bd22`) called the +caller-supplied `on_attempt_complete` observer callback and then +unconditionally called `future.set_result(value)` on the success path -- but +if the callback itself raised, that exception propagated up through the +worker thread and the `Future` was left permanently `RUNNING`, so `wait()`/ +`future.result()` on an unbounded (`deadline_seconds=None`) race could hang +forever, silently reintroducing the exact class of bug the `ThreadPoolExecutor` +removal had just fixed (`ThreadPoolExecutor` used to settle a `Future` with a +raised worker-callback's exception automatically; the bare-`Future` rewrite +had to reimplement that contract explicitly and initially missed the +success-path case). Fixed in commit `a3afc800`: both the failure-path and +success-path callback invocations are now wrapped in `try`/`except +BaseException`, and any callback exception settles the `Future` via +`future.set_exception(...)` on whichever path it occurred, so an observer +failure can never strand a `Future` mid-race. Regression coverage added in +`tests/test_endpoint_race_callback_settlement.py` (commit `83582ad7`) +exercises callback failures after both a successful and a failed provider +attempt, with `deadline_seconds=None`. Traced/documented in +`CHANGELOG.d/endpoint-race-callback-settlement.md` (commit `742b5d52`). + +### Hardening: `_DaemonWorkerPool.submit()` now fails closed after `shutdown()` + +A further Devin Review pass on the new `_DaemonWorkerPool` (added above in +`59b2fc87`) found an Info-severity gap: `shutdown()` set no closed flag, so a +concurrent direct `submit()` could enqueue real work behind the shutdown +sentinels every worker exits on -- work no worker would ever pick up again. +Stock `ThreadPoolExecutor.submit()` raises `RuntimeError` once `shutdown()` +has run; `_DaemonWorkerPool` did not replicate that fail-fast contract. +Traced every current call site: `ProviderEmbeddingBatchBackend` only calls +`submit()` from `__init__` (before any external reference to `self` exists) +and from `start()`, and both `start()` and `close()` already serialize +through the backend's own `_executor_lock` with `start()` checking +`self._closed` first -- so today's actual production risk was narrow -- but +a standalone primitive should not depend on every future caller reproducing +that locking discipline. Fixed in commit `18a76eb5`: a `self._shutdown` flag, +set under the pool's existing `_workers_lock` inside `shutdown()` and checked +under the same lock at the top of `submit()` (which now also enqueues under +that lock, not before acquiring it), so the check and the +enqueue/shutdown transition can never interleave; `submit()` now raises +`RuntimeError("cannot schedule new work after shutdown")` instead of +silently queuing behind the sentinels. New regression +`tests/test_provider_embedding_batch_backend.py::test_daemon_worker_pool_submit_after_shutdown_raises_instead_of_stranding_work` +and a follow-on standalone regression `tests/test_daemon_worker_pool_shutdown.py` +(commit `02389e2f`) both pass; the broader 14-file affected suite: 210 +passed, the same 4 pre-existing tokenizer-unavailable ZDR failures. +Traced/documented in `CHANGELOG.d/provider-embedding-daemon-worker-pool.md` +(commit `558c8470`). + +### False-positive analysis preserved as evidence: a proposed `provider_routing` equality guard is non-causal + +A separately, concurrently pushed self-fix workflow on this same branch +(`.github/workflows/source-fix-971-live-review-quality.yml` + +`scripts/ci/pr971_live_review_quality_repair.py`) proposed two repairs: (1) +bounding the OpenRouter free-endpoint worker fetch to eight daemon workers -- +already independently correct and already landed in production (commit +`0d774450`, this file's "unbounded per-model OS thread allocation" entry +below), so this half was fully superseded before the workflow could ever run; +and (2) adding `or request.provider_routing != first.provider_routing` to +`_run_provider_embeddings`'s (`cost_router.py`) existing +`model`/`agent_id`/`zdr_only` homogeneity guard, on the theory that equal +`model`/`agent_id`/`zdr_only` fields could still coalesce two requests +carrying different persisted `provider_routing` metadata into one batch. + +Traced before applying anything: `EmbeddingBatchRequest.provider_routing` +(`batch_routing.py`) is read in exactly one place in the whole codebase, +`EmbeddingBatchRequest.to_jsonl_line()`, which serializes it into the OpenAI +Batch API JSONL request body (`body["provider"] = dict(self.provider_routing)`) +for the separate JSONL-batch-submission path. `_run_provider_embeddings` +never calls `to_jsonl_line()` and never reads `provider_routing` anywhere in +its body -- confirmed by reading the function in full: its homogeneity check +and its downstream `_run_embedding_shard`/client calls only ever reference +`model`, `agent_id`, and `zdr_only`. The proposed guard was therefore +non-causal for the execution path it targeted: no request routed through +`_run_provider_embeddings` can have its provider behavior affected by +`provider_routing` at all, matching or mismatching. **Not applied** -- kept +as verified false-positive evidence rather than blindly implementing an +external suggestion. The stale workflow and its helper script were removed +directly (commits `0247eca9`, `d388564c`) per this branch's standing "no +purpose-complete self-modifying/source-fix workflows" rule, since the +worker-bound half could never re-apply against the new head (its +`replace_once` target text no longer exists there) and the `provider_routing` +half was confirmed to fix nothing real. + +### Verification + +Directly re-run in this session against current head `558c8470`: +`tests/test_review_gateway.py`, `tests/test_discovery_bootstrap_selection.py`, +`tests/test_model_discovery_boundaries.py`, +`tests/test_endpoint_race_callback_settlement.py`, and +`tests/test_daemon_worker_pool_shutdown.py` together -- 53 passed, 0 failed. +Each individual fix above additionally carries its own RED-before/GREEN-after +subprocess or in-process regression and its own contemporaneous full-suite +run (3430-3433 passed, 6 failed -- the same named pre-existing/out-of-scope +sandbox gaps throughout this file's 2026-09-02 entries: tokenizer-unavailable +`test_batch_embeddings.py` ZDR tests, the unavailable `fast_mlsirm` module, +and one `usage_source` spend-analytics assertion -- 2 skipped), recorded in +each commit message and reproduced here rather than re-typed from memory. +`git fetch origin fix/model-group-timeout-openrouter` immediately before +writing this entry showed head unchanged at `558c8470`; the PR's live +`mergeable_state` was `blocked` at the same check. + +### Remaining open work carried forward + +1. **Bootstrap admission's hand-authored diversity/tie-break heuristics** + (`model_discovery.py`'s `select_bootstrap_discovered_agents`: + representative request weights, lexical tie-breakers, + provider/model-group pass ordering, incomplete/equal-evidence fallbacks) + still need a research-/evidence-backed decision model, or a documented + fail-closed replacement when evidence cannot uniquely justify a decision + -- not a repair by changing constants, weights, quotas, or tie-break + strings. +2. **The hourly OpenCode workflow's `cancel-in-progress: false` concurrency + gap**: a single wedged run can occupy the `opencode-hourly-loop` + concurrency group for up to GitHub's implicit ~360-minute hosted-runner + ceiling, serializing every later hourly trigger behind it rather than + running it. This needs a durable/resumable contextual-orchestrator-owned + execution/checkpoint/re-dispatch boundary preserving exact-head identity + across external runner termination -- not a leaf-authored elapsed-time + cutoff, which would violate this org's `timeout=null` model-inference + policy. A related but narrower fix has already been opened separately as + PR #1027 ("fix(ci): remove elapsed-time job cap on the hourly loop; pin + model to orchestrator/free", head `ad9c23c2`, open/draft as of this + entry): it removes the `loop` job's `timeout-minutes` cap entirely and + pins `orchestrator/auto` -> `orchestrator/free`, but its own description + explicitly does not attempt the checkpointing/resumability piece, which + remains open. +3. **Legacy bootstrap identifier-generation mixing**: bootstrap reporting can + expose a fingerprinted `selected_agent_ids` identity while + `enabled_agent_ids` retains the migrated legacy persisted identity for the + same endpoint. Consumer semantics need one identity-consistent contract, + or explicitly typed generated-vs-persisted identifiers. + +These are large-scope architectural items, deliberately left untouched by +today's session rather than patched narrowly. + +## 2026-09-02 PR #971: unbounded per-model OS thread allocation in OpenRouter free-endpoint discovery + +Observation time: 2026-09-02 Asia/Seoul. Follow-up correction to the +"unbounded provider discovery and single-provider bootstrap concentration" +fix recorded below: that fix's own `_openrouter_free_model_endpoints` +rewrite (raw `threading.Thread(daemon=True)` workers instead of a +`ThreadPoolExecutor`, to avoid blocking interpreter shutdown) left a +separate, distinct gap, independently reported by a Devin Review comment on +PR #971 and independently re-verified against current-head code before any +change was made. + +### Finding (Devin Review, PR #971, `contextual_orchestrator/model_discovery.py`) + +> Free-model discovery creates unbounded threads +> +> When OpenRouter lists many free models, `workers` allocates one thread per +> row. Large catalogs can exhaust memory or prevent provider discovery from +> starting. + +### Root cause + +`_openrouter_free_model_endpoints` built one `threading.Thread` object per +free model up front (`workers = [threading.Thread(...) for model_id in +model_ids]`) and started every one of them immediately. A +`threading.Semaphore(min(8, len(model_ids) or 1))` bounded how many of those +threads could do real fetch *work* concurrently, but did nothing to bound +how many native OS threads were *created and started* in the first place -- +each with real kernel/stack allocation overhead. A catalog of hundreds or +thousands of free models (OpenRouter's actual free-tier catalog size is not +contractually bounded) would therefore still allocate and start that many +threads at once, before the semaphore ever limited anything, risking memory +exhaustion or stalling discovery before a single fetch could begin -- exactly +the finding. + +### Fix + +Replaced the one-thread-per-model construction with a fixed pool of at most +8 daemon worker threads that each pull model IDs from a `queue.Queue` until +it is empty, so the live thread count for this fetch stays bounded (`<= 8`) +regardless of catalog size. The already-established daemon-only, +abandon-on-hang behavior (no `ThreadPoolExecutor`, no interpreter-shutdown +block) is unchanged; each worker still simply stops draining the queue if +its current fetch hangs, while the other workers keep making independent +progress. + +### Verification + +New regression `test_openrouter_free_model_endpoints_caps_concurrent_thread_creation` +in `tests/test_model_discovery.py`: submits 40 free models with a +fetch that blocks on a shared `threading.Event`, samples +`threading.enumerate()` for live `openrouter-endpoints`-named threads while +every fetch that will ever start is blocked, and asserts the count is `<= 8` +(not 40). RED confirmed against the pre-fix one-thread-per-model +implementation (40 live threads observed for 40 models); GREEN after the +fixed-pool fix (`<= 8`). The pre-existing hang-safety regression +(`test_openrouter_free_model_endpoints_hang_does_not_block_process_exit`) +continues to pass unchanged. Full suite: 3384 passed, 7 failed (all +pre-existing/out-of-scope: 4x missing native `_token_packer` extension in +`test_batch_embeddings.py`, the still-in-progress terminal-embedding-failover +finding tracked separately, missing `fast_mlsirm` module, and the +tokenizer-mismatch `test_spend_analytics.py` sandbox artifact), 2 skipped. +`interrogate -f 100 contextual_orchestrator/model_discovery.py`: 100%. + +## 2026-09-02 PR #971: terminal-failure batch embedding documents restored endpoint health + +Observation time: 2026-09-02 Asia/Seoul. Closes the remaining still-open +review blocker carried in PR #971's own description ("terminal +failed/cancelled/rejected batch documents must not clear endpoint circuit +failures, and incomplete synchronous documents must be recorded through the +shared embedding-failure path before failover"): the synchronous-document +half of that sentence was already fixed (see the "synchronous `/v1/embeddings` +member result ... bypassed `orchestrator._record_embedding_failure`" entry in +`CHANGELOG.md`'s `## [0.2.0] - Unreleased`); this entry is the terminal-batch +half. Independently re-verified against current-head code before any change +was made (never trusted as stated) -- confirmed genuinely still broken, not +already fixed or stale. + +### Finding (Devin Review, PR #971, `contextual_orchestrator/server.py`) + +> Failed embedding jobs restore endpoint health +> +> When a batch returns a terminal failure, `observe_success` records success +> and clears its circuit. Later requests keep selecting the failed endpoint. + +### Root cause + +The `/v1/batch/embeddings` HTTP handler's member-failover loop (the `for +embedding_agent in embedding_agents:` loop calling +`coordinator.complete_embeddings_batch(...)`) recorded success +unconditionally on any call that returned without raising: + +```python +except Exception as exc: + last_embedding_error = exc + orchestrator._record_embedding_failure(embedding_agent, "/v1/batch/embeddings", exc) + continue +orchestrator._group_router.observe_success(embedding_agent.id, ...) +orchestrator._record_success(embedding_agent.id) +break +... +is_complete = document.get("status") == "completed" # only decides the HTTP status code +``` + +`complete_embeddings_batch` submits the batch and returns whatever document +`CostRoutingCoordinator.embeddings_batch_document` produces for it -- which +can be a **terminal-failure document** (`status` of `"failed"`, +`"cancelled"`, or `"rejected"` -- the exact set +`embeddings_batch_document` itself checks to stop emitting a polling +cadence) returned *normally*, with no exception raised. The bug was +ordering, not a missing check: `observe_success`/`_record_success` ran and +the loop `break`-ed *before* the terminal-status check the same function +already computed one line later (`is_complete`) -- that check only ever +gated the HTTP response code (200 vs 202), never success recording or +failover. So a genuinely failed batch member still marked its endpoint +healthy and cleared its circuit breaker; later requests kept selecting that +broken endpoint instead of failing over to a healthy one. The sibling +synchronous `/v1/embeddings` handler in the same file does not have this +bug: it already gates success recording on +`document.get("status") == "completed" and document.get("embeddings") is +not None` and routes anything else through `_record_embedding_failure` +before continuing -- confirmed the only other `observe_success`/ +`_record_success` call site in `server.py`, so this was the one remaining +instance of the pattern in this file. + +### Fix + +`contextual_orchestrator/server.py`, `/v1/batch/embeddings` handler: + +- Added a module-level `_TERMINAL_EMBEDDING_BATCH_FAILURE_STATUSES = + frozenset({"failed", "cancelled", "rejected"})`, matching + `CostRoutingCoordinator.embeddings_batch_document`'s own terminal-status + vocabulary (`cost_router.py`). +- The loop now checks `document.get("status")` against that set + immediately after a successful (non-raising) call and before recording + any success: a terminal-failure document builds a synthetic + `RuntimeError(f"embedding batch member ended with {document.get('status')}")`, + routes it through the same shared `orchestrator._record_embedding_failure` + recorder an exception would use, discards the document, and `continue`s + to the next candidate agent -- the same failover shape a raised exception + already got. Only a document whose status is *not* a terminal failure + (`"completed"`, or an in-flight status such as `"queued"`, `"validating"`, + `"running"`) still records success and `break`s. +- No other file changed; `model_discovery.py` and the rest of + `cost_router.py` are out of scope for this fix (separate in-flight work on + this branch). + +### Verification + +RED-before/GREEN-after. A pre-existing test in +`tests/test_pr971_review_quality_regressions.py` -- +`test_terminal_embedding_batch_document_fails_over_before_marking_health` +(mocks `coordinator.complete_embeddings_batch` to return a `status: "failed"` +document for the first, higher-priority agent and a non-terminal +`status: "validating"` document for the second, then posts to +`/v1/batch/embeddings`) -- failed pre-fix (asserted `attempted == [first.id, +second.id]`; pre-fix code `break`-ed after the first agent's false success, +so only `[first.id]` was ever attempted and the response carried the failed +`batch_id`) and passes post-fix (both agents attempted, 202 response carries +the second agent's `"accepted-batch"` id). Directly confirmed with an +additional ad hoc spy harness (not committed) around +`orchestrator._group_router.observe_success`, `orchestrator._record_success`, +and `orchestrator._record_embedding_failure` for the same scenario: +post-fix, `observe_success`/`_record_success` are called only with the +second (healthy) agent's id, and `_record_embedding_failure` is called +exactly once with the first (failed) agent's id and a `RuntimeError` +("embedding batch member ended with failed") -- i.e. the failed endpoint's +circuit is never falsely cleared, and it is routed through the same +recorder an exception would use. Full suite and `interrogate --fail-under +100` confirmed green apart from named pre-existing sandbox-only failures +unrelated to this change. + +## 2026-09-02 PR #971: recovered ZDR embedding batch bypassed request-policy enforcement + +Observation time: 2026-09-02 Asia/Seoul. Follow-up correction to the +"recovered ZDR batch not re-validated" fix recorded in the +"embedding recovery/deadline and legacy-id quarantine review" entry below: +that fix's own re-validation left a gap, independently reported by a Devin +Review security-level comment on PR #971 and independently re-verified +against current-head code before any change was made (never trusted as +stated). + +### Finding (Devin Review, PR #971, `contextual_orchestrator/cost_router.py`) + +> Recovered embeddings bypass ZDR enforcement +> +> Recovered `zdr_only` jobs never restore `request_policy`, so OpenRouter +> omits `provider.zdr`. Mixed privacy identities can also execute under the +> first request's policy. + +### Root cause + +`_run_provider_embeddings` is the replay entry point +`ProviderEmbeddingBatchBackend` calls when a durably-queued embedding job is +recovered after a process restart and executed on a background worker +thread. The earlier fix (see "embedding recovery/deadline and legacy-id +quarantine review" below) added a check that the resolved agent still +carries the `privacy:zdr` tag, but that check is necessary and not +sufficient: + +1. **The `request_policy` contextvar was never restored.** OpenRouter's + enforcing `provider.zdr: true` request field is applied by + `_pin_openrouter_zdr` (`orchestrator.py`), which branches on the + `_REQUEST_ZDR_ONLY` contextvar set by `TaskOrchestrator.request_policy(...)` + -- not on the request's own `zdr_only` field. The submission-time + `request_policy(zdr_only)` scope used by every synchronous call site + (`cost_router.py` lines ~636, ~770, ~1038, ~1417) is a `ContextVar`, + which does not cross the thread boundary `ProviderEmbeddingBatchBackend` + replays a recovered job across. `_run_provider_embeddings` called + `self._run_embedding_shard(agent, shard)` -> `self.orchestrator.client.embed` + /`embed_with_usage` -> `_send_raw` -> `_pin_openrouter_zdr` with no + `request_policy` scope active on that thread at all, so the pin's branch + condition was always false for a recovered batch regardless of + `first.zdr_only` -- the actual HTTP request to OpenRouter silently + omitted `provider.zdr`, even though the tag check already re-validated + the agent and the code's own comment believed privacy safety was + handled. +2. **No per-request `zdr_only` homogeneity check.** The batch's existing + consistency check only asserted every request shared `model` and + `agent_id`: + `EmbeddingBatchRequest` carries `zdr_only` per request (`batch_routing.py`), + but nothing compared it across the batch. A batch mixing + `zdr_only=True` and `zdr_only=False` requests under the same + `agent_id` executed entirely under `first`'s policy -- either + over-restricting a non-ZDR request or, the real risk, silently + under-restricting a ZDR request that was not first in the list. + +### Fix + +`_run_provider_embeddings` (`contextual_orchestrator/cost_router.py`): + +- Wraps the sharded `_run_embedding_shard` execution loop in + `with self.orchestrator.request_policy(first.zdr_only): ...` -- the same + context-manager pattern already used at every other client call site in + this file -- so the OpenRouter ZDR pin is correctly re-armed for a + recovered batch's actual client call(s), not just checked-and-trusted at + the tag level. +- Extends the existing route-homogeneity check to also require every + request in the batch share `first.zdr_only`, alongside the pre-existing + `model`/`agent_id` check, raising the same `RuntimeError` (fail closed, + same style as the adjacent tag-mismatch check) when they diverge, instead + of silently running the whole batch under `first`'s policy. + +Both changes are additive and scoped to `_run_provider_embeddings`; no +other call site or the batch-submission path changed. + +### Why this is fail-closed and ZDR-first + +This repo's stated policy is ZDR-first: every LLM path routes through the +`orchestrator/free` pool and privacy-scoped requests must be provably +zero-retention, not merely assumed so. Before this fix, a *recovered* +`zdr_only` request could silently execute without the provider-side +enforcement its caller explicitly requested -- a privacy guarantee that +looked re-validated (the tag check ran and passed) while the request that +actually left the process carried no enforcement of it. Both parts of the +fix restore the same fail-closed shape used everywhere else in this file: +mismatched privacy identity now raises before any request is sent, and a +matching identity now genuinely carries its enforcement through to the +provider, rather than being asserted once and then dropped on the +replay path. + +### Verification + +RED-before/GREEN-after, both already present in +`tests/test_pr971_review_quality_regressions.py` (added ahead of this fix +landing) and confirmed to fail against pre-fix code, pass against the fix: + +- `test_recovered_zdr_batch_reenters_request_privacy_scope` -- monkeypatches + `orchestrator.request_policy` to record every `zdr_only` value it is + entered with, runs a recovered single-request ZDR batch through + `_run_provider_embeddings`, and asserts `request_policy` was entered with + `True` exactly once. Failed pre-fix (`entries == []`, the scope was never + entered); passes post-fix. +- `test_provider_embedding_batch_rejects_mixed_privacy_identity` -- builds + a two-request batch sharing `model`/`agent_id` but differing + `zdr_only`, and asserts `_run_provider_embeddings` raises `RuntimeError` + matching `"privacy policy"`. Pre-fix, the batch executed without raising; + post-fix it fails closed. + +## 2026-09-02 PR #971: unbounded provider discovery and single-provider bootstrap concentration + +Observation time: 2026-09-02 Asia/Seoul. This closes the two remaining +unaddressed items from PR #971's review-blocker list, both independently +verified against current-head code rather than trusted as stated. + +### Summary + +- **Model discovery had no separately bounded/cancellable per-provider + deadline.** `discover_all_models` (`contextual_orchestrator/model_discovery.py`) + ran a plain sequential `for source in sources: discover_provider_models(...)` + loop with no thread, no `asyncio.wait_for`, and no join timeout around any + individual provider's call -- confirmed genuinely unbounded, not merely + a stale claim. `DISCOVERY_TIMEOUT_SECONDS` (the per-HTTP-call socket + timeout passed *into* each fetch) defaults to `None` per #971's own + no-inference-deadline design boundary, and even set to a finite value it + only bounds one socket read at a time inside a provider's multi-fetch + discovery attempt -- it cannot bound a hang that ignores that parameter + entirely (confirmed with a mock discovery call blocking on an `Event` + nothing ever sets). A stalled provider therefore blocked discovery of + every later, healthy provider forever. Fixed: `discover_all_models` now + runs each provider's discovery on its own daemon thread and stops + waiting once a new, separately configured + `PROVIDER_DISCOVERY_DEADLINE_SECONDS` (default 30.0s) elapses, recording + a `ProviderDiscoveryError(error_code="discovery_timeout")` for that + provider and continuing to the next one; the abandoned thread is + daemonized so it cannot block interpreter shutdown, and its eventual + result (if any) is discarded. This constant is independent of both + `DISCOVERY_TIMEOUT_SECONDS` and `ModelClient.timeout` -- it is never + reused or repurposed from either -- and an explicit + `discovery_deadline=None` still opts back into the pre-fix unbounded + wait. RED-before/GREEN-after: + `tests/test_model_discovery.py::test_discover_all_models_bounds_a_stalled_provider_so_later_providers_still_complete` + (mocked `discover_provider_models` hangs forever for one source; asserted + the second source's model still arrives and the whole call returns in + well under 5s) hung the test process indefinitely against the pre-fix + sequential loop (had to be killed by an outer `timeout`) and passes in + under 2s with the fix. +- **Bootstrap diversity was model-group-diverse only, not genuinely + provider-diverse, despite claiming otherwise.** This repo's own + `CLAUDE.md` and `tests/test_discovery_bootstrap_selection.py`'s title/ + assertions ("missing provider-diverse bootstrap selector") both describe + `select_bootstrap_discovered_agents` as layering provider-diverse + selection on top of cheapest-price ranking. The actual first pass + (`model_discovery.py`) admitted at most one endpoint per **model group** + (the provider-declared exact model identity) and never checked + `provider_name` at all, so several cheap, distinctly-named models from + one provider could -- and, in the existing + `test_bootstrap_selector_prefers_model_group_diversity` fixture, did -- + fill most or all of a bootstrap pool before a genuinely independent, + viable alternative provider was ever tried: an apparently diverse pool + (distinct model names) that was actually concentrated on one provider's + continued availability. `contextual_orchestrator/provider_bootstrap.py`'s + separate `select_model_group_diverse_models` was deliberately left + unchanged: it never claims provider diversity (its own name and + docstring say "model group" only), and its own test suite has an + explicit, deliberate price-honesty-over-diversity contract (an + unknown-priced model must never outrank a same-provider model with a + known price) that a blanket provider constraint would have broken -- + exactly the "unless evidence/contract explicitly chooses otherwise" case + the review finding itself carved out. Fixed (`select_bootstrap_discovered_agents` + only): a new first pass now admits at most one endpoint per provider + *and* per model group; once every viable provider has contributed once + (or capacity runs out), a second pass fills remaining slots from + still-untried model groups regardless of provider; a final pass, as + before, falls back to duplicate model-group endpoints only once real + diversity is exhausted. Two downstream tests that had pinned the old, + single-provider-concentrated outcome as correct + (`tests/test_discovery_bootstrap_selection.py::test_bootstrap_selector_prefers_model_group_diversity`, + `tests/test_review_gateway.py::test_build_review_orchestrator_uses_model_group_diversity`) + were updated to the corrected, genuinely cross-provider expectation. + RED-before/GREEN-after: the new + `test_bootstrap_selector_spans_multiple_providers_before_repeating_one` + (three individually-cheaper same-provider models plus one pricier + independent-provider model, pool size 2) failed against the pre-fix + algorithm with the pool collapsed onto the single cheaper provider + (`{'openrouter'} == {'openai', 'openrouter'}`) and passes with the fix. + **2026-09-08 no-heuristics correction (Proposed):** provider identity and + exact model-group identity are observations, not an outage probability, + expected utility, or psychometric quality model. The three-pass proposal + could therefore displace a lower-cost candidate or reorder a complete pool + solely because another candidate had a distinct label. Both bootstrap + selectors now compare every diversity proposal with the price-evidenced + candidate sequence and fail closed when they differ. Equal or incomplete + evidence remains fail-closed as before. New RED-before/GREEN-after fixtures + cover bounded model-group/provider displacement and complete-pool + reordering; the two earlier fixtures that required an unmodeled, + more-expensive provider were corrected to require rejection. + Raw candidates remain available for a future released allocation contract; + this branch does not invent outage weights, provider quotas, or a fallback + tie-break. +- While landing the above, a concurrently-pushed, staged (not-yet-applied) + self-modifying repair workflow was discovered on the same branch -- + `.github/workflows/source-fix-971-review-quality.yml` and + `scripts/ci/source_fix_971_review_quality.py`, added by a parallel + session targeting these same two findings. Its proposed fix was weaker + and, for the second finding, targeted the wrong function: it would have + (a) simply set the pre-existing `DISCOVERY_TIMEOUT_SECONDS` to a fixed + 15.0s per-HTTP-call socket timeout -- not a separately bounded/ + cancellable per-provider mechanism, so it would not have caught a hang + that ignores that parameter (verified: this branch's own new + `test_discover_all_models_bounds_a_stalled_provider_so_later_providers_still_complete` + mocks exactly that and would still fail against a mere socket-timeout + fix); and (b) added provider-diversity logic to + `provider_bootstrap.py`'s `select_model_group_diverse_models` -- the + function this repo's own tests deliberately keep price-honesty-over- + diversity for -- without updating its existing + `test_diverse_selection_prefers_known_cost_without_treating_unknown_as_free` + contract, which its own proposed algorithm would have broken. Since its + own "revalidate exact unchanged writer head" step (queued run + `33618086791` on `170103c0`) fails closed the moment the branch head + moves, the ordinary commits above were pushed immediately to win that + race safely, and the now-fully-superseded temp workflow/script were + then deleted per the branch's standing "no purpose-complete self- + modifying/source-fix workflows" rule (it could never successfully run + again against the new head regardless) -- restoring `interrogate` to + 100% (the deleted script's undocumented `main()` had briefly dropped it + to 99.9% after the merge that brought the staged files in). + +## 2026-09-02 PR #971: shared discovery metadata fetches bypassed the discovery deadline + +Observation time: 2026-09-02 Asia/Seoul. Closes a Devin Review finding +(bug id `BUG_pr-review-job-93783e6ce7a2440ab487ebce4076fe6f_0002`, +`contextual_orchestrator/model_discovery.py` line 52) raised against this same +branch after the per-provider `PROVIDER_DISCOVERY_DEADLINE_SECONDS` bound +above landed, independently verified against current-head code rather than +trusted as stated. A related CodeRabbit finding on the same file, surfaced +while this fix was in progress, is folded into the same entry below. + +### Finding and root cause + +`discover_all_models`'s per-provider `discovery_deadline` bound +(`_discover_provider_models_bounded`, documented in the section above) covers +only `discover_provider_models` inside the per-provider loop. Three *shared* +metadata fetches the same function makes outside that loop were confirmed +still wholly unbounded, each receiving only `timeout` (the per-HTTP-call +socket timeout, `None`/unbounded by #971's own design default): + +- `_fetch_models_dev_metadata(timeout=timeout)` -- called once, *before* the + per-provider loop starts, whenever any registered source declares + `models_dev_provider_id` (`opencode_zen`, `nvidia_nim`, `nvidia_nim_sub`, + `openai`). +- `_openrouter_zdr_model_ids(timeout=timeout)` -- called unconditionally + *after* the per-provider loop finishes. +- `openrouter_paid_inference_available(timeout=timeout)` -- called *after* + the loop, only once an `OPENROUTER_API_KEY` credential is registered. + +None of these three ran on the bounded daemon thread the per-provider loop +already used; a connection accepted but never answered (or any other hang +the per-request socket timeout cannot catch -- the same class of bug the +per-provider fix above already addressed) on any one of them could block +`discover_all_models`, and therefore first-boot pool bootstrapping, +indefinitely regardless of `discovery_deadline`. + +### Fix + +Reuses the exact mechanism already reviewed favorably for the per-provider +bound rather than inventing a second one: `_discover_provider_models_bounded` +is generalized into a shared primitive, `_run_bounded_by_deadline` (runs the +wrapped call on its own daemon thread, `worker.join(timeout=discovery_deadline)`, +abandons a still-alive worker past the deadline -- Python threads cannot be +forcibly killed, so "cancellable" means "the caller stops waiting"). All four +call sites -- the per-provider attempt plus the three shared fetches -- now +go through this one helper under the same `discovery_deadline` parameter +`discover_all_models` already accepted and threaded through; an explicit +`discovery_deadline=None` still opts every one of them back into the pre-fix +unbounded wait. + +### Fail-closed reasoning for each timeout fallback + +The per-provider path keeps its existing behavior (raises +`ProviderDiscoveryError(error_code="discovery_timeout")`, recorded in the +caller's `errors` list). Each shared fetch has no such per-provider error +list to append to; on timeout, `_run_bounded_by_deadline`'s `on_timeout` +callback returns the *exact same fallback value the wrapped function already +returns for an ordinary fetch failure* -- never a new, more permissive value +-- so the fail-closed posture already established for a network error is +identical for a network hang. Traced downstream for each: + +- `_fetch_models_dev_metadata` ordinary-failure return is `None`. + `_merge_models_dev_metadata` (`model_discovery.py`) treats non-dict + metadata as "no evidence" and returns the provider's own catalog payload + unenriched (no cost/modality enrichment, not "verified free" or "verified + priced") -- confirmed by reading the function: `provider_row = + metadata.get(provider) if isinstance(metadata, dict) else None`, then + `if not isinstance(rows, list) or not isinstance(models, dict): return + payload`. On timeout, the bounded wrapper returns `None` -- not the + `_NOT_FETCHED` sentinel -- so `discover_provider_models` treats the + metadata as "already fetched, unavailable" and does not re-attempt the + same stalled fetch a second time inside the (separately bounded) + per-provider thread; the provider's own catalog listing still completes. +- `_openrouter_zdr_model_ids` ordinary-failure return is an empty `set()`. + `_apply_discovered_model_evidence` short-circuits on an empty set + (`if not zdr_model_ids: return discovered`) and leaves every row's + `zdr_capable` exactly as `discover_provider_models` already set it -- + never *adds* ZDR-capable status on missing/timed-out evidence, only ever + on a positive, exact model-id match. On timeout, the bounded wrapper + returns `set()`. +- `openrouter_paid_inference_available` ordinary-failure/"could not + determine" return is `None`. `apply_openrouter_spend_admission`'s existing + rule is `spend_admitted = provider != "openrouter" or is_free or + paid_available is True` -- a paid (non-free) OpenRouter row is + `spend_admitted=False` for anything other than `paid_available is True`, + so `None` (timeout) and `False` (attested no credit) are both already + fail-closed today; the timeout fallback changes nothing about that + contract. On timeout, the bounded wrapper returns `None`. + +### Verification + +RED-before/GREEN-after, one regression test per shared fetch plus the +existing per-provider one, all mocking the target function itself (not the +transport layer) to block on a `threading.Event` nothing ever sets, mirroring +`test_discover_all_models_bounds_a_stalled_provider_so_later_providers_still_complete`'s +style: +`tests/test_model_discovery.py::test_discover_all_models_bounds_a_stalled_models_dev_metadata_fetch`, +`::test_discover_all_models_bounds_a_stalled_openrouter_zdr_fetch`, +`::test_discover_all_models_bounds_a_stalled_openrouter_paid_inference_fetch`. +Each hung the test process indefinitely against the pre-fix code (verified by +temporarily reverting the fix and killing the hung run with an outer +`timeout` -- exit 143 on all three) and passes in well under 5s with the fix. +`tests/test_model_discovery.py`, `tests/test_discovery_bootstrap_selection.py`, +`tests/test_review_gateway.py`, and every other test file importing +`model_discovery` (`test_auto_discovery_server.py`, +`test_chat_model_capability_isolation.py`, `test_ci_gateway_bootstrap.py`, +`test_discover_models_cli.py`, `test_model_discovery_boundaries.py`, +`test_openrouter_free_canary.py`, `test_privacy_policy_analysis.py`, +`test_provider_bootstrap*.py`, `test_provider_catalog_*.py`) pass unchanged. +`interrogate` remains 100%. + +### Related CodeRabbit finding: a hung endpoint fetch could block process exit + +`_openrouter_free_model_endpoints` (called from `discover_provider_models`'s +OpenRouter branch, itself already inside the per-provider bounded thread +above) fanned its per-model endpoint fetch out across a +`concurrent.futures.ThreadPoolExecutor`. Verified with a local repro before +changing anything: even wrapping the whole call in an already-`daemon=True` +outer thread does not stop a still-hung `ThreadPoolExecutor` worker from +blocking process shutdown -- `concurrent.futures.thread` registers its own +interpreter-exit hook that unconditionally joins every still-running worker +it created, independent of the daemon status of whichever thread constructed +the executor. A bare script reproducing this (one daemon thread, one +`ThreadPoolExecutor` with a permanently blocked worker, then a normal, +unforced fall-through to script exit) hung for a bounded outer `timeout` +command's full 10s and was killed (exit 124) against the pre-fix code, and +exited cleanly in well under 1s once the fetch fan-out was rewritten to use +plain `threading.Thread(daemon=True)` workers (concurrency capped at 8 via a +semaphore, matching the prior `max_workers`) instead of a `ThreadPoolExecutor` +-- daemon threads carry no such exit-blocking registration, so a hung fetch +is abandoned exactly like every other stalled discovery-time network call in +this module. This is a distinct failure mode from the discovery-deadline +finding above (it is not about `discover_all_models`'s *caller* waiting too +long -- that was already bounded by the outer per-provider thread -- it is +about the *process* being unable to exit at all while an abandoned +`ThreadPoolExecutor` worker is still running), so it is documented here +rather than folded silently into the fix above. +Verified: +`tests/test_model_discovery.py::test_openrouter_free_model_endpoints_hang_does_not_block_process_exit` +(spawns a real, separate interpreter, since an in-process thread-introspection +assertion cannot distinguish "still hanging in the background" from "would +actually block this process's shutdown" -- the whole point of the finding); +RED-before (killed by the test's own bound against the reverted, +`ThreadPoolExecutor`-based code) / GREEN-after (exits well under the bound). + +### Note on a concurrently-pushed, broader automated repair + +The branch owner pushed `.github/workflows/source-fix-971-live-review-quality.yml` +and `tests/test_pr971_review_quality_regressions.py` directly (not through +this session) while this fix was in progress, targeting this same finding +plus three unrelated Devin Review findings (durably recovered `zdr_only` +embedding work not re-entering request-policy scope; coalesced +provider-embedding batches able to mix privacy/routing identity; terminal +failed/cancelled provider batch documents recorded as endpoint success). +This branch's `cost_router.py` fix (commit `e3fa6d9b`, see the "recovered ZDR +embedding batch bypassed request-policy enforcement" section above) landed +independently, ahead of the workflow's own repair step, and fixed the first +two of those three (both are the same `_run_provider_embeddings` gap: +missing `request_policy` re-entry and missing per-request `zdr_only` +homogeneity) -- confirmed by re-running +`tests/test_pr971_review_quality_regressions.py` after that commit landed: +`test_recovered_zdr_batch_reenters_request_privacy_scope` and +`test_provider_embedding_batch_rejects_mixed_privacy_identity` are both now +GREEN. Only the third (terminal failed/cancelled provider batch documents +recorded as endpoint success, +`test_terminal_embedding_batch_document_fails_over_before_marking_health`) +remains unfixed as of this entry -- untouched by this fix, in `server.py`. +This workflow's own regression file was verified test-by-test against the +fix in this section: +`test_discover_all_models_bounds_every_shared_metadata_fetch` (the +discovery-deadline finding this section fixes) passes against this fix -- +confirming it is genuinely superseded for that one finding -- but, as of +this entry, the workflow's repair step has *not* fully served its purpose +for the remaining `server.py` finding, so it was deliberately left in +place rather than deleted, per this branch's own precedent's "delete only +once fully superseded" rule; its `model_discovery.py` `replace_once` steps +target the pre-fix literal source text this section replaces, so if ever +dispatched against this fix's head those specific steps will now fail closed +(`SystemExit`, no partial write survives past the failing step) rather than +silently reapplying a weaker, duplicate bound -- consistent with the +workflow's own "exact writer head" / "smallest causal GREEN repair" design +intent, not a bypass of it. + +## 2026-09-02 PR #971: embedding recovery/deadline and legacy-id quarantine review + +Observation time: 2026-09-02 Asia/Seoul. + +### Summary + +Bot-reported (CodeRabbit/Devin) findings on PR #971 were independently +verified against the actual current-head code (never trusted as ground +truth) and, where confirmed real, fixed with a RED-before/GREEN-after +regression test: + +- **Docstring coverage** (CodeRabbit: "48.87%, threshold 80%"): not + reproducible against this repo's own gate. `interrogate` (pinned + `requirements-opencode-review-ci.txt`, config in `pyproject.toml`: + `fail-under = 100`, `exclude = ["tests"]`) reports **100.0%** for the + entire `contextual_orchestrator` package, including every file this PR's + diff touches. Even scored with every `ignore-*` flag off (the strictest + reading short of including `tests/`), coverage is 81.7% -- still above an + 80% bar. CodeRabbit's own "38 files" count matches this diff's combined + source *and* test `.py` files, which strongly indicates its check scores + test functions too; this repo deliberately excludes `tests/` from the + docstring gate and does not conventionally docstring pytest test + functions, which would explain the low externally-reported number. No + docstring changes were needed. +- **Default-timeout embeddings OverflowError, failed legacy-id gateway + probe never disabling, recovered ZDR batch not re-validated, incomplete + sync embedding results evading the circuit breaker, durable claim-lease + crash + unbounded-execution-timeout substitution, and the OpenRouter + uptime collector's unbounded background fetch**: all confirmed real + against current-head code and fixed; see the dated `CHANGELOG.md` + entries under PR #971 for the exact mechanism, fix, and regression test + per item. +- The branch's self-modifying one-shot repair workflows + (`.github/workflows/source-fix-971-exact-head-review.yml`, + `.github/workflows/source-fix-971-runtime-context-green.yml`, + `scripts/source_fix_971_runtime_context_green.py`) were deleted per the + standing "no purpose-complete self-modifying/source-fix workflows" rule, + after the source/test fixes above landed as ordinary commits. + +## 2026-09-02 PR #971: no-heuristics `ModelClient` default retry correction + +Observation time: 2026-09-02 Asia/Seoul. + +### Summary + +- A fresh audit of PR #971 (`fix/model-group-timeout-openrouter`) found that + `ModelClient` still defaulted `max_retries` to a hand-picked `2` with no + cited standard, paper, or the org's own research (Fugu, Conductor, + TRINITY) establishing that number. +- RED regression commit `c1dcff6e903a7cfd7dab4584d628a5fbf57cf789` + (`tests/test_no_heuristic_default_transport_retry.py`) asserted a default + `ModelClient` must allocate zero automatic retries, independent of + provider/model/reasoning identity; it failed against the `max_retries=2` + default (`2 == 0`), confirming the RED state before the fix. +- Fix applied: `ModelClient.__init__`'s `max_retries` default is now `0` + (`contextual_orchestrator/orchestrator.py`). `docs/adr/0001-tool-execution-fallback-policy.md` + gained a 2026-09-02 amendment recording that RFC 9110 and NIST SP 800-204 + constrain *when* retry/circuit-breaking is safe but name no specific + numeric allocation, so the allocation is no longer library-authored; + explicit nonzero retry budgets remain caller-owned configuration. The RED + regression test now passes, and the focused provider/transport regression + suites (`tests/test_provider_gateway_resilience.py`, + `tests/test_provider_integration.py`, `tests/test_provider_error_taxonomy.py`, + `tests/test_no_heuristic_default_transport_retry.py`) were re-run green. +- The PR's own one-shot repair machinery + (`.github/workflows/source-fix-971-default-retry-policy.yml` and its + trigger/script) sat `queued` for 100+ minutes under org-wide Actions + capacity congestion and was completed manually instead of waiting further; + the now-superseded queued run was cancelled and the one-shot machinery + removed from the branch per the standing self-removal convention. + ## 2026-09-01 Autonomous Commercialization Loop: PR #970 Merge, Token Accounting & Cost Gateway Harmonization Observation time: 2026-09-01 Asia/Seoul. @@ -143,7 +1145,7 @@ design was not actually serving as a useful signal. **Bytez code path checked for a false-positive bug** (`contextual_orchestrator/model_discovery.py` `PROVIDER_MODEL_SOURCES`/`_parse_bytez`/`discover_provider_models`): URL -(`https://api.bytez.com/models/v2/list/models?task=chat`), `Authorization: Key ` header, and +(`https://api.bytez.com/models/v2/list/models?task=chat`), prefix-free `Authorization: ` header, and response parsing all look correct and match this repo's stdlib `urllib` discovery convention used by every other provider; nothing there would unconditionally reject every response. No `BYTEZ_API_KEY` is available in this sandbox to replay the exact authenticated call, but an unauthenticated live probe @@ -463,9 +1465,9 @@ completion finishes within any particular bound — the gateway preflight's separate curl timeout (originally 30s) was itself later found to be too tight for real reasoning-model latency and raised to 120s in `ContextualWisdomLab/.github#1440` (see that entry above); the two timeouts -are independent and this entry originally conflated them. Per owner review -on that PR, source correctness alone does not establish -operational acceptance: the fix also carries a RED→GREEN parity test +are independent and this entry originally conflated them. Source correctness +alone does not establish operational acceptance: the fix also carries a +RED→GREEN parity test (`test_gateway_preflight_max_tokens_is_synchronized_with_the_routing_probe`, confirmed to fail on the pre-fix `16` literal and pass once synchronized) and a negative control @@ -2661,22 +3663,139 @@ failed after the verdict was already correctly published and enforced — a downstream status-publish step, not the review pipeline itself; not yet investigated. -**Separate, newly found, NOT YET FIXED bug** (traced via `.github#1276`'s +**Superseded timeout diagnosis** (traced via `.github#1276`'s `noema-review` run, flagged by the user as "still not working" after the above fixes landed): `TaskOrchestrator._invoke()` (`contextual_orchestrator/orchestrator.py:6353-6537`, the per-agent -call/retry/failover loop) has no overall wall-clock deadline. Each -attempt is capped at `ModelClient.timeout = 90s`, but a hanging (not -erroring) candidate plus one same-agent retry can chain past 90s + -backoff + 90s = 180s+ before any response — well past -`contextual_orchestrator_review_sidecar.sh`'s 120s outer -`curl --max-time` bound on its own gateway-preflight self-check, so the -sidecar sees exactly "0 bytes received after 120002ms" even though -nothing is truly hung, just still working through an unbounded internal -retry chain. Reproduced once (`.github` run `33312258611`, job -`99259327051`); org-wide evidence since the pingora/Strix fixes landed -shows this is now occasional, not the dominant failure mode (most -`.github`-hosted PRs' `noema-review`/dispatch runs succeed). Correct fix -is an overall deadline on `_invoke`'s candidate/retry loop, not another -timeout increase on the sidecar's client side — deferred rather than -rushed into this heavily-tested core file without dedicated validation. +call/retry/failover loop) could run longer than the sidecar's former +120-second client ceiling. That did not prove an internal hang; it proved +the outer limit could terminate a healthy slow model. The former proposal +to add an overall `_invoke` deadline is withdrawn. PR #971 removes fixed +wall-clock limits from inference, discovery, OpenRouter ZDR lookup, and +local readiness paths; only operator cancellation or a superseded PR head +may terminate that work. + +## 2026-08-31 OpenRouter is a normal, routable provider again + +Supersedes the 2026-08-30 entry above's characterization of `evidence_only=True` +on `openrouter` (commit `952996ec`) as settled ZDR hardening: that +characterization was false, as the entry above now records. On direct review +this pass, "ZDR eligibility is grounds to block a whole provider account" +turns out to be backwards -- ZDR is a route/model-level property, never a +provider-account-level one. `PROVIDER_MODEL_SOURCES`'s `openrouter` entry no +longer sets `evidence_only=True`. Concretely this fixes two bugs at once: + +1. **`orchestrator/free` structural emptiness (ADR 0041's own finding).** + OpenRouter is the one provider source with genuinely reliable native + pricing/`is_free` evidence; excluding it from serving regardless of that + evidence directly caused the "structurally empty in practice" state ADR + 0041 documented. OpenRouter can now serve like any other discovered + provider. +2. **A backwards ZDR-evidence exclusion.** `_apply_discovered_model_evidence` + computed `zdr_capable=not model.evidence_only and matches(...)`, which + meant OpenRouter's own rows could never be marked ZDR-capable even when + they exactly matched OpenRouter's own declared ZDR feed + (`https://openrouter.ai/api/v1/endpoints/zdr`). That exclusion is gone; + OpenRouter's own matching rows are now credited exactly like every other + provider's. + +**What is preserved, not removed**: the underlying reason a "provider-neutral, +not OpenRouter-only" evidence-application contract was insisted on during +PR #901's review (matching model ids from OpenRouter's feed onto *other* +providers' discovered rows, not just OpenRouter's own) is completely +untouched -- `_apply_discovered_model_evidence` still applies evidence to +every provider's rows identically; OpenRouter's own rows simply stop being +the one arbitrary exception to that rule. + +**The genuine technical risk this raises, and how it is closed**: OpenRouter +can multiplex one model id across several backing providers, so a +discovery-time ZDR feed snapshot proves a route *was* attested when fetched, +not which provider serves a *later* request. Client-side endpoint tracking +to predict this would only be as reliable as the last snapshot. Instead, +`ModelClient` now applies OpenRouter's own documented request-time +enforcement -- `"provider": {"zdr": true}` in the request body +(https://openrouter.ai/docs/features/provider-routing) -- via +`_pin_openrouter_zdr`, called from every wire-level transport an OpenRouter +agent can reach under an active `zdr_only` request scope: `_send` (the +`route`/`conduct` chat path), `_stream_send` (SSE streaming), and +`_send_raw` (the tools/structured-output passthrough path both +`proxy_send` and `proxy_send_once` funnel through), plus `proxy_send_bytes` +(binary speech responses whose request body is still JSON). This is OpenRouter's own +server-side enforcement for the request being sent right now, not a +client-side prediction — strictly stronger than what discovery-time +filtering could ever guarantee. + +**The asynchronous Batch API path is pinned too**: `_batch_run` (JSONL file +upload then a separate `/batches` job) does not go through the three +transport functions above, but it independently calls `_pin_openrouter_zdr` +on each request body it serializes into the uploaded JSONL, so a `zdr_only` +batch request against OpenRouter gets the same `"provider": {"zdr": true}` +enforcement as the synchronous paths. + +Embedding Batch JSONL follows the same contract. After `zdr_only` resolves an +attested OpenRouter embedding agent, `CostRoutingCoordinator` records the +provider-routing pin on each `EmbeddingBatchRequest`; its JSONL body emits +`"provider": {"zdr": true}` without exposing the internal `zdr_only` field. + +Verified: `tests/test_orchestrator_client_boundaries.py` adds direct unit +coverage of `_pin_openrouter_zdr` (no-op outside `zdr_only`, no-op for +non-OpenRouter agents, adds/merges the pin correctly) plus wiring-verification +tests on `_send`/`_stream_send`/`_send_raw`/`_batch_run` that capture the actual +outgoing JSON body. `tests/test_model_discovery.py`, +`tests/test_auto_discovery_server.py`, and `tests/test_review_gateway.py` +were updated where they asserted the old, now-reversed +`openrouter` + `evidence_only=True` behavior; the general `evidence_only` +mechanism itself (for any future provider that might legitimately need it) +is untouched and still tested, just no longer applied to OpenRouter by +default. Full suite green; `interrogate` 100% on the touched modules. + +### GAP RESOLVED ON PR HEAD — 2026-08-31: model groups, free discovery, and measured capacity + +[PR #971](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/971) +implementation parent `a36770179a695b3825a0fc2ca45eace09b5e3b8f` implements the +requested provider-neutral contract. The live PR head must be read from GitHub +because this baseline commit necessarily advances it. + +- Routing identity is an exact `model_group`; provider-family grouping is not + part of the serving contract. Collision-resistant group ids preserve model + punctuation rather than conflating distinct upstream model names. +- Authenticated OpenRouter discovery admits concrete zero-price models and + excludes the aggregate `openrouter/free` router from serving candidates. + ZDR evidence is evaluated per discovered model rather than disabling the + entire OpenRouter account. +- Discovery, ZDR lookup, provider policy/credit reads, inference, and racing + default to no fixed wall-clock timeout. +- The measured group ledger reports observed peak RPM and TPM from real + completed requests; missing token usage remains missing rather than being + invented. +- Bytez's discovery parser and `Key` authentication path are implemented. The + observed Bytez HTTP 500 remains upstream/account evidence, not proof that the + provider should be silently excluded. + +Verification on the current PR working tree: +local full suite `2913 passed, 1 skipped`; the no-fixed-timeout revert-focused +suite passed `15 passed, 1 skipped`. Hosted Security, unit, fuzz, OpenCode, +Noema, and Strix checks for exact head +`39c80927f89c1f1f955f42af0b9d1f14b1527b70` are queued, so earlier-head hosted +success is not current-head merge evidence. Therefore this remains verified +PR-head behavior, not yet protected-main or deployed evidence. Central +`.github` PR #1546 merged as `5686de41660d51a7a7f22b8840dfa6ccfe5ff3f1`, +delivering the shared no-timeout, independent-Noema, stacked-PR, and exact-head +review fixes; a fresh central scheduler run has been dispatched for this PR and +is waiting for a GitHub-hosted runner. + +## 2026-09-02 canonical immutable release + resumable long-running execution — cross-repo consumer evidence (keyverse#132, EgressWeave#235) + +**Observed gap (owner-lane, `contextual-orchestrator`).** Fresh consumer evidence surfaced on PR #971 (`ContextualWisdomLab/keyverse#132`, migrating its hourly model-backed workflow to `orchestrator/free`): this repository has no GitHub `latest` release endpoint (`/releases/latest` returns 404), so a consumer that wants a durable, pinnable contract has no choice but to vendor/pin a raw source revision (currently `464da4715b495b5eaaa593eba3796e2d976ee0c9`). That violates the org-wide CWL boundary convention documented in `ContextualWisdomLab/.github`'s `docs/CWL-MASTER-CONTEXT.md` §7 (this repository has no local copy of that doc): a consumer must consume a released API/client/schema contract, not a mutable sibling source commit. `CLAUDE.md`'s own commands section already documents "Semantic Versioning where the repository publishes a release" as the intended model; today nothing publishes one. + +**Observed gap (owner-lane, runtime).** The same comment reports Keyverse inherits `OPENCODE_RUN_TIMEOUT_SECONDS=2100` and that `EgressWeave#235` independently hit a 45-minute Actions job timeout around the same gateway-backed OpenCode pattern. Both are consumer-side leaf wall-clock wrappers reappearing around the org's `timeout=null` model-inference contract (this repository's own `docs/planning/adrs/0032-model-group-cost-aware-discovery.md`: "Model inference has no fixed wall-clock timeout") because this repository has never supplied the owner-side alternative: a way for a long-running (hours-scale) `orchestrator/free` request to survive a host/runner-level execution boundary without being misclassified as a model failure, and a way for an interrupted maintenance execution to resume/re-dispatch from a checkpoint instead of restarting or serializing the hourly lane forever. Every leaf keeps re-inventing its own timeout because the owner has not yet drawn the line between "the model is slow" (never a failure) and "the execution environment ended" (an infrastructure/admin/user-cancellation event with its own terminal state). + +**Why this is owner-lane, not per-consumer.** Per the org-wide CWL boundary convention (`ContextualWisdomLab/.github`'s `docs/CWL-MASTER-CONTEXT.md` §7): a boundary gap common to multiple consumers is fixed at the canonical owner, never duplicated/worked around per leaf. Two independent consumers (Keyverse, EgressWeave) hitting the identical class of gap in the same window is exactly the "genuine common demand" signal that rules out excluding this as consumer-specific. + +**Scoped action items (tracked here, not yet started as of this entry).** +1. *Canonical immutable release.* Publish a tagged, versioned GitHub Release for `contextual-orchestrator` (SemVer, per `CLAUDE.md`'s already-stated intent) with an immutable client/API/schema surface consumers can pin via `/releases/latest` instead of a source SHA. No paid/provider-specific fallback should be required to consume it. +2. *Resumable long-running execution.* A dedicated ADR (this is architecturally significant, not a leaf fix) defining: explicit, distinguishable terminal states for user cancellation, provider termination, audited admin timeout, and infrastructure/runner loss; and a checkpoint/exact-head re-dispatch mechanism so an interrupted maintenance execution resumes rather than restarting from zero or blocking the hourly lane indefinitely. Must preserve the existing `timeout=null` inference contract — this is explicitly not a return to a sidecar-wide fixed deadline. + +**Owner RED/GREEN acceptance (as stated by the repository owner).** A long-lived `orchestrator/free` request survives beyond the former leaf timeout without synthetic model failure; explicit user cancellation/provider termination/audited admin timeout/infrastructure loss remain distinguishable; an interrupted maintenance execution can resume/re-dispatch with exact-head/checkpoint identity; the resulting released API/client/schema is immutable enough for consumers to pin without vendoring this repository's source. + +**Relationship to already-tracked items above.** This extends, with concrete cross-repo evidence, the "hourly-loop durable/resumable execution boundary" item already deferred out of PR #971 (the narrower job-timeout piece of that item is tracked separately in PR #1027). The immutable-release item is new to this baseline. diff --git a/tests/test_auto_discovery_server.py b/tests/test_auto_discovery_server.py index 031f4bb06..116d5d74e 100644 --- a/tests/test_auto_discovery_server.py +++ b/tests/test_auto_discovery_server.py @@ -15,6 +15,8 @@ DiscoveredModel, agent_from_discovered, agent_id_for, + legacy_agent_id_for, + model_group_name_for, ) from contextual_orchestrator.orchestrator import ModelAgent, TaskOrchestrator @@ -137,7 +139,7 @@ def probe(_orchestrator, model): result = _auto_discover_runtime_agents(orchestrator) assert probes == ["stale-model", "live-model"] - assert result["added"] == ["configured_gateway_live_model"] + assert result["added"] == [agent_id_for(models[1])] assert all(agent.model != "stale-model" for agent in orchestrator.agents) @@ -331,6 +333,56 @@ def test_failed_gateway_probe_disables_persisted_discovered_agent_after_restart( assert "structured:blocked" in restarted.candidates[0].tags +def test_failed_gateway_probe_disables_legacy_id_persisted_agent( + monkeypatch, tmp_path +) -> None: + """A persisted agent kept under the pre-fingerprint legacy id is disabled too. + + A failed configured-gateway structured probe is recorded only under the + new fingerprinted id (``agent_id_for``), but a persisted agent from + before model-group fingerprinting keeps its legacy id + (``legacy_agent_id_for``). The ``runtime_models`` membership check must + accept either id form -- matching the ``existing_by_id`` fallback lookup + used later in the same function -- otherwise a legacy-id agent whose + model now fails the probe is silently dropped from ``runtime_models``, + ``sync_discovered_agents`` is never called for it, and it is left + enabled (and unpersisted) indefinitely. + """ + model = DiscoveredModel( + provider_name="configured_gateway", + model_id="stale-legacy-model", + credential_name="LLM_GATEWAY_API_KEY", + chat_base_url="https://gateway.synthetic.example/v1", + auth_scheme="Bearer", + capabilities=("chat",), + ) + existing = replace( + agent_from_discovered(model), + id=legacy_agent_id_for(model), + disabled=False, + ) + agents_db = str(tmp_path / "agents.db") + monkeypatch.setattr( + "contextual_orchestrator.__main__.get_credential", lambda _name: "present" + ) + monkeypatch.setattr( + "contextual_orchestrator.__main__.discover_all_models", + lambda *_args, **_kwargs: ([model], []), + ) + monkeypatch.setattr( + "contextual_orchestrator.__main__._probe_configured_gateway_structured_chat", + lambda *_args: False, + ) + orchestrator = TaskOrchestrator([existing], agents_db=agents_db) + + result = _auto_discover_runtime_agents(orchestrator) + + assert result["updated"] == [existing.id] + persisted = next(agent for agent in orchestrator.candidates if agent.id == existing.id) + assert persisted.disabled is True + assert "structured:blocked" in persisted.tags + + def test_failed_gateway_probe_keeps_persisted_embedding_capability( monkeypatch, tmp_path ) -> None: @@ -480,9 +532,9 @@ def test_auto_discovery_activates_bare_chat_but_not_embedding_ids(monkeypatch) - [ModelAgent("bootstrap_agent", "bootstrap-model", tags=("bootstrap_seed",))] ) result = _auto_discover_runtime_agents(orchestrator) - assert result["added"] == ["configured_gateway_gpt_chat_7x"] + assert result["added"] == [agent_id_for(bare_chat)] agents = orchestrator.agents - assert any(agent.id == "configured_gateway_gpt_chat_7x" for agent in agents) + assert any(agent.id == agent_id_for(bare_chat) for agent in agents) assert all(agent.model != "text-embedding-5" for agent in agents) @@ -507,7 +559,7 @@ def test_auto_discovery_activates_provider_catalog_rows(monkeypatch) -> None: result = _auto_discover_runtime_agents(orchestrator) assert result == { - "added": ["nvidia_nim_provider_nim_chat"], + "added": [agent_id_for(provider_row)], "updated": ["bootstrap_agent"], } agent = orchestrator.candidates[-1] @@ -515,13 +567,13 @@ def test_auto_discovery_activates_provider_catalog_rows(monkeypatch) -> None: assert agent.disabled is False -def test_auto_discovery_never_activates_openrouter_evidence_rows(monkeypatch) -> None: - """OpenRouter catalog rows provide evidence but never serving agents.""" +def test_auto_discovery_never_activates_evidence_only_rows(monkeypatch) -> None: + """A row explicitly marked evidence_only never becomes a serving agent.""" evidence = DiscoveredModel( - provider_name="openrouter", + provider_name="example_evidence_provider", model_id="provider/router-chat", - credential_name="OPENROUTER_API_KEY", - chat_base_url="https://openrouter.ai/api/v1", + credential_name="EXAMPLE_EVIDENCE_PROVIDER_API_KEY", + chat_base_url="https://example-evidence-provider.example/v1", auth_scheme="Bearer", capabilities=("chat", "response_format"), evidence_only=True, @@ -580,7 +632,7 @@ def test_auto_discovery_keeps_metadata_free_general_chat_models(monkeypatch) -> result = _auto_discover_runtime_agents(orchestrator) - assert result["added"] == ["openai_gpt_5_4"] + assert result["added"] == [agent_id_for(discovered)] assert orchestrator.agents[-1].model == discovered.model_id @@ -607,7 +659,7 @@ def test_auto_discovery_removes_the_configured_gateway_placeholder(monkeypatch) result = _auto_discover_runtime_agents(orchestrator) - assert result["added"] == ["configured_gateway_chat_capable_model"] + assert result["added"] == [agent_id_for(discovered)] assert [agent.model for agent in orchestrator.agents] == ["chat-capable-model"] assert placeholder.id not in orchestrator._group_router.snapshot() @@ -642,6 +694,30 @@ def test_auto_discovery_removes_placeholder_for_existing_gateway_model(monkeypat assert orchestrator.agents == [existing] +def test_auto_discovery_assigns_group_to_legacy_discovered_agent(monkeypatch) -> None: + discovered = DiscoveredModel( + "openrouter", "Vendor/Model", "OPENROUTER_API_KEY", + "https://openrouter.ai/api/v1", "Bearer", capabilities=("chat",), + ) + legacy = replace( + agent_from_discovered(discovered), + id="openrouter_vendor_model", + group_name="", + disabled=False, + ) + orchestrator = TaskOrchestrator([legacy]) + monkeypatch.setattr( + "contextual_orchestrator.__main__.discover_all_models", + lambda *_args, **_kwargs: ([discovered], []), + ) + + result = _auto_discover_runtime_agents(orchestrator) + + assert result == {"added": [], "updated": [legacy.id]} + assert orchestrator.candidates[0].id == legacy.id + assert orchestrator.candidates[0].group_name == model_group_name_for(discovered) + + def test_auto_discovery_keeps_last_enabled_placeholder_for_disabled_gateway_model( monkeypatch, ) -> None: @@ -694,9 +770,9 @@ def test_auto_discovery_adds_embedding_without_disabling_configured_chat_pool(mo orchestrator = TaskOrchestrator([ModelAgent("bootstrap_agent", "bootstrap-model")]) result = _auto_discover_runtime_agents(orchestrator) - assert result == {"added": ["openai_embedding_capable_model"], "updated": []} + assert result == {"added": [agent_id_for(embedding)], "updated": []} assert {agent.id for agent in orchestrator.agents} == { - "bootstrap_agent", "openai_embedding_capable_model" + "bootstrap_agent", agent_id_for(embedding) } @@ -723,7 +799,7 @@ def test_unrelated_discovery_keeps_configured_gateway_placeholder(monkeypatch) - result = _auto_discover_runtime_agents(orchestrator) - assert result["added"] == ["openai_chat_capable_model"] + assert result["added"] == [agent_id_for(discovered)] assert placeholder in orchestrator.agents @@ -744,7 +820,7 @@ def test_auto_discovery_uses_explicit_capabilities_before_model_id_heuristics(mo orchestrator = TaskOrchestrator([ModelAgent("bootstrap_agent", "bootstrap-model")]) result = _auto_discover_runtime_agents(orchestrator) - assert result["added"] == ["openai_generic_deployment"] + assert result["added"] == [agent_id_for(generic_non_chat)] agent = orchestrator.select_capability_agent("embedding") assert agent.model == "generic-deployment" assert "chat" not in agent.tags @@ -916,7 +992,9 @@ def test_auto_discovery_disables_existing_discovered_paid_openrouter_without_cre ) _auto_discover_runtime_agents(orchestrator) - assert orchestrator.candidates[0] == existing + assert orchestrator.candidates[0] == replace( + existing, group_name=model_group_name_for(recovered) + ) def test_auto_discovery_recovers_model_first_discovered_while_spend_blocked( @@ -983,7 +1061,9 @@ def test_auto_discovery_preserves_operator_disable_across_spend_recovery( ) _auto_discover_runtime_agents(orchestrator) - assert orchestrator.candidates[0] == existing + assert orchestrator.candidates[0] == replace( + existing, group_name=model_group_name_for(recovered) + ) def test_runtime_auto_discovery_does_not_read_gateway_environment(monkeypatch) -> None: @@ -1055,8 +1135,13 @@ def test_auto_discovery_retires_mock_seed_when_real_agent_already_exists( result = _auto_discover_runtime_agents(orchestrator) - assert result == {"added": [], "updated": ["mock_seed_agent"]} - assert orchestrator.agents == [real_agent] + assert result == { + "added": [], + "updated": [real_agent.id, "mock_seed_agent"], + } + assert orchestrator.agents == [ + replace(real_agent, group_name=model_group_name_for(discovered)) + ] def test_auto_discovery_retires_mock_seed_when_current_discovery_is_empty( diff --git a/tests/test_batch_embeddings.py b/tests/test_batch_embeddings.py index 153581df0..dbfbcdde2 100644 --- a/tests/test_batch_embeddings.py +++ b/tests/test_batch_embeddings.py @@ -495,6 +495,110 @@ def test_batch_embeddings_zdr_only_omitted_model_selects_zdr_capable_embedding_a server.shutdown() +def test_openrouter_zdr_embedding_batch_pins_provider_routing() -> None: + agent = ModelAgent( + "zdr_embedding", + "text-embedding-3-small", + provider_name="openrouter", + tags=("embedding", "privacy:zdr"), + ) + backend = _RecordingEmbeddingBackend() + coordinator = CostRoutingCoordinator( + TaskOrchestrator([agent]), + InMemoryConfigStore(), + embedding_batch_backend=backend, + embedding_token_counter=_ExactTestCounter(), + ) + + coordinator.submit_embeddings_batch( + ["private"], + model=agent.model, + zdr_only=True, + agent_id=agent.id, + ) + + assert backend.requests[0].to_jsonl_line()["body"]["provider"] == {"zdr": True} + + +def test_openrouter_zdr_embedding_batch_infers_legacy_provider_name() -> None: + agent = ModelAgent( + "legacy_zdr_embedding", + "text-embedding-3-small", + base_url="https://openrouter.ai/api/v1", + tags=("embedding", "privacy:zdr"), + ) + backend = _RecordingEmbeddingBackend() + coordinator = CostRoutingCoordinator( + TaskOrchestrator([agent]), + InMemoryConfigStore(), + embedding_batch_backend=backend, + embedding_token_counter=_ExactTestCounter(), + ) + + coordinator.submit_embeddings_batch( + ["private"], + model=agent.model, + zdr_only=True, + agent_id=agent.id, + ) + + assert backend.requests[0].provider_routing == {"zdr": True} + + +def test_openrouter_zdr_embedding_batch_overrides_mistyped_provider_name() -> None: + """The batch ZDR pin is applied even for a nonempty but wrong ``provider_name``. + + Mirrors ``orchestrator._resolved_openrouter_provider``'s fix for the same + pattern: an agent whose ``base_url`` is OpenRouter's own endpoint but + whose ``provider_name`` is a typo/mislabel ("openai") must still resolve + to "openrouter" for the ZDR-pin decision, since ``base_url`` — not the + free-text ``provider_name`` — determines the actual outbound destination + (CodeRabbit review on #953, discussion_r3898659143). + """ + agent = ModelAgent( + "mistyped_zdr_embedding", + "text-embedding-3-small", + provider_name="openai", + base_url="https://openrouter.ai/api/v1", + tags=("embedding", "privacy:zdr"), + ) + backend = _RecordingEmbeddingBackend() + coordinator = CostRoutingCoordinator( + TaskOrchestrator([agent]), + InMemoryConfigStore(), + embedding_batch_backend=backend, + embedding_token_counter=_ExactTestCounter(), + ) + + coordinator.submit_embeddings_batch( + ["private"], + model=agent.model, + zdr_only=True, + agent_id=agent.id, + ) + + assert backend.requests[0].provider_routing == {"zdr": True} + + +def test_openrouter_zdr_embedding_batch_uses_atomic_target_snapshot(monkeypatch) -> None: + backend = _RecordingEmbeddingBackend() + coordinator = CostRoutingCoordinator( + TaskOrchestrator([ModelAgent("removed_agent", "text-embedding-3-small")]), + InMemoryConfigStore(), + embedding_batch_backend=backend, + embedding_token_counter=_ExactTestCounter(), + ) + monkeypatch.setattr( + coordinator, + "_resolve_embedding_target", + lambda *_args: ("text-embedding-3-small", "removed_agent", "openrouter"), + ) + + coordinator.submit_embeddings_batch(["private"], zdr_only=True) + + assert backend.requests[0].provider_routing == {"zdr": True} + + def test_pending_batch_preserves_resolved_model_identity() -> None: orchestrator = TaskOrchestrator([ModelAgent("embedding_worker", "resolved-embedding")]) coordinator = CostRoutingCoordinator( diff --git a/tests/test_batch_routing_boundaries.py b/tests/test_batch_routing_boundaries.py index 4b7f69171..447b92a4f 100644 --- a/tests/test_batch_routing_boundaries.py +++ b/tests/test_batch_routing_boundaries.py @@ -94,10 +94,15 @@ def test_embedding_request_jsonl_line_shape() -> None: def test_embedding_request_jsonl_preserves_zdr_policy() -> None: - request = EmbeddingBatchRequest(input_text="private", zdr_only=True) + request = EmbeddingBatchRequest( + input_text="private", + zdr_only=True, + provider_routing={"zdr": True}, + ) assert request.zdr_only is True assert "zdr_only" not in request.to_jsonl_line()["body"] + assert request.to_jsonl_line()["body"]["provider"] == {"zdr": True} def test_chat_request_jsonl_preserves_zdr_policy() -> None: diff --git a/tests/test_chat_model_capability_isolation.py b/tests/test_chat_model_capability_isolation.py index 5d67e1ac2..f6b967658 100644 --- a/tests/test_chat_model_capability_isolation.py +++ b/tests/test_chat_model_capability_isolation.py @@ -250,7 +250,7 @@ def test_generic_media_catalog_rows_stay_out_of_bootstrap_and_review_chat_pool( ] == ["text-free-model"] assert [ model.model_id - for model in provider_bootstrap.select_provider_diverse_models( + for model in provider_bootstrap.select_model_group_diverse_models( discovered, limit=10 ) ] == ["text-free-model"] diff --git a/tests/test_cost_router.py b/tests/test_cost_router.py index 82ca8c413..a1b3e90b1 100644 --- a/tests/test_cost_router.py +++ b/tests/test_cost_router.py @@ -833,7 +833,9 @@ def test_virtual_embedding_model_binds_concrete_tokenizer_before_accounting() -> "contextual-orchestrator", False, None ) - assert resolved == ("text-embedding-3-small", "remote_embedding") + assert resolved == ( + "text-embedding-3-small", "remote_embedding", "provider.synthetic.invalid" + ) def test_non_zdr_batch_preserves_an_explicit_model_outside_the_pool() -> None: @@ -1188,12 +1190,13 @@ def test_non_zdr_embedding_batch_preserves_explicit_model_outside_the_pool() -> ) coordinator = CostRoutingCoordinator(orchestrator, InMemoryConfigStore()) - resolved_model, resolved_agent_id = coordinator._resolve_embedding_target( + resolved_model, resolved_agent_id, resolved_provider = coordinator._resolve_embedding_target( "unconfigured-upstream-model", zdr_only=False, agent_id=None ) assert resolved_model == "unconfigured-upstream-model" assert resolved_agent_id is None + assert resolved_provider is None def test_non_zdr_batch_preserves_an_explicit_model_outside_the_pool() -> None: diff --git a/tests/test_daemon_worker_pool_shutdown.py b/tests/test_daemon_worker_pool_shutdown.py new file mode 100644 index 000000000..38529b6bd --- /dev/null +++ b/tests/test_daemon_worker_pool_shutdown.py @@ -0,0 +1,100 @@ +"""Regression coverage for daemon worker pool shutdown ordering.""" + +from __future__ import annotations + +import queue +import threading + +import pytest + +from contextual_orchestrator.batch_routing import _DaemonWorkerPool + + +def test_daemon_worker_pool_rejects_submit_after_shutdown() -> None: + """Shutdown must close admission before worker sentinels are queued.""" + pool = _DaemonWorkerPool(max_workers=1) + pool.shutdown(wait=False, cancel_futures=True) + + with pytest.raises(RuntimeError, match="shutdown"): + pool.submit(lambda: None) + + +def test_daemon_worker_pool_shutdown_is_idempotent() -> None: + """A second ``shutdown()`` call must not error or re-cancel/re-queue. + + Regression for ContextualWisdomLab/contextual-orchestrator#971: the + admission flag now gates the drain and sentinel steps, so repeating + ``shutdown()`` after it already ran must be a harmless no-op beyond the + (already-terminated) worker join. + """ + pool = _DaemonWorkerPool(max_workers=1) + pool.submit(lambda: None) + + pool.shutdown(wait=True, cancel_futures=True) + # Must not raise, hang, or attempt to cancel/queue anything a second + # time -- the pool is already fully torn down. + pool.shutdown(wait=True, cancel_futures=True) + + +def test_daemon_worker_pool_submit_racing_shutdown_cannot_execute() -> None: + """A ``submit()`` that races ``shutdown(cancel_futures=True)`` must lose. + + Regression for the Devin finding "Concurrent shutdown admits cancelled + work" (ContextualWisdomLab/contextual-orchestrator#971): before the fix, + ``shutdown()`` drained the queue *before* closing admission, so a + ``submit()`` landing in that gap could enqueue work that survived the + cancellation drain and later ran on a worker despite + ``cancel_futures=True``. + + This test forces that exact interleaving deterministically -- no real + thread-timing luck involved -- by hooking the queue drain's empty-check + (the point where the old code was about to leave the vulnerable window) + and holding it open with an ``Event`` until a concurrent ``submit()`` + has had a chance to run. Under the fix, admission is already closed by + the time the drain even starts, so the racing ``submit()`` must observe + the closed pool and raise instead of enqueuing -- and its work must + never execute. + """ + pool = _DaemonWorkerPool(max_workers=1) + + drain_found_empty = threading.Event() + release_submit = threading.Event() + executed = threading.Event() + original_get_nowait = pool._queue.get_nowait + + def instrumented_get_nowait() -> object: + try: + return original_get_nowait() + except queue.Empty: + # The drain has just found the queue empty and is about to + # return control to ``shutdown()`` -- exactly the window the + # unfixed implementation left open before closing admission. + # Hold it open long enough for the racing submit() below to run. + drain_found_empty.set() + release_submit.wait(timeout=5.0) + raise + + pool._queue.get_nowait = instrumented_get_nowait + + submit_errors: list[BaseException] = [] + + def racing_submit() -> None: + assert drain_found_empty.wait(timeout=5.0), "drain never reached the empty check" + try: + pool.submit(executed.set) + except BaseException as exc: # noqa: BLE001 - captured for assertion below + submit_errors.append(exc) + finally: + release_submit.set() + + racer = threading.Thread(target=racing_submit) + racer.start() + try: + pool.shutdown(wait=True, cancel_futures=True) + finally: + racer.join(timeout=5.0) + + assert not racer.is_alive(), "racing submit() thread never completed" + assert len(submit_errors) == 1, "racing submit() must observe the closed pool" + assert isinstance(submit_errors[0], RuntimeError) + assert not executed.is_set(), "cancelled work must never execute" diff --git a/tests/test_discover_models_cli.py b/tests/test_discover_models_cli.py index a431e386a..09d36f89b 100644 --- a/tests/test_discover_models_cli.py +++ b/tests/test_discover_models_cli.py @@ -13,6 +13,11 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from contextual_orchestrator.__main__ import main # noqa: E402 +from contextual_orchestrator.model_discovery import ( # noqa: E402 + DiscoveredModel, + agent_from_discovered, + agent_id_for, +) from contextual_orchestrator.credentials import ( # noqa: E402 InMemoryCredentialBackend, get_credential, @@ -183,7 +188,7 @@ def urlopen(request, timeout=None, **_kwargs): assert report["discovered_count"] == 1 assert report["models"] == [ { - "provider": "openai", "model": "gpt-5.5", "agent_id": "openai_gpt_5_5", + "provider": "openai", "model": "gpt-5.5", "agent_id": agent_id_for(DiscoveredModel("openai", "gpt-5.5", "OPENAI_API_KEY", "https://api.openai.com/v1", "Bearer")), "is_free": False, "max_output_tokens": None, "context_window": None, @@ -334,7 +339,7 @@ def urlopen(request, timeout=None, **_kwargs): set_backend(None) reloaded = TaskOrchestrator([ModelAgent("seed_agent", "seed-model")], agents_db=db_path) - assert any(agent.id == "openai_gpt_5_5" for agent in reloaded.candidates) + assert any(agent.model == "gpt-5.5" for agent in reloaded.candidates) def test_discover_models_closes_temporary_agents_db_orchestrator(tmp_path) -> None: @@ -423,11 +428,176 @@ def urlopen(request, timeout=None, **_kwargs): set_backend(None) report = json.loads(stdout.getvalue()) - assert report["enabled_agent_ids"] == ["openrouter_cheap_model"] + assert report["enabled_agent_ids"] == [ + agent_id_for(DiscoveredModel("openrouter", "cheap-model", "OPENROUTER_API_KEY", "https://openrouter.ai/api/v1", "Bearer")) + ] reloaded = TaskOrchestrator([ModelAgent("seed_agent", "seed-model")], agents_db=db_path) by_id = {agent.id: agent for agent in reloaded.candidates} - assert by_id["openrouter_cheap_model"].disabled is False + assert by_id[report["enabled_agent_ids"][0]].disabled is False + + +def test_enable_cheapest_reuses_and_activates_legacy_discovered_id(tmp_path) -> None: + from dataclasses import replace + from contextual_orchestrator import TaskOrchestrator + + model = DiscoveredModel( + "openrouter", "cheap-model", "OPENROUTER_API_KEY", + "https://openrouter.ai/api/v1", "Bearer", is_free=True, + ) + db_path = str(tmp_path / "legacy.db") + seeded = TaskOrchestrator([], agents_db=db_path, allow_empty_agents=True) + seeded.sync_discovered_agents([ + replace(agent_from_discovered(model), id="openrouter_cheap_model") + ]) + seeded.close() + set_backend(InMemoryCredentialBackend()) + register_credential("OPENROUTER_API_KEY", "sk-router") + stdout = StringIO() + + try: + with ( + patch.object(sys, "argv", [ + "contextual-orchestrator", "discover-models", "--agents-db", db_path, + "--enable-cheapest", "1", + ]), + patch.object(sys, "stdout", stdout), + patch( + "contextual_orchestrator.model_discovery._open_trusted_discovery_request", + return_value=_Response({ + "data": [{ + "id": "cheap-model", + "pricing": {"prompt": "0", "completion": "0"}, + }] + }), + ), + ): + main() + finally: + set_backend(None) + + report = json.loads(stdout.getvalue()) + assert report["enabled_agent_ids"] == ["openrouter_cheap_model"] + reloaded = TaskOrchestrator([], agents_db=db_path, allow_empty_agents=True) + assert [agent.id for agent in reloaded.agents] == ["openrouter_cheap_model"] + assert all(agent.id != agent_id_for(model) for agent in reloaded.candidates) + reloaded.close() + + +def test_enable_cheapest_does_not_activate_matching_manual_agent(tmp_path) -> None: + from dataclasses import replace + from contextual_orchestrator import TaskOrchestrator + + model = DiscoveredModel( + "openrouter", "cheap-model", "OPENROUTER_API_KEY", + "https://openrouter.ai/api/v1", "Bearer", is_free=True, + ) + db_path = str(tmp_path / "manual-and-discovered.db") + seeded = TaskOrchestrator([], agents_db=db_path, allow_empty_agents=True) + legacy = replace(agent_from_discovered(model), id="openrouter_cheap_model") + seeded.sync_discovered_agents([legacy]) + manual = replace(legacy, id="manual_cheap_model", tags=("operator",)) + seeded._pool_store.save(manual) + seeded.close() + set_backend(InMemoryCredentialBackend()) + register_credential("OPENROUTER_API_KEY", "sk-router") + stdout = StringIO() + + try: + with ( + patch.object(sys, "argv", [ + "contextual-orchestrator", "discover-models", "--agents-db", db_path, + "--enable-cheapest", "1", + ]), + patch.object(sys, "stdout", stdout), + patch( + "contextual_orchestrator.model_discovery._open_trusted_discovery_request", + return_value=_Response({"data": [{ + "id": "cheap-model", "pricing": {"prompt": "0", "completion": "0"}, + }]}), + ), + ): + main() + finally: + set_backend(None) + + report = json.loads(stdout.getvalue()) + assert report["enabled_agent_ids"] == [legacy.id] + reloaded = TaskOrchestrator([], agents_db=db_path, allow_empty_agents=True) + by_id = {agent.id: agent for agent in reloaded.candidates} + assert by_id[legacy.id].disabled is False + assert by_id[manual.id].disabled is True + reloaded.close() + + +def test_enable_cheapest_preserves_manual_legacy_id_without_crashing( + tmp_path, +) -> None: + """A manual owner of the old generated ID remains intact without a crash.""" + from dataclasses import replace + from contextual_orchestrator import TaskOrchestrator + + model = DiscoveredModel( + "openrouter", + "cheap-model", + "OPENROUTER_API_KEY", + "https://openrouter.ai/api/v1", + "Bearer", + is_free=True, + ) + db_path = str(tmp_path / "manual-legacy-collision.db") + manual = replace( + agent_from_discovered(model), + id="openrouter_cheap_model", + tags=("operator",), + ) + seeded = TaskOrchestrator([], agents_db=db_path, allow_empty_agents=True) + assert seeded._pool_store is not None + seeded._pool_store.save(manual) + seeded.close() + set_backend(InMemoryCredentialBackend()) + register_credential("OPENROUTER_API_KEY", "sk-router") + stdout = StringIO() + + try: + with ( + patch.object( + sys, + "argv", + [ + "contextual-orchestrator", + "discover-models", + "--agents-db", + db_path, + "--enable-cheapest", + "1", + ], + ), + patch.object(sys, "stdout", stdout), + patch( + "contextual_orchestrator.model_discovery._open_trusted_discovery_request", + return_value=_Response( + { + "data": [ + { + "id": "cheap-model", + "pricing": {"prompt": "0", "completion": "0"}, + } + ] + } + ), + ), + ): + main() + finally: + set_backend(None) + + assert json.loads(stdout.getvalue())["enabled_agent_ids"] == [] + reloaded = TaskOrchestrator([], agents_db=db_path, allow_empty_agents=True) + by_id = {agent.id: agent for agent in reloaded.candidates} + assert by_id[manual.id] == manual + assert list(by_id) == [manual.id] + reloaded.close() def test_enable_cheapest_bootstraps_independent_provider_accounts(tmp_path) -> None: @@ -467,9 +637,12 @@ def urlopen(request, timeout=None, **_kwargs): report = json.loads(stdout.getvalue()) assert report["enabled_agent_ids"] == [ - "openrouter_router_model", - "nvidia_nim_nim_model", - "openai_openai_model", + agent_id_for(DiscoveredModel(provider, model, credential, url, "Bearer")) + for provider, model, credential, url in ( + ("openrouter", "router-model", "OPENROUTER_API_KEY", "https://openrouter.ai/api/v1"), + ("nvidia_nim", "nim-model", "NVIDIA_NIM_API_KEY", "https://integrate.api.nvidia.com/v1"), + ("openai", "openai-model", "OPENAI_API_KEY", "https://api.openai.com/v1"), + ) ] reloaded = TaskOrchestrator([ModelAgent("seed_agent", "seed-model")], agents_db=db_path) enabled = {agent.id for agent in reloaded.candidates if not agent.disabled} diff --git a/tests/test_discovery_bootstrap_selection.py b/tests/test_discovery_bootstrap_selection.py index 2070e9ed7..dfea65707 100644 --- a/tests/test_discovery_bootstrap_selection.py +++ b/tests/test_discovery_bootstrap_selection.py @@ -304,8 +304,8 @@ def test_conflicting_duplicate_prices_are_withheld_as_ambiguous() -> None: ) == [complete] -def test_bootstrap_selector_prefers_provider_diversity_before_duplicates() -> None: - """The initial failover pool must span providers before repeating one.""" +def test_bootstrap_selector_rejects_unmodeled_model_group_diversity() -> None: + """Provider/model diversity cannot substitute for an outage utility model.""" selector = getattr( model_discovery, "select_bootstrap_discovered_agents", @@ -315,21 +315,48 @@ def test_bootstrap_selector_prefers_provider_diversity_before_duplicates() -> No price_book = PriceBook(InMemoryConfigStore()) router_cheapest = _model("openrouter", "router-cheapest") - router_second = _model("openrouter", "router-second") + router_second = _model("openrouter", "shared-model") + nim_duplicate = _model("nvidia_nim", "shared-model") nim_model = _model("nvidia_nim", "nim-model") openai_model = _model("openai", "openai-model") _set_price(price_book, router_cheapest, 0.01) _set_price(price_book, router_second, 0.02) + _set_price(price_book, nim_duplicate, 0.03) _set_price(price_book, nim_model, 0.5) _set_price(price_book, openai_model, 1.0) - selected = selector( - [router_second, openai_model, nim_model, router_cheapest], - price_book, - 3, - ) + with pytest.raises(ValueError, match="decision model"): + selector( + [ + router_second, + nim_duplicate, + openai_model, + nim_model, + router_cheapest, + ], + price_book, + 3, + ) - assert selected == [router_cheapest, nim_model, openai_model] + +def test_bootstrap_selector_rejects_unmodeled_provider_diversity() -> None: + """A provider label alone cannot justify displacing lower-cost evidence.""" + price_book = PriceBook(InMemoryConfigStore()) + router_a = _model("openrouter", "router-a") + router_b = _model("openrouter", "router-b") + router_c = _model("openrouter", "router-c") + openai_model = _model("openai", "openai-only") + _set_price(price_book, router_a, 0.01) + _set_price(price_book, router_b, 0.02) + _set_price(price_book, router_c, 0.03) + _set_price(price_book, openai_model, 0.5) + + with pytest.raises(ValueError, match="decision model"): + model_discovery.select_bootstrap_discovered_agents( + [router_a, router_b, router_c, openai_model], + price_book, + 2, + ) def test_bootstrap_selector_keeps_nim_primary_and_sub_credential_accounts_independent() -> None: @@ -366,6 +393,29 @@ def test_bootstrap_selector_keeps_nim_primary_and_sub_credential_accounts_indepe assert selected == [nim_primary, nim_sub] +def test_bootstrap_selector_falls_back_to_duplicate_model_group_when_capacity_remains() -> None: + """Genuine duplicate model-group endpoints still fill leftover capacity. + + Once every provider has contributed and every distinct model group is + exhausted, a still-open slot falls back to a second endpoint for a model + group already selected (a real failover path for that one model, not a + new independently-failing provider) rather than leaving capacity idle. + """ + price_book = PriceBook(InMemoryConfigStore()) + router_shared = _model("openrouter", "shared-model") + nim_shared = _model("nvidia_nim", "shared-model") + _set_price(price_book, router_shared, 0.01) + _set_price(price_book, nim_shared, 0.02) + + selected = model_discovery.select_bootstrap_discovered_agents( + [nim_shared, router_shared], + price_book, + 2, + ) + + assert selected == [router_shared, nim_shared] + + def test_bootstrap_selector_is_deterministic_when_every_model_is_unpriced() -> None: """All-unpriced discovery remains usable but never order-dependent.""" selector = getattr( diff --git a/tests/test_embeddings_model_pool_http_honesty.py b/tests/test_embeddings_model_pool_http_honesty.py index 80e12fd02..ffcc6839c 100644 --- a/tests/test_embeddings_model_pool_http_honesty.py +++ b/tests/test_embeddings_model_pool_http_honesty.py @@ -105,6 +105,244 @@ def test_select_capability_agent_skips_disabled_and_excluded_agents() -> None: raise AssertionError("an unavailable capability must fail closed") +def test_embedding_capability_skips_circuit_open_endpoint_until_half_open_probe() -> None: + """Repeated endpoint failures quarantine one member without losing recovery.""" + first = ModelAgent("first_embedding", "embed-v1", tags=("embedding",)) + second = ModelAgent("second_embedding", "embed-v2", tags=("embedding",)) + orchestrator = TaskOrchestrator([first, second]) + + for _ in range(orchestrator.circuit_failure_threshold): + orchestrator._record_failure(first.id) + + assert [agent.id for agent in orchestrator._capability_agents("embedding")] == [ + second.id + ] + + for _ in range(orchestrator.circuit_failure_threshold): + orchestrator._record_failure(second.id) + + try: + orchestrator._capability_agents("embedding") + except RuntimeError as exc: + assert str(exc) == "all enabled agents temporarily unavailable for capability=embedding" + else: + raise AssertionError("open embedding circuits must remain quarantined until cooldown") + + +def test_embedding_request_too_large_does_not_quarantine_endpoint() -> None: + """A caller-specific 413 remains observable without penalizing endpoint health.""" + agent = ModelAgent("embedding_agent", "embed-v1", tags=("embedding",)) + orchestrator = TaskOrchestrator([agent]) + + for _ in range(orchestrator.circuit_failure_threshold + 1): + orchestrator._record_embedding_failure( + agent, + "/v1/embeddings", + urllib.error.HTTPError("https://provider.invalid", 413, "too large", {}, None), + ) + + assert orchestrator._circuit_open(agent.id) is False + failures = [ + event + for event in orchestrator._analytics_events + if event["event_name"] == "embedding_endpoint_failed" + ] + assert all(event["event_detail"]["provider_status"] == 413 for event in failures) + + +def test_explicit_embedding_model_returns_503_while_all_circuits_are_open() -> None: + agent = ModelAgent("embedding_agent", "embed-v1", tags=("embedding",)) + orchestrator = TaskOrchestrator([agent]) + for _ in range(orchestrator.circuit_failure_threshold): + orchestrator._record_failure(agent.id) + server = build_server( + orchestrator, + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN), + coordinator=CostRoutingCoordinator(orchestrator), + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + for path in ("/v1/embeddings", "/v1/batch/embeddings"): + status, body = _post( + server.server_address[1], + path, + {"model": agent.model, "input": "invoice search chunk"}, + ) + assert status == 503, body + assert body["error"]["code"] == "embeddings_unavailable" + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_quarantines_repeated_400_endpoint_with_safe_evidence() -> None: + """Stop selecting one repeatedly rejected endpoint and retain safe diagnostics. + + The `/v1/embeddings` failover order is now cost-ordered (see + `CostRoutingCoordinator._cost_ordered_capability_candidates`): a single + recorded failure already drops the failing member's measured health + below the healthy threshold, demoting it behind every still-healthy + candidate on the very next request -- a strictly faster, price-aware + replacement for the older fixed `circuit_failure_threshold`-strikes + breaker this test originally pinned. The endpoint is still quarantined + (never selected again once a healthy alternative exists) and the + recorded failure evidence stays exception-message-free either way. + """ + first = ModelAgent( + "rejected_embedding", "embed-v1", tags=("embedding",), priority=1 + ) + second = ModelAgent("healthy_embedding", "embed-v1", tags=("embedding",)) + orchestrator = TaskOrchestrator([first, second]) + coordinator = CostRoutingCoordinator(orchestrator) + attempted: list[str] = [] + + def complete_embeddings_batch(_inputs, *, agent_id, **_kwargs): + attempted.append(agent_id) + if agent_id == first.id: + raise urllib.error.HTTPError( + "https://provider.invalid/embeddings", 400, "Bad Request", {}, None + ) + return { + "status": "completed", + "embeddings": [{"index": 0, "embedding": [0.25]}], + "total_tokens": 1, + } + + coordinator.complete_embeddings_batch = complete_embeddings_batch + server = build_server( + orchestrator, + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN), + coordinator=coordinator, + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + for _ in range(orchestrator.circuit_failure_threshold + 1): + status, body = _post( + server.server_address[1], + "/v1/embeddings", + {"model": "embed-v1", "input": "invoice search chunk"}, + ) + assert status == 200, body + + # The first request attempts the rejected member once (demoting it), + # then falls back to the healthy member; every later request in this + # loop selects only the now-healthier member and never retries the + # quarantined one. + assert attempted == [first.id] + [second.id] * (orchestrator.circuit_failure_threshold + 1) + failures = [ + event + for event in orchestrator._analytics_events + if event["event_name"] == "embedding_endpoint_failed" + ] + assert len(failures) == 1 + assert all(event["event_detail"]["provider_status"] == 400 for event in failures) + assert all("provider.invalid" not in json.dumps(event) for event in failures) + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_http_embeddings_quarantines_repeated_incomplete_document_with_failure_evidence() -> None: + """A non-completed/embedding-less sync result must quarantine too. + + Only a *raised* member exception previously reached + ``orchestrator._record_embedding_failure``; a synchronous document that + comes back without raising (``status`` not ``completed``, or + ``embeddings`` is ``None``) bypassed it, so the circuit breaker never + opened and no ``embedding_endpoint_failed`` analytics event was ever + recorded for that failure mode -- a repeatedly incomplete member would + be retried forever. Mirrors + ``test_http_embeddings_quarantines_repeated_400_endpoint_with_safe_evidence`` + but fails the first member by returning an incomplete document instead + of raising. + """ + first = ModelAgent( + "incomplete_embedding", "embed-v1", tags=("embedding",), priority=1 + ) + second = ModelAgent("healthy_embedding", "embed-v1", tags=("embedding",)) + orchestrator = TaskOrchestrator([first, second]) + coordinator = CostRoutingCoordinator(orchestrator) + attempted: list[str] = [] + + def complete_embeddings_batch(_inputs, *, agent_id, **_kwargs): + attempted.append(agent_id) + if agent_id == first.id: + return {"status": "failed", "embeddings": None} + return { + "status": "completed", + "embeddings": [{"index": 0, "embedding": [0.25]}], + "total_tokens": 1, + } + + coordinator.complete_embeddings_batch = complete_embeddings_batch + server = build_server( + orchestrator, + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN), + coordinator=coordinator, + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + for _ in range(orchestrator.circuit_failure_threshold + 1): + status, body = _post( + server.server_address[1], + "/v1/embeddings", + {"model": "embed-v1", "input": "invoice search chunk"}, + ) + assert status == 200, body + + # Same demotion contract as the raised-exception case: one recorded + # failure already drops the failing member behind the healthy one. + assert attempted == [first.id] + [second.id] * (orchestrator.circuit_failure_threshold + 1) + failures = [ + event + for event in orchestrator._analytics_events + if event["event_name"] == "embedding_endpoint_failed" + ] + assert len(failures) == 1 + assert failures[0]["event_detail"]["agent_id"] == first.id + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_batch_embedding_submission_success_clears_prior_endpoint_failures() -> None: + """An accepted asynchronous job proves the endpoint is responsive.""" + agent = ModelAgent("embedding_agent", "embed-v1", tags=("embedding",)) + orchestrator = TaskOrchestrator([agent]) + orchestrator._record_failure(agent.id) + coordinator = CostRoutingCoordinator(orchestrator) + coordinator.complete_embeddings_batch = lambda *_args, **_kwargs: { + "status": "validating", + "batch_id": "batch_accepted", + "backend": "remote", + } + server = build_server( + orchestrator, + port=0, + security=SecurityConfig(auth_token=_TEST_AUTH_TOKEN), + coordinator=coordinator, + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + status, body = _post( + server.server_address[1], + "/v1/batch/embeddings", + {"model": "embed-v1", "input": "invoice search chunk"}, + ) + assert status == 202, body + assert agent.id not in orchestrator._circuit + finally: + server.shutdown() + thread.join(timeout=5) + + def test_http_embeddings_rejects_model_outside_agent_pool() -> None: server, thread, port = _server() try: diff --git a/tests/test_endpoint_race.py b/tests/test_endpoint_race.py index b1f859ad2..70601f23d 100644 --- a/tests/test_endpoint_race.py +++ b/tests/test_endpoint_race.py @@ -2,6 +2,7 @@ import threading import time +from concurrent.futures import ALL_COMPLETED, wait as wait_futures from contextvars import ContextVar import pytest @@ -12,6 +13,7 @@ EndpointEquivalenceContract, race_first_valid, ) +from contextual_orchestrator.orchestrator import _ProviderRequestCancelled def contract(**changes: object) -> EndpointEquivalenceContract: @@ -57,6 +59,179 @@ def slow() -> str: assert outcome.cancellation_outcomes == (("primary_endpoint", "safe_drain"),) +def test_winner_cancels_and_reaps_running_loser_without_generation_deadline() -> None: + release = threading.Event() + loser_started = threading.Event() + + def loser() -> str: + loser_started.set() + release.wait() + return "late" + + outcome = race_first_valid( + [ + EndpointAttempt( + "slow_endpoint", contract(cancellation_supported=True), loser, + cancellation_supported=True, cancel=release.set, + ), + EndpointAttempt( + "fast_endpoint", contract(cancellation_supported=True), + lambda: loser_started.wait(1) and "complete", + ), + ], + validate=bool, + deadline_seconds=None, + max_concurrency=2, + ) + + assert outcome.cancellation_outcomes == (("slow_endpoint", "cancellation_requested"),) + deadline = time.monotonic() + 1 + while any(thread.name.startswith("equivalent_endpoint_race") for thread in threading.enumerate()): + assert time.monotonic() < deadline + time.sleep(0.01) + + +def test_winner_requests_running_loser_cancellation_exactly_once() -> None: + release = threading.Event() + started = threading.Event() + calls = 0 + + def slow() -> str: + started.set() + release.wait(timeout=1) + return "slow" + + def cancel() -> None: + nonlocal calls + calls += 1 + if calls > 1: + raise AssertionError("cancellation callback repeated") + release.set() + + race_first_valid( + [ + EndpointAttempt( + "slow_endpoint", contract(cancellation_supported=True), slow, + cancellation_supported=True, cancel=cancel, + ), + EndpointAttempt( + "fast_endpoint", contract(cancellation_supported=True), + lambda: started.wait(1) and "fast", + ), + ], + validate=bool, + deadline_seconds=None, + max_concurrency=2, + ) + + assert calls == 1 + + +def test_throwing_loser_cancellation_cannot_discard_valid_winner() -> None: + started = threading.Event() + + def blocked() -> str: + started.set() + time.sleep(0.1) + return "late" + + outcome = race_first_valid( + [ + EndpointAttempt( + "slow_endpoint", contract(cancellation_supported=True), blocked, + cancellation_supported=True, + cancel=lambda: (_ for _ in ()).throw(OSError("close failed")), + ), + EndpointAttempt( + "fast_endpoint", contract(cancellation_supported=True), + lambda: started.wait(1) and "winner", + ), + ], + validate=bool, + deadline_seconds=None, + max_concurrency=2, + ) + + assert outcome.value == "winner" + assert outcome.cancellation_outcomes == (("slow_endpoint", "safe_drain"),) + + +def test_cancelled_race_attempt_does_not_penalize_provider() -> None: + orchestrator = TaskOrchestrator([ModelAgent("cancelled_endpoint", "shared/model")]) + + orchestrator._record_race_attempt( + "cancelled_endpoint", + None, + _ProviderRequestCancelled("cancelled"), + capability="text", + ) + + assert orchestrator._circuit == {} + assert orchestrator._group_router.member_report("cancelled_endpoint")["failure_count"] == 0 + event = orchestrator.list_recent_audit_events()[-1] + assert event["event_detail"]["validation_outcome"] == "cancelled" + + +def test_already_completed_loser_is_reported_as_completed(monkeypatch) -> None: + second_done = threading.Event() + + def first() -> str: + assert second_done.wait(timeout=1) + return "first" + + def second() -> str: + second_done.set() + return "second" + + monkeypatch.setattr( + "contextual_orchestrator.endpoint_race.wait", + lambda futures, **_kwargs: (wait_futures(futures, return_when=ALL_COMPLETED)[0], set()), + ) + outcome = race_first_valid( + [ + EndpointAttempt("first_endpoint", contract(), first), + EndpointAttempt("second_endpoint", contract(), second), + ], + validate=bool, + deadline_seconds=1, + max_concurrency=2, + ) + + assert outcome.winner_endpoint_id == "first_endpoint" + assert outcome.cancellation_outcomes == (("second_endpoint", "completed"),) + + +def test_unsupported_cancellation_is_safe_drain_without_callback() -> None: + release = threading.Event() + started = threading.Event() + cancellation_called = threading.Event() + + def slow() -> str: + started.set() + release.wait(timeout=1) + return "slow" + + outcome = race_first_valid( + [ + EndpointAttempt( + "slow_endpoint", contract(cancellation_supported=False), slow, + cancellation_supported=True, cancel=cancellation_called.set, + ), + EndpointAttempt( + "fast_endpoint", contract(cancellation_supported=False), + lambda: started.wait(1) and "fast", + ), + ], + validate=bool, + deadline_seconds=None, + max_concurrency=2, + ) + release.set() + + assert outcome.cancellation_outcomes == (("slow_endpoint", "safe_drain"),) + assert not cancellation_called.is_set() + + def test_fast_invalid_completion_does_not_suppress_valid_result() -> None: def valid() -> str: time.sleep(0.01) diff --git a/tests/test_endpoint_race_callback_settlement.py b/tests/test_endpoint_race_callback_settlement.py new file mode 100644 index 000000000..eeffaca9b --- /dev/null +++ b/tests/test_endpoint_race_callback_settlement.py @@ -0,0 +1,106 @@ +"""Regression tests for endpoint-race observer failure settlement.""" + +from __future__ import annotations + +import threading + +from contextual_orchestrator.endpoint_race import ( + EndpointAttempt, + EndpointEquivalenceContract, + race_first_valid, +) + + +def _contract() -> EndpointEquivalenceContract: + """Return one complete equivalence contract for callback-settlement tests.""" + return EndpointEquivalenceContract( + contract_id="callback_settlement_contract", + model_revision="revision_2026_09", + reasoning_effort_profile="worker_medium", + capability_set=("text",), + structured_output_contract="openai_response_v1", + accuracy_class="full_precision", + data_residency_policy="kr_region_only", + retention_policy="zero_retention", + context_limit=128_000, + pricing_evidence_id="catalog_snapshot_2026_09_02", + hedge_eligible=True, + cancellation_supported=False, + execution_policy="immediate_race", + ) + + +def _run_unbounded_race( + attempts: list[EndpointAttempt[str]], + callback, +) -> BaseException: + """Run a no-deadline race and prove it terminates even if observers fail.""" + completed = threading.Event() + observed: dict[str, BaseException] = {} + + def run() -> None: + try: + race_first_valid( + attempts, + validate=bool, + deadline_seconds=None, + max_concurrency=2, + on_attempt_complete=callback, + ) + except BaseException as exc: + observed["exception"] = exc + finally: + completed.set() + + thread = threading.Thread(target=run, daemon=True) + thread.start() + assert completed.wait(1), "callback failure left a no-deadline race unsettled" + assert "exception" in observed + return observed["exception"] + + +def test_success_observer_failure_settles_managed_future() -> None: + """A callback failure after a valid result must become a settled race failure.""" + shared = _contract() + + def callback(_endpoint: str, _value: str | None, _error: BaseException | None) -> None: + raise RuntimeError("observer failed after success") + + error = _run_unbounded_race( + [ + EndpointAttempt("first_endpoint", shared, lambda: "first"), + EndpointAttempt("second_endpoint", shared, lambda: "second"), + ], + callback, + ) + + assert isinstance(error, RuntimeError) + assert isinstance(error.__cause__, RuntimeError) + assert str(error.__cause__) == "observer failed after success" + + +def test_failure_observer_failure_settles_managed_future() -> None: + """A callback failure while reporting a provider error must not strand the race.""" + shared = _contract() + provider_errors: list[type[BaseException]] = [] + + def provider_failure() -> str: + raise OSError("provider failed") + + def callback(_endpoint: str, _value: str | None, error: BaseException | None) -> None: + assert error is not None + provider_errors.append(type(error)) + raise LookupError("observer failed while reporting provider failure") + + error = _run_unbounded_race( + [ + EndpointAttempt("first_endpoint", shared, provider_failure), + EndpointAttempt("second_endpoint", shared, provider_failure), + ], + callback, + ) + + assert provider_errors == [OSError, OSError] + assert isinstance(error, RuntimeError) + assert isinstance(error.__cause__, LookupError) + assert str(error.__cause__) == "observer failed while reporting provider failure" diff --git a/tests/test_endpoint_race_process_exit.py b/tests/test_endpoint_race_process_exit.py new file mode 100644 index 000000000..5240cea00 --- /dev/null +++ b/tests/test_endpoint_race_process_exit.py @@ -0,0 +1,110 @@ +"""Regression: an uncancellable losing race participant must not block exit.""" + +from __future__ import annotations + +import subprocess +import sys +import textwrap +import time +from pathlib import Path + + +def test_uncancellable_loser_never_returning_does_not_block_process_exit() -> None: + """A losing race attempt stuck in an unbounded call must not join at shutdown. + + Regression for a Devin Review finding ("Endpoint races block process + shutdown", ContextualWisdomLab/contextual-orchestrator#971): with this + org's default no-deadline ``ModelClient.timeout=None`` policy, + ``race_first_valid`` used to fan its attempts out across a + ``concurrent.futures.ThreadPoolExecutor``. That executor's worker + threads register with ``concurrent.futures.thread``'s own + interpreter-exit hook, which unconditionally *joins* every + still-running worker at shutdown regardless of that worker's own + daemon status. A losing attempt whose equivalence contract does not + support cancellation (``cancellation_supported=False``, the + ``cancel_loser`` "safe_drain" path) and whose blocking call never + returns would therefore hang the whole process at exit, forever, even + though the winner already answered the caller and ``race_first_valid`` + already returned. ``race_first_valid`` now drives each attempt from a + plain ``threading.Thread(daemon=True)`` -- never a + ``ThreadPoolExecutor`` -- so an abandoned loser carries no such + registration and the process can still exit. + + Verified end-to-end in a real, separate interpreter (an in-process + thread-introspection assertion cannot distinguish "still hanging in + the background" from "would actually block this process's shutdown" + -- the whole point of the finding): a helper script calls + ``race_first_valid`` with one attempt that blocks on an ``Event`` that + is never set and one that returns immediately, then -- after + ``race_first_valid`` has already returned the winner -- lets the + script's ``__main__`` fall through to a normal, unforced exit with no + explicit ``sys.exit()``/``os._exit()``. RED-before/GREEN-after against + the pre-fix ``ThreadPoolExecutor`` version: the same script hung for + the full outer bound and was killed; it exits cleanly, well under that + bound, with this fix. + """ + script = textwrap.dedent( + """ + import sys + import threading + + sys.path.insert(0, %(repo_root)r) + from contextual_orchestrator.endpoint_race import ( + EndpointAttempt, + EndpointEquivalenceContract, + race_first_valid, + ) + + never_set = threading.Event() + + def hung_loser(): + never_set.wait() # Hangs forever -- nothing ever sets this event. + raise AssertionError("unreachable: the stalled loser must never return") + + def fast_winner(): + return "winner" + + contract = EndpointEquivalenceContract( + contract_id="process_exit_regression", + model_revision="revision_2026_09", + reasoning_effort_profile="worker_medium", + capability_set=("text",), + structured_output_contract="openai_response_v1", + accuracy_class="full_precision", + data_residency_policy="kr_region_only", + retention_policy="zero_retention", + context_limit=128_000, + pricing_evidence_id="catalog_snapshot_2026_09_02", + hedge_eligible=True, + cancellation_supported=False, + execution_policy="immediate_race", + ) + + outcome = race_first_valid( + [ + EndpointAttempt("stuck_endpoint", contract, hung_loser), + EndpointAttempt("fast_endpoint", contract, fast_winner), + ], + validate=bool, + deadline_seconds=None, + max_concurrency=2, + ) + assert outcome.value == "winner" + assert outcome.cancellation_outcomes == (("stuck_endpoint", "safe_drain"),) + # No explicit sys.exit()/os._exit(): a genuinely non-blocking fix + # must let normal interpreter shutdown proceed on its own, with the + # hung loser thread still blocked in the background. + """ + ) % {"repo_root": str(Path(__file__).resolve().parents[1])} + + started = time.monotonic() + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + timeout=15, + ) + elapsed = time.monotonic() - started + + assert result.returncode == 0, result.stderr + assert elapsed < 5.0, f"process took {elapsed:.1f}s to exit with an uncancellable loser outstanding" diff --git a/tests/test_endpoint_race_terminal_provenance.py b/tests/test_endpoint_race_terminal_provenance.py new file mode 100644 index 000000000..4be081e0c --- /dev/null +++ b/tests/test_endpoint_race_terminal_provenance.py @@ -0,0 +1,62 @@ +"""Regression coverage for exact terminal provenance in endpoint races.""" + +from __future__ import annotations + +import threading +from concurrent.futures import ALL_COMPLETED, wait as wait_futures + +from contextual_orchestrator.endpoint_race import ( + EndpointAttempt, + EndpointEquivalenceContract, + race_first_valid, +) + + +def _contract() -> EndpointEquivalenceContract: + """Return one complete equivalence contract shared by both attempts.""" + return EndpointEquivalenceContract( + contract_id="terminal_provenance_contract", + model_revision="revision_2026_09", + reasoning_effort_profile="worker_medium", + capability_set=("text",), + structured_output_contract="openai_response_v1", + accuracy_class="full_precision", + data_residency_policy="kr_region_only", + retention_policy="zero_retention", + context_limit=128_000, + pricing_evidence_id="catalog_snapshot_2026_09_02", + hedge_eligible=True, + cancellation_supported=False, + execution_policy="immediate_race", + ) + + +def test_failed_completed_loser_is_not_reported_as_successful_completion(monkeypatch) -> None: + """RaceOutcome must preserve a loser's failed terminal state after it finishes.""" + loser_finished = threading.Event() + + def winner() -> str: + assert loser_finished.wait(timeout=1) + return "winner" + + def loser() -> str: + loser_finished.set() + raise RuntimeError("synthetic provider failure") + + monkeypatch.setattr( + "contextual_orchestrator.endpoint_race.wait", + lambda futures, **_kwargs: (wait_futures(futures, return_when=ALL_COMPLETED)[0], set()), + ) + + outcome = race_first_valid( + [ + EndpointAttempt("winner_endpoint", _contract(), winner), + EndpointAttempt("failed_endpoint", _contract(), loser), + ], + validate=bool, + deadline_seconds=1, + max_concurrency=2, + ) + + assert outcome.winner_endpoint_id == "winner_endpoint" + assert outcome.cancellation_outcomes == (("failed_endpoint", "failed"),) diff --git a/tests/test_hourly_opencode_loop_contract.py b/tests/test_hourly_opencode_loop_contract.py index af0d4cdfb..a5beb61a3 100644 --- a/tests/test_hourly_opencode_loop_contract.py +++ b/tests/test_hourly_opencode_loop_contract.py @@ -24,6 +24,20 @@ def test_hourly_loop_uses_the_local_free_orchestrator_without_copilot_token() -> assert "COPILOT_GITHUB_TOKEN" not in workflow assert "node scripts/ci/install_locked_opencode.mjs" in workflow assert "python -m pip install --require-hashes -r requirements.lock" in workflow + assert "while :; do" in workflow + assert "gateway_pid=$!" in workflow + assert 'kill -0 "$gateway_pid"' in workflow + assert "gateway exited before becoming healthy" in workflow + loop_header = workflow.split(" loop:\n", 1)[1].split(" steps:\n", 1)[0] + gateway_step = workflow.split( + " - name: Start the contextual-orchestrator gateway with auto-discovery\n", 1 + )[1].split(" - name:", 1)[0] + maintenance_step = workflow.split( + " - name: Run the hourly loop agent\n", 1 + )[1].split(" - name:", 1)[0] + assert "timeout-minutes" not in loop_header + assert "timeout-minutes" not in gateway_step + assert "timeout-minutes" not in maintenance_step assert "--auth-token-key CONTEXTUAL_ORCHESTRATOR_TOKEN" in workflow assert "--auth-token=" not in workflow assert "--auth-token " not in workflow @@ -114,3 +128,4 @@ def test_adr_0007_body_and_index_status_agree_and_stay_proposed_while_open() -> "ADR-0007 is unmerged: both body and index status must be `Proposed` " "until ordinary protected-branch merge grants exact-head acceptance" ) + diff --git a/tests/test_local_mlx.py b/tests/test_local_mlx.py index 66acb48aa..ef5e22888 100644 --- a/tests/test_local_mlx.py +++ b/tests/test_local_mlx.py @@ -2,12 +2,14 @@ from __future__ import annotations +import errno import json import socket import sys +import threading import urllib.request from pathlib import Path -from unittest.mock import patch +from unittest.mock import Mock, patch import pytest @@ -17,6 +19,9 @@ from contextual_orchestrator.credentials import NotConfigured # noqa: E402 from contextual_orchestrator.orchestrator import ( # noqa: E402 ModelClient, + _ProviderCancellation, + _PROVIDER_DNS_SLOTS, + _ProviderRequestCancelled, _chat_to_responses_payload, _is_local_provider_url, _responses_to_chat_payload, @@ -152,7 +157,7 @@ def test_local_gateway_credential_cannot_be_attached_to_mlx_worker() -> None: ) -def test_provider_probe_verifies_registry_then_uses_one_bounded_completion_without_retry() -> None: +def test_provider_probe_leaves_registry_and_model_inference_unbounded() -> None: agent = ModelAgent("local_agent", "local-model", base_url="mlx://127.0.0.1:8080/v1") client = ModelClient(max_retries=2, local_max_retries=2, chat_template_args={"enable_thinking": False}) seen: list[tuple[object, float | None]] = [] @@ -167,15 +172,16 @@ def open_provider(request, _destination=None, *, timeout=None): }) with patch.object(client, "_open_provider", side_effect=open_provider): - report = client.probe(agent, timeout=1.25) + report = client.probe(agent) assert report["status"] == "ready" assert report["usage"]["total_tokens"] == 7 assert len(seen) == 2 assert seen[0][0].get_method() == "GET" assert seen[0][0].full_url == "http://127.0.0.1:8080/v1/models" + assert seen[0][1] is None assert seen[1][0].get_method() == "POST" - assert seen[1][1] == 1.25 + assert seen[1][1] is None import json payload = json.loads(seen[1][0].data) @@ -184,6 +190,15 @@ def open_provider(request, _destination=None, *, timeout=None): assert payload["chat_template_kwargs"] == {"enable_thinking": False} +def test_legacy_timeout_argument_positions_remain_compatible() -> None: + client = ModelClient(0.25, 321, 3, connect_timeout=0.5) + + assert client.timeout == 0.25 + assert client.connect_timeout == 0.5 + assert client.max_output_tokens == 321 + assert client.max_retries == 3 + + def test_provider_probe_rejects_a_local_model_registry_mismatch() -> None: agent = ModelAgent("local_agent", "requested-model", base_url="mlx://127.0.0.1:8080/v1") client = ModelClient(max_retries=0) @@ -192,7 +207,7 @@ def test_provider_probe_rejects_a_local_model_registry_mismatch() -> None: "_open_provider", return_value=_Response({"object": "list", "data": [{"id": "other-model"}]}), ) as open_provider: - report = client.probe(agent, timeout=0.5) + report = client.probe(agent) assert report["status"] == "not_ready" assert report["error_type"] == "RuntimeError" @@ -205,7 +220,7 @@ def test_provider_probe_reports_timeout_without_retry() -> None: agent = ModelAgent("local_agent", "local-model", base_url="mlx://127.0.0.1:8080/v1") client = ModelClient(max_retries=2, local_max_retries=2) with patch.object(client, "_open_provider", side_effect=TimeoutError("probe timeout")) as open_provider: - report = client.probe(agent, timeout=0.5) + report = client.probe(agent) assert report["status"] == "not_ready" assert report["error_type"] == "TimeoutError" @@ -218,7 +233,7 @@ def test_provider_probe_does_not_serialize_provider_exception_text() -> None: agent = ModelAgent("local_agent", "local-model", base_url="mlx://127.0.0.1:8080/v1") client = ModelClient(max_retries=0) with patch.object(client, "_open_provider", side_effect=RuntimeError("provider-output-secret")): - report = client.probe(agent, timeout=0.5) + report = client.probe(agent) serialized = json.dumps(report) assert "provider-output-secret" not in serialized @@ -234,14 +249,14 @@ def test_provider_readiness_report_keeps_liveness_unprobed_until_refresh() -> No ], client=client) with patch.object(client, "probe", return_value={"status": "ready", "agent_id": "ready_agent", "model": "ready-model"}) as probe: unprobed = orchestrator.provider_readiness_report() - refreshed = orchestrator.provider_readiness_report(refresh=True, timeout=2.0) + refreshed = orchestrator.provider_readiness_report(refresh=True) assert unprobed["status"] == "unprobed" assert unprobed["items"][0]["status"] == "unprobed" assert refreshed["status"] == "ready" assert refreshed["ready_agent_count"] == 1 assert refreshed["items"][1]["status"] == "disabled" - probe.assert_called_once_with(orchestrator.agents[0], timeout=2.0) + probe.assert_called_once_with(orchestrator.agents[0]) def test_provider_readiness_refresh_serializes_concurrent_probes() -> None: @@ -252,10 +267,10 @@ def test_provider_readiness_refresh_serializes_concurrent_probes() -> None: entered = threading.Event() release = threading.Event() counters = {"active": 0, "max_active": 0} + reports = [] counter_lock = threading.Lock() - def probe(_agent, *, timeout): - del timeout + def probe(_agent): with counter_lock: counters["active"] += 1 counters["max_active"] = max(counters["max_active"], counters["active"]) @@ -267,7 +282,9 @@ def probe(_agent, *, timeout): with patch.object(client, "probe", side_effect=probe): first = threading.Thread(target=lambda: orchestrator.provider_readiness_report(refresh=True)) - second = threading.Thread(target=lambda: orchestrator.provider_readiness_report(refresh=True)) + second = threading.Thread( + target=lambda: reports.append(orchestrator.provider_readiness_report(refresh=True)) + ) first.start() assert entered.wait(timeout=2) second.start() @@ -276,9 +293,17 @@ def probe(_agent, *, timeout): second.join(timeout=2) assert counters["max_active"] == 1 + assert reports == [{ + "status": "refresh_in_progress", + "probe": "refresh", + "checked_at": None, + "agent_count": 1, + "ready_agent_count": 0, + "items": [], + }] -def test_local_provider_serializes_model_switches_and_bounds_waiters() -> None: +def test_local_provider_serializes_model_switches_and_bounds_explicit_waiters() -> None: import threading first_agent = ModelAgent("first_agent", "model-a", base_url="mlx://127.0.0.1:8080/v1") @@ -655,6 +680,10 @@ def __init__(self, *args, **kwargs): self.args = args self.kwargs = kwargs self.request_args = None + self.sock = None + + def connect(self): + self.sock = Mock() def request(self, *args, **kwargs): self.request_args = (args, kwargs) @@ -679,11 +708,256 @@ def close(self): assert response.status == 200 https_connection.assert_called_once_with( - "provider.example", 443, timeout=client.timeout, context=client._ssl_context + "provider.example", 443, timeout=None, context=client._ssl_context ) + connection.sock.settimeout.assert_called_once_with(None) assert connection.request_args[0][0] == "POST" +def test_cancellable_provider_call_closes_blocked_transport_without_generation_timeout() -> None: + entered = threading.Event() + released = threading.Event() + + class Socket: + def settimeout(self, value): + assert value is None + + def shutdown(self, _how): + released.set() + + class Connection: + def __init__(self, *_args, **_kwargs): + self.sock = Socket() + self.closed = False + + def connect(self): + return None + + def request(self, *_args, **_kwargs): + return None + + def getresponse(self): + entered.set() + assert released.wait(timeout=1) + raise OSError("cancelled") + + def close(self): + self.closed = True + + client = ModelClient(timeout=None) + connection = Connection() + request = urllib.request.Request("http://provider.example/v1/chat/completions") + call, cancel = client.cancellable_call( + lambda: client._open_provider(request, (socket.AF_INET, ("127.0.0.1", 80))) + ) + errors = [] + + def run(): + try: + call() + except _ProviderRequestCancelled as exc: + errors.append(exc) + + with patch( + "contextual_orchestrator.orchestrator.http.client.HTTPConnection", + return_value=connection, + ): + thread = threading.Thread(target=run) + thread.start() + assert entered.wait(timeout=1) + cancel() + thread.join(timeout=1) + + assert not thread.is_alive() + assert errors + assert connection.closed + + +def test_cancellable_provider_call_stops_nonblocking_tcp_establishment() -> None: + entered = threading.Event() + + class Socket: + closed = False + + def setblocking(self, _enabled): + return None + + def connect_ex(self, _sockaddr): + entered.set() + return errno.EINPROGRESS + + def close(self): + self.closed = True + + client = ModelClient() + connection = Socket() + call, cancel = client.cancellable_call( + lambda: client._connect_validated( + (socket.AF_INET, ("127.0.0.1", 443)), None, None + ) + ) + errors = [] + + def run(): + try: + call() + except _ProviderRequestCancelled as exc: + errors.append(exc) + + with patch("contextual_orchestrator.orchestrator.socket.socket", return_value=connection), patch( + "contextual_orchestrator.orchestrator.select.select", return_value=((), (), ()) + ): + thread = threading.Thread(target=run) + thread.start() + assert entered.wait(timeout=1) + cancel() + thread.join(timeout=1) + + assert not thread.is_alive() + assert errors + assert connection.closed + + +def test_cancellable_tcp_establishment_honors_explicit_caller_deadline() -> None: + class Socket: + def setblocking(self, _enabled): + return None + + def connect_ex(self, _sockaddr): + return errno.EINPROGRESS + + def close(self): + return None + + client = ModelClient() + call, _cancel = client.cancellable_call( + lambda: client._connect_validated( + (socket.AF_INET, ("127.0.0.1", 443)), 0.01, None + ) + ) + with patch("contextual_orchestrator.orchestrator.socket.socket", return_value=Socket()), patch( + "contextual_orchestrator.orchestrator.select.select", return_value=((), (), ()) + ): + with pytest.raises(TimeoutError, match="explicit deadline"): + call() + + +def test_cancellable_provider_call_stops_waiting_for_dns() -> None: + entered = threading.Event() + released = threading.Event() + + def resolve(*_args, **_kwargs): + entered.set() + released.wait(timeout=1) + return [] + + client = ModelClient() + call, cancel = client.cancellable_call( + lambda: client._resolve_addresses("provider.example", 443) + ) + errors = [] + + def run(): + try: + call() + except _ProviderRequestCancelled as exc: + errors.append(exc) + + with patch("contextual_orchestrator.orchestrator.socket.getaddrinfo", side_effect=resolve): + thread = threading.Thread(target=run) + thread.start() + assert entered.wait(timeout=1) + cancel() + thread.join(timeout=1) + released.set() + + assert not thread.is_alive() + assert errors + + +def test_cancelled_dns_lookups_have_bounded_worker_ownership() -> None: + all_workers_entered = threading.Event() + release = threading.Event() + lock = threading.Lock() + calls = 0 + + def resolve(*_args, **_kwargs): + nonlocal calls + with lock: + calls += 1 + if calls == 4: + all_workers_entered.set() + release.wait(timeout=2) + return [] + + request_threads = [] + cancels = [] + errors = [] + + def start_request(): + call, cancel = ModelClient().cancellable_call( + lambda: ModelClient._resolve_addresses("provider.example", 443) + ) + cancels.append(cancel) + + def run(): + try: + call() + except _ProviderRequestCancelled as exc: + errors.append(exc) + + thread = threading.Thread(target=run) + request_threads.append(thread) + thread.start() + + with patch("contextual_orchestrator.orchestrator.socket.getaddrinfo", side_effect=resolve): + for _ in range(4): + start_request() + assert all_workers_entered.wait(timeout=1) + for cancel in cancels: + cancel() + for thread in request_threads: + thread.join(timeout=1) + + start_request() + fifth_cancel = cancels[-1] + fifth_thread = request_threads[-1] + fifth_cancel() + fifth_thread.join(timeout=1) + with lock: + assert calls == 4 + release.set() + + assert all(not thread.is_alive() for thread in request_threads) + assert len(errors) == 5 + + +def test_dns_slot_returns_when_resolver_thread_cannot_start() -> None: + call, _cancel = ModelClient().cancellable_call( + lambda: ModelClient._resolve_addresses("provider.example", 443) + ) + + with patch.object(threading.Thread, "start", side_effect=RuntimeError("no thread")), patch.object( + _PROVIDER_DNS_SLOTS, "release", wraps=_PROVIDER_DNS_SLOTS.release + ) as release: + with pytest.raises(RuntimeError, match="no thread"): + call() + + release.assert_called_once_with() + + +def test_cancellable_provider_call_ignores_connection_close_failure() -> None: + class Connection: + sock = None + + def close(self): + raise OSError("close failed") + + scope = _ProviderCancellation() + scope.register(Connection()) + scope.cancel() + + def test_validated_connect_binds_source_address() -> None: class FakeSocket: def __init__(self): diff --git a/tests/test_mixed_pool_role_effort_selection.py b/tests/test_mixed_pool_role_effort_selection.py index 4ef9adeb9..b96846226 100644 --- a/tests/test_mixed_pool_role_effort_selection.py +++ b/tests/test_mixed_pool_role_effort_selection.py @@ -42,6 +42,11 @@ def _mixed_pool() -> tuple[ModelAgent, ModelAgent]: base_url=_SUPPORTED_BASE_URL, priority=1, reasoning_effort_supported=True, + # Tagged "discovered" so the discovery_sync mutation case below is + # actually reachable: sync_discovered_agents now protects any + # existing candidate that is not tagged "discovered" from being + # silently overwritten by a same-id incoming discovery row. + tags=("discovered",), ) return unsupported, supported diff --git a/tests/test_model_discovery.py b/tests/test_model_discovery.py index 1cd2b5e2d..e50809996 100644 --- a/tests/test_model_discovery.py +++ b/tests/test_model_discovery.py @@ -5,7 +5,11 @@ import io import json import logging +import subprocess import sys +import textwrap +import threading +import time import urllib.error import urllib.parse from contextlib import contextmanager @@ -28,6 +32,7 @@ from contextual_orchestrator.cost_ledger import PriceBook # noqa: E402 from contextual_orchestrator.kv_config import InMemoryConfigStore # noqa: E402 from contextual_orchestrator.model_discovery import ( # noqa: E402 + PROVIDER_DISCOVERY_DEADLINE_SECONDS, PROVIDER_MODEL_SOURCES, DiscoveredModel, ModelUnitPrice, @@ -37,12 +42,14 @@ _OPENROUTER_PROVIDER_POLICIES_URL, _OPENROUTER_ZDR_ENDPOINTS_URL, _bytez_meter_price_is_free, + _apply_discovered_model_evidence, _deduplicate_discovered_models, _fetch_json, _merge_configured_gateway_metadata, _merge_models_dev_metadata, _merge_openrouter_provider_privacy, _merge_openrouter_zdr_metadata, + _openrouter_free_model_endpoints, _price_per_1k, _parse_openai_compatible, _positive_int_metadata, @@ -54,6 +61,7 @@ free_discovered_models, general_free_serving_candidates, is_routable_discovered_model, + model_group_name_for, openrouter_paid_inference_available, refresh_price_book, _response_contains_parallel_probe_tool_calls, @@ -894,7 +902,7 @@ def test_openrouter_discovery_preserves_every_declared_modality() -> None: embedding = next(model for model in discovered if "embedding" in model.capabilities) assert embedding.output_modalities == ("embeddings",) assert {"input:text", "output:embeddings"} <= set( - agent_from_discovered(replace(embedding, evidence_only=False)).tags + agent_from_discovered(embedding).tags ) @@ -932,6 +940,161 @@ def test_openrouter_skips_model_endpoint_fetches_when_provider_policies_fail() - assert [model.model_id for model in discovered] == ["free/model"] endpoint_fetch.assert_not_called() + + +def test_openrouter_free_model_endpoints_hang_does_not_block_process_exit() -> None: + """A hung per-model endpoint fetch must not prevent interpreter shutdown. + + Regression for a CodeRabbit finding (re-confirming the #971 "shared + metadata fetches bypass discovery deadline" class of bug from a + different angle, verified with a local repro before this fix landed): + ``_openrouter_free_model_endpoints`` used to fan its per-model fetch out + across a ``concurrent.futures.ThreadPoolExecutor``. That executor's + worker threads register with an interpreter-exit hook + (``concurrent.futures.thread``'s own ``atexit`` handler) that + unconditionally joins every still-running worker at shutdown -- + regardless of whether the thread that *created* the executor is itself + ``daemon=True``. A single hung fetch therefore blocked process shutdown + even from inside this module's already-daemonized, already-bounded + per-provider discovery thread. The fetch fan-out now uses plain + ``threading.Thread(daemon=True)`` workers, which carry no such + registration, so a hung fetch is abandoned like every other stalled + discovery-time network call in this module and the process can still + exit. + + Verified end-to-end in a real, separate interpreter (an in-process + thread-introspection assertion cannot distinguish "still hanging in the + background" from "would actually block this process's shutdown" -- + the whole point of the finding): a helper script imports the real + function, patches ``_fetch_json`` to hang forever, runs the function on + its own daemon thread exactly as ``_discover_provider_models_bounded`` + does, then lets the script's ``__main__`` fall through to a normal, + unforced exit. RED-before/GREEN-after against the pre-fix + ``ThreadPoolExecutor`` version: the same script hung for the full + outer-`timeout`-command bound and was killed (exit 124); it exits + cleanly, well under that bound, with this fix. + """ + script = textwrap.dedent( + """ + import sys + import threading + from unittest.mock import patch + + sys.path.insert(0, %(repo_root)r) + from contextual_orchestrator.model_discovery import _openrouter_free_model_endpoints + + never_set = threading.Event() + + def hung_fetch_json(url, *, api_key="", auth_scheme="Bearer", timeout=None): + never_set.wait() # Hangs forever -- nothing ever sets this event. + raise AssertionError("unreachable: the stalled fetch must never return") + + payload = { + "data": [ + {"id": "free/model-a", "pricing": {"prompt": "0", "completion": "0"}}, + {"id": "free/model-b", "pricing": {"prompt": "0", "completion": "0"}}, + ] + } + + def outer_daemon_work(): + with patch( + "contextual_orchestrator.model_discovery._fetch_json", + side_effect=hung_fetch_json, + ): + _openrouter_free_model_endpoints(payload, api_key="k", timeout=None) + + worker = threading.Thread(target=outer_daemon_work, daemon=True) + worker.start() + worker.join(timeout=0.5) + assert worker.is_alive() + # No explicit sys.exit()/os._exit(): a genuinely non-blocking fix + # must let normal interpreter shutdown proceed on its own. + """ + ) % {"repo_root": str(Path(__file__).resolve().parents[1])} + + started = time.monotonic() + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + timeout=15, + ) + elapsed = time.monotonic() - started + + assert result.returncode == 0, result.stderr + assert elapsed < 5.0, f"process took {elapsed:.1f}s to exit with a hung endpoint fetch outstanding" + + +def test_openrouter_free_model_endpoints_caps_concurrent_thread_creation() -> None: + """A large free-model catalog must not allocate one OS thread per model. + + Regression for a Devin Review finding: the per-model endpoint fetch fan-out + used to build one ``threading.Thread`` object per free model and start all + of them immediately, gating only *work* (not thread creation itself) behind + an 8-slot semaphore. A catalog of hundreds or thousands of free models would + therefore still allocate and start that many native OS threads at once -- + each with real kernel/stack overhead -- before any semaphore-bounded + concurrency limit ever applied, risking memory exhaustion or stalling + discovery before a single fetch could even begin. The fan-out now uses a + fixed pool of at most 8 daemon worker threads pulling model IDs from a + queue, so the live thread count stays bounded regardless of catalog size. + """ + model_count = 40 + payload = { + "data": [ + {"id": f"free/model-{i}", "pricing": {"prompt": "0", "completion": "0"}} + for i in range(model_count) + ] + } + release = threading.Event() + entered = threading.Event() + concurrent_entries = 0 + max_concurrent_entries = 0 + entries_lock = threading.Lock() + + def blocking_fetch_json(url, *, api_key="", auth_scheme="Bearer", timeout=None): + nonlocal concurrent_entries, max_concurrent_entries + with entries_lock: + concurrent_entries += 1 + max_concurrent_entries = max(max_concurrent_entries, concurrent_entries) + entered.set() + release.wait(timeout=5) + with entries_lock: + concurrent_entries -= 1 + return {"data": []} + + with patch( + "contextual_orchestrator.model_discovery._fetch_json", + side_effect=blocking_fetch_json, + ): + runner = threading.Thread( + target=_openrouter_free_model_endpoints, + args=(payload,), + kwargs={"api_key": "k", "timeout": None}, + daemon=True, + ) + runner.start() + assert entered.wait(timeout=5), "no fetch ever started" + # Give every worker that will ever start a chance to do so before + # sampling -- the whole point is proving a ceiling holds, not a + # transient snapshot. + time.sleep(0.2) + live_worker_threads = [ + thread + for thread in threading.enumerate() + if thread.name.startswith("openrouter-endpoints") + ] + release.set() + runner.join(timeout=5) + assert not runner.is_alive() + + assert len(live_worker_threads) <= 8, ( + f"{len(live_worker_threads)} live 'openrouter-endpoints' threads for " + f"{model_count} models -- expected a fixed pool of at most 8" + ) + assert max_concurrent_entries <= 8 + + def test_non_text_model_does_not_gain_structured_response_capability() -> None: """A provider parameter alone cannot make an image-only model a synthesizer.""" register_credential("OPENROUTER_API_KEY", "sk-router") @@ -1024,7 +1187,7 @@ def test_discovery_retains_full_catalog_and_marks_free_models() -> None: assert [model.model_id for model in discovered] == ["vendor/free-model", "paid/model", "request-fee/model"] assert [model.model_id for model in free_discovered_models(discovered)] == ["vendor/free-model"] - assert agent_from_discovered(replace(discovered[0], evidence_only=False)).group_name == "" + assert agent_from_discovered(discovered[0]).group_name == "model_vendor_free_model_7959c29fc9" def _nim_vision_model() -> DiscoveredModel: @@ -1496,7 +1659,30 @@ def urlopen(request, timeout=None, **_kwargs): assert discovered[0].input_modalities == ("text", "image") assert discovered[1].prompt_price_per_1k == pytest.approx(0.002) assert discovered[1].completion_price_per_1k == pytest.approx(0.012) - assert agent_from_discovered(discovered[0]).group_name == "" + assert agent_from_discovered(discovered[0]).group_name == "model_provider_example_free_681f6a3471" + + +def test_model_group_name_preserves_distinct_exact_model_identities() -> None: + first = DiscoveredModel( + "openai", + "vendor/model-a", + "OPENAI_API_KEY", + "https://api.openai.com/v1", + "Bearer", + ) + second = replace(first, model_id="vendor/model_a") + + assert model_group_name_for(first) != model_group_name_for(second) + + +def test_model_group_name_preserves_case_sensitive_model_identities() -> None: + first = DiscoveredModel( + "openai", "Vendor/Model", "OPENAI_API_KEY", "https://api.openai.com/v1", "Bearer" + ) + + assert model_group_name_for(first) != model_group_name_for( + replace(first, model_id="vendor/model") + ) def test_opencode_zen_metadata_failure_keeps_availability_but_not_free_suffix() -> None: @@ -2051,10 +2237,263 @@ def urlopen(request, timeout=None, **_kwargs): assert errors[0].__cause__ is None -def test_discover_all_models_applies_model_zdr_evidence_to_other_sources() -> None: +def test_provider_discovery_deadline_default_is_bounded_and_independent() -> None: + """The discovery deadline is a finite default, distinct from other timeouts. + + #971's design boundary keeps model *inference* (``ModelClient.timeout``) + and the per-HTTP-call discovery socket timeout (``DISCOVERY_TIMEOUT_SECONDS``) + unbounded by default. The separate per-provider discovery deadline this + finding requires must not silently inherit that -- it needs its own + finite bound so a stalled provider is ever actually abandoned. + """ + assert PROVIDER_DISCOVERY_DEADLINE_SECONDS is not None + assert 0 < PROVIDER_DISCOVERY_DEADLINE_SECONDS < float("inf") + + +def test_discover_all_models_bounds_a_stalled_provider_so_later_providers_still_complete() -> None: + """One provider's catalog fetch hanging forever must not starve the rest. + + Regression for the #971 review finding: "model discovery must not allow + one stalled provider catalog request to block discovery of all later + healthy providers forever; this requires a separately bounded/cancellable + discovery mechanism, not a model-inference timeout." Before this fix, + ``discover_all_models``'s per-provider loop called + ``discover_provider_models`` directly and in-line -- nothing bounded or + cancelled that call, so a hang there blocked every later source forever + (this test would time out the whole suite without the fix). Patches + ``discover_provider_models`` itself, not just the HTTP layer, with a call + that blocks on an ``Event`` nothing ever sets -- proving the new bound + catches a hang the per-request socket ``timeout=`` kwarg could never + catch, since this mock does not even look at it. + """ + register_credential("OPENAI_API_KEY", "sk-openai") + register_credential("OPENROUTER_API_KEY", "sk-router") + never_set = threading.Event() + + def fake_discover_provider_models(source, *, timeout=None, ca_bundle=None, models_dev_metadata=None): + if source.provider_name == "openai": + never_set.wait() # Hangs forever -- nothing ever sets this event. + raise AssertionError("unreachable: the stalled provider must never return") + return [ + DiscoveredModel( + provider_name=source.provider_name, + model_id="meta/llama-3.3", + credential_name=source.credential_name, + chat_base_url=source.chat_base_url, + auth_scheme=source.auth_scheme, + capabilities=("chat",), + ) + ] + + started = time.monotonic() + with patch( + "contextual_orchestrator.model_discovery.discover_provider_models", + side_effect=fake_discover_provider_models, + ): + discovered, errors = discover_all_models( + (OPENAI_SOURCE, OPENROUTER_SOURCE), + discovery_deadline=0.2, + ) + elapsed = time.monotonic() - started + + # Generous bound for CI jitter -- what matters is that this is nowhere + # near "forever" and is driven by the 0.2s deadline, not the test runner. + assert elapsed < 5.0, f"discover_all_models blocked for {elapsed:.1f}s on a stalled provider" + assert [m.model_id for m in discovered] == ["meta/llama-3.3"] + assert len(errors) == 1 + assert errors[0].provider_name == "openai" + assert errors[0].error_code == "discovery_timeout" + + +def test_discover_all_models_discovery_deadline_none_opts_into_unbounded_wait() -> None: + """An explicit ``discovery_deadline=None`` bypasses the bounding thread entirely. + + Covers :func:`_discover_provider_models_bounded`'s unbounded branch: a + caller that explicitly wants the pre-#971-fix unbounded wait back (no + daemon thread, no join deadline) can still get it by passing + ``discovery_deadline=None``, and ordinary discovery still succeeds. + """ + register_credential("OPENAI_API_KEY", "sk-openai") + + def urlopen(request, timeout=None, **_kwargs): + return _Response({"data": [{"id": "gpt-review"}]}) + + with patch( + "contextual_orchestrator.model_discovery._open_trusted_discovery_request", + side_effect=urlopen, + ): + discovered, errors = discover_all_models( + (OPENAI_SOURCE,), + discovery_deadline=None, + ) + + assert errors == [] + assert [m.model_id for m in discovered] == ["gpt-review"] + + +def test_discover_all_models_bounds_a_stalled_models_dev_metadata_fetch() -> None: + """A hung shared Models.dev metadata fetch must not block discovery forever. + + Regression for the #971 review finding "shared metadata fetches bypass + discovery deadline" (Devin bug id + ``BUG_pr-review-job-93783e6ce7a2440ab487ebce4076fe6f_0002``): before this + fix, ``discover_all_models`` called ``_fetch_models_dev_metadata`` inline + *before* the per-provider loop even started, wholly outside + ``discovery_deadline`` -- this test would hang the whole suite without + the fix. Patches ``_fetch_models_dev_metadata`` itself with a call that + blocks on an ``Event`` nothing ever sets, mirroring + ``test_discover_all_models_bounds_a_stalled_provider_so_later_providers_still_complete``'s + style. Also proves the timeout fallback is threaded through as an + already-fetched ``None`` (not the ``_NOT_FETCHED`` sentinel): the + per-provider catalog fetch below must not itself retry the same stalled + fetch a second time. + """ + models_dev_source = replace(OPENAI_SOURCE, models_dev_provider_id="openai") + register_credential("OPENAI_API_KEY", "sk-openai") + never_set = threading.Event() + + def fake_fetch_models_dev_metadata(*, timeout=None): + never_set.wait() # Hangs forever -- nothing ever sets this event. + raise AssertionError("unreachable: the stalled fetch must never return") + + def urlopen(request, timeout=None, **_kwargs): + return _Response({"data": [{"id": "gpt-review"}]}) + + started = time.monotonic() + with ( + patch( + "contextual_orchestrator.model_discovery._fetch_models_dev_metadata", + side_effect=fake_fetch_models_dev_metadata, + ), + patch( + "contextual_orchestrator.model_discovery._open_trusted_discovery_request", + side_effect=urlopen, + ), + ): + discovered, errors = discover_all_models( + (models_dev_source,), + discovery_deadline=0.2, + ) + elapsed = time.monotonic() - started + + assert elapsed < 5.0, f"discover_all_models blocked for {elapsed:.1f}s on a stalled Models.dev fetch" + # The stalled shared fetch degrades to the same "no evidence" fallback + # (None) _fetch_models_dev_metadata already returns for an ordinary + # failure -- _merge_models_dev_metadata passes rows through unchanged -- + # rather than blocking; the provider's own catalog discovery still + # succeeds untouched. + assert errors == [] + assert [m.model_id for m in discovered] == ["gpt-review"] + + +def test_discover_all_models_bounds_a_stalled_openrouter_zdr_fetch() -> None: + """A hung shared OpenRouter ZDR evidence fetch must not block discovery forever. + + Regression for the #971 review finding "shared metadata fetches bypass + discovery deadline": before this fix, ``discover_all_models`` called + ``_openrouter_zdr_model_ids`` inline *after* the per-provider loop + finished, wholly outside ``discovery_deadline`` -- this test would hang + the whole suite without the fix. + """ + register_credential("OPENAI_API_KEY", "sk-openai") + never_set = threading.Event() + + def fake_openrouter_zdr_model_ids(*, timeout=None): + never_set.wait() # Hangs forever -- nothing ever sets this event. + raise AssertionError("unreachable: the stalled fetch must never return") + + def urlopen(request, timeout=None, **_kwargs): + return _Response({"data": [{"id": "gpt-review"}]}) + + started = time.monotonic() + with ( + patch( + "contextual_orchestrator.model_discovery._openrouter_zdr_model_ids", + side_effect=fake_openrouter_zdr_model_ids, + ), + patch( + "contextual_orchestrator.model_discovery._open_trusted_discovery_request", + side_effect=urlopen, + ), + ): + discovered, errors = discover_all_models( + (OPENAI_SOURCE,), + discovery_deadline=0.2, + ) + elapsed = time.monotonic() - started + + assert elapsed < 5.0, f"discover_all_models blocked for {elapsed:.1f}s on a stalled OpenRouter ZDR fetch" + assert errors == [] + assert [m.model_id for m in discovered] == ["gpt-review"] + # The stalled fetch degrades to the same empty-set fallback + # _openrouter_zdr_model_ids already returns for an ordinary failure -- + # never marks a model ZDR-capable on missing/timed-out evidence. + assert discovered[0].zdr_capable is False + + +def test_discover_all_models_bounds_a_stalled_openrouter_paid_inference_fetch() -> None: + """A hung shared OpenRouter credits fetch must not block discovery forever. + + Regression for the #971 review finding "shared metadata fetches bypass + discovery deadline": before this fix, ``discover_all_models`` called + ``openrouter_paid_inference_available`` inline *after* the per-provider + loop finished (only once an OpenRouter credential is registered), wholly + outside ``discovery_deadline`` -- this test would hang the whole suite + without the fix. + """ + register_credential("OPENROUTER_API_KEY", "sk-openrouter") + never_set = threading.Event() + + def fake_openrouter_paid_inference_available(*, timeout=None): + never_set.wait() # Hangs forever -- nothing ever sets this event. + raise AssertionError("unreachable: the stalled fetch must never return") + + def fake_discover_provider_models(source, *, timeout=None, ca_bundle=None, models_dev_metadata=None): + return [ + DiscoveredModel( + provider_name=source.provider_name, + model_id="paid/model", + credential_name=source.credential_name, + chat_base_url=source.chat_base_url, + auth_scheme=source.auth_scheme, + capabilities=("chat",), + is_free=False, + ) + ] + + started = time.monotonic() + with ( + patch( + "contextual_orchestrator.model_discovery.openrouter_paid_inference_available", + side_effect=fake_openrouter_paid_inference_available, + ), + patch( + "contextual_orchestrator.model_discovery.discover_provider_models", + side_effect=fake_discover_provider_models, + ), + ): + discovered, errors = discover_all_models( + (OPENROUTER_SOURCE,), + discovery_deadline=0.2, + ) + elapsed = time.monotonic() - started + + assert elapsed < 5.0, f"discover_all_models blocked for {elapsed:.1f}s on a stalled OpenRouter credits fetch" + assert errors == [] + assert [m.model_id for m in discovered] == ["paid/model"] + # The stalled fetch degrades to the same "could not determine" fallback + # (None) openrouter_paid_inference_available already returns for an + # ordinary failure -- apply_openrouter_spend_admission's fail-closed rule + # never admits spend for a paid row without positive evidence. + assert discovered[0].spend_admitted is False + + +@pytest.mark.parametrize("provider_name", ["nvidia_nim", "experiential_labs"]) +def test_discover_all_models_keeps_zdr_evidence_provider_scoped(provider_name) -> None: + """An OpenRouter model match cannot attest another provider's retention.""" register_credential("OPENROUTER_API_KEY", "sk-openrouter") other_source = ProviderModelSource( - provider_name="nvidia_nim", + provider_name=provider_name, credential_name="NVIDIA_NIM_API_KEY", list_url="https://integrate.api.nvidia.com/v1/models", chat_base_url="https://integrate.api.nvidia.com/v1", @@ -2082,10 +2521,25 @@ def urlopen(request, timeout=None, **_kwargs): assert errors == [] assert [(model.provider_name, model.zdr_capable) for model in discovered] == [ ("openrouter", True), - ("nvidia_nim", True), + (provider_name, False), ] +def test_openrouter_evidence_preserves_other_provider_attestation() -> None: + """An unrelated feed cannot erase independently supplied ZDR evidence.""" + attested_model = DiscoveredModel( + provider_name="experiential_labs", + model_id="independently-attested-model", + credential_name="EXPERIENTAL_LABS_API_KEY", + chat_base_url="https://api.experientiallabs.ai/v1", + auth_scheme="Bearer", + zdr_capable=True, + ) + assert _apply_discovered_model_evidence( + [attested_model], {"unrelated/openrouter-model"} + ) == [attested_model] + + def test_openrouter_zdr_evidence_uses_the_registered_kv_credential() -> None: register_credential("OPENROUTER_API_KEY", "sk-openrouter") seen_calls = [] @@ -2387,7 +2841,7 @@ def urlopen(request, timeout=None, **_kwargs): discovered = discover_provider_models(OPENAI_SOURCE) assert len(attempt_timeouts) == 2 - assert attempt_timeouts[1] < attempt_timeouts[0] # retry uses the shortened timeout + assert attempt_timeouts == [None, None] mock_sleep.assert_called_once() assert [model.model_id for model in discovered] == ["gpt-test"] @@ -2484,7 +2938,7 @@ def test_agent_id_for_is_two_word_snake_case() -> None: chat_base_url="https://openrouter.ai/api/v1", auth_scheme="Bearer", ) - assert agent_id_for(discovered) == "openrouter_meta_llama_3_3_70b" + assert agent_id_for(discovered).startswith("openrouter_meta_llama_3_3_70b_") def test_agent_from_discovered_builds_disabled_agent_with_correct_auth() -> None: @@ -2496,7 +2950,7 @@ def test_agent_from_discovered_builds_disabled_agent_with_correct_auth() -> None auth_scheme=AUTH_SCHEME_RAW_TOKEN, ) agent = agent_from_discovered(discovered, priority=3) - assert agent.id == "bytez_0_hero_matter_0_1_slim_7b_c" + assert agent.id.startswith("bytez_0_hero_matter_0_1_slim_7b_c_") assert agent.disabled is True assert agent.auth_scheme == AUTH_SCHEME_RAW_TOKEN assert agent.credential_key == "BYTEZ_API_KEY" @@ -2505,11 +2959,12 @@ def test_agent_from_discovered_builds_disabled_agent_with_correct_auth() -> None def test_agent_from_discovered_rejects_evidence_only_rows() -> None: + """Any row explicitly marked evidence_only stays unroutable, regardless of provider.""" discovered = DiscoveredModel( - provider_name="openrouter", + provider_name="example_evidence_provider", model_id="provider/evidence-model", - credential_name="OPENROUTER_API_KEY", - chat_base_url="https://openrouter.ai/api/v1", + credential_name="EXAMPLE_EVIDENCE_PROVIDER_API_KEY", + chat_base_url="https://example-evidence-provider.example/v1", auth_scheme="Bearer", evidence_only=True, ) @@ -2716,22 +3171,22 @@ def test_sync_discovered_agents_adds_and_updates_idempotently() -> None: agent_v1 = agent_from_discovered(discovered, priority=0) result = orchestrator.sync_discovered_agents([agent_v1]) - assert result == {"added": ["openrouter_meta_llama_3_3"], "updated": []} - assert {a.id for a in orchestrator.candidates} == {"seed_agent", "openrouter_meta_llama_3_3"} + assert result == {"added": [agent_v1.id], "updated": []} + assert {a.id for a in orchestrator.candidates} == {"seed_agent", agent_v1.id} agent_v2 = agent_from_discovered(discovered, priority=7) result = orchestrator.sync_discovered_agents([agent_v2]) - assert result == {"added": [], "updated": ["openrouter_meta_llama_3_3"]} - stored = next(a for a in orchestrator.candidates if a.id == "openrouter_meta_llama_3_3") + assert result == {"added": [], "updated": [agent_v1.id]} + stored = next(a for a in orchestrator.candidates if a.id == agent_v1.id) assert stored.priority == 7 # No duplicate rows were appended on the update pass. assert len(orchestrator.candidates) == 2 orchestrator.set_model_group( - "shared_reasoning_model", ["openrouter_meta_llama_3_3"] + "shared_reasoning_model", [agent_v1.id] ) orchestrator.sync_discovered_agents([agent_v1]) - stored = next(a for a in orchestrator.candidates if a.id == "openrouter_meta_llama_3_3") + stored = next(a for a in orchestrator.candidates if a.id == agent_v1.id) assert stored.group_name == "shared_reasoning_model" @@ -2772,7 +3227,72 @@ def test_sync_discovered_agents_persists_when_agents_db_is_set(tmp_path) -> None first.sync_discovered_agents([agent]) second = TaskOrchestrator([ModelAgent("seed_agent", "seed-model")], agents_db=db_path) - assert any(a.id == "openai_gpt_5_5" for a in second.candidates) + assert any(a.id == agent.id for a in second.candidates) + + +def test_durable_legacy_discovered_agent_adopts_generated_group_and_id(tmp_path) -> None: + db_path = str(tmp_path / "legacy-pool.db") + discovered = DiscoveredModel( + "openrouter", "Vendor/Model", "OPENROUTER_API_KEY", + "https://openrouter.ai/api/v1", "Bearer", + ) + incoming = agent_from_discovered(discovered) + legacy = replace(incoming, id="openrouter_vendor_model", group_name="") + seeded = TaskOrchestrator([], agents_db=db_path, allow_empty_agents=True) + seeded.sync_discovered_agents([legacy]) + seeded.close() + + restarted = TaskOrchestrator([], agents_db=db_path, allow_empty_agents=True) + result = restarted.sync_discovered_agents([incoming]) + stored = next(agent for agent in restarted.candidates if agent.id == legacy.id) + + assert result == {"added": [], "updated": [legacy.id]} + assert stored.group_name == incoming.group_name + assert all(agent.id != incoming.id for agent in restarted.candidates) + restarted.close() + + +def test_legacy_operator_agent_is_not_duplicated_or_overwritten() -> None: + discovered = DiscoveredModel( + "openai", "Vendor/Model", "OPENAI_API_KEY", + "https://api.openai.com/v1", "Bearer", + ) + incoming = agent_from_discovered(discovered) + operator = replace( + incoming, + id="openai_vendor_model", + tags=("operator-tag",), + disabled=True, + ) + orchestrator = TaskOrchestrator([operator], allow_empty_agents=True) + + assert orchestrator.sync_discovered_agents([incoming]) == {"added": [], "updated": []} + assert orchestrator.candidates == [operator] + + +def test_exact_model_id_collisions_persist_as_distinct_discovered_agents(tmp_path) -> None: + base = DiscoveredModel( + "openrouter", "vendor/model-a", "OPENROUTER_API_KEY", + "https://openrouter.ai/api/v1", "Bearer", + ) + models = [ + base, + replace(base, model_id="vendor/model_a"), + replace(base, model_id="Vendor/Model"), + replace(base, model_id="vendor/model"), + ] + agents = [agent_from_discovered(model) for model in models] + orchestrator = TaskOrchestrator( + [], agents_db=str(tmp_path / "collisions.db"), allow_empty_agents=True + ) + + orchestrator.sync_discovered_agents(agents) + + assert len({agent.id for agent in orchestrator.candidates}) == len(models) + assert {agent.model for agent in orchestrator.candidates} == { + model.model_id for model in models + } + orchestrator.close() _MODEL_DISCOVERY_LOGGER_NAME = "contextual_orchestrator.model_discovery" diff --git a/tests/test_model_discovery_boundaries.py b/tests/test_model_discovery_boundaries.py index 5e193da25..bbac2d5f3 100644 --- a/tests/test_model_discovery_boundaries.py +++ b/tests/test_model_discovery_boundaries.py @@ -479,18 +479,90 @@ def test_bootstrap_fills_remainder_from_deferred_same_family_models() -> None: assert all(m.provider_name == "nvidia_nim" for m in selected[2:]) -def test_bootstrap_early_return_stops_at_limit_within_loop() -> None: - """A limit below the distinct-family count returns without a second pass.""" +def test_bootstrap_selection_fails_closed_at_unpriced_boundary() -> None: + """A capacity boundary cannot admit lexically chosen unpriced candidates.""" book = PriceBook(InMemoryConfigStore()) models = [ _chat_model("openai", "openai-model"), _chat_model("openrouter", "openrouter-model"), _chat_model("bytez", "bytez-model"), ] - selected = select_bootstrap_discovered_agents(models, book, 2) - # Unpriced ties rank by provider name: bytez < openai < openrouter. - assert len(selected) == 2 - assert [m.provider_name for m in selected] == ["bytez", "openai"] + + with pytest.raises(ValueError, match="ambiguous"): + select_bootstrap_discovered_agents(models, book, 2) + + +def test_bootstrap_selection_fails_closed_at_equal_known_price_boundary() -> None: + """Equal comparable cost cannot be resolved by provider/model names.""" + book = PriceBook(InMemoryConfigStore()) + models = [ + replace( + _chat_model(provider, f"{provider}-model"), + prompt_price_per_1k=0.5, + completion_price_per_1k=0.5, + currency_code="USD", + ) + for provider in ("openai", "openrouter", "bytez") + ] + + with pytest.raises(ValueError, match="ambiguous"): + select_bootstrap_discovered_agents(models, book, 2) + + +def test_bootstrap_selection_rejects_unmodeled_cost_displacement() -> None: + """Provider diversity cannot displace cheaper evidence without a utility model.""" + book = PriceBook(InMemoryConfigStore()) + models = [ + replace( + _chat_model("openrouter", "cheap-model"), + prompt_price_per_1k=0.5, + completion_price_per_1k=0.5, + currency_code="USD", + ), + replace( + _chat_model("openrouter", "next-cheapest-model"), + prompt_price_per_1k=0.75, + completion_price_per_1k=0.75, + currency_code="USD", + ), + replace( + _chat_model("bytez", "expensive-model"), + prompt_price_per_1k=1.0, + completion_price_per_1k=1.0, + currency_code="USD", + ), + ] + + with pytest.raises(ValueError, match="decision model"): + select_bootstrap_discovered_agents(models, book, 2) + + +def test_bootstrap_selection_rejects_unmodeled_full_pool_reordering() -> None: + """Admitting every candidate cannot make diversity an implicit route order.""" + book = PriceBook(InMemoryConfigStore()) + models = [ + replace( + _chat_model("openrouter", "cheap-model"), + prompt_price_per_1k=0.5, + completion_price_per_1k=0.5, + currency_code="USD", + ), + replace( + _chat_model("openrouter", "next-cheapest-model"), + prompt_price_per_1k=0.75, + completion_price_per_1k=0.75, + currency_code="USD", + ), + replace( + _chat_model("bytez", "expensive-model"), + prompt_price_per_1k=1.0, + completion_price_per_1k=1.0, + currency_code="USD", + ), + ] + + with pytest.raises(ValueError, match="decision model"): + select_bootstrap_discovered_agents(models, book, 3) if __name__ == "__main__": # pragma: no cover diff --git a/tests/test_multimodal_model_group_http.py b/tests/test_multimodal_model_group_http.py index ac7ffb970..e133ebd69 100644 --- a/tests/test_multimodal_model_group_http.py +++ b/tests/test_multimodal_model_group_http.py @@ -129,6 +129,21 @@ def test_speech_endpoint_preserves_binary_media_response() -> None: server.shutdown() +def test_speech_endpoint_rejects_non_object_provider_routing() -> None: + agent = ModelAgent("speech_member", "provider/speech", tags=("speech",)) + server = build_server(TaskOrchestrator([agent]), port=0, security=SecurityConfig(auth_token=TOKEN)) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + status, body = _post_error( + server.server_address[1], + "/v1/audio/speech", + {"input": "hello", "voice": "alloy", "provider": 1}, + ) + assert status == 400 and body["error"]["code"] == "invalid_provider" + finally: + server.shutdown() + + def test_video_poll_and_content_use_the_submission_provider() -> None: """Async video follow-ups stay bound to the measured submission winner.""" first = ModelAgent( diff --git a/tests/test_no_heuristic_default_transport_retry.py b/tests/test_no_heuristic_default_transport_retry.py new file mode 100644 index 000000000..e56ee675f --- /dev/null +++ b/tests/test_no_heuristic_default_transport_retry.py @@ -0,0 +1,37 @@ +"""Fail-closed contracts for default provider transport retry allocation.""" + +from __future__ import annotations + +from contextual_orchestrator.orchestrator import ModelAgent, ModelClient + + +def test_model_client_default_allocates_no_unproven_retry_attempts() -> None: + """The library default cannot invent a retry count for provider inference.""" + client = ModelClient() + agent = ModelAgent( + "provider_route", + "arbitrary-chat-model", + base_url="https://provider.example/v1", + provider_name="provider", + ) + + assert client.max_retries == 0 + assert client.local_max_retries == 0 + assert client._retry_limit(agent) == 0 + + +def test_default_retry_policy_is_independent_of_model_or_provider_identity() -> None: + """No name/capability branch may manufacture a default retry budget.""" + client = ModelClient() + agents = ( + ModelAgent("a_route", "model-a", base_url="https://a.example/v1", provider_name="a"), + ModelAgent( + "b_route", + "model-b", + base_url="https://b.example/v1", + provider_name="b", + reasoning_effort_supported=True, + ), + ) + + assert {client._retry_limit(agent) for agent in agents} == {0} diff --git a/tests/test_openrouter_uptime.py b/tests/test_openrouter_uptime.py index 1bdf6abed..b8998108a 100644 --- a/tests/test_openrouter_uptime.py +++ b/tests/test_openrouter_uptime.py @@ -2,10 +2,13 @@ from __future__ import annotations +import json +import socket import sys import threading import time from pathlib import Path +from unittest.mock import patch sys.path.insert(0, str(Path(__file__).resolve().parents[1])) @@ -87,6 +90,89 @@ def test_unavailable_uptime_poll_is_a_no_op() -> None: assert collector.window_evidence(agent.id) == (0.0, 0.0) +def test_uptime_fetch_keeps_a_fixed_network_deadline_independent_of_inference() -> None: + """The background uptime GET keeps its own bound, unlike inference calls. + + #971 removes the *inference* client's fixed wall-clock deadline (a user + is actively waiting on a model completion). This collector's HTTP GET is + unrelated background telemetry on one dedicated sequential sweep thread + that ``stop()`` cannot interrupt mid-request (Python threads cannot be + forcibly cancelled): an unbounded fetch would let one unresponsive + OpenRouter endpoint hang that thread forever, leaking it and + indefinitely starving every later member of an uptime update. The fetch + must stay bounded regardless of #971's inference-deadline policy. + """ + class _Response: + def __enter__(self): + return self + + def __exit__(self, *_args: object) -> None: + return None + + def read(self) -> bytes: + return json.dumps({"data": {"endpoints": []}}).encode() + + collector, _, _, _ = _collectors(None) + del collector._fetch_uptime + with patch( + "contextual_orchestrator.openrouter_uptime.urllib.request.urlopen", + return_value=_Response(), + ) as opened: + assert collector._fetch_uptime("org/model-a") is None + + timeout = opened.call_args.kwargs["timeout"] + assert timeout is not None + assert 0 < timeout <= 30 + + +def test_uptime_fetch_does_not_hang_forever_on_an_unresponsive_endpoint(monkeypatch) -> None: + """A stalled connection is bounded end-to-end, not blocked forever. + + Complements the kwarg-level assertion above by proving the timeout is + actually enforced: a real TCP listener that accepts the connection and + never responds must still return within a short bound instead of + hanging the sweep thread indefinitely (which would leak that thread and + starve every later member of an uptime update). + """ + import contextual_orchestrator.openrouter_uptime as uptime_module + + server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server.bind(("127.0.0.1", 0)) + server.listen(1) + host, port = server.getsockname() + accepted = threading.Event() + + def accept_and_stall() -> None: + try: + conn, _ = server.accept() + accepted.set() + time.sleep(2.0) # Accept the connection but never respond. + conn.close() + except OSError: + pass + + acceptor = threading.Thread(target=accept_and_stall, daemon=True) + acceptor.start() + try: + monkeypatch.setattr(uptime_module, "_UPTIME_FETCH_TIMEOUT_SECONDS", 0.3) + monkeypatch.setattr( + uptime_module, "_OPENROUTER_UPTIME_ORIGIN", f"http://{host}:{port}/api/v1" + ) + collector, _, _, _ = _collectors(None) + del collector._fetch_uptime + + started = time.monotonic() + result = collector._fetch_uptime("org/model-a") + elapsed = time.monotonic() - started + + assert result is None + assert elapsed < 2.0 + assert accepted.wait(timeout=1.0) + finally: + server.close() + acceptor.join(timeout=3.0) + + def test_background_loop_accumulates_and_stop_joins() -> None: """The sweep thread runs until stop(), and stop() returns quickly.""" collector, _, _, _ = _collectors(80.0) diff --git a/tests/test_orchestrator_client_boundaries.py b/tests/test_orchestrator_client_boundaries.py index 9bb489b41..1527e7011 100644 --- a/tests/test_orchestrator_client_boundaries.py +++ b/tests/test_orchestrator_client_boundaries.py @@ -7,12 +7,12 @@ import threading import types import urllib.error +from typing import Any from unittest.mock import patch import pytest from contextual_orchestrator.orchestrator import ( - MAX_PROVIDER_PROBE_TIMEOUT, ModelAgent, ModelClient, ProviderResponseError, @@ -22,9 +22,11 @@ _coerce_message_content_text, _local_provider_slot, _local_provider_state, + _pin_openrouter_zdr, + _REQUEST_ZDR_ONLY, _resolve_fast_mlsirm_components, + _resolved_openrouter_provider, _validate_batch_results, - _validate_provider_probe_timeout, ) from contextual_orchestrator.provider_errors import ProviderUpstreamError @@ -43,22 +45,6 @@ def _agent(agent_id: str = "planner_agent", **overrides) -> ModelAgent: return ModelAgent(**fields) -# -- probe timeout validation ------------------------------------------------- - - -@pytest.mark.parametrize("bad", [True, "5", None]) -def test_probe_timeout_rejects_non_numeric_types(bad) -> None: - with pytest.raises(ValueError, match="finite number"): - _validate_provider_probe_timeout(bad) - - -@pytest.mark.parametrize("bad", [float("nan"), float("inf"), 0.05, 31.0]) -def test_probe_timeout_rejects_out_of_range_values(bad) -> None: - with pytest.raises(ValueError, match="between 0.1 and"): - _validate_provider_probe_timeout(bad) - assert MAX_PROVIDER_PROBE_TIMEOUT == 30.0 - - # -- optional fast-mlsirm adapter seam ---------------------------------------- @@ -345,7 +331,7 @@ def test_probe_reports_not_ready_for_empty_mock_content() -> None: orch = _orch(_agent()) agent = orch.candidates[0] with patch.object(ModelClient, "_mock", return_value=" "): - report = orch.client.probe(agent, timeout=1.0) + report = orch.client.probe(agent) assert report["status"] == "not_ready" assert report["failure_code"] == "provider_empty_probe_response" assert report["error_type"] == "RuntimeError" @@ -362,7 +348,7 @@ def test_probe_against_https_provider_skips_registry_and_reports_ready() -> None with patch.object(client, "_validate_provider", return_value=None), patch.object( client, "_send", return_value="OK" ): - report = client.probe(agent, timeout=1.0) + report = client.probe(agent) assert report["status"] == "ready" assert report["latency_ms"] >= 0 @@ -403,7 +389,7 @@ def test_probe_registry_check_passes_for_registered_local_model( with patch.object(client, "_validate_provider", return_value=None), patch.object( client, "_open_provider", return_value=_RegistryResponse(registry_payload) ), patch.object(client, "_send", return_value="OK"): - report = client.probe(agent, timeout=1.0) + report = client.probe(agent) assert report["status"] == "ready" missing_client = ModelClient() @@ -412,7 +398,7 @@ def test_probe_registry_check_passes_for_registered_local_model( "_open_provider", return_value=_RegistryResponse({"data": [{"id": "other"}]}), ): - missing = missing_client.probe(agent, timeout=1.0) + missing = missing_client.probe(agent) assert missing["status"] == "not_ready" assert missing["failure_code"] == "provider_model_not_registered" @@ -547,6 +533,273 @@ def __iter__(self): assert "connection reset" not in str(excinfo.value) +# -- OpenRouter request-time ZDR pin ------------------------------------------------ + + +def _openrouter_agent(**overrides) -> ModelAgent: + fields = { + "id": "openrouter_agent", + "model": "some-vendor/some-model", + "base_url": "https://openrouter.ai/api/v1", + "provider_name": "openrouter", + "credential_key": "OPENROUTER_API_KEY", + } + fields.update(overrides) + return ModelAgent(**fields) + + +def test_pin_openrouter_zdr_is_noop_outside_zdr_only_context() -> None: + """Only an active zdr_only request scope may add the provider.zdr pin.""" + agent = _openrouter_agent() + payload = {"model": agent.model, "messages": []} + assert _pin_openrouter_zdr(agent, payload) is payload + + +def test_pin_openrouter_zdr_is_noop_for_non_openrouter_agents() -> None: + """The pin is OpenRouter-specific; every other provider is untouched.""" + agent = _agent(provider_name="openai", base_url="https://api.openai.com/v1") + payload = {"model": agent.model, "messages": []} + token = _REQUEST_ZDR_ONLY.set(True) + try: + assert _pin_openrouter_zdr(agent, payload) is payload + finally: + _REQUEST_ZDR_ONLY.reset(token) + + +def test_pin_openrouter_zdr_infers_provider_from_base_url_when_name_is_empty() -> None: + """An OpenRouter agent with an unset ``provider_name`` still gets pinned. + + ``provider_name`` is free-text and unvalidated at construction; a + hand-authored or auto-discovered agent can carry ``base_url`` pointing at + OpenRouter's own endpoint while ``provider_name`` stays empty (its + default). Trusting ``provider_name`` verbatim here would silently skip + the ``provider.zdr=true`` enforcement pin under an active ``zdr_only`` + scope even though the request still routes to OpenRouter (CodeRabbit + review on #953, discussion_r3898471887). + """ + agent = _openrouter_agent(id="legacy_openrouter_agent", provider_name="") + assert _resolved_openrouter_provider(agent) == "openrouter" + payload = {"model": agent.model, "messages": []} + token = _REQUEST_ZDR_ONLY.set(True) + try: + pinned = _pin_openrouter_zdr(agent, payload) + finally: + _REQUEST_ZDR_ONLY.reset(token) + assert pinned["provider"] == {"zdr": True} + + +def test_pin_openrouter_zdr_overrides_mistyped_provider_name_from_exact_host() -> None: + """The actual OpenRouter destination wins over unvalidated provider text.""" + agent = _openrouter_agent(provider_name="open_router") + payload = {"model": agent.model, "messages": []} + token = _REQUEST_ZDR_ONLY.set(True) + try: + pinned = _pin_openrouter_zdr(agent, payload) + finally: + _REQUEST_ZDR_ONLY.reset(token) + + assert pinned["provider"] == {"zdr": True} + + +def test_pin_openrouter_zdr_adds_provider_zdr_flag() -> None: + """A zdr_only request to an OpenRouter agent gets OpenRouter's own enforcement pin.""" + agent = _openrouter_agent() + payload = {"model": agent.model, "messages": []} + token = _REQUEST_ZDR_ONLY.set(True) + try: + pinned = _pin_openrouter_zdr(agent, payload) + finally: + _REQUEST_ZDR_ONLY.reset(token) + assert pinned["provider"] == {"zdr": True} + assert "provider" not in payload # the original payload is never mutated in place + + +def test_pin_openrouter_zdr_preserves_caller_supplied_provider_routing() -> None: + """An explicit caller provider-routing preference keeps its other keys.""" + agent = _openrouter_agent() + payload = { + "model": agent.model, + "messages": [], + "provider": {"order": ["mistral"], "allow_fallbacks": False}, + } + token = _REQUEST_ZDR_ONLY.set(True) + try: + pinned = _pin_openrouter_zdr(agent, payload) + finally: + _REQUEST_ZDR_ONLY.reset(token) + assert pinned["provider"] == { + "order": ["mistral"], + "allow_fallbacks": False, + "zdr": True, + } + + +@pytest.mark.parametrize("malformed_provider", [5, True, ["order"], "openrouter", 0, ""]) +def test_pin_openrouter_zdr_rejects_non_mapping_provider(malformed_provider: Any) -> None: + """A non-object ``provider`` under zdr_only fails closed with a named + validation error, not the bare ``TypeError`` ``dict()`` would raise + (Devin review on #953: malformed speech routing returned server errors). + """ + agent = _openrouter_agent() + payload = {"model": agent.model, "messages": [], "provider": malformed_provider} + token = _REQUEST_ZDR_ONLY.set(True) + try: + with pytest.raises(ValueError, match="provider must be an object"): + _pin_openrouter_zdr(agent, payload) + finally: + _REQUEST_ZDR_ONLY.reset(token) + + +def _capture_request_body(sink: dict) -> Any: + """Return an ``_open_provider`` stand-in that records the outgoing JSON body.""" + + def _fake_open_provider(request, *_args, **_kwargs): + sink["body"] = json.loads(request.data.decode("utf-8")) + return _RegistryResponse({"choices": [{"message": {"content": "OK"}}]}) + + return _fake_open_provider + + +def _capture_binary_request_body(sink: dict) -> Any: + def _fake_open_provider(request, *_args, **_kwargs): + sink["body"] = json.loads(request.data.decode("utf-8")) + response = _RegistryResponse({}) + response.headers = types.SimpleNamespace(get_content_type=lambda: "audio/mpeg") + return response + + return _fake_open_provider + + +def test_send_pins_openrouter_zdr_on_the_wire() -> None: + """``_send`` (the normal chat transport) actually applies the pin, not just the helper.""" + agent = _openrouter_agent() + client = ModelClient() + captured: dict[str, Any] = {} + with patch.object(client, "_open_provider", side_effect=_capture_request_body(captured)): + token = _REQUEST_ZDR_ONLY.set(True) + try: + client._send(agent, {"model": agent.model, "messages": []}) + finally: + _REQUEST_ZDR_ONLY.reset(token) + assert captured["body"]["provider"] == {"zdr": True} + + +def test_send_pins_openrouter_zdr_on_the_wire_for_legacy_provider_name() -> None: + """``_send`` still applies the pin for an agent with a missing ``provider_name``. + + Proves the fix end-to-end on the real transport, not just against the + ``_pin_openrouter_zdr`` helper in isolation: an agent whose ``base_url`` + is OpenRouter's own endpoint but whose ``provider_name`` was left at its + empty default must not reach the wire without ``provider.zdr=true`` under + an active ``zdr_only`` scope (CodeRabbit review on #953, + discussion_r3898471887). + """ + agent = _openrouter_agent(id="legacy_openrouter_agent", provider_name="") + client = ModelClient() + captured: dict[str, Any] = {} + with patch.object(client, "_open_provider", side_effect=_capture_request_body(captured)): + token = _REQUEST_ZDR_ONLY.set(True) + try: + client._send(agent, {"model": agent.model, "messages": []}) + finally: + _REQUEST_ZDR_ONLY.reset(token) + assert captured["body"]["provider"] == {"zdr": True} + + +def test_send_pins_openrouter_zdr_on_the_wire_for_mistyped_provider_name() -> None: + """``_send`` still applies the pin when ``provider_name`` is nonempty but wrong. + + Closes the gap CodeRabbit and Devin Review both flagged against + ``df97709a``: ``_resolved_openrouter_provider`` used to return any + *nonempty* ``agent.provider_name`` before ever checking ``base_url``, so + an agent with ``provider_name="openai"`` and ``base_url`` actually + pointing at OpenRouter still reported ``"openai"`` and silently skipped + the ``provider.zdr=true`` enforcement pin — even though ``base_url``, not + the free-text ``provider_name`` label, decides where the request + actually goes. Proved end-to-end on the real ``_send`` transport (the + captured outgoing JSON body), not just against the + ``_resolved_openrouter_provider``/``_pin_openrouter_zdr`` helpers in + isolation, per CodeRabbit's explicit ask for on-wire coverage of this + exact case (CodeRabbit review on #953, discussion_r3898659143; Devin + review on #953, discussion_r3898661634). + """ + agent = _openrouter_agent(id="mistyped_openrouter_agent", provider_name="openai") + client = ModelClient() + captured: dict[str, Any] = {} + with patch.object(client, "_open_provider", side_effect=_capture_request_body(captured)): + token = _REQUEST_ZDR_ONLY.set(True) + try: + client._send(agent, {"model": agent.model, "messages": []}) + finally: + _REQUEST_ZDR_ONLY.reset(token) + assert captured["body"]["provider"] == {"zdr": True} + + +def test_stream_send_pins_openrouter_zdr_on_the_wire() -> None: + """``_stream_send`` applies the same pin as the non-streaming transport.""" + agent = _openrouter_agent() + client = ModelClient() + captured: dict[str, Any] = {} + + def _fake_open_provider(request, *_args, **_kwargs): + captured["body"] = json.loads(request.data.decode("utf-8")) + return _StreamResponse([b"data: [DONE]"]) + + with patch.object(client, "_open_provider", side_effect=_fake_open_provider): + token = _REQUEST_ZDR_ONLY.set(True) + try: + list(client._stream_send(agent, {"model": agent.model, "messages": [], "stream": True})) + finally: + _REQUEST_ZDR_ONLY.reset(token) + assert captured["body"]["provider"] == {"zdr": True} + + +def test_send_raw_pins_openrouter_zdr_on_the_wire() -> None: + """``_send_raw`` (the passthrough transport) applies the same pin.""" + agent = _openrouter_agent() + client = ModelClient() + captured: dict[str, Any] = {} + with patch.object(client, "_open_provider", side_effect=_capture_request_body(captured)): + token = _REQUEST_ZDR_ONLY.set(True) + try: + client._send_raw(agent, "chat/completions", {"model": agent.model, "messages": []}) + finally: + _REQUEST_ZDR_ONLY.reset(token) + assert captured["body"]["provider"] == {"zdr": True} + + +def test_send_does_not_pin_zdr_outside_zdr_only_context() -> None: + """A normal (non-zdr_only) request to OpenRouter is sent unmodified.""" + agent = _openrouter_agent() + client = ModelClient() + captured: dict[str, Any] = {} + with patch.object(client, "_open_provider", side_effect=_capture_request_body(captured)): + client._send(agent, {"model": agent.model, "messages": []}) + assert "provider" not in captured["body"] + + +def test_proxy_send_bytes_pins_openrouter_zdr_only_in_policy_scope() -> None: + """Binary speech transport applies the same request-time ZDR boundary.""" + agent = _openrouter_agent() + client = ModelClient() + captured: dict[str, Any] = {} + with patch.object(client, "_validate_provider", return_value=None), patch.object( + client, "_open_provider", side_effect=_capture_binary_request_body(captured) + ): + token = _REQUEST_ZDR_ONLY.set(True) + try: + client.proxy_send_bytes(agent, "audio/speech", {"input": "hello"}) + finally: + _REQUEST_ZDR_ONLY.reset(token) + assert captured["body"]["provider"] == {"zdr": True} + + with patch.object(client, "_validate_provider", return_value=None), patch.object( + client, "_open_provider", side_effect=_capture_binary_request_body(captured) + ): + client.proxy_send_bytes(agent, "audio/speech", {"input": "hello"}) + assert "provider" not in captured["body"] + + # -- batch success paths ------------------------------------------------------------ @@ -665,6 +918,36 @@ def batch_json(_agent, method, _path, payload=None, destination=None): assert results["task_0"]["usage"] == {"prompt_tokens": 3} +def test_batch_run_pins_openrouter_zdr_in_uploaded_jsonl() -> None: + agent = _openrouter_agent() + client = ModelClient() + captured: dict[str, Any] = {} + raw = b'{"custom_id":"task_0","response":{"body":{"choices":[{"message":{"content":"ok"}}]}}}\n' + + def capture_upload(_agent, content, _destination): + captured["line"] = json.loads(content.decode("utf-8")) + return "file_1" + + def batch_json(_agent, method, _path, payload=None, destination=None): + del payload, destination + return {"id": "batch_1"} if method == "POST" else { + "status": "completed", "output_file_id": "file_9" + } + + with patch.object(client, "_batch_upload", side_effect=capture_upload), patch.object( + client, "_batch_json", side_effect=batch_json + ), patch.object(client, "_batch_raw", return_value=raw): + token = _REQUEST_ZDR_ONLY.set(True) + try: + client._batch_run( + agent, {"task_0": [{"role": "user", "content": "hi"}]}, None, 0.01, 5.0 + ) + finally: + _REQUEST_ZDR_ONLY.reset(token) + + assert captured["line"]["body"]["provider"] == {"zdr": True} + + def test_batch_run_clamps_known_provider_output_ceiling() -> None: client = ModelClient(max_output_tokens=256) agent = ModelAgent( diff --git a/tests/test_pr971_review_quality_regressions.py b/tests/test_pr971_review_quality_regressions.py new file mode 100644 index 000000000..6c09bdd5c --- /dev/null +++ b/tests/test_pr971_review_quality_regressions.py @@ -0,0 +1,255 @@ +"""Review regressions for PR #971 privacy, health, and discovery boundaries.""" + +from __future__ import annotations + +import json +import threading +import urllib.error +import urllib.request +from contextlib import contextmanager +from dataclasses import replace +from unittest.mock import patch + +import pytest + +from contextual_orchestrator import CostRoutingCoordinator, ModelAgent, TaskOrchestrator +from contextual_orchestrator.batch_routing import EmbeddingBatchRequest +from contextual_orchestrator.credentials import ( + InMemoryCredentialBackend, + register_credential, + set_backend, +) +from contextual_orchestrator.model_discovery import discover_all_models +from contextual_orchestrator.server import SecurityConfig, build_server +from tests.test_model_discovery import OPENAI_SOURCE, OPENROUTER_SOURCE + + +_AUTH_TOKEN = "pr971_review_quality_token" + + +def _embedding_coordinator() -> tuple[CostRoutingCoordinator, TaskOrchestrator, ModelAgent]: + """Build one deterministic ZDR-capable embedding route.""" + agent = ModelAgent( + "zdr_embedding_agent", + "embedding-model", + "https://provider.synthetic.invalid/v1", + tags=("embedding", "privacy:zdr"), + ) + orchestrator = TaskOrchestrator([agent]) + orchestrator.client.embed = lambda _agent, texts: [[1.0] for _ in texts] + coordinator = CostRoutingCoordinator(orchestrator) + return coordinator, orchestrator, agent + + +def test_recovered_zdr_batch_reenters_request_privacy_scope(monkeypatch) -> None: + """Persisted ZDR execution must restore request policy on the worker thread.""" + coordinator, orchestrator, agent = _embedding_coordinator() + entries: list[bool] = [] + + @contextmanager + def observed_policy(zdr_only: bool): + entries.append(zdr_only) + yield + + monkeypatch.setattr(orchestrator, "request_policy", observed_policy) + coordinator._run_provider_embeddings( + [ + EmbeddingBatchRequest( + input_text="private input", + model=agent.model, + token_count=2, + zdr_only=True, + agent_id=agent.id, + ) + ] + ) + + assert entries == [True] + + +def test_provider_embedding_batch_rejects_mixed_privacy_identity() -> None: + """One coalesced execution cannot mix privacy/routing identities.""" + coordinator, _orchestrator, agent = _embedding_coordinator() + private = EmbeddingBatchRequest( + input_text="private", + model=agent.model, + token_count=1, + zdr_only=True, + agent_id=agent.id, + provider_routing={"zdr": True}, + ) + ordinary = EmbeddingBatchRequest( + input_text="ordinary", + model=agent.model, + token_count=1, + zdr_only=False, + agent_id=agent.id, + provider_routing=None, + ) + + with pytest.raises(RuntimeError, match="privacy policy"): + coordinator._run_provider_embeddings([private, ordinary]) + + +def _post(port: int, path: str, payload: dict) -> tuple[int, dict]: + request = urllib.request.Request( + f"http://127.0.0.1:{port}{path}", + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "authorization": f"Bearer {_AUTH_TOKEN}", + "connection": "close", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=5) as response: + return response.status, json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + return exc.code, json.loads(exc.read().decode("utf-8")) + + +def test_terminal_embedding_batch_document_fails_over_before_marking_health() -> None: + """Terminal provider documents are failures, not endpoint-health successes.""" + first = ModelAgent( + "first_embedding_agent", + "embed-v1", + tags=("embedding",), + priority=1, + ) + second = ModelAgent( + "second_embedding_agent", + "embed-v1", + tags=("embedding",), + priority=0, + ) + orchestrator = TaskOrchestrator([first, second]) + coordinator = CostRoutingCoordinator(orchestrator) + attempted: list[str] = [] + + def complete(_inputs, *, agent_id=None, **_kwargs): + attempted.append(str(agent_id)) + if agent_id == first.id: + return { + "batch_id": "failed-batch", + "status": "failed", + "backend": "provider", + "model": first.model, + "embeddings": None, + } + return { + "batch_id": "accepted-batch", + "status": "validating", + "backend": "provider", + "model": second.model, + "embeddings": None, + } + + coordinator.complete_embeddings_batch = complete + server = build_server( + orchestrator, + port=0, + security=SecurityConfig(auth_token=_AUTH_TOKEN), + coordinator=coordinator, + ) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + status, body = _post( + server.server_address[1], + "/v1/batch/embeddings", + {"model": "embed-v1", "input": "hello"}, + ) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + assert status == 202 + assert body["batch_id"] == "accepted-batch" + assert attempted == [first.id, second.id] + + +def _assert_discovery_finishes(call) -> None: + """Require one discovery call to abandon a hung shared metadata fetch.""" + errors: list[BaseException] = [] + + def run() -> None: + try: + call() + except BaseException as exc: # pragma: no cover - surfaced below + errors.append(exc) + + worker = threading.Thread(target=run, daemon=True) + worker.start() + worker.join(timeout=0.5) + assert not worker.is_alive(), "shared discovery metadata bypassed the discovery deadline" + assert errors == [] + + +def test_discover_all_models_bounds_every_shared_metadata_fetch() -> None: + """Models.dev, OpenRouter ZDR, and credit metadata share the control-plane bound.""" + set_backend(InMemoryCredentialBackend()) + try: + register_credential("OPENAI_API_KEY", "sk-openai") + never_models_dev = threading.Event() + source = replace(OPENAI_SOURCE, models_dev_provider_id="openai") + with ( + patch( + "contextual_orchestrator.model_discovery._fetch_models_dev_metadata", + side_effect=lambda **_kwargs: never_models_dev.wait(), + ), + patch( + "contextual_orchestrator.model_discovery.discover_provider_models", + return_value=[], + ), + patch( + "contextual_orchestrator.model_discovery._openrouter_zdr_model_ids", + return_value=set(), + ), + ): + _assert_discovery_finishes( + lambda: discover_all_models((source,), discovery_deadline=0.05) + ) + + set_backend(InMemoryCredentialBackend()) + register_credential("OPENROUTER_API_KEY", "sk-router") + never_zdr = threading.Event() + with ( + patch( + "contextual_orchestrator.model_discovery.discover_provider_models", + return_value=[], + ), + patch( + "contextual_orchestrator.model_discovery._openrouter_zdr_model_ids", + side_effect=lambda **_kwargs: never_zdr.wait(), + ), + patch( + "contextual_orchestrator.model_discovery.openrouter_paid_inference_available", + return_value=None, + ), + ): + _assert_discovery_finishes( + lambda: discover_all_models((OPENROUTER_SOURCE,), discovery_deadline=0.05) + ) + + never_credit = threading.Event() + with ( + patch( + "contextual_orchestrator.model_discovery.discover_provider_models", + return_value=[], + ), + patch( + "contextual_orchestrator.model_discovery._openrouter_zdr_model_ids", + return_value=set(), + ), + patch( + "contextual_orchestrator.model_discovery.openrouter_paid_inference_available", + side_effect=lambda **_kwargs: never_credit.wait(), + ), + ): + _assert_discovery_finishes( + lambda: discover_all_models((OPENROUTER_SOURCE,), discovery_deadline=0.05) + ) + finally: + set_backend(None) diff --git a/tests/test_privacy_policy_analysis.py b/tests/test_privacy_policy_analysis.py index 4b98c7a74..f1ce9e937 100644 --- a/tests/test_privacy_policy_analysis.py +++ b/tests/test_privacy_policy_analysis.py @@ -205,6 +205,7 @@ def read(self, _limit: int) -> bytes: set_backend(None) request = opened.call_args.args[0] + assert opened.call_args.kwargs["timeout"] is None assert request.full_url == "http://127.0.0.1:8080/api/outbound/fetch" assert json.loads(request.data) == { "url": "https://provider.example/privacy", diff --git a/tests/test_provider_bootstrap.py b/tests/test_provider_bootstrap.py index 442bfa46f..3d66f15b6 100644 --- a/tests/test_provider_bootstrap.py +++ b/tests/test_provider_bootstrap.py @@ -18,7 +18,9 @@ from contextual_orchestrator.model_discovery import ( DiscoveredModel, PROVIDER_MODEL_SOURCES, + agent_id_for, agent_from_discovered, + model_group_name_for, ) from contextual_orchestrator import provider_bootstrap @@ -135,11 +137,11 @@ def test_diverse_selection_prefers_known_cost_without_treating_unknown_as_free() _model("openrouter", "OPENROUTER_API_KEY", "mistral-router", 2.0), _model("bytez", "BYTEZ_API_KEY", "llama-unknown", None), ] - selected = provider_bootstrap.select_provider_diverse_models(models, limit=3) + selected = provider_bootstrap.select_model_group_diverse_models(models, limit=3) assert [(item.provider_name, item.model_id) for item in selected] == [ ("openai", "gpt-cheap"), ("openrouter", "mistral-router"), - ("bytez", "llama-unknown"), + ("openai", "gpt-expensive"), ] @@ -151,11 +153,63 @@ def test_diverse_selection_prefers_provider_declared_free_over_unknown() -> None is_free=True, ) - assert provider_bootstrap.select_provider_diverse_models( + assert provider_bootstrap.select_model_group_diverse_models( [unknown, free], limit=1 ) == [free] +def test_diverse_selection_fails_closed_at_unpriced_boundary() -> None: + """A bounded pool cannot admit one of two equally unsupported candidates.""" + models = [ + _model("bytez", "BYTEZ_API_KEY", "bytez-unknown", None), + _model("openrouter", "OPENROUTER_API_KEY", "router-unknown", None), + ] + + with pytest.raises(provider_bootstrap.ProviderBootstrapError, match="ambiguous"): + provider_bootstrap.select_model_group_diverse_models(models, limit=1) + + +def test_diverse_selection_fails_closed_at_equal_known_price_boundary() -> None: + """Equal comparable cost cannot be resolved by provider/model names.""" + models = [ + _model("bytez", "BYTEZ_API_KEY", "bytez-priced", 1.0), + _model("openrouter", "OPENROUTER_API_KEY", "router-priced", 1.0), + ] + + with pytest.raises(provider_bootstrap.ProviderBootstrapError, match="ambiguous"): + provider_bootstrap.select_model_group_diverse_models(models, limit=1) + + +def test_diverse_selection_rejects_unmodeled_cost_displacement() -> None: + """Model-group diversity cannot displace cheaper evidence without a utility model.""" + models = [ + _model("nvidia_nim", "NVIDIA_NIM_API_KEY", "shared-model", 1.0), + _model("nvidia_nim_sub", "NVIDIA_NIM_API_KEY_SUB", "shared-model", 1.5), + _model("openrouter", "OPENROUTER_API_KEY", "distinct-model", 2.0), + ] + + with pytest.raises( + provider_bootstrap.ProviderBootstrapError, + match="decision model", + ): + provider_bootstrap.select_model_group_diverse_models(models, limit=2) + + +def test_diverse_selection_rejects_unmodeled_full_pool_reordering() -> None: + """Admitting every candidate cannot make diversity an implicit route order.""" + models = [ + _model("nvidia_nim", "NVIDIA_NIM_API_KEY", "shared-model", 1.0), + _model("nvidia_nim_sub", "NVIDIA_NIM_API_KEY_SUB", "shared-model", 1.5), + _model("openrouter", "OPENROUTER_API_KEY", "distinct-model", 2.0), + ] + + with pytest.raises( + provider_bootstrap.ProviderBootstrapError, + match="decision model", + ): + provider_bootstrap.select_model_group_diverse_models(models, limit=3) + + def test_partial_price_is_unknown_in_provider_bootstrap_ranking(): """A missing prompt or completion price cannot become an invented zero.""" partial = replace( @@ -164,7 +218,7 @@ def test_partial_price_is_unknown_in_provider_bootstrap_ranking(): ) complete = _model("nvidia_nim", "NVIDIA_NIM_API_KEY", "complete-model", 1.0) - selected = provider_bootstrap.select_provider_diverse_models( + selected = provider_bootstrap.select_model_group_diverse_models( [partial, complete], limit=2 ) @@ -182,7 +236,7 @@ def test_non_usd_price_cannot_outrank_a_comparable_usd_price(): ) priced_usd = _model("openai", "OPENAI_API_KEY", "priced-usd", 1.0) - selected = provider_bootstrap.select_provider_diverse_models( + selected = provider_bootstrap.select_model_group_diverse_models( [cheap_foreign, priced_usd], limit=2 ) @@ -210,22 +264,37 @@ def test_provider_bootstrap_reuses_shared_chat_capability_policy(model_id, eligi assert eligible is is_general_chat_agent_model_id(model_id) -def test_provider_bootstrap_keeps_nim_credentials_as_independent_accounts(): - """Each credential account competes independently, even at the same vendor.""" - nim_primary = _model("nvidia_nim", "NVIDIA_NIM_API_KEY", "primary-model", 0.01) - nim_secondary = _model("nvidia_nim_sub", "NVIDIA_NIM_API_KEY_SUB", "secondary-model", 0.02) - independent = _model("bytez", "BYTEZ_API_KEY", "independent-model", 0.5) +def test_provider_bootstrap_first_pass_is_model_group_diverse(): + nim_primary = _model("nvidia_nim", "NVIDIA_NIM_API_KEY", "deepseek/shared", 0.01) + nim_secondary = _model("nvidia_nim_sub", "NVIDIA_NIM_API_KEY_SUB", "deepseek/shared", 0.02) + independent = _model("openrouter", "OPENROUTER_API_KEY", "qwen/concrete:free", 0.0) - selected = provider_bootstrap.select_provider_diverse_models( + selected = provider_bootstrap.select_model_group_diverse_models( [nim_secondary, independent, nim_primary], limit=2 ) assert [(item.provider_name, item.model_id) for item in selected] == [ - ("nvidia_nim", "primary-model"), - ("nvidia_nim_sub", "secondary-model"), + ("openrouter", "qwen/concrete:free"), + ("nvidia_nim", "deepseek/shared"), ] +def test_openrouter_discovery_keeps_concrete_free_models_not_free_meta_router(): + concrete = replace( + _model("openrouter", "OPENROUTER_API_KEY", "qwen/concrete:free", 0.0), + is_free=True, + ) + meta_router = replace( + _model("openrouter", "OPENROUTER_API_KEY", "openrouter/free", 0.0), + is_free=True, + ) + + assert provider_bootstrap.select_model_group_diverse_models( + [meta_router, concrete], limit=2 + ) == [concrete] + assert agent_from_discovered(concrete).group_name == model_group_name_for(concrete) + + def test_non_chat_catalog_rows_are_never_selected_for_chat_service(): """Embeddings, rerankers, speech, image, moderation, and realtime rows stay inert.""" models = [ @@ -246,7 +315,7 @@ def test_non_chat_catalog_rows_are_never_selected_for_chat_service(): 2.0, ), ] - selected = provider_bootstrap.select_provider_diverse_models(models, limit=10) + selected = provider_bootstrap.select_model_group_diverse_models(models, limit=10) assert [(item.provider_name, item.model_id) for item in selected] == [ ("openai", "openai/gpt-4.1-mini") ] @@ -376,7 +445,7 @@ def fake_discover_all_models(): assert report.discovered_model_count == 1 assert report.eligible_model_count == 1 - assert report.selected_agent_ids == ("openai_gpt_test",) + assert report.selected_agent_ids == (agent_id_for(_model("openai", "OPENAI_API_KEY", "gpt-test", 1.0)),) assert report.enabled_agent_ids == () assert report.durable_agent_pool is False assert all( @@ -465,8 +534,9 @@ def test_durable_pool_withdraws_bootstrap_and_stale_discovered_agents( assert report.discovered_model_count == 1 assert report.eligible_model_count == 1 - assert report.selected_agent_ids == ("openrouter_qwen_current_coder",) - assert report.enabled_agent_ids == ("openrouter_qwen_current_coder",) + expected_id = agent_id_for(new_model) + assert report.selected_agent_ids == (expected_id,) + assert report.enabled_agent_ids == (expected_id,) assert report.durable_agent_pool is True restarted = TaskOrchestrator( @@ -474,7 +544,7 @@ def test_durable_pool_withdraws_bootstrap_and_stale_discovered_agents( agents_db=agents_db, ) assert {agent.id for agent in restarted.agents} == { - "openrouter_qwen_current_coder" + expected_id } assert restarted.agents[0].tags == ( "discovered", @@ -506,7 +576,9 @@ def test_cli_report_never_contains_secret_values(monkeypatch, capsys): report = json.loads(output) assert "OPENAI_API_KEY" in output assert report["eligible_model_count"] == 1 - assert report["selected_agent_ids"] == ["openai_gpt_test"] + assert report["selected_agent_ids"] == [ + agent_id_for(_model("openai", "OPENAI_API_KEY", "gpt-test", 1.0)) + ] assert report["enabled_agent_ids"] == [] assert report["durable_agent_pool"] is False for value in environment.values(): diff --git a/tests/test_provider_bootstrap_boundaries.py b/tests/test_provider_bootstrap_boundaries.py index 453377774..70d5fc6d5 100644 --- a/tests/test_provider_bootstrap_boundaries.py +++ b/tests/test_provider_bootstrap_boundaries.py @@ -20,7 +20,7 @@ ProviderBootstrapError, collect_provider_credentials, register_provider_credentials_atomically, - select_provider_diverse_models, + select_model_group_diverse_models, ) @@ -166,12 +166,12 @@ class AlienBackend: set_backend(None) -# --- select_provider_diverse_models ------------------------------------------------- +# --- select_model_group_diverse_models --------------------------------------------- def test_select_rejects_non_positive_limit() -> None: with pytest.raises(ValueError, match="must be positive"): - select_provider_diverse_models([_model("openai", "OPENAI_API_KEY", "gpt-x")], limit=0) + select_model_group_diverse_models([_model("openai", "OPENAI_API_KEY", "gpt-x")], limit=0) def test_select_unpriced_and_foreign_currency_models_sort_last_but_still_fill() -> None: @@ -179,7 +179,7 @@ def test_select_unpriced_and_foreign_currency_models_sort_last_but_still_fill() unpriced = _model("openrouter", "OPENROUTER_API_KEY", "qwen-free", prompt=None) eur = _model("nvidia_nim", "NVIDIA_NIM_API_KEY", "nim-eur", prompt=0.01, currency="EUR") - selected = select_provider_diverse_models( + selected = select_model_group_diverse_models( [unpriced, eur, priced], limit=3 ) # Known USD pricing wins the diversity slot; unknown/incomparable fill after. @@ -194,7 +194,7 @@ def test_select_keeps_independent_credential_accounts() -> None: same_family_sub = _model( "nvidia_nim_sub", "NVIDIA_NIM_API_KEY_SUB", "nim-delta", prompt=4.0 ) - selected = select_provider_diverse_models([primary, same_family_sub], limit=2) + selected = select_model_group_diverse_models([primary, same_family_sub], limit=2) # A shared vendor endpoint does not collapse independent credential accounts. assert [m.model_id for m in selected] == ["nim-gamma", "nim-delta"] @@ -202,7 +202,7 @@ def test_select_keeps_independent_credential_accounts() -> None: def test_select_skips_non_chat_candidates_entirely() -> None: guard = _model("openai", "OPENAI_API_KEY", "llama-guard-4b", prompt=0.1) chat = _model("openrouter", "OPENROUTER_API_KEY", "qwen-chat", prompt=9.0) - selected = select_provider_diverse_models([guard, chat], limit=5) + selected = select_model_group_diverse_models([guard, chat], limit=5) assert [m.model_id for m in selected] == ["qwen-chat"] @@ -267,7 +267,7 @@ def test_select_returns_partial_pool_when_family_exhausted_below_limit() -> None ) # Only one outage family exists, so diversity yields one slot, the filler # adds the second, and the pool legitimately ends below ``limit``. - selected = select_provider_diverse_models([primary, sibling], limit=5) + selected = select_model_group_diverse_models([primary, sibling], limit=5) assert [m.model_id for m in selected] == ["nim-primary", "nim-sibling"] @@ -301,8 +301,9 @@ def test_durable_pool_sync_keeps_manual_agents_and_disabled_leftovers( ] enabled = _synchronize_durable_agent_pool(agents_db, selected_models) + expected_ids = tuple(sorted(pb.agent_id_for(model) for model in selected_models)) - assert enabled == ("openai_gpt_current", "openrouter_qwen_current") + assert enabled == expected_ids restarted = TaskOrchestrator([], agents_db=agents_db) ids_by_state = { agent.id: ("disabled" if agent.disabled else "enabled") @@ -312,14 +313,144 @@ def test_durable_pool_sync_keeps_manual_agents_and_disabled_leftovers( # is neither activated nor deleted. assert ids_by_state["manual_operator_agent"] == "enabled" assert ids_by_state["openai_retired_model"] == "disabled" - assert ids_by_state["openai_gpt_current"] == "enabled" - assert ids_by_state["openrouter_qwen_current"] == "enabled" + assert all(ids_by_state[agent_id] == "enabled" for agent_id in expected_ids) + + +def test_durable_pool_does_not_activate_manual_legacy_id_collision(tmp_path: Any) -> None: + """Discovery must not override an operator-disabled manual endpoint.""" + from dataclasses import replace + from contextual_orchestrator import TaskOrchestrator + + agents_db = str(tmp_path / "manual_collision.db") + model = _model("openrouter", "OPENROUTER_API_KEY", "Vendor/Model") + generated = pb._active_agent_from_discovered(model) + manual = replace( + generated, + id="openrouter_vendor_model", + base_url="https://manual.example/v1", + disabled=True, + priority=77, + tags=("manual",), + ) + seeded = TaskOrchestrator([], agents_db=agents_db, allow_empty_agents=True) + assert seeded._pool_store is not None + seeded._pool_store.save(manual) + seeded.close() + + with pytest.raises(pb.ProviderBootstrapError, match="operator-managed agent identities"): + pb._synchronize_durable_agent_pool(agents_db, [model]) + restarted = TaskOrchestrator([], agents_db=agents_db, allow_empty_agents=True) + stored = next(agent for agent in restarted.candidates if agent.id == manual.id) + assert stored.base_url == manual.base_url + assert stored.disabled is True + assert stored.priority == 77 + assert stored.tags == ("manual",) + restarted.close() + + +def test_durable_pool_does_not_replace_manual_exact_id_collision(tmp_path: Any) -> None: + """The generated discovery id cannot overwrite an operator-owned row.""" + from dataclasses import replace + from contextual_orchestrator import TaskOrchestrator + + agents_db = str(tmp_path / "manual_exact_collision.db") + model = _model("openrouter", "OPENROUTER_API_KEY", "Vendor/Model") + generated = pb._active_agent_from_discovered(model) + manual = replace( + generated, + base_url="https://manual.example/v1", + disabled=True, + priority=77, + tags=("manual",), + ) + seeded = TaskOrchestrator([], agents_db=agents_db, allow_empty_agents=True) + assert seeded._pool_store is not None + seeded._pool_store.save(manual) + seeded.close() + + with pytest.raises(pb.ProviderBootstrapError, match="operator-managed agent identities"): + pb._synchronize_durable_agent_pool(agents_db, [model]) + restarted = TaskOrchestrator([], agents_db=agents_db, allow_empty_agents=True) + assert restarted.candidates == [manual] + restarted.close() + + +def test_durable_pool_collision_preflight_prevents_partial_activation(tmp_path: Any) -> None: + """A mixed valid/collision selection must write nothing before failing.""" + from dataclasses import replace + from contextual_orchestrator import TaskOrchestrator + + agents_db = str(tmp_path / "mixed_collision.db") + collision = _model("openrouter", "OPENROUTER_API_KEY", "Vendor/Model") + manual = replace( + pb._active_agent_from_discovered(collision), + id="openrouter_vendor_model", + base_url="https://manual.example/v1", + disabled=True, + tags=("manual",), + ) + seeded = TaskOrchestrator([], agents_db=agents_db, allow_empty_agents=True) + assert seeded._pool_store is not None + seeded._pool_store.save(manual) + seeded.close() + + valid = _model("nvidia_nim", "NVIDIA_NIM_API_KEY", "valid-model") + with pytest.raises(pb.ProviderBootstrapError, match="operator-managed agent identities"): + pb._synchronize_durable_agent_pool(agents_db, [valid, collision]) + + restarted = TaskOrchestrator([], agents_db=agents_db, allow_empty_agents=True) + assert restarted.candidates == [manual] + restarted.close() + + +@pytest.mark.parametrize( + ("model_id", "legacy_id", "capabilities"), + ( + ("Straße/Model", "openrouter_stra_e_model", ()), + ("模型", "openrouter_model", ("image",)), + ), +) +def test_unicode_legacy_collision_preflight_leaves_pool_unchanged( + tmp_path: Any, + model_id: str, + legacy_id: str, + capabilities: tuple[str, ...], +) -> None: + """Historical Unicode legacy IDs must collide before any selected row is saved.""" + from dataclasses import replace + + from contextual_orchestrator import TaskOrchestrator + + agents_db = str(tmp_path / "unicode_collision.db") + collision = replace( + _model("openrouter", "OPENROUTER_API_KEY", model_id), + capabilities=capabilities, + ) + manual = replace( + pb._active_agent_from_discovered(collision), + id=legacy_id, + base_url="https://manual.example/v1", + disabled=True, + tags=("manual",), + ) + seeded = TaskOrchestrator([], agents_db=agents_db, allow_empty_agents=True) + assert seeded._pool_store is not None + seeded._pool_store.save(manual) + seeded.close() + + valid = _model("nvidia_nim", "NVIDIA_NIM_API_KEY", "valid-model") + with pytest.raises(pb.ProviderBootstrapError, match="operator-managed agent identities"): + pb._synchronize_durable_agent_pool(agents_db, [valid, collision]) + + restarted = TaskOrchestrator([], agents_db=agents_db, allow_empty_agents=True) + assert restarted.candidates == [manual] + restarted.close() def test_durable_pool_sync_closes_temporary_orchestrator( monkeypatch: pytest.MonkeyPatch, tmp_path: Any, -) -> None: + ) -> None: """A refresh must stop telemetry owned by its temporary orchestrator.""" from contextual_orchestrator import TaskOrchestrator from contextual_orchestrator.provider_bootstrap import ( @@ -339,5 +470,55 @@ def close(self) -> None: [_model("openrouter", "OPENROUTER_API_KEY", "qwen-current")], ) - assert enabled == ("openrouter_qwen_current",) + assert enabled == (pb.agent_id_for(_model("openrouter", "OPENROUTER_API_KEY", "qwen-current")),) assert closed == [True] + + +def test_durable_pool_migrates_legacy_selected_endpoint_without_duplicate(tmp_path: Any) -> None: + from dataclasses import replace + from contextual_orchestrator import TaskOrchestrator + + agents_db = str(tmp_path / "legacy_selected.db") + model = _model("openrouter", "OPENROUTER_API_KEY", "Vendor/Model") + generated = pb._active_agent_from_discovered(model) + legacy = replace(generated, id="openrouter_vendor_model", disabled=False) + seeded = TaskOrchestrator([], agents_db=agents_db, allow_empty_agents=True) + seeded.sync_discovered_agents([legacy]) + seeded.close() + + enabled = pb._synchronize_durable_agent_pool(agents_db, [model]) + restarted = TaskOrchestrator([], agents_db=agents_db, allow_empty_agents=True) + matching = [ + agent + for agent in restarted.candidates + if agent.provider_name == generated.provider_name + and agent.credential_name == generated.credential_name + and agent.model == generated.model + ] + + assert enabled == (legacy.id,) + assert [agent.id for agent in matching] == [legacy.id] + assert matching[0].disabled is False + restarted.close() + + +def test_durable_pool_activates_refreshed_duplicate_legacy_endpoint(tmp_path: Any) -> None: + from dataclasses import replace + from contextual_orchestrator import TaskOrchestrator + + agents_db = str(tmp_path / "duplicate_legacy_selected.db") + model = _model("openrouter", "OPENROUTER_API_KEY", "Vendor/Model") + generated = pb._active_agent_from_discovered(model) + seeded = TaskOrchestrator([], agents_db=agents_db, allow_empty_agents=True) + assert seeded._pool_store is not None + seeded._pool_store.save(replace(generated, id="legacy_first", priority=1)) + seeded._pool_store.save(replace(generated, id="legacy_refreshed", priority=2)) + seeded.close() + + enabled = pb._synchronize_durable_agent_pool(agents_db, [model]) + restarted = TaskOrchestrator([], agents_db=agents_db, allow_empty_agents=True) + + assert enabled == ("legacy_refreshed",) + assert [agent.id for agent in restarted.agents] == ["legacy_refreshed"] + assert restarted.agents[0].priority == generated.priority + restarted.close() diff --git a/tests/test_provider_bootstrap_report_identity.py b/tests/test_provider_bootstrap_report_identity.py new file mode 100644 index 000000000..097eeb451 --- /dev/null +++ b/tests/test_provider_bootstrap_report_identity.py @@ -0,0 +1,102 @@ +"""Regression coverage for bootstrap report identity consistency.""" + +from __future__ import annotations + +from dataclasses import replace + +import pytest + +from contextual_orchestrator import TaskOrchestrator +from contextual_orchestrator.credentials import InMemoryCredentialBackend, set_backend +from contextual_orchestrator.model_discovery import DiscoveredModel, legacy_agent_id_for +from contextual_orchestrator import provider_bootstrap + + +@pytest.fixture(autouse=True) +def isolated_credential_backend(): + """Give this report-boundary regression an isolated credential registry.""" + set_backend(InMemoryCredentialBackend()) + yield + set_backend(None) + + +def test_durable_bootstrap_report_uses_persisted_legacy_identity(monkeypatch, tmp_path) -> None: + """Selected and enabled IDs must name the same persisted durable agent.""" + model = DiscoveredModel( + provider_name="openrouter", + model_id="vendor/model-a", + credential_name="OPENROUTER_API_KEY", + chat_base_url="https://openrouter.ai/api/v1", + auth_scheme="Bearer", + prompt_price_per_1k=0.0, + completion_price_per_1k=0.0, + is_free=True, + ) + current_agent = provider_bootstrap._active_agent_from_discovered(model) + legacy_id = legacy_agent_id_for(model) + assert current_agent.id != legacy_id + + agents_db = str(tmp_path / "agents.db") + seeded = TaskOrchestrator( + [replace(current_agent, id=legacy_id)], + agents_db=agents_db, + ) + seeded.sync_discovered_agents([replace(current_agent, id=legacy_id)]) + seeded.close() + + monkeypatch.setattr( + provider_bootstrap, + "discover_all_models", + lambda: ([model], []), + ) + + report = provider_bootstrap.bootstrap_provider_runtime( + environ={"OPENROUTER_API_KEY": "test-secret"}, + require_all_credentials=False, + agents_db=agents_db, + model_limit=1, + ) + + assert report.enabled_agent_ids == (legacy_id,) + assert report.selected_agent_ids == report.enabled_agent_ids + assert report.as_dict()["selected_agent_ids"] == [legacy_id] + + +def test_durable_bootstrap_preserves_selected_model_order(tmp_path) -> None: + """Durable report order must match the selector instead of agent-ID sorting.""" + selected = [ + DiscoveredModel( + provider_name="z_provider", + model_id="cheap-model", + credential_name="Z_PROVIDER_API_KEY", + chat_base_url="https://z-provider.example/v1", + auth_scheme="Bearer", + prompt_price_per_1k=0.0, + completion_price_per_1k=0.0, + is_free=True, + ), + DiscoveredModel( + provider_name="a_provider", + model_id="expensive-model", + credential_name="A_PROVIDER_API_KEY", + chat_base_url="https://a-provider.example/v1", + auth_scheme="Bearer", + prompt_price_per_1k=1.0, + completion_price_per_1k=1.0, + ), + ] + expected_ids = tuple( + provider_bootstrap.agent_id_for(model) for model in selected + ) + assert expected_ids != tuple(sorted(expected_ids)) + + enabled_ids = provider_bootstrap._synchronize_durable_agent_pool( + str(tmp_path / "ordered-agents.db"), + selected, + ) + + assert enabled_ids == expected_ids + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(pytest.main([__file__])) diff --git a/tests/test_provider_catalog_bootstrap.py b/tests/test_provider_catalog_bootstrap.py index 46cc38ba6..19d991b0e 100644 --- a/tests/test_provider_catalog_bootstrap.py +++ b/tests/test_provider_catalog_bootstrap.py @@ -19,6 +19,7 @@ DiscoveredModel, ProviderDiscoveryError, ProviderModelSource, + agent_id_for, ) from contextual_orchestrator.privacy_policy_analysis import PrivacyPolicyAssessment from contextual_orchestrator.provider_bootstrap import PROVIDER_CREDENTIAL_NAMES @@ -112,8 +113,8 @@ def test_failed_provider_uses_persisted_last_known_good_model() -> None: assert refreshes[1]["error_code"] is None assert all(row["finished_at"].endswith("+00:00") for row in refreshes) assert set(second.selected_agent_ids) == { - "openai_gpt_live", - "openrouter_router_new", + agent_id_for(_model(openai, "gpt-live")), + agent_id_for(_model(openrouter, "router-new")), } assert "secret-bearing detail" not in str(second.as_dict()) finally: @@ -158,7 +159,7 @@ def test_openrouter_fallback_rechecks_spend_before_last_known_good_selection( ) assert report.last_known_good_model_count == 2 - assert report.selected_agent_ids == ("openrouter_provider_free",) + assert report.selected_agent_ids == (agent_id_for(free),) finally: set_backend(None) diff --git a/tests/test_provider_catalog_bootstrap_boundaries.py b/tests/test_provider_catalog_bootstrap_boundaries.py index a1ac57bc3..08b28814a 100644 --- a/tests/test_provider_catalog_bootstrap_boundaries.py +++ b/tests/test_provider_catalog_bootstrap_boundaries.py @@ -18,6 +18,7 @@ from contextual_orchestrator.model_discovery import ( DiscoveredModel, ProviderModelSource, + agent_id_for, ) from contextual_orchestrator.provider_bootstrap import ( PROVIDER_CREDENTIAL_NAMES, @@ -223,7 +224,7 @@ def test_runtime_skips_sources_without_registered_credential() -> None: ) # The unregistered source contributes no model and no refresh evidence. assert report.catalog_model_count == 1 - assert report.selected_agent_ids == ("openai_gpt_live",) + assert report.selected_agent_ids == (agent_id_for(_model(openai, "gpt-live")),) assert report.providers_with_errors == () assert report.catalog_refresh_failure_count == 0 refreshes = report.as_dict()["catalog_refreshes"] diff --git a/tests/test_provider_catalog_bootstrap_report_identity.py b/tests/test_provider_catalog_bootstrap_report_identity.py new file mode 100644 index 000000000..ff779cca9 --- /dev/null +++ b/tests/test_provider_catalog_bootstrap_report_identity.py @@ -0,0 +1,77 @@ +"""Regression coverage for provider-catalog bootstrap report identities.""" + +from __future__ import annotations + +from dataclasses import replace + +import pytest + +from contextual_orchestrator import TaskOrchestrator +from contextual_orchestrator.credentials import InMemoryCredentialBackend, set_backend +from contextual_orchestrator.model_discovery import ( + DiscoveredModel, + ProviderModelSource, + legacy_agent_id_for, +) +from contextual_orchestrator.provider_bootstrap import _active_agent_from_discovered +from contextual_orchestrator.provider_catalog_bootstrap import ( + bootstrap_provider_catalog_runtime, +) +from contextual_orchestrator.provider_catalog_store import InMemoryProviderCatalogStore + + +@pytest.fixture(autouse=True) +def isolated_credential_backend(): + """Give the catalog bootstrap regression an isolated credential registry.""" + set_backend(InMemoryCredentialBackend()) + yield + set_backend(None) + + +def test_durable_catalog_report_uses_persisted_legacy_identity(tmp_path) -> None: + """Selected and enabled catalog IDs must name the same durable agent.""" + source = ProviderModelSource( + provider_name="openrouter", + credential_name="OPENROUTER_API_KEY", + list_url="https://openrouter.example/v1/models", + chat_base_url="https://openrouter.example/v1", + ) + model = DiscoveredModel( + provider_name=source.provider_name, + model_id="vendor/model-a", + credential_name=source.credential_name, + chat_base_url=source.chat_base_url, + auth_scheme=source.auth_scheme, + prompt_price_per_1k=0.0, + completion_price_per_1k=0.0, + is_free=True, + ) + current_agent = _active_agent_from_discovered(model) + legacy_id = legacy_agent_id_for(model) + assert current_agent.id != legacy_id + + agents_db = str(tmp_path / "agents.db") + seeded = TaskOrchestrator( + [replace(current_agent, id=legacy_id)], + agents_db=agents_db, + ) + seeded.sync_discovered_agents([replace(current_agent, id=legacy_id)]) + seeded.close() + + report = bootstrap_provider_catalog_runtime( + environ={"OPENROUTER_API_KEY": "test-secret"}, + require_all_credentials=False, + agents_db=agents_db, + model_limit=1, + catalog_store=InMemoryProviderCatalogStore(), + sources=(source,), + discovery=lambda _sources: ([model], []), + ) + + assert report.enabled_agent_ids == (legacy_id,) + assert report.selected_agent_ids == report.enabled_agent_ids + assert report.as_dict()["selected_agent_ids"] == [legacy_id] + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(pytest.main([__file__])) diff --git a/tests/test_provider_catalog_credential_promotion.py b/tests/test_provider_catalog_credential_promotion.py index 2a8e019ac..c5b792290 100644 --- a/tests/test_provider_catalog_credential_promotion.py +++ b/tests/test_provider_catalog_credential_promotion.py @@ -14,6 +14,7 @@ DiscoveredModel, ProviderDiscoveryError, ProviderModelSource, + agent_id_for, ) from contextual_orchestrator.provider_bootstrap import ProviderBootstrapError from contextual_orchestrator.provider_catalog_bootstrap import ( @@ -89,7 +90,7 @@ def failing_discovery(_sources): ) assert get_credential(source.credential_name) == "old-working-secret" - assert report.selected_agent_ids == ("openai_gpt_last_known_good",) + assert report.selected_agent_ids == (agent_id_for(_model(source, "gpt-last-known-good")),) assert report.restored_credentials == (source.credential_name,) @@ -110,7 +111,7 @@ def test_empty_refresh_restores_previous_credential_before_using_lkg() -> None: ) assert get_credential(source.credential_name) == "old-working-secret" - assert report.selected_agent_ids == ("openai_gpt_last_known_good",) + assert report.selected_agent_ids == (agent_id_for(_model(source, "gpt-last-known-good")),) assert report.restored_credentials == (source.credential_name,) @@ -188,7 +189,7 @@ def test_successful_refresh_promotes_the_candidate_credential() -> None: ) assert get_credential(source.credential_name) == "new-working-secret" - assert report.selected_agent_ids == ("openai_gpt_new_live",) + assert report.selected_agent_ids == (agent_id_for(live),) assert report.restored_credentials == () diff --git a/tests/test_provider_embedding_batch_backend.py b/tests/test_provider_embedding_batch_backend.py index 0eb661fbb..6c2bca137 100644 --- a/tests/test_provider_embedding_batch_backend.py +++ b/tests/test_provider_embedding_batch_backend.py @@ -8,6 +8,7 @@ from contextual_orchestrator.batch_routing import ( EmbeddingBatchRequest, ProviderEmbeddingBatchBackend, + _DaemonWorkerPool, ) from contextual_orchestrator.batch_job_registry import JobRegistryFactory from contextual_orchestrator import ( @@ -18,6 +19,7 @@ PriceEntry, TaskOrchestrator, ) +from contextual_orchestrator.cost_router import _DEFAULT_EMBEDDING_CLAIM_LEASE_SECONDS from contextual_orchestrator.orchestrator import ModelClient from contextual_orchestrator.provider_errors import ProviderUpstreamError from contextual_orchestrator.server import SecurityConfig, build_server @@ -148,6 +150,31 @@ def runner(requests): backend.close() +def test_provider_batch_wait_survives_infinite_deadline() -> None: + """A caller with no wall-clock deadline (``timeout=inf``) still completes. + + ``/v1/embeddings`` computes an ``inf`` remaining timeout when the client + has no configured deadline (contextual-orchestrator's no-implicit-deadline + default since #971). ``threading.Event.wait`` raises ``OverflowError`` for + a non-finite timeout on CPython/Linux, so the backend must translate an + infinite deadline into an unbounded (``None``) wait instead of passing it + straight through. + """ + release = threading.Event() + + def runner(requests): + release.wait(timeout=1) + return [[float(len(request.input_text))] for request in requests], 2 + + backend = ProviderEmbeddingBatchBackend(runner) + request = EmbeddingBatchRequest(input_text="synthetic input", model="synthetic-model") + job = backend.submit([request]) + release.set() + assert backend.wait(job, timeout=float("inf"))["status"] == "completed" + assert backend.retrieve(job)[0].embedding == [15.0] + backend.close() + + def test_queued_document_exposes_backend_poll_and_registry_retention_contract() -> None: """Queued HTTP documents carry owned cadence/retention, not caller guesses.""" release = threading.Event() @@ -285,6 +312,29 @@ def shutdown(self, **_kwargs): assert shutdown_called.is_set() +def test_daemon_worker_pool_submit_after_shutdown_raises_instead_of_stranding_work() -> None: + """A post-shutdown ``submit`` must fail fast, not enqueue behind sentinels. + + Regression for a Devin Review finding ("Daemon pool accepts + post-shutdown work", ContextualWisdomLab/contextual-orchestrator#971): + ``_DaemonWorkerPool.shutdown`` used to set no closed state, so a + concurrent direct ``submit`` could enqueue real work behind the + shutdown sentinels every worker exits on -- work no worker would ever + pick up again. Real ``ThreadPoolExecutor.submit`` raises + ``RuntimeError`` once ``shutdown()`` has run; ``_DaemonWorkerPool`` now + matches that contract instead of silently stranding the job. + """ + pool = _DaemonWorkerPool(2) + ran = threading.Event() + pool.submit(ran.set) + assert ran.wait(timeout=1) + + pool.shutdown() + + with pytest.raises(RuntimeError, match="shutdown"): + pool.submit(lambda: None) + + def test_server_shutdown_closes_embedding_workers() -> None: class ClosingBackend: name = "closing" @@ -414,6 +464,106 @@ def test_local_startup_registers_provider_backend_for_recovered_jobs() -> None: ) +class _FakeValkeyClient: + """The minimal hash/lock surface ``ValkeyJsonMapping``/``JobRegistryFactory`` use.""" + + def hset(self, *_args, **_kwargs): + """Accept a hash-field write; no data is actually persisted.""" + return 1 + + def hget(self, *_args, **_kwargs): + """Report every field as absent, matching a fresh empty hash.""" + return None + + def hgetall(self, *_args, **_kwargs): + """Report an empty hash for any key.""" + return {} + + def hdel(self, *_args, **_kwargs): + """Accept a hash-field delete; no data is actually persisted.""" + return 1 + + def hkeys(self, *_args, **_kwargs): + """Report no fields for any hash.""" + return [] + + def hlen(self, *_args, **_kwargs): + """Report an empty hash length.""" + return 0 + + def expire(self, *_args, **_kwargs): + """Accept a TTL refresh as a no-op.""" + return True + + def lock(self, *_args, **_kwargs): + """Return a lock object that always acquires immediately.""" + + class _Lock: + def acquire(self, *_args, **_kwargs): + """Always succeed synchronously.""" + return True + + def release(self): + """No-op release.""" + + return _Lock() + + +def test_durable_provider_embedding_backend_survives_unbounded_client_timeout() -> None: + """A durable job registry must not require ``ModelClient.timeout`` to be set. + + Constructing the coordinator previously raised ``ValueError: durable + provider backend claim lease must be positive`` whenever the client had + no configured wall-clock timeout (contextual-orchestrator's + no-implicit-deadline default since #971) and the job registry was + durable (Valkey-backed) -- a startup crash caused by deriving the claim + lease, an internal locking heartbeat, from that unrelated optional + client attribute. The lease must fall back to a fixed positive default + instead. + """ + registry = JobRegistryFactory(client=_FakeValkeyClient()) + assert registry.durable is True + agent = ModelAgent( + "remote_embedding", + "embed-v1", + base_url="https://synthetic.invalid/v1", + tags=("embedding",), + ) + orchestrator = TaskOrchestrator([agent]) + assert orchestrator.client.timeout is None + + coordinator = CostRoutingCoordinator(orchestrator, job_registry=registry) + + backend = coordinator._embedding_backends["provider"] + assert backend._claim_lease_seconds == _DEFAULT_EMBEDDING_CLAIM_LEASE_SECONDS + backend.close() + + +def test_unbounded_execution_timeout_never_substitutes_registry_retention() -> None: + """``execution_timeout_seconds=None`` stays genuinely unbounded. + + It previously fell back to the job registry's storage retention window + (7 days by default), silently expiring an intentionally unbounded + embedding job once that window elapsed -- contradicting #971's + no-implicit-deadline policy. The per-job deadline must be ``+inf``, and + a job that outlives the registry's default retention window must still + complete rather than being force-failed as expired. + """ + release = threading.Event() + + def runner(requests): + release.wait(timeout=1) + return [[float(len(request.input_text))] for request in requests], 2 + + backend = ProviderEmbeddingBatchBackend(runner, execution_timeout_seconds=None) + request = EmbeddingBatchRequest(input_text="synthetic input", model="synthetic-model") + job = backend.submit([request]) + assert backend._execution_deadline(job.job_id) == float("inf") + release.set() + assert backend.wait(job, timeout=1)["status"] == "completed" + backend.close() + + def test_server_closes_provider_backend_added_after_startup() -> None: orchestrator = TaskOrchestrator([], allow_empty_agents=True) coordinator = CostRoutingCoordinator( @@ -433,6 +583,40 @@ def test_server_closes_provider_backend_added_after_startup() -> None: assert closed.is_set() +def test_recovered_privacy_scoped_embedding_batch_revalidates_current_agent_tags() -> None: + """A pinned ``zdr_only`` route must still satisfy ZDR at execution time. + + ``ProviderEmbeddingBatchBackend`` replays a durably-queued job's pinned + ``agent_id`` after a process restart recovers it -- an arbitrarily long + gap in which an operator could remove the agent's ``privacy:zdr`` tag or + repoint it to a non-ZDR route. Executing a request whose stored + ``zdr_only=True`` against an agent that no longer carries that tag must + fail closed instead of silently sending the batch through an unverified + route. + """ + agent = ModelAgent( + "reconfigured_embedding", + "reconfigured-embedding-model", + base_url="https://synthetic.invalid/v1", + tags=("embedding",), # privacy:zdr was revoked since this batch was submitted + ) + client = _SyntheticProviderClient() + coordinator = CostRoutingCoordinator( + TaskOrchestrator([agent], client=client), + embedding_token_counter=_SyntheticExactCounter(), + ) + stale_request = EmbeddingBatchRequest( + input_text="synthetic input", + model=agent.model, + zdr_only=True, + agent_id=agent.id, + ) + + with pytest.raises(RuntimeError, match="zdr_only"): + coordinator._run_provider_embeddings([stale_request]) + assert client.embedding_calls == [] + + def test_provider_embedding_requests_are_sharded_by_the_existing_token_limit() -> None: agent = ModelAgent( "synthetic_embedding", diff --git a/tests/test_provider_embedding_batch_backend_process_exit.py b/tests/test_provider_embedding_batch_backend_process_exit.py new file mode 100644 index 000000000..daab8933a --- /dev/null +++ b/tests/test_provider_embedding_batch_backend_process_exit.py @@ -0,0 +1,100 @@ +"""Regression: an unbounded provider embedding runner must not block exit.""" + +from __future__ import annotations + +import subprocess +import sys +import textwrap +import time +from pathlib import Path + + +def test_hung_provider_embedding_runner_does_not_block_process_exit_after_close() -> None: + """A never-returning embedding runner must not block interpreter shutdown. + + Regression for a Devin Review finding ("Unbounded embeddings block + process shutdown", ContextualWisdomLab/contextual-orchestrator#971): + ``ProviderEmbeddingBatchBackend`` used to fan its durable, pollable job + queue out across a ``concurrent.futures.ThreadPoolExecutor``. That + executor's worker threads register with ``concurrent.futures.thread``'s + own interpreter-exit hook, which unconditionally *joins* every + still-running worker at shutdown regardless of that worker's own daemon + status. ``cost_router.py``'s ``_provider_embedding_backend`` passes + ``execution_timeout_seconds=None`` by default (this org's deliberate + no-deadline ``ModelClient.timeout=None`` policy), and even a finite + timeout there is only a *cooperative* check performed after the + runner call returns -- never a preemptive cancellation of an in-flight + call. A worker permanently blocked inside a provider embedding runner + that never returns would therefore hang the join, and therefore process + shutdown, forever -- even after ``ProviderEmbeddingBatchBackend.close()`` + had already been called. ``ProviderEmbeddingBatchBackend`` now drives its + bounded worker pool from a private ``_DaemonWorkerPool`` built on plain + ``threading.Thread(daemon=True)`` workers, never + ``ThreadPoolExecutor``, so an abandoned worker carries no such + registration and the process can still exit. + + Verified end-to-end in a real, separate interpreter (an in-process + thread-introspection assertion cannot distinguish "still hanging in the + background" from "would actually block this process's shutdown" -- the + whole point of the finding): a helper script submits one embedding job + whose runner blocks on an ``Event`` that is never set, waits for the job + to actually reach the "running" state (so the runner is genuinely + in-flight, not merely queued), calls ``backend.close()`` -- mirroring a + real server shutdown -- and then lets the script's ``__main__`` fall + through to a normal, unforced exit with no explicit + ``sys.exit()``/``os._exit()``. RED-before/GREEN-after against the + pre-fix ``ThreadPoolExecutor`` version: the same script hung for the + full outer bound and was killed; it exits cleanly, well under that + bound, with this fix. + """ + script = textwrap.dedent( + """ + import sys + import threading + import time + + sys.path.insert(0, %(repo_root)r) + from contextual_orchestrator.batch_routing import ( + EmbeddingBatchRequest, + ProviderEmbeddingBatchBackend, + ) + + never_set = threading.Event() + runner_entered = threading.Event() + + def hung_runner(requests): + runner_entered.set() + never_set.wait() # Hangs forever -- nothing ever sets this event. + raise AssertionError("unreachable: the stalled runner must never return") + + backend = ProviderEmbeddingBatchBackend(hung_runner, max_concurrency=1) + job = backend.submit( + [EmbeddingBatchRequest(input_text="synthetic input", model="synthetic-model")] + ) + assert runner_entered.wait(timeout=5), "runner never started" + # The runner is now genuinely blocked inside the worker thread -- + # not merely queued -- exactly the scenario the finding describes. + deadline = time.monotonic() + 2 + while backend.poll(job)["status"] != "running" and time.monotonic() < deadline: + time.sleep(0.01) + assert backend.poll(job)["status"] == "running" + + backend.close() + assert backend.poll(job)["status"] == "running" + # No explicit sys.exit()/os._exit(): a genuinely non-blocking fix + # must let normal interpreter shutdown proceed on its own, with the + # hung runner thread still blocked in the background. + """ + ) % {"repo_root": str(Path(__file__).resolve().parents[1])} + + started = time.monotonic() + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + timeout=15, + ) + elapsed = time.monotonic() - started + + assert result.returncode == 0, result.stderr + assert elapsed < 5.0, f"process took {elapsed:.1f}s to exit with a hung embedding runner outstanding" diff --git a/tests/test_provider_error_taxonomy.py b/tests/test_provider_error_taxonomy.py index 17f5ebd95..79605e1d3 100644 --- a/tests/test_provider_error_taxonomy.py +++ b/tests/test_provider_error_taxonomy.py @@ -20,8 +20,14 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import pytest # noqa: E402 + from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 -from contextual_orchestrator.orchestrator import ModelClient, is_transient_error # noqa: E402 +from contextual_orchestrator.orchestrator import ( # noqa: E402 + ModelClient, + _REQUEST_ZDR_ONLY, + is_transient_error, +) from contextual_orchestrator.provider_errors import ( # noqa: E402 MAX_PROVIDER_ERROR_BODY_BYTES, MAX_SAFE_MESSAGE_CHARS, @@ -313,6 +319,26 @@ def test_binary_passthrough_classifies_provider_transport_failure() -> None: raise AssertionError("binary provider failure must be classified") +def test_speech_passthrough_rejects_malformed_zdr_provider_routing() -> None: + """A non-object ``provider`` on the speech/audio bytes path fails closed. + + Devin review on #953: under ``zdr_only`` scope, ``_pin_openrouter_zdr`` + used to build ``dict(payload["provider"])`` unconditionally, so a + caller-supplied non-mapping ``provider`` (an int, bool, list, or string) + raised an uncaught ``TypeError`` from inside ``proxy_send_bytes`` before + any provider transport or failure classification ran. It must instead + raise the same named validation error the helper raises everywhere else. + """ + client = ModelClient(max_retries=0) + agent = ModelAgent("audio_agent", "audio-model", provider_name="openrouter") + token = _REQUEST_ZDR_ONLY.set(True) + try: + with pytest.raises(ValueError, match="provider must be an object"): + client.proxy_send_bytes(agent, "audio/speech", {"input": "hello", "provider": 5}) + finally: + _REQUEST_ZDR_ONLY.reset(token) + + def test_detail_and_transport_are_preserved_for_callers() -> None: """The structured detail names agent/model/status/retryability/transport.""" classified = classify_provider_failure( diff --git a/tests/test_provider_gateway_resilience.py b/tests/test_provider_gateway_resilience.py new file mode 100644 index 000000000..f65c8f87a --- /dev/null +++ b/tests/test_provider_gateway_resilience.py @@ -0,0 +1,110 @@ +"""Regression contracts for provider retry and inference deadlines.""" + +from __future__ import annotations + +import io +import urllib.error + +import pytest + +from contextual_orchestrator import ModelAgent +from contextual_orchestrator.orchestrator import ModelClient +from contextual_orchestrator.provider_errors import ProviderUpstreamError + + +def _http_error(status: int) -> urllib.error.HTTPError: + """Build one deterministic OpenAI-compatible provider failure.""" + return urllib.error.HTTPError( + "https://provider.example/v1/chat/completions", + status, + "provider failure", + {}, + io.BytesIO(b"{}"), + ) + + +def _agent(*, reasoning_effort_supported: bool | None) -> ModelAgent: + """Return a provider-neutral chat route with explicit capability evidence.""" + return ModelAgent( + "provider_route", + "arbitrary-chat-model", + base_url="https://provider.example/v1", + provider_name="provider", + reasoning_effort_supported=reasoning_effort_supported, + ) + + +class _GatewayClient(ModelClient): + """Expose deterministic raw-provider outcomes through the public proxy seam.""" + + def __init__(self, outcomes: list[object], *, max_retries: int) -> None: + """Store provider outcomes and disable real retry sleeping.""" + super().__init__(max_retries=max_retries, retry_backoff=0.0) + self._outcomes = iter(outcomes) + self.attempts = 0 + + def _validate_provider(self, agent: ModelAgent): # type: ignore[override] + """Bypass DNS validation because this test never opens a socket.""" + del agent + return None + + def _send_raw( # type: ignore[override] + self, + agent: ModelAgent, + endpoint: str, + payload: dict[str, object], + destination=None, + ) -> dict[str, object]: + """Return or raise the next transport outcome.""" + del agent, endpoint, payload, destination + self.attempts += 1 + outcome = next(self._outcomes) + if isinstance(outcome, BaseException): + raise outcome + assert isinstance(outcome, dict) + return outcome + + +@pytest.mark.parametrize("reasoning_effort_supported", [None, False, True]) +def test_proxy_send_recovers_transient_502_independent_of_reasoning_capability( + reasoning_effort_supported: bool | None, +) -> None: + """Transport retry depends on failure taxonomy, never a model family or capability flag.""" + response = { + "choices": [{"finish_reason": "stop", "message": {"content": "OK"}}] + } + client = _GatewayClient([_http_error(502), response], max_retries=1) + agent = _agent(reasoning_effort_supported=reasoning_effort_supported) + + result = client.proxy_send( + agent, + "chat/completions", + {"model": agent.model, "messages": []}, + ) + + assert result == response + assert client.attempts == 2 + + +def test_proxy_send_does_not_retry_permanent_auth_failure() -> None: + """A 401 remains terminal even when the client has a retry budget.""" + client = _GatewayClient([_http_error(401)], max_retries=3) + agent = _agent(reasoning_effort_supported=True) + + with pytest.raises(ProviderUpstreamError) as excinfo: + client.proxy_send( + agent, + "chat/completions", + {"model": agent.model, "messages": []}, + ) + + assert excinfo.value.provider_status == 401 + assert client.attempts == 1 + + +def test_model_client_has_no_default_inference_deadline() -> None: + """Every model may run until explicit caller or workflow cancellation.""" + client = ModelClient() + + assert client.timeout is None + assert client.connect_timeout is None diff --git a/tests/test_review_gateway.py b/tests/test_review_gateway.py index edbf23435..0379f67a1 100644 --- a/tests/test_review_gateway.py +++ b/tests/test_review_gateway.py @@ -108,7 +108,7 @@ def test_build_review_orchestrator_never_routes_evidence_only_models(monkeypatch """Evidence-only catalog rows are never review upstreams.""" discovered = [ _discovered( - "openrouter", + "bytez", "router-review", "OPENROUTER_API_KEY", evidence_only=True,