fix(routing): select concrete free model groups

.github/workflows/opencode-hourly-loop.yml

@@ -51,7 +51,6 @@ jobs:
   loop:
     name: Run OpenCode maintenance agent through the local gateway
     runs-on: ubuntu-latest
-    timeout-minutes: 55
     permissions:
       contents: write # required for reviewed branch fixes; scoped to this job
       pull-requests: write
@@ -68,29 +67,35 @@ jobs:
       NVIDIA_NIM_API_KEY_SUB: ${{ secrets.NVIDIA_NIM_API_KEY_SUB }}
     steps:
       - name: Checkout repository
+        timeout-minutes: 5
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7
         with:
           persist-credentials: true # the agent needs to push fixes/branches
 
       - name: Set up Python
+        timeout-minutes: 5
         uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6
         with:
           python-version: "3.12"
 
       - name: Set up Node for the OpenCode CLI
+        timeout-minutes: 5
         uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # actions/setup-node@v4
         with:
           node-version: "22"
 
       - name: Install pinned runtime dependencies
+        timeout-minutes: 10
         run: python -m pip install --require-hashes -r requirements.lock
 
       - name: Install OpenCode CLI
+        timeout-minutes: 10
         run: |
           npm ci --ignore-scripts
           node scripts/ci/install_locked_opencode.mjs
 
       - name: Generate ephemeral loopback gateway token
+        timeout-minutes: 1
         run: |
           gateway_token=$(python -c 'import secrets; print(secrets.token_urlsafe(32))')
           echo "::add-mask::$gateway_token"
@@ -106,17 +111,24 @@ jobs:
             --auth-token-key CONTEXTUAL_ORCHESTRATOR_TOKEN \
             --host 127.0.0.1 --port 8000 \
             > /tmp/gateway.log 2>&1 &
-          for _attempt in $(seq 1 30); do
+          gateway_pid=$!
+          while :; do
             if curl -fsS -H "Authorization: Bearer ${CONTEXTUAL_ORCHESTRATOR_TOKEN}" \
               http://127.0.0.1:8000/healthz >/dev/null 2>&1; then
               echo "gateway ready"
               exit 0
             fi
+            if ! kill -0 "$gateway_pid" 2>/dev/null; then
+              wait "$gateway_pid" || true
+              echo "::error::gateway exited before becoming healthy"
+              tail -50 /tmp/gateway.log
+              exit 1
+            fi
             sleep 2
           done
-          echo "::error::gateway did not become healthy"; tail -50 /tmp/gateway.log; exit 1
 
       - name: Point OpenCode at the local gateway
+        timeout-minutes: 1
         run: |
           set -euo pipefail
           umask 077
@@ -142,6 +154,7 @@ jobs:
           JSON
 
       - name: Build trusted same-repository PR allowlist
+        timeout-minutes: 5
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
@@ -157,5 +170,6 @@ jobs:
             "$(cat .github/opencode/hourly-loop-prompt.md)"
 
       - name: Dump gateway log on failure
+        timeout-minutes: 5
         if: failure()
         run: tail -100 /tmp/gateway.log

.github/workflows/source-fix-971-live-review-quality.yml

@@ -0,0 +1,105 @@
+name: PR 971 live review quality repair
+
+on:
+  push:
+    branches:
+      - fix/model-group-timeout-openrouter
+    paths:
+      - .github/workflows/source-fix-971-live-review-quality.yml
+  workflow_dispatch:
+
+permissions:
+  contents: write
+
+concurrency:
+  group: pr-971-live-review-quality-repair
+  cancel-in-progress: false
+
+jobs:
+  repair:
+    name: RED to GREEN owner repair
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout exact writer head
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7
+        with:
+          fetch-depth: 0
+          persist-credentials: true
+
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
+        with:
+          version: "0.12.5"
+
+      - name: Guard exact writer head
+        shell: bash
+        run: |
+          set -euo pipefail
+          remote_head="$(git ls-remote origin "refs/heads/${GITHUB_REF_NAME}" | cut -f1)"
+          test "$remote_head" = "$GITHUB_SHA"
+
+      - name: Verify four causal RED regressions
+        shell: bash
+        run: |
+          set -euo pipefail
+          nodes=(
+            "tests/test_pr971_review_quality_regressions.py::test_recovered_zdr_batch_reenters_request_privacy_scope"
+            "tests/test_pr971_review_quality_regressions.py::test_provider_embedding_batch_rejects_mixed_privacy_identity"
+            "tests/test_pr971_review_quality_regressions.py::test_terminal_embedding_batch_document_fails_over_before_marking_health"
+            "tests/test_pr971_review_quality_regressions.py::test_discover_all_models_bounds_every_shared_metadata_fetch"
+          )
+          for node in "${nodes[@]}"; do
+            set +e
+            uv run --locked --extra api --extra db --extra queue --group dev python -m pytest -q "$node"
+            rc=$?
+            set -e
+            if [ "$rc" -ne 1 ]; then
+              echo "Expected pytest RED exit 1 for ${node}, got ${rc}" >&2
+              exit 1
+            fi
+          done
+
+      - name: Apply causal owner repairs
+        run: python scripts/ci/pr971_live_review_quality_repair.py
+
+      - name: Verify focused GREEN
+        run: >-
+          uv run --locked --extra api --extra db --extra queue --group dev
+          python -m pytest -q
+          tests/test_pr971_review_quality_regressions.py
+          tests/test_provider_embedding_batch_backend.py
+          tests/test_embeddings_model_pool_http_honesty.py
+          tests/test_model_discovery.py
+
+      - name: Verify broader GREEN
+        run: |
+          set -euo pipefail
+          uv run --locked --extra api --extra db --extra queue --group dev python -m pytest -q
+          python -m compileall -q contextual_orchestrator tests/test_pr971_review_quality_regressions.py
+          git diff --check
+
+      - name: Remove temporary repair machinery and publish
+        shell: bash
+        run: |
+          set -euo pipefail
+          remote_head="$(git ls-remote origin "refs/heads/${GITHUB_REF_NAME}" | cut -f1)"
+          test "$remote_head" = "$GITHUB_SHA"
+          rm .github/workflows/source-fix-971-live-review-quality.yml
+          rm scripts/ci/pr971_live_review_quality_repair.py
+          if find .github/workflows -maxdepth 1 -type f -name '*source-fix-971*' -print -quit | grep -q .; then
+            echo "A source-fix-971 workflow would remain in the publishable tree" >&2
+            exit 1
+          fi
+          git diff --check
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git add -A
+          git commit -m "fix(review): enforce embedding privacy health and discovery bounds"
+          remote_head="$(git ls-remote origin "refs/heads/${GITHUB_REF_NAME}" | cut -f1)"
+          test "$remote_head" = "$GITHUB_SHA"
+          git push origin "HEAD:${GITHUB_REF_NAME}"

CHANGELOG.d/openrouter-not-evidence-only.md

@@ -0,0 +1 @@
+Stopped treating OpenRouter as a whole-account, ZDR-motivated serving exclusion. `PROVIDER_MODEL_SOURCES`'s `openrouter` entry no longer sets `evidence_only=True`: ZDR eligibility is a route/model-level property, never grounds to block an entire provider account from serving, and OpenRouter was the one provider source with genuinely reliable native pricing/`is_free` evidence, so excluding it directly caused `orchestrator/free`'s previously-documented structural emptiness (ADR 0041). Also fixed a backwards side effect of the old flag: `_apply_discovered_model_evidence` could never mark OpenRouter's own rows `zdr_capable=True` even when they exactly matched OpenRouter's own declared ZDR feed. The provider-neutral evidence-application contract from PR #901 (OpenRouter's feed also crediting matching rows from every other provider) is unchanged. Since OpenRouter can multiplex one model id across several backing providers, `ModelClient` now pins every OpenRouter request made under an active `zdr_only` scope with OpenRouter's own documented `"provider": {"zdr": true}` request-time enforcement, applied at the shared `_send`/`_stream_send`/`_send_raw` transport chokepoints, and at the async Batch API path (`_batch_run`) via the same helper on each uploaded JSONL request body.

CHANGELOG.md

@@ -408,6 +408,19 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html)
   retried as if it were a network blip. Fixes the shared classifier itself
   (not just the discovery retry call site), so every current and future
   caller of `is_transient_error` benefits.
+- (Devin review on #953) `_pin_openrouter_zdr` no longer raises a bare
+  `TypeError` from `dict()` when a caller-supplied `provider` field is
+  present but not an object (an int, bool, list, or string) under an active
+  `zdr_only` scope. It now validates the field and raises a named `ValueError`
+  ("provider must be an object with optional OpenRouter routing keys")
+  instead, matching this codebase's existing convention for malformed
+  caller-input fields. Every call site sharing this one choke point (chat,
+  streaming, tools/binary-media passthrough, and the batch JSONL path)
+  benefits; a valid `provider` object or an absent/`None` one keep their
+  existing behavior unchanged.
+- OpenRouter embedding Batch JSONL now carries the same request-scoped
+  `provider.zdr=true` enforcement when `zdr_only` selects an attested
+  OpenRouter embedding agent.
 - `batch_route` no longer fabricates a hardcoded, ungated
   `{"accepted": True, "verifier_output": ""}` verification verdict for every
   batched answer regardless of `policy.realtime_judge`. It now calls the same
@@ -854,6 +867,110 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html)
   otherwise. `_write_sse` relies on `_begin_sse`'s already-set marker rather
   than touching it itself, since it is only ever called after a prior
   successful header flush.
+- (2026-09-02, PR #971) `ModelClient`'s constructor no longer defaults
+  `max_retries` to a hand-picked `2`: no cited standard, paper, or the org's
+  own research (Fugu, Conductor, TRINITY) establishes that number, and a
+  fresh audit found it was never justified. RFC 9110 §9.2.2 constrains *when*
+  replay can be safe for idempotent semantics, and NIST SP 800-204 discusses
+  retry/circuit-breaker resilience as a pattern, but neither identifies a
+  specific numeric retry allocation for this library. The default is now
+  `0`: a default `ModelClient` allocates zero automatic provider transport
+  retries, independent of provider, model, or reasoning identity
+  (`tests/test_no_heuristic_default_transport_retry.py`). Explicit nonzero
+  retry budgets remain caller-owned configuration, never a library-authored
+  default (ADR 0001 amendment, 2026-09-02).
+- (2026-09-02, PR #971) A default (no configured deadline) synchronous
+  `/v1/embeddings` request against a provider-backed member returned 503:
+  `embedding_deadline` collapsed to `+inf`, and `ProviderEmbeddingBatchBackend.wait`
+  passed that straight into `threading.Event.wait(timeout=...)`, which raises
+  `OverflowError` for a non-finite timeout on CPython -- directly contradicting
+  #971's own no-implicit-deadline policy. `wait()` now translates a non-finite
+  timeout to `None` (block indefinitely) instead
+  (`tests/test_provider_embedding_batch_backend.py::test_provider_batch_wait_survives_infinite_deadline`).
+- (2026-09-02, PR #971) A failed configured-gateway structured-chat probe
+  recorded the failing model only under its new fingerprinted id
+  (`agent_id_for`); a persisted agent kept under its pre-fingerprint legacy
+  id (`legacy_agent_id_for`) was silently dropped from `runtime_models` and
+  never reached the disable path, leaving a failed legacy endpoint enabled
+  indefinitely. `_auto_discover_runtime_agents` now accepts either id form
+  when checking for an existing persisted agent
+  (`tests/test_auto_discovery_server.py::test_failed_gateway_probe_disables_legacy_id_persisted_agent`).
+- (2026-09-02, PR #971) A privacy-scoped (`zdr_only=True`) embedding batch's
+  pinned `agent_id` was replayed verbatim by `ProviderEmbeddingBatchBackend`
+  after a process restart recovers a durably queued job, with no
+  re-validation that the agent still carried the `privacy:zdr` tag -- an
+  operator could remove ZDR support or repoint the agent between submission
+  and a resumed execution and the batch would still run. `_run_provider_embeddings`
+  now re-checks the request's own recorded `zdr_only` against the resolved
+  agent's current tags at execution time and fails closed if they no longer
+  match
+  (`tests/test_provider_embedding_batch_backend.py::test_recovered_privacy_scoped_embedding_batch_revalidates_current_agent_tags`).
+- (2026-09-02, PR #971) A synchronous `/v1/embeddings` member result that
+  came back without raising but was not `completed` (or had no embeddings)
+  bypassed `orchestrator._record_embedding_failure`, so the circuit breaker
+  never opened and no `embedding_endpoint_failed` analytics event was
+  recorded for that failure mode -- a repeatedly incomplete member was
+  retried forever instead of being quarantined like a raised exception. It
+  now routes through the same failure recorder
+  (`tests/test_embeddings_model_pool_http_honesty.py::test_http_embeddings_quarantines_repeated_incomplete_document_with_failure_evidence`).
+- (2026-09-02, PR #971) `CostRoutingCoordinator` derived the durable
+  provider-embedding claim lease from `ModelClient.timeout`, so a durable
+  (Valkey-backed) job registry raised `ValueError: durable provider backend
+  claim lease must be positive` at startup whenever the client had no
+  configured deadline (the default since #971) -- a crash directly caused by
+  conflating an internal locking heartbeat with the caller's request
+  deadline. The claim lease now falls back to a fixed, positive default
+  independent of `ModelClient.timeout`. Separately, `execution_timeout_seconds=None`
+  previously fell back to the job registry's storage retention window (7
+  days), silently expiring an intentionally unbounded embedding job; it now
+  stays genuinely unbounded (`+inf` deadline)
+  (`tests/test_provider_embedding_batch_backend.py::test_durable_provider_embedding_backend_survives_unbounded_client_timeout`,
+  `::test_unbounded_execution_timeout_never_substitutes_registry_retention`).
+- (2026-09-02, PR #971) `OpenRouterUptimeCollector._fetch_uptime` adopted
+  #971's inference no-fixed-deadline policy (`timeout=None`) even though it
+  is unrelated background telemetry polled sequentially on one dedicated
+  sweep thread that `stop()` cannot interrupt mid-request: one unresponsive
+  OpenRouter endpoint would hang that thread forever, leaking it and
+  indefinitely starving every later member of an uptime update. The fetch
+  now keeps its own fixed, independent bound
+  (`tests/test_openrouter_uptime.py::test_uptime_fetch_does_not_hang_forever_on_an_unresponsive_endpoint`).
+- (2026-09-02, PR #971) `discover_all_models`'s per-provider loop
+  (`model_discovery.py`) called `discover_provider_models` directly and
+  in-line, with no separate bound or cancellation mechanism -- a stalled
+  provider catalog request (a connection accepted but never answered, or
+  in tests a mock that never returns) blocked discovery of every later,
+  healthy provider forever, regardless of `DISCOVERY_TIMEOUT_SECONDS`
+  (which only bounds one socket read at a time and stays `None` by
+  default). Each provider's discovery attempt now runs on its own daemon
+  thread bounded by a new, separately configured
+  `PROVIDER_DISCOVERY_DEADLINE_SECONDS` (default 30.0s, independent of
+  both `DISCOVERY_TIMEOUT_SECONDS` and `ModelClient.timeout` -- neither is
+  reused or repurposed): once the deadline elapses the caller stops
+  waiting, records a `ProviderDiscoveryError(error_code="discovery_timeout")`
+  for that provider, and moves on; the abandoned thread is daemonized so
+  it cannot block interpreter shutdown
+  (`tests/test_model_discovery.py::test_discover_all_models_bounds_a_stalled_provider_so_later_providers_still_complete`).
+- (2026-09-02, PR #971) `select_bootstrap_discovered_agents`'s first pass
+  admitted at most one endpoint per model group (the provider-declared
+  exact model identity) but never checked provider identity, so several
+  cheap, distinctly-named models from one provider could fill most or all
+  of a bootstrap pool before a genuinely independent alternative provider
+  was ever tried -- an apparently diverse pool (distinct model names) that
+  was actually one provider's outage away from total failure, contrary to
+  this repo's own documented "provider-diverse selection" claim for this
+  function. The first pass now admits at most one endpoint per provider
+  *and* per model group; once every viable provider has contributed once
+  (or capacity runs out), a second pass fills remaining slots from
+  still-untried model groups regardless of provider, and a final pass
+  falls back to duplicate model-group endpoints only once real diversity
+  is exhausted. `contextual_orchestrator/provider_bootstrap.py`'s separate,
+  honestly-named `select_model_group_diverse_models` (which never claimed
+  provider diversity, and whose own tests deliberately rank a known
+  same-provider price ahead of diversity) was left unchanged
+  (`tests/test_discovery_bootstrap_selection.py::test_bootstrap_selector_spans_multiple_providers_before_repeating_one`,
+  `::test_bootstrap_selector_prefers_model_group_diversity`,
+  `::test_bootstrap_selector_falls_back_to_duplicate_model_group_when_capacity_remains`,
+  `tests/test_review_gateway.py::test_build_review_orchestrator_uses_model_group_diversity`).
 
 ### Added
 

README.md

@@ -290,8 +290,9 @@ is read from a **KV config store**, never `os.getenv`.
 - **Health.** `GET /healthz` is an unauthenticated liveness probe that returns
   only service identity and process status; it never discloses worker topology,
   backend names, usage volume, or upstream readiness. Admins can use
-  `GET /api/v1/provider_readiness/latest?refresh=true` for one bounded,
-  non-retrying chat probe per enabled worker.
+  `GET /api/v1/provider_readiness/latest?refresh=true` for one explicitly
+  cancellable, non-retrying chat probe per enabled worker. Concurrent refreshes
+  return `refresh_in_progress` instead of blocking behind a slow provider.
 - **Standalone + optional pg-llm-batch integration.** The hub runs standalone
   with the in-memory config store and local batch backend; wiring a Postgres DSN
   and an installed/deployed `pg_llm_batch` client activates the KV/secret stores,