diff --git a/.github/opencode/hourly-loop-prompt.md b/.github/opencode/hourly-loop-prompt.md index 53a91d280..0f2da8301 100644 --- a/.github/opencode/hourly-loop-prompt.md +++ b/.github/opencode/hourly-loop-prompt.md @@ -9,6 +9,18 @@ completion, explicit user/operator cancellation, and the hosting platform's own execution contract are the only time-based termination authorities available to this agent. Never post intermediate progress reports. +PR 0 only via merge or verified-successor full-delta inheritance. Never +force-push, never close without evidence (user-explicit, no valid delta, +malicious change, or verified complete inheritance only). A wrong close is +recovered through reopen or successor and never left closed. Single-writer +deltas are integrated, never discarded. Before and after long runs record +`git rev-parse HEAD` and `git diff --stat origin/main...HEAD -- +contextual_orchestrator/ tests/`; docs-only drift does not invalidate code +evidence. Prefer isolated worktrees, preserve live execution handles, and +never rebase or push another session's branch. Synthetic recovery is unit +evidence only; customer accuracy and decision latency require observed +outcomes with declared denominators and uncertainty. + Before changing code, read `docs/product_planning.md`, the applicable PRD in `docs/model-group-product-technical-spec.md`, and `docs/product-technical-gap-baseline.md`. Treat current files and exact GitHub @@ -44,10 +56,25 @@ contract rather than an inferred ranking: dependency model do not identify an order, do not invent an ordering; record the unresolved decision evidence and fail closed rather than using an informal leverage score. Implement an identified gap with tests + docstrings and open a - PR that updates the baseline file. + PR that updates the baseline file. When the PR queue is exhausted, continue + with gap development and merges plus ContextualWisdomLab repository and + connector linkage under responsibility boundaries, without waiting on + review or Checks for unrelated work. A failing check never idles the + loop: fix and rerun what you own immediately while continuing safe + independent work; codify manual workarounds into code or config and + root-cause PYTHONPATH, Actions, and execution errors from logs. Never make + a full foundation or a mutual official release a precondition: cut the + owner/consumer cycle with a minimal contract, port, or ACL and complete + independently verifiable functionality first. Rules: - Keep each change minimal and reviewable; stack dependent PRs when natural. + Record merge and delete rationale before committing; remove + self-modifying or source-fix workflows whose purpose is done. + Integrate single-writer deltas, never discard them; restack or retarget with + normal merges only, never force-push, and never flip another session's Draft + readiness. Fix stacked-PR review gaps in `.github`, coordinating through + agent dialogue or spawn. Preserve PRD and TRD naming case exactly. - Delete existing code, tests, or documentation only with a clear redundancy or root-cause rationale and after verifying that no supported consumer needs it. - Never expose internal implementation boundaries in customer-facing copy. Every diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 746067c9f..e54a3deaf 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -44,8 +44,16 @@ jobs: with: version: "0.12.5" + - name: Build locked native decision measurement + run: | + set -euo pipefail + rustup show active-toolchain + uv sync --locked --extra api --extra db --extra queue --group dev --group native-build + uv run --no-sync maturin develop --locked --release --features pyo3/extension-module --manifest-path rust/decision_receipt/Cargo.toml + uv run --no-sync maturin build --locked --release --features pyo3/extension-module --manifest-path rust/decision_receipt/Cargo.toml --out "$RUNNER_TEMP/decision-wheels" + - name: Run full test suite - run: uv run --locked --extra api --extra db --extra queue --group dev python -m pytest -q + run: uv run --no-sync python -m pytest -q - name: Install hash-locked quality tools run: | @@ -74,12 +82,15 @@ jobs: set -euo pipefail rm -rf dist "$RUNNER_TEMP/nim-wheel-site" python -m pip wheel --no-deps --no-build-isolation . --wheel-dir dist + python scripts/verify_decision_wheel_manifest.py dist/contextual_orchestrator-*.whl "$RUNNER_TEMP"/decision-wheels/*.whl python -m pip install --no-deps \ --target "$RUNNER_TEMP/nim-wheel-site" \ - dist/contextual_orchestrator-*.whl + dist/contextual_orchestrator-*.whl "$RUNNER_TEMP"/decision-wheels/*.whl cd "$RUNNER_TEMP" PYTHONPATH="$RUNNER_TEMP/nim-wheel-site" \ - python -c "import contextual_orchestrator; import contextual_orchestrator.nim_benchmark" + python -c "import pathlib, contextual_orchestrator as core; import contextual_orchestrator.nim_benchmark; import contextual_orchestrator._decision_receipt as native; root = pathlib.Path('nim-wheel-site').resolve(); assert pathlib.Path(core.__file__).is_relative_to(root); assert pathlib.Path(native.__file__).is_relative_to(root)" + PYTHONPATH="$RUNNER_TEMP/nim-wheel-site" \ + python -m pytest --noconftest --import-mode=importlib "$GITHUB_WORKSPACE/tests/test_decision_receipts.py" -q fuzz: name: Property and coverage-guided fuzzing diff --git a/AGENTS.md b/AGENTS.md index cbfe8e152..6e559d3de 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,8 +1,33 @@ # AGENTS.md +Cache reuse is an item observation, not a request terminal until close. Preserve +mixed-batch selection timing, explicit failures and ContextVar cleanup. See +`docs/doctoring/kpi_stack_integration.md` for the RED cases and separate source +versus installed-core verification commands; do not weaken the native invariant. + +Deferred batch lineage: read `docs/doctoring/batch_request_lineage.md` for the +HTTP reproduction, atomic submission-event projection, and remote/local failure +boundary. Do not retry a remotely submitted job after local lineage failure. + +Workflow origin identity and persistence limitations are documented in +`docs/doctoring/workflow_request_link.md`; preserve origin on replacements and reload. + Cross-agent conventions for `contextual-orchestrator`, readable by any coding agent (Claude, Codex, Cursor, opencode, …). Keep this file tool-agnostic. +## Autonomous research handoff + +Identifier checks do not discover title-only citations. Verify their persistent +identifier and register it in both the citing document and paper inventory; +retain read-depth and reuse limits. See the KPI runbook's citation reconciliation. + +Read [the KPI runbook](docs/doctoring/autonomous_kpi_runbook.md) before numerical +experiments and update its verified evidence before handoff. Choose KPI scope +autonomously under `docs/analytics_spec.md`. Preserve live execution handles; +high host load and silent numerical work are not proof of deadlock. Synthetic +recovery is unit evidence, not customer accuracy. Break owner/consumer release +cycles with isolated exact-revision contracts, never production source copies. + ## Agent guidance (CWL governance) @@ -181,6 +206,20 @@ push or open a PR. `production_default_change_allowed` is true. Temperature is not effort. +## Stacked quality checks + +For Noema incidents, `caller attempts=1` does not count internal provider +attempts. Match request identifiers and deployed revision before attributing +fallback; preflight failures are not review-request evidence. Keep ambiguous +timeout/502 replay separate from explicit rejection. See the +[attribution runbook](docs/doctoring/autonomous_kpi_runbook.md#noema-terminal-failure-attribution-2026-09-09). + +Zero check runs on a stacked PR can mean its base was excluded by +`pull_request.branches: [main]`, not that checks passed. Keep the repository +quality trigger unfiltered and validate `tests/test_repository_security_metadata.py` +plus actionlint. After a new head, verify actual hosted execution; previous-head +results are historical. See the [reproduction runbook](docs/doctoring/autonomous_kpi_runbook.md#stacked-quality-trigger-repair). + ## Tool-call handoffs Return worker tool calls before text-answer judging or later workflow roles; diff --git a/CLAUDE.md b/CLAUDE.md index 6429d7832..394112022 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,9 +1,33 @@ # CLAUDE.md +Read `docs/doctoring/kpi_stack_integration.md` before changing cache measurements. +One admission may contain cached and uncached items: finalize cache-only status +at request close, retain failures and isolate request context. The runbook owns +reproduction, native prerequisites and separate installed-package evidence. + +Batch request lineage evidence and unresolved registry failure semantics live +in `docs/doctoring/batch_request_lineage.md`; HTTP 201 alone does not establish +durable lineage. Preserve job-scoped item IDs and original submission identity. + +See `docs/doctoring/workflow_request_link.md` for request-to-workflow correlation +tests, cache semantics, and the distinction between in-memory and durable outcomes. + This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. ## Read AGENTS.md first +For title-only research citations, verify and register the persistent identifier +in the source document and paper inventory. Passing DOI discovery does not prove +full bibliography coverage; see the KPI runbook's citation reconciliation. + +For autonomous experiments, also read and maintain +[the single KPI runbook](docs/doctoring/autonomous_kpi_runbook.md). +It records commands, environment limits, failed interpretations, and evidence +boundaries. Do not restart a live numerical test because polling is silent or +claim customer improvement from synthetic recovery. Use the independently +verifiable owner/consumer sequence recorded there instead of waiting for all +foundation releases before developing a port. + Equivalent model-group endpoints may race only through the normalized, explicit endpoint-equivalence contract. Preserve modality validation, bounded concurrency, deadline, cancellation/drain provenance, and honest duplicate-cost evidence. @@ -113,6 +137,18 @@ Agent pools are **data, not code**: `examples/agents.mock.json` and `examples/ag ## Key conventions +- **Failure attribution**: one Noema caller attempt is not one internal provider + attempt. Verify deployed revision and request identity; do not mix preflight + failures with the review request or automatically replay ambiguous timeout/502. + See [the incident runbook](docs/doctoring/autonomous_kpi_runbook.md#noema-terminal-failure-attribution-2026-09-09). + +- **Stacked quality checks**: zero check runs can result from the former + `pull_request.branches: [main]` filter excluding the PR base. Keep the quality + trigger unfiltered, run `tests/test_repository_security_metadata.py` and actionlint, + and verify hosted execution after each new head. Earlier-head results are + historical, not current approval. Reproduction and limits: + [owner runbook](docs/doctoring/autonomous_kpi_runbook.md#stacked-quality-trigger-repair). + - **TDD from papers**: paper claims (Fugu, TRINITY, Conductor — see `docs/architecture.md`) become executable contracts in `tests/` *before* implementation changes. Many tests assert doc/API contracts, so behavior changes usually require updating the matching `docs/*.md` in the same PR. - **Naming**: configurable, API, and DB object names must be lower snake_case with **two or more semantic words** (`agent_pool`, `workflow_run`; never `agent` or `agentPool`). Enforced by `conventions.require_object_name()` and `tests/test_conventions.py`. Paper role values (`thinker`, `worker`, `verifier`, `synthesizer`) are deliberate exceptions. - **Ponytail design gate**: before adding a dependency or designing a subsystem, research existing libraries and record the decision in `docs/library_research.md`. No new dependency when the stdlib or an already selected library covers the need; no interface or factory until a second real implementation exists. diff --git a/contextual_orchestrator/batch_job_registry.py b/contextual_orchestrator/batch_job_registry.py index 21c7fc8a7..25a8a56a9 100644 --- a/contextual_orchestrator/batch_job_registry.py +++ b/contextual_orchestrator/batch_job_registry.py @@ -199,6 +199,11 @@ def __init__(self, client: Any = None, *, retention_seconds: int = DEFAULT_RETEN ) self._local_locks_guard = threading.Lock() + @property + def retention_seconds(self) -> int: + """Return the configured registry retention for durable recovery expiry.""" + return self._retention_seconds + def lock( self, name: str, diff --git a/contextual_orchestrator/batch_routing.py b/contextual_orchestrator/batch_routing.py index 27e2c1c2d..0fe1a55da 100644 --- a/contextual_orchestrator/batch_routing.py +++ b/contextual_orchestrator/batch_routing.py @@ -218,6 +218,15 @@ class BatchJob: # Prompt-token fallback estimates are safe metadata, stored atomically with # the job handle rather than retaining submitted prompt text. prompt_token_estimates: Dict[str, int] = field(default_factory=dict) + # Finalized before the registry snapshot is written; the append-only event + # supplies the durable request association independently of that registry. + request_link_status: str = "unavailable" + recovery_status: str = "unavailable" + # Deliberately not a dataclass field: HSET may succeed before expiry fails, + # so an operation result must never be serialized into its own snapshot. + registry_persistence_status = "unavailable" + backend_registry_persistence_status = "unavailable" + recovered_request_metadata = None @dataclass @@ -453,17 +462,75 @@ def __init__( endpoint: str = "/v1/chat/completions", payload_assembler: Any = None, job_registry: Any = None, + recovery_identity: str | None = None, ) -> None: self._client = client self._endpoint_alias = endpoint_alias self._endpoint = endpoint self._assembler = payload_assembler + if recovery_identity is not None and (not isinstance(recovery_identity, str) or not recovery_identity.strip()): + raise ValueError("recovery identity must be a nonempty operator-controlled identifier") + self._recovery_identity = recovery_identity # Tracked requests survive a restart when a Valkey-backed registry # is injected; a plain dict preserves the historical behavior. self._jobs: Dict[str, Dict[str, Any]] = ( job_registry.mapping("pg_llm_batch_jobs") if job_registry is not None else {} ) + @property + def recovery_enabled(self) -> bool: + """Whether the operator supplied a stable deployment/account binding.""" + return self._recovery_identity is not None + + def has_job_metadata(self, job: BatchJob) -> bool: + """Check whether active registry metadata supports the existing job.""" + try: + document = self._jobs.get(job.job_id) + except Exception: + return False + return (isinstance(document, dict) + and document.get("endpoint_alias") == self._endpoint_alias + and document.get("recovery_identity") == self._recovery_identity + and (document.get("endpoint") == self._endpoint + or ("endpoint" not in document and self._recovery_identity is None)) + and isinstance(document.get("requests"), dict) + and len(document["requests"]) == job.request_count) + + def recovery_descriptor(self, requests: List[BatchRequest]) -> Dict[str, Any]: + """Describe exact target and item metadata without submitted prompt text.""" + from .cost_ledger import AttributionDimensions + return { + "recovery_identity": self._recovery_identity, + "backend_name": self.name, + "endpoint_alias": self._endpoint_alias, + "endpoint": self._endpoint, + "items": [{"custom_id": item.custom_id, "model": item.model, + "mode": item.mode, "attribution": AttributionDimensions.from_mapping(item.attribution).as_dict()} + for item in requests], + } + + def restore_descriptor(self, job: BatchJob, descriptor: Dict[str, Any]) -> None: + """Restore prompt-free item identity only for this exact configured target.""" + if (not isinstance(descriptor, dict) or descriptor.get("backend_name") != self.name + or self._recovery_identity is None + or descriptor.get("recovery_identity") != self._recovery_identity + or descriptor.get("endpoint_alias") != self._endpoint_alias + or descriptor.get("endpoint") != self._endpoint): + raise ValueError("batch target mismatch") + items = descriptor.get("items") + if not isinstance(items, list) or len(items) != job.request_count: + raise ValueError("batch item count mismatch") + restored = {} + for item in items: + if not isinstance(item, dict) or set(item) != {"custom_id", "model", "mode", "attribution"}: + raise ValueError("invalid batch item descriptor") + if any(not isinstance(item[field], str) or not item[field] for field in ("custom_id", "model", "mode")): + raise ValueError("invalid batch item identity") + if item["custom_id"] in restored or not isinstance(item["attribution"], dict): + raise ValueError("invalid batch item metadata") + restored[item["custom_id"]] = {**item, "messages": []} + job.recovered_request_metadata = restored + def _assemble_payload(self, requests: List[BatchRequest]) -> str: if self._assembler is not None: return self._assembler.assemble( @@ -496,18 +563,26 @@ async def _submit() -> Dict[str, Any]: # Tracked requests are stored as JSON primitives (not dataclass # instances) so the registry can be a JSON-backed Valkey mapping; # retrieve() rebuilds the dataclass view it needs. - self._jobs[batch_id] = { - "endpoint_alias": self._endpoint_alias, - "requests": { - request.custom_id: dataclasses.asdict(request) for request in requests - }, - } - return BatchJob( + registry_status = "stored" + try: + self._jobs[batch_id] = { + "endpoint_alias": self._endpoint_alias, + "recovery_identity": self._recovery_identity, + "endpoint": self._endpoint, + "requests": { + request.custom_id: dataclasses.asdict(request) for request in requests + }, + } + except Exception: + registry_status = "write_failed" + job = BatchJob( job_id=batch_id, backend=self.name, status=job_payload.get("status", "validating"), request_count=len(requests), ) + job.backend_registry_persistence_status = registry_status + return job def poll(self, job: BatchJob) -> Dict[str, Any]: """Poll batch status via the pg-llm-batch client.""" @@ -542,7 +617,8 @@ async def _download() -> Dict[str, Any]: reason, ) raise BatchDownloadError(job.job_id, reason) - tracked = self._jobs.get(job.job_id, {}).get("requests", {}) + tracked = (job.recovered_request_metadata if job.recovered_request_metadata is not None + else self._jobs.get(job.job_id, {}).get("requests", {})) responses = _validated_download_responses( payload, expected_custom_ids=set(tracked), diff --git a/contextual_orchestrator/benchmark_priors.py b/contextual_orchestrator/benchmark_priors.py index fe795bdee..21fc326fc 100644 --- a/contextual_orchestrator/benchmark_priors.py +++ b/contextual_orchestrator/benchmark_priors.py @@ -1,40 +1,37 @@ """Benchmark-quality priors for the model-group Beta ledgers. -Two public measurements feed each member's prior success probability: -LMSYS Chatbot Arena Elo (Bradley–Terry rating scale) and Artificial -Analysis' Quality Index. Both are *published measurements*; this module -never invents a numeric weight. Everything else is derived from either - -1. those measurements themselves, -2. an existing repository constant (``model_group``'s Laplace prior - budget), or -3. arithmetic over the items above. +Legacy heuristic priors combine shipped Chatbot Arena ratings and Artificial +Analysis Quality Index values. Their snapshot provenance has not been verified +against archived source data. The transform below is deterministic but is not a +calibrated probability of task success or an identified psychometric estimate. Derivation contract (auditable, deterministic): - Each shipped rating ``r_i`` is centered on the median of the shipped set and scaled by the set's own median absolute deviation (MAD), giving ``z_i = (r_i - median) / MAD_i``. -- The two instruments are averaged after normalization (they measure - overlapping-but-distinct constructs; equal weight is the maximum - entropy choice across exactly two sources, not a tuned parameter). -- ``p_hat = logistic(z)`` is then a posterior-style membership value in - ``(0, 1)`` measured from ratings alone. +- The two instruments are averaged after normalization. Equal weighting is + an implementation choice, not a fitted weight justified by these references. +- ``p_hat = logistic(z)`` maps the composite to ``(0, 1)``. That range alone + does not establish posterior or predictive calibration. - The prior is *mass preserving*: ``(alpha0, beta0)`` splits the exact unobserved-evidence budget that ``model_group`` already spends on any unknown member (its Laplace counts), so a known member never receives more evidence than an unknown one — it only receives that identical budget distributed according to measurement instead of uniformly. -Failure denominator: members absent from every shipped instrument fall -back to the unchanged repository Laplace prior. +Members missing from either instrument fall back to the existing Laplace prior. +Neither the fixed evidence budget nor that fallback proves validity. Replacing +this legacy behavior requires observed-task calibration and release evidence; +do not treat these values as customer accuracy evidence. References (APA 7th): Bradley, R. A., & Terry, M. E. (1952). Rank analysis of incomplete block designs: I. The method of paired comparisons. *Biometrika, 39*(3/4), 324–345. https://doi.org/10.1093/biomet/39.3-4.324 - Chiang, W., Zheng, L., Ma, Z., Li, Y., Sheng, Z., Wu, X., ... Zhang, - H. (2024). *Chatbot Arena: An open platform for evaluating LLMs + Chiang, W.-L., Zheng, L., Sheng, Y., Angelopoulos, A. N., Li, T., + Li, D., Zhang, H., Zhu, B., Jordan, M. I., Gonzalez, J. E., & + Stoica, I. (2024). *Chatbot Arena: An open platform for evaluating LLMs by human preference* [Preprint]. arXiv. https://doi.org/10.48550/arXiv.2403.04132 """ @@ -123,7 +120,7 @@ def _normalized_membership(name: str) -> float | None: def measured_quality_probability(member_id: str) -> float | None: - """Return the measurement-derived prior success probability, if known.""" + """Return the legacy heuristic prior fraction, if both scores are known.""" lowered = member_id.lower() for key in _ARENA_ELO: if key in lowered: @@ -134,7 +131,7 @@ def measured_quality_probability(member_id: str) -> float | None: def resolve_quality_prior(member_id: str) -> tuple[float, float]: - """Resolve the benchmark-measured ``(alpha, beta)`` prior for one member. + """Resolve the legacy benchmark-derived ``(alpha, beta)`` prior. Unknown members receive the repository's unchanged Laplace pair, so behaviour for unmeasured identifiers is bit-for-bit the pre-existing diff --git a/contextual_orchestrator/cost_router.py b/contextual_orchestrator/cost_router.py index f7fd48dec..121c65817 100644 --- a/contextual_orchestrator/cost_router.py +++ b/contextual_orchestrator/cost_router.py @@ -20,10 +20,12 @@ import hashlib import re +import time from contextvars import ContextVar -from dataclasses import replace +from dataclasses import asdict, replace from threading import Lock from typing import Any, Dict, List, Optional +from .decision_receipts import record_initial_selection from .batch_routing import ( BatchBackend, @@ -37,6 +39,7 @@ LocalBatchBackend, LocalEmbeddingBatchBackend, ProviderEmbeddingBatchBackend, + PgLlmBatchBackend, RoutingHints, RoutingPolicy, ) @@ -983,6 +986,7 @@ def submit_batch( requests: List[BatchRequest], metadata: Optional[Dict[str, Any]] = None, owner_id: Optional[str] = None, + request_id: Optional[str] = None, ) -> BatchJob: """Submit a batch, resolve its targets, and bind its authenticated owner.""" try: @@ -1006,7 +1010,36 @@ def submit_batch( job = self.batch_backend.submit(prepared_requests, metadata=metadata) job.owner_id = owner_id job.prompt_token_estimates = prompt_token_estimates - self._batch_jobs[job.job_id] = job + if request_id is not None and self.orchestrator._store is not None: + try: + # One append-only submission envelope commits all item links + # together. A later retrieval never rewrites this origin. + self.orchestrator._store.save("batch_request_link", job.job_id, { + "request_id": request_id, + "batch_job_id": job.job_id, + "custom_ids": [request.custom_id for request in prepared_requests], + "owner_id": owner_id, + "recovery_descriptor": ({ + "job": asdict(job), + "expires_at": job.submitted_at + self._job_registry.retention_seconds, + "backend": self.batch_backend.recovery_descriptor(prepared_requests), + } if isinstance(self.batch_backend, PgLlmBatchBackend) else None), + }, durable=True) + except Exception: + # The upstream submission already happened. Preserve its handle + # and report incomplete lineage instead of inviting a resubmit. + job.request_link_status = "write_failed" + else: + job.request_link_status = "durable" + if isinstance(self.batch_backend, PgLlmBatchBackend) and self.batch_backend.recovery_enabled: + job.recovery_status = "durable_descriptor" + job.registry_persistence_status = "stored" + try: + self._batch_jobs[job.job_id] = job + except Exception: + # Submission already applied remotely; an HSET/expiry failure may + # itself be partially applied. Return the handle without replay. + job.registry_persistence_status = "write_failed" return job def _resolve_batch_request(self, request: BatchRequest) -> BatchRequest: @@ -1057,7 +1090,7 @@ def retrieve_batch(self, job_id: str, *, owner_id: Optional[str] = None) -> Dict not self._batch_item_usage_valid(item) and item.custom_id not in prompt_token_estimates for item in items - ) + ) and job.recovered_request_metadata is None request_by_custom_id = ( self._legacy_batch_requests(job) if needs_legacy_lookup else {} ) @@ -1304,7 +1337,53 @@ def _resolve_batch_provider_model(self, item: BatchResultItem) -> tuple[str, str return provider, item.model def _require_job(self, job_id: str, *, owner_id: Optional[str] = None) -> BatchJob: - job = self._batch_jobs.get(job_id) + try: + job = self._batch_jobs.get(job_id) + except Exception: + job = None + if job is not None and job.owner_id != owner_id: + raise KeyError(f"batch job {job_id!r} not found") + if job is not None and isinstance(self.batch_backend, PgLlmBatchBackend): + if self.batch_backend.has_job_metadata(job): + return job + # Missing or differently bound metadata cannot use the ordinary + # retrieval path; only a validated durable descriptor may recover. + job = None + if (owner_id is not None and self.orchestrator._store is not None + and (job is None or (isinstance(self.batch_backend, PgLlmBatchBackend) + and self.batch_backend.recovery_enabled))): + record = self.orchestrator._store.load_latest_key("batch_request_link", job_id) + if (isinstance(record, dict) and record.get("owner_id") == owner_id + and record.get("batch_job_id") == job_id + and isinstance(self.batch_backend, PgLlmBatchBackend)): + descriptor = record.get("recovery_descriptor") + try: + if not isinstance(descriptor, dict) or type(descriptor.get("expires_at")) is not int: + raise ValueError("invalid descriptor") + if descriptor["expires_at"] <= time.time(): + raise ValueError("expired descriptor") + recovered = BatchJob(**descriptor["job"]) + if recovered.job_id != job_id or recovered.owner_id != owner_id or recovered.backend != self.batch_backend.name: + raise ValueError("mismatched descriptor") + custom_ids = record.get("custom_ids") + if (type(recovered.request_count) is not int or recovered.request_count < 1 + or not isinstance(custom_ids, list) + or any(not isinstance(item, str) or not item for item in custom_ids) + or len(custom_ids) != recovered.request_count + or len(set(custom_ids)) != recovered.request_count): + raise ValueError("invalid recovery item identities") + estimates = recovered.prompt_token_estimates + if (not isinstance(estimates, dict) or not set(estimates).issubset(custom_ids) + or any(type(value) is not int or value < 0 for value in estimates.values())): + raise ValueError("invalid recovery estimates") + self.batch_backend.restore_descriptor(recovered, descriptor["backend"]) + if set(recovered.recovered_request_metadata) != set(custom_ids): + raise ValueError("mismatched recovery items") + recovered.request_link_status = "durable" + recovered.recovery_status = "durable_descriptor" + job = recovered + except (KeyError, TypeError, ValueError): + job = None if job is None or job.owner_id != owner_id: raise KeyError(f"batch job {job_id!r} not found") return job @@ -1350,6 +1429,8 @@ def submit_embeddings_batch( if callable(reserve) and callable(start): job = reserve(requests, metadata=metadata) else: + if resolved_agent_id is not None: + record_initial_selection([resolved_agent_id], "embedding_submission") job = backend.submit(requests, metadata=metadata) self._embedding_models[job.job_id] = resolved_model self._embedding_owners[job.job_id] = owner_id @@ -1359,6 +1440,8 @@ def submit_embeddings_batch( self._embedding_part_limits[job.job_id] = part_limits self._embedding_jobs[job.job_id] = job if callable(reserve) and callable(start): + if resolved_agent_id is not None: + record_initial_selection([resolved_agent_id], "embedding_submission") start(job) return job diff --git a/contextual_orchestrator/decision_receipts.py b/contextual_orchestrator/decision_receipts.py new file mode 100644 index 000000000..ed6bd09d5 --- /dev/null +++ b/contextual_orchestrator/decision_receipts.py @@ -0,0 +1,227 @@ +"""Persistence boundary for Rust-owned initial-decision measurements.""" + +from contextvars import ContextVar +from contextlib import contextmanager +import logging +import threading +import uuid +import hashlib +import json + +_CURRENT_DECISION = ContextVar("initial_decision", default=None) +_LOGGER = logging.getLogger(__name__) + + +class DecisionMeasurement: + """Bind one native clock to an HTTP request and its durable store.""" + + def __init__(self, store, *, policy=None, route_mode="unclassified", request_id=None, + endpoint_path=None, request_method=None, + admission_boundary="explicit_scope"): + """Require the native module only when measurement is explicitly enabled.""" + from ._decision_receipt import DecisionReceipt + + self.receipt = DecisionReceipt() + self.store = store + self.request_id = request_id or uuid.uuid4().hex + self.identity_source = "http_request" if request_id else "measurement_scope" + policy_snapshot = policy.as_dict() if policy is not None else None + self.policy_hash = hashlib.sha256(json.dumps( + policy_snapshot, sort_keys=True, separators=(",", ":") + ).encode()).hexdigest() if policy_snapshot is not None else None + self.route_mode = route_mode + self.endpoint_path = endpoint_path + self.request_method = request_method + self.admission_boundary = admission_boundary + self.selected_agent_ids = [] + self.selection_attempt_count = 0 + self._race_attempt_ids = set() + self.first_provider_phase = None + self._answer_cache_observed = False + self._lock = threading.Lock() + self._token = _CURRENT_DECISION.set(self) + try: + if self.store is None: + raise RuntimeError("no durable measurement store") + self.store.save("accepted_request", None, self.snapshot(), durable=True) + except Exception as exc: + _CURRENT_DECISION.reset(self._token) + _LOGGER.warning("Decision admission write failed error_type=%s", type(exc).__name__) + raise RuntimeError("decision measurement admission could not be persisted") from None + + def snapshot(self): + """Return no prompts, provider credentials, or fabricated missing timings.""" + return { + "request_id": self.request_id, + "identity_source": self.identity_source, + "policy_snapshot_hash": self.policy_hash, + "route_mode": self.route_mode, + "endpoint_path": self.endpoint_path, + "request_method": self.request_method, + "measurement_unit": "http_request" if self.endpoint_path else "explicit_scope", + "admission_boundary": self.admission_boundary, + "status": self.receipt.status, + "selected_agent_ids": list(self.selected_agent_ids), + "selection_attempt_count": self.selection_attempt_count, + "selection_elapsed_ns": self.receipt.selection_elapsed_ns, + "durable_ack_elapsed_ns": self.receipt.durable_ack_elapsed_ns, + "first_provider_elapsed_ns": self.receipt.first_provider_elapsed_ns, + "first_provider_phase": self.first_provider_phase, + "first_provider_boundary": "provider_ready_before_diagnostic_commit", + "metric_scope": "initial_task_route_decision", + } + + def select(self, agent_ids, route_mode, *, attempt_id=None): + """Acknowledge the first decision synchronously before provider dispatch.""" + with self._lock: + if attempt_id is not None: + if attempt_id in self._race_attempt_ids: + return + self._race_attempt_ids.add(attempt_id) + self.selection_attempt_count += 1 + if self.receipt.status != "accepted": + try: + self.store.save("selection_attempt", None, { + "request_id": self.request_id, + "attempt_number": self.selection_attempt_count, + "selected_agent_ids": list(agent_ids), "route_mode": route_mode, + }, durable=True) + except Exception as exc: + _LOGGER.warning("Decision attempt export failed error_type=%s", type(exc).__name__) + return + self.receipt.record_selection() + self.route_mode = route_mode + self.selected_agent_ids = list(agent_ids) + if self.store is None: + self.receipt.record_failure("store_unavailable") + return + try: + self.store.save("initial_decision", None, self.snapshot(), durable=True) + except Exception as exc: + self.receipt.record_failure("write_failed") + _LOGGER.warning("Initial decision measurement write failed error_type=%s", type(exc).__name__) + return + self.receipt.record_durable_ack() + self._record_provider_locked(agent_ids, "task_execution") + + def _record_provider_locked(self, agent_ids, phase): + """Keep a first-provider diagnostic independent of the task-route clock.""" + if self.first_provider_phase is not None: + return + self.receipt.record_provider_dispatch() + self.first_provider_phase = phase + try: + self.store.save("provider_dispatch", self.request_id, { + **self.snapshot(), "provider_agent_ids": list(agent_ids), + }, durable=True) + except Exception as exc: + _LOGGER.warning("Provider diagnostic write failed error_type=%s", type(exc).__name__) + + @contextmanager + def auxiliary_call(self, agent_ids, phase): + """Retain auxiliary work without subtracting it from the task-route interval.""" + with self._lock: + if phase == "routing_evidence_embedding" and self.receipt.selection_elapsed_ns is not None: + phase = "post_decision_evidence_embedding" + self._record_provider_locked(agent_ids, phase) + started = self.receipt.current_elapsed_ns() + outcome = "completed" + try: + yield + except BaseException: + outcome = "failed" + raise + finally: + with self._lock: + finished = self.receipt.current_elapsed_ns() + try: + self.store.save("auxiliary_dispatch", self.request_id, { + "request_id": self.request_id, "phase": phase, + "provider_agent_ids": list(agent_ids), "outcome": outcome, + "started_elapsed_ns": started, "finished_elapsed_ns": finished, + }, durable=True) + except Exception as exc: + _LOGGER.warning("Auxiliary diagnostic write failed error_type=%s", type(exc).__name__) + + def close(self, reason="unfinished"): + """Persist the post-commit measurement separately; never claim its own ack.""" + try: + with self._lock: + if (self.receipt.status == "accepted" + and self._answer_cache_observed and reason == "unfinished"): + self.receipt.record_cache_hit() + elif self.receipt.status in ("accepted", "selected"): + self.receipt.record_failure(reason) + if self.store is not None: + try: + self.store.save("decision_receipt", None, self.snapshot(), durable=True) + except Exception as exc: + _LOGGER.warning("Decision receipt export failed error_type=%s", type(exc).__name__) + finally: + _CURRENT_DECISION.reset(self._token) + + +def record_initial_selection(agent_ids, route_mode="unclassified", *, attempt_id=None): + """Record selection only within an explicitly enabled request measurement.""" + measurement = _CURRENT_DECISION.get() + if measurement is not None: + measurement.select(agent_ids, route_mode, attempt_id=attempt_id) + + +def record_answer_cache_hit(): + """Defer cache-only classification until all request items have finished.""" + measurement = _CURRENT_DECISION.get() + if measurement is not None: + with measurement._lock: + measurement._answer_cache_observed = True + + +@contextmanager +def observe_auxiliary_dispatch(agent_ids, phase): + """Use the request's native clock for an actual auxiliary provider call.""" + measurement = _CURRENT_DECISION.get() + if measurement is None: + yield + else: + with measurement.auxiliary_call(agent_ids, phase): + yield + + +def export_decision_receipts(store, limit=256): + """Join retained admissions without dropping missing final acknowledgements. + + This is a local ledger view, not an all-ingress census. Storage outages need + external ingress reconciliation. Concurrent finalization may appear on the + next read; missing values remain unfinished rather than successful zeros. + """ + cohort = store.load_decision_window(limit) + accepted = cohort["accepted"] + decisions = {row["request_id"]: row for row in cohort["decisions"]} + receipts = {row["request_id"]: row for row in cohort["receipts"]} + diagnostics = {} + for diagnostic in cohort["diagnostics"]: + diagnostics.setdefault(diagnostic["request_id"], []).append(diagnostic) + observations = [] + for admission in accepted: + request_id = admission["request_id"] + row = dict(admission) + row["status"] = "unfinished" + if request_id in decisions: + row.update(decisions[request_id]) + row["status"] = "acknowledgement_unobserved" + if request_id in receipts: + row.update(receipts[request_id]) + request_diagnostics = diagnostics.get(request_id, []) + row["auxiliary_dispatches"] = [entry for entry in request_diagnostics + if entry["record_kind"] == "auxiliary_dispatch"] + row["provider_diagnostics"] = [entry for entry in request_diagnostics + if entry["record_kind"] == "provider_dispatch"] + observations.append(row) + return { + "schema_version": 1, + "measurement_complete": False, + "reconciliation_required": True, + "scope": "retained_local_admissions", + "window": cohort["window"], + "observations": observations, + } diff --git a/contextual_orchestrator/nim_benchmark.py b/contextual_orchestrator/nim_benchmark.py index db4eb064a..b74640daa 100644 --- a/contextual_orchestrator/nim_benchmark.py +++ b/contextual_orchestrator/nim_benchmark.py @@ -2940,6 +2940,9 @@ def assemble_benchmark_report( cells, evaluation["locked_task_count"], ) + if run_mode == "dry_run": + evidence_summary["evidence_status"] = "synthetic_diagnostic_only" + evidence_summary["decision_use"] = "benchmark_smoke_only" report = { "benchmark_schema_version": BENCHMARK_SCHEMA_VERSION, "provenance": build_provenance( diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 0481829dc..cb397f393 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -6,6 +6,7 @@ from collections.abc import Iterable, Mapping from contextlib import contextmanager, nullcontext from contextvars import ContextVar, copy_context +from .decision_receipts import observe_auxiliary_dispatch, record_answer_cache_hit, record_initial_selection from concurrent.futures import ThreadPoolExecutor import copy import hashlib @@ -3858,6 +3859,7 @@ class _StateStore: _LEGACY_INDEX_NAME = "records_kind_seq" _INDEX_NAME = "orchestration_records_kind_seq" _STREAM_LIMITS = {"audit": 256, "authorization": 256, "analytics": 256} + _MEASUREMENT_KINDS = ("accepted_request", "initial_decision", "decision_receipt", "selection_attempt") _CREATE_RECORDS_SQL = ( "CREATE TABLE IF NOT EXISTS orchestration_records (" "seq INTEGER PRIMARY KEY AUTOINCREMENT, kind TEXT NOT NULL, key TEXT, payload TEXT NOT NULL)" @@ -3882,6 +3884,18 @@ def __init__(self, path: str) -> None: self._migrate_legacy_table() self._conn.execute(self._CREATE_RECORDS_SQL) self._conn.execute(self._CREATE_RECORDS_KIND_SEQ_INDEX_SQL) + self._conn.execute( + "CREATE INDEX IF NOT EXISTS orchestration_records_kind_key_seq " + "ON orchestration_records(kind, key, seq)" + ) + self._conn.execute( + "UPDATE orchestration_records SET key = json_extract(payload, '$.request_id') " + "WHERE kind IN (?, ?, ?, ?) AND key IS NULL " + "AND CASE WHEN json_valid(payload) THEN " + "json_type(payload, '$.request_id') = 'text' " + "AND length(json_extract(payload, '$.request_id')) > 0 ELSE 0 END", + self._MEASUREMENT_KINDS, + ) self._conn.commit() except Exception: self._conn.rollback() @@ -3937,16 +3951,76 @@ def save(self, kind: str, key: str | None, payload: dict[str, Any], *, durable: return self._save_sync(kind, key, payload) + def load_decision_window(self, limit: int = 256) -> dict[str, Any]: + """Read a bounded shared admission cohort without deleting historical rows.""" + if type(limit) is not int or not 1 <= limit <= 1000: + raise ValueError("decision window limit must be between 1 and 1000") + with self._lock: + admissions = self._conn.execute( + "SELECT seq, key, payload FROM orchestration_records WHERE kind = ? ORDER BY seq DESC LIMIT ?", + ("accepted_request", limit + 1), + ).fetchall() + truncated = len(admissions) > limit + admissions = list(reversed(admissions[:limit])) + accepted = [json.loads(payload) for _, _, payload in admissions] + if any(key != row.get("request_id") for (_, key, _), row in zip(admissions, accepted)): + raise ValueError("measurement admission identity mismatch") + request_ids = [row["request_id"] for row in accepted] + phases = [] + diagnostics = [] + if request_ids: + placeholders = ",".join("?" for _ in request_ids) + phases = self._conn.execute( + "SELECT kind, key, payload FROM orchestration_records " + "WHERE kind IN ('initial_decision', 'decision_receipt') " + "AND key IN (" + placeholders + ") " + "ORDER BY seq DESC LIMIT ?", + (*request_ids, 2 * limit + 1), + ).fetchall() + diagnostics = self._conn.execute( + "SELECT kind, key, payload FROM orchestration_records " + "WHERE kind IN ('provider_dispatch', 'auxiliary_dispatch') " + "AND key IN (" + placeholders + ") ORDER BY seq DESC LIMIT ?", + (*request_ids, 8 * limit + 1), + ).fetchall() + diagnostic_truncated = len(diagnostics) > 8 * limit + diagnostics = list(reversed(diagnostics[:8 * limit])) + if any(key != json.loads(payload).get("request_id") for _, key, payload in diagnostics): + raise ValueError("measurement diagnostic identity mismatch") + phase_truncated = len(phases) > 2 * limit + phases = list(reversed(phases[:2 * limit])) + if any(key != json.loads(payload).get("request_id") for _, key, payload in phases): + raise ValueError("measurement phase identity mismatch") + unresolved_legacy = self._conn.execute( + "SELECT 1 FROM orchestration_records WHERE kind IN (?, ?, ?, ?) " + "AND key IS NULL LIMIT 1", self._MEASUREMENT_KINDS, + ).fetchone() is not None + return { + "accepted": accepted, + "decisions": [json.loads(payload) for kind, _, payload in phases if kind == "initial_decision"], + "receipts": [json.loads(payload) for kind, _, payload in phases if kind == "decision_receipt"], + "diagnostics": [{"record_kind": kind, **json.loads(payload)} for kind, _, payload in diagnostics], + "window": {"limit": limit, "truncated": truncated, "phase_truncated": phase_truncated, + "diagnostic_limit": 8 * limit, "diagnostic_truncated": diagnostic_truncated, + "unresolved_legacy_identity": unresolved_legacy, + "first_admission_seq": admissions[0][0] if admissions else None, + "last_admission_seq": admissions[-1][0] if admissions else None}, + } + def _save_sync(self, kind: str, key: str | None, payload: dict[str, Any]) -> None: + if kind in self._MEASUREMENT_KINDS: + request_id = payload.get("request_id") + if not isinstance(request_id, str) or not request_id or (key is not None and key != request_id): + raise ValueError("measurement record requires a consistent request identity") + key = request_id blob = json.dumps(payload, ensure_ascii=False) - with self._lock: + with self._lock, self._conn: if kind in self._KEYED: self._conn.execute(self._DELETE_KEYED_SQL, (kind, key)) self._conn.execute(self._INSERT_SQL, (kind, key, blob)) if kind in self._STREAM_LIMITS: limit = self._STREAM_LIMITS[kind] self._conn.execute(self._PRUNE_STREAM_SQL, (kind, kind, limit)) - self._conn.commit() def _drain_stream_queue(self) -> None: while True: @@ -3995,6 +4069,15 @@ def load(self, kind: str, limit: int | None = None) -> list[dict[str, Any]]: rows = list(reversed(rows)) return [json.loads(row[0]) for row in rows] + def load_latest_key(self, kind: str, key: str) -> dict[str, Any] | None: + """Read one exact-key durable event using the existing identity index.""" + with self._lock: + row = self._conn.execute( + "SELECT payload FROM orchestration_records WHERE kind = ? AND key = ? " + "ORDER BY seq DESC LIMIT 1", (kind, key), + ).fetchone() + return json.loads(row[0]) if row is not None else None + def prune_keyed(self, kind: str, retained_keys: set[str]) -> None: """Delete keyed rows outside the caller's bounded in-memory set.""" if kind not in self._KEYED: @@ -4345,7 +4428,7 @@ def _reload_state(self) -> None: observation.get("irt_row", ()), ) for record in self._store.load("workflow_run"): - self._replace_workflow_run(record) + self._replace_workflow_run(record, restored=True) # A batch_route row persisted before judging (see batch_route's # own pending-record comment) carries an explicit # "pending_verification" marker and intentionally never reaches @@ -4544,6 +4627,7 @@ def proxy_completion( agent, upstream, effort_profile, api_surface=api_surface ) measured = bool(agent.group_name or requested_model == self.FREE_MODEL) + record_initial_selection([agent.id], "explicit_proxy") started_at = time.perf_counter() try: result = self.client.proxy_send(agent, endpoint, upstream) @@ -4627,6 +4711,7 @@ def proxy_completion( send_once = getattr(self.client, "proxy_send_once", None) if not callable(send_once): send_once = self.client.proxy_send + record_initial_selection([candidate.id], "automatic_proxy") result = send_once(candidate, endpoint, candidate_payload) except Exception as exc: # noqa: BLE001 - provider trust boundary if not _is_passthrough_failover_error(exc): @@ -5764,6 +5849,7 @@ def complete( ): result = copy.deepcopy(dict(cached)) result["cache_status"] = "hit" + record_answer_cache_hit() return result route_decision = self._resolved_route_decision(messages, mode, model_name, cheap_decision) result = self._dispatch(messages, mode, model_name, route_decision=route_decision) @@ -5889,6 +5975,7 @@ def stream_route( emitted = False started_at = time.perf_counter() try: + record_initial_selection([agent.id], "stream_route") for delta in self.client.stream_chat(agent, messages, **stream_kwargs): emitted = True parts.append(delta) @@ -5999,6 +6086,8 @@ def stream_route( record["owner_id"] = owner_id self._replace_workflow_run(record) self._run_order.appendleft(record["workflow_run_id"]) + if self._store is not None: + self._store.save("workflow_run", record["workflow_run_id"], record) self._append_audit_event( "workflow_run_created", { @@ -7530,11 +7619,12 @@ def _plan_generated(self, task: str) -> list[WorkflowStep]: {"role": "user", "content": task}, ] effort_profile = self._role_effort_profile("planner") - raw = ( - self.client.chat(planner, planner_messages, effort_profile=effort_profile) - if effort_profile is not None - else self.client.chat(planner, planner_messages) - ) + with observe_auxiliary_dispatch([planner.id], "generated_planner"): + raw = ( + self.client.chat(planner, planner_messages, effort_profile=effort_profile) + if effort_profile is not None + else self.client.chat(planner, planner_messages) + ) return self._parse_workflow_plan(raw) def _parse_workflow_plan(self, raw: str) -> list[WorkflowStep]: @@ -7965,7 +8055,8 @@ def _embed_cached(self, text: str) -> list[float] | None: if embedding_member is None: return None try: - vectors = self.client.embed(self._agent(embedding_member), [text]) + with observe_auxiliary_dispatch([embedding_member], "routing_evidence_embedding"): + vectors = self.client.embed(self._agent(embedding_member), [text]) except Exception: # noqa: BLE001 - similarity is best-effort evidence return None vector = vectors[0] if vectors else None @@ -7992,9 +8083,10 @@ def _descriptor_vector_cached(self, agent: ModelAgent) -> list[float] | None: if embedding_member is None: return None try: - vectors = self.client.embed( - self._agent(embedding_member), [self._agent_descriptor_text(agent)] - ) + with observe_auxiliary_dispatch([embedding_member], "routing_evidence_embedding"): + vectors = self.client.embed( + self._agent(embedding_member), [self._agent_descriptor_text(agent)] + ) except Exception: # noqa: BLE001 - similarity is best-effort evidence return None vector = vectors[0] if vectors else None @@ -8080,7 +8172,8 @@ def _compute_triage_verdict(self, text: str) -> bool: {"role": "user", "content": text}, ] try: - reply = self.client.chat(triage_agent, messages, temperature=0.0) + with observe_auxiliary_dispatch([triage_agent.id], "structured_triage"): + reply = self.client.chat(triage_agent, messages, temperature=0.0) return _parse_triage_reply(reply) except Exception: # noqa: BLE001 - fail closed toward verified orchestration return True @@ -8367,6 +8460,8 @@ def call(agent: ModelAgent) -> dict[str, Any] | tuple[bytes, str]: if agent.provider_name == "openrouter" and endpoint == "images/generations" else endpoint ) + record_initial_selection([member.id for member in race_members], "capability_race", + attempt_id=decision_attempt_id) return ( self.client.proxy_send_bytes(agent, provider_endpoint, payload) if binary else self.client.proxy_send(agent, provider_endpoint, payload) @@ -8374,6 +8469,7 @@ def call(agent: ModelAgent) -> dict[str, Any] | tuple[bytes, str]: contract = EndpointEquivalenceContract(**race_members[0].endpoint_equivalence) # type: ignore[arg-type] attempt_completed, finalize_attempts = self._race_attempt_collector(capability) + decision_attempt_id = uuid.uuid4().hex try: outcome = race_first_valid( [ @@ -8425,6 +8521,7 @@ def call(agent: ModelAgent) -> dict[str, Any] | tuple[bytes, str]: ) started_at = time.perf_counter() try: + record_initial_selection([agent.id], "capability_proxy") result = ( self.client.proxy_send_bytes(agent, provider_endpoint, payload) if binary @@ -8547,6 +8644,8 @@ def call( else nullcontext() ) with self.client.request_settings(**request_settings), tool_scope: + record_initial_selection([member.id for member in race_members], "text_race", + attempt_id=decision_attempt_id) output = ( self.client.chat(agent, messages, effort_profile=effort_profile) if effort_profile is not None @@ -8562,6 +8661,7 @@ def call( contract = EndpointEquivalenceContract(**race_members[0].endpoint_equivalence) # type: ignore[arg-type] attempt_completed, finalize_attempts = self._race_attempt_collector("text") + decision_attempt_id = uuid.uuid4().hex try: outcome = race_first_valid( [ @@ -8638,6 +8738,7 @@ def call( else nullcontext() ) with transport_scope, tool_scope: + record_initial_selection([agent.id], "invocation_" + role) output = ( self.client.chat(agent, messages, effort_profile=effort_profile) if effort_profile is not None @@ -9504,7 +9605,7 @@ def _run_budget_output_by_model( ) return output_by_model, True - def _replace_workflow_run(self, record: dict[str, Any]) -> None: + def _replace_workflow_run(self, record: dict[str, Any], *, restored: bool = False) -> None: """Store one run and update its constant-time budget meter atomically.""" model_by_agent = {agent.id: agent.model for agent in self.candidates} for step in record.get("trace", []): @@ -9514,6 +9615,13 @@ def _replace_workflow_run(self, record: dict[str, Any]) -> None: run_id = record["workflow_run_id"] with self._budget_spend_lock: previous = self._workflow_runs.get(run_id) + origin_request_id = (previous.get("request_id") if previous is not None + else record.get("request_id") if restored + else current_request_id()) + if origin_request_id is not None: + record["request_id"] = origin_request_id + else: + record.pop("request_id", None) for sign, run in ((-1, previous), (1, record)): if run is None: continue diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index 1b871fa58..3fa2c42c5 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -15,6 +15,7 @@ import secrets import socket import struct +import sys import tempfile import threading import time @@ -25,6 +26,7 @@ import uuid from .admin import ADMIN_HTML, ADMIN_TRANSLATIONS +from .decision_receipts import DecisionMeasurement, export_decision_receipts from .api_contract import OPENAPI_SPEC from .cost_ledger import ATTRIBUTION_DIMENSIONS, dimension_catalog from .cost_router import ( @@ -5482,6 +5484,7 @@ def build_server( clearfolio_url: str | None = None, coordinator: CostRoutingCoordinator | None = None, release_authority: Mapping[str, Any] | None = None, + decision_receipts: bool = False, ) -> ThreadingHTTPServer: """Build, but do not start, the orchestration HTTP server. @@ -5490,6 +5493,12 @@ def build_server( every completion is priced, recorded, and sync/batch routed. """ security = security or SecurityConfig() + if type(decision_receipts) is not bool: + raise TypeError("decision_receipts must be a boolean") + if decision_receipts: + from ._decision_receipt import DecisionReceipt # noqa: F401 + if orchestrator._store is None: + raise ValueError("decision receipts require a durable state store") security.check_bind(host) release_authority = verify_release_authority_snapshot(release_authority) coordinator = coordinator or CostRoutingCoordinator(orchestrator) @@ -5506,6 +5515,7 @@ class Handler(BaseHTTPRequestHandler): """Handle authenticated orchestration, administration, and health routes.""" _session_token = None _trace_token = None + _classification_slot_held = False def _bind_session(self, session_id: str | None) -> None: """Bind validated request correlation to this handler context.""" @@ -5635,10 +5645,18 @@ def handle_one_request(self) -> None: self.command = None self.path = None self._request_started = None + self._decision_measurement = None + self._decision_failure_reason = "unfinished" + self._classification_slot_held = False with request_identity(): try: super().handle_one_request() finally: + if self._classification_slot_held: + self._release_measured_slot() + if self._decision_measurement is not None: + self._decision_measurement.close(self._decision_failure_reason) + self._decision_measurement = None self._log_request_summary(self._request_started) self._reset_session() # A request that declared a body it never delivered (unsupported @@ -5987,7 +6005,10 @@ def do_GET(self) -> None: # noqa: N802 self._send(orchestrator.provider_readiness_report(refresh=raw_refresh == "true")) return if path == "/api/v1/analytics_snapshots/latest": - self._send(orchestrator.analytics_snapshot(locale_bundles=ADMIN_TRANSLATIONS)) + snapshot = orchestrator.analytics_snapshot(locale_bundles=ADMIN_TRANSLATIONS) + if decision_receipts: + snapshot["initial_decision_measurements"] = export_decision_receipts(orchestrator._store) + self._send(snapshot) return if path == "/api/v1/spend_analytics/latest": self._send(orchestrator.spend_analytics()) @@ -7141,24 +7162,6 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di return messages = _validate_messages(body.get("messages")) mode = _validate_mode(body.get("orchestration") or body.get("orchestration_mode") or body.get("mode") or "auto") - # Tools on a virtual selector stay on route/conduct so - # _invoke can re-select a worker before any SSE byte is - # committed. True stream_route is the no-tools Fugu path. - route_stream = bool( - stream - and not tools_list - and orchestrator.would_route(messages, mode, model_name) - ) - if route_stream: - if explicit_trace: - raise RequestError( - 400, - "unsupported_trace_disclosure", - "remove include_orchestration_trace or use Responses streaming", - ) - include_trace = False - elif include_trace: - self._authorize_trace_access() # stream + stream_options already coerced/validated before passthrough. attribution = _validate_attribution(body.get("attribution")) # Require model — silent default to contextual-orchestrator hid @@ -7179,6 +7182,26 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di # sampling/controls already validated before passthrough branch. if "metadata" in body: _validate_openai_metadata(body) + if explicit_trace: + self._authorize_trace_access() + self._ensure_decision_measurement("validated_endpoint") + if stream: + self._acquire_measured_slot() + self._classification_slot_held = True + route_stream = bool( + stream and not tools_list + and orchestrator.would_route(messages, mode, model_name) + ) + if route_stream: + if explicit_trace: + raise RequestError( + 400, + "unsupported_trace_disclosure", + "remove include_orchestration_trace or use Responses streaming", + ) + include_trace = False + elif include_trace and not explicit_trace: + self._authorize_trace_access() started_at = time.perf_counter() model_client = orchestrator.client request_settings = { @@ -7202,6 +7225,7 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di messages, model_name, include_usage=include_usage, + slot_acquired=True, ) orchestrator.record_analytics_event( "chat_completion_requested", @@ -7281,13 +7305,6 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di ) # Same pool honesty as chat/Completions: do not silently serve # a different embedding deployment than the client requested. - embedding_agents = orchestrator._capability_agents( - "embedding", - TaskOrchestrator.AUTO_MODEL if model_was_omitted else model_name, - ) - embedding_agents = coordinator._cost_ordered_capability_candidates( - embedding_agents - ) encoding_format = _validate_embeddings_encoding_format(body) _validate_embeddings_dimensions(body) end_user_id = _validate_completions_user(body) @@ -7337,6 +7354,12 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di attribution["model_name"] = model_name if not attribution.get("service"): attribution["service"] = "embeddings_api" + self._ensure_decision_measurement("validated_endpoint") + embedding_agents = coordinator._cost_ordered_capability_candidates( + orchestrator._capability_agents( + "embedding", TaskOrchestrator.AUTO_MODEL if model_was_omitted else model_name + ) + ) started_at = time.perf_counter() configured_timeout = orchestrator.client.timeout embedding_deadline = ( @@ -7430,13 +7453,6 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di _require_pool_model( orchestrator, model_name, required_capability="embedding" ) - embedding_agents = orchestrator._capability_agents( - "embedding", - TaskOrchestrator.AUTO_MODEL if model_was_omitted else model_name, - ) - embedding_agents = coordinator._cost_ordered_capability_candidates( - embedding_agents - ) _validate_embeddings_encoding_format(body) _validate_embeddings_dimensions(body) # OpenAI ``user`` end-user id — same fail-closed shape as sync embeddings. @@ -7455,6 +7471,12 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di endpoint_alias = _validate_batch_embeddings_endpoint(body) if endpoint_alias is not None: submit_metadata["endpoint_alias"] = endpoint_alias + self._ensure_decision_measurement("validated_endpoint") + embedding_agents = coordinator._cost_ordered_capability_candidates( + orchestrator._capability_agents( + "embedding", TaskOrchestrator.AUTO_MODEL if model_was_omitted else model_name + ) + ) document = None last_embedding_error: Exception | None = None for embedding_agent in embedding_agents: @@ -7513,6 +7535,7 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di batch_requests, metadata=metadata, owner_id=security.principal_id(self.headers), + request_id=current_request_id(), ) ) except InvalidBatchModelError as exc: @@ -7533,6 +7556,10 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di "backend": job.backend, "status": job.status, "request_count": job.request_count, + "request_link_status": job.request_link_status, + "registry_persistence_status": job.registry_persistence_status, + "recovery_status": job.recovery_status, + "backend_registry_persistence_status": job.backend_registry_persistence_status, }, 201) return if path.startswith("/api/v1/batch_routing_jobs/") and path.endswith("/results"): @@ -8196,11 +8223,60 @@ def _validate_trace_request(self, body: dict[str, Any]) -> bool: return include_trace def _run(self, callback: Any) -> dict[str, Any]: - security.acquire_run_slot() + if not self._classification_slot_held: + self._acquire_measured_slot() try: return callback() + finally: + self._release_measured_slot() + + def _acquire_measured_slot(self) -> None: + """Retain accepted admission before the nonblocking capacity decision.""" + self._ensure_decision_measurement() + try: + security.acquire_run_slot() + except Exception: + self._decision_failure_reason = "capacity_rejected" + raise + + def _ensure_decision_measurement(self, boundary="first_execution_slot") -> None: + """Open one request scope, with explicit timing eligibility at this boundary.""" + if decision_receipts and self._decision_measurement is None: + endpoint = urllib.parse.urlsplit(self.path).path + if endpoint not in { + "/v1/chat/completions", "/v1/completions", "/v1/responses", + "/v1/embeddings", "/v1/batch/embeddings", + }: + endpoint = "other_execution_endpoint" + try: + self._decision_measurement = DecisionMeasurement( + orchestrator._store, policy=orchestrator.policy, + endpoint_path=endpoint, request_method=self.command, + admission_boundary=boundary, + request_id=current_request_id(), + ) + except RuntimeError: + raise RequestError( + 503, "measurement_unavailable", + "Request measurement is unavailable; retry after service recovery.", + {"measurement_complete": False, "reconciliation_required": True}, + ) from None + + def _release_measured_slot(self) -> None: + """Finalize measurement independently of releasing the execution slot.""" + try: + measurement = self._decision_measurement if decision_receipts else None + if measurement is not None: + active_error = sys.exc_info()[1] + reason = self._decision_failure_reason + if isinstance(active_error, (ConnectionError, BrokenPipeError, GeneratorExit)): + reason = "cancelled" + elif active_error is not None: + reason = "selection_failed" + self._decision_failure_reason = reason finally: security.release_run_slot() + self._classification_slot_held = False def _parse_positive_int(self, raw: str | None, field_name: str, default: int, max_value: int | None = None) -> int: value = default if raw is None else int(raw) @@ -8260,6 +8336,12 @@ def _send_error( message: str, detail: dict[str, Any] | None = None, ) -> None: + if decision_receipts and status >= 400: + measurement = self._decision_measurement + if (measurement is not None + and measurement.receipt.status in ("accepted", "selected") + and self._decision_failure_reason == "unfinished"): + self._decision_failure_reason = "selection_failed" request_id = current_request_id() or uuid.uuid4().hex _LOGGER.warning( "request_failed status=%s code=%s request_id=%s", status, code, request_id @@ -8313,6 +8395,8 @@ def _write_response(self, writer: Callable[[], None]) -> bool: writer() return True except (BrokenPipeError, ConnectionError, OSError): + if decision_receipts: + self._decision_failure_reason = "cancelled" _LOGGER.debug("client_disconnected") # `_send*`/`_begin_sse` writers record their *intended* # status in `self._last_status` before calling this method @@ -8544,7 +8628,10 @@ def progress(role: str, status: str, output: str = "") -> None: part={"type": "reasoning_text", "text": output}, ) - security.acquire_run_slot() + if decision_receipts: + self._acquire_measured_slot() + else: + security.acquire_run_slot() try: if not self._begin_sse(): return False @@ -8592,8 +8679,10 @@ def progress(role: str, status: str, output: str = "") -> None: conduct_kwargs["workflow_run_id"] = f"run_{uuid.uuid4().hex}" result = orchestrator.conduct(messages, **conduct_kwargs) except ConnectionAbortedError: + self._decision_failure_reason = "cancelled" raise except ProviderUpstreamError as exc: + self._decision_failure_reason = "selection_failed" failed = { **created_response, "status": "failed", @@ -8607,6 +8696,7 @@ def progress(role: str, status: str, output: str = "") -> None: self._write_sse("data: [DONE]\n\n") return False except Exception: # noqa: BLE001 - headers sent; terminate with a valid Responses event + self._decision_failure_reason = "selection_failed" failed = { **created_response, "status": "failed", @@ -8706,9 +8796,13 @@ def progress(role: str, status: str, output: str = "") -> None: self._write_sse("data: [DONE]\n\n") return True except ConnectionAbortedError: + self._decision_failure_reason = "cancelled" return False finally: - security.release_run_slot() + if decision_receipts: + self._release_measured_slot() + else: + security.release_run_slot() def _stream_route_completion( self, @@ -8718,6 +8812,7 @@ def _stream_route_completion( model_name: str, *, include_usage: bool = False, + slot_acquired: bool = False, ) -> None: """Pipe live provider deltas as OpenAI chat-completion SSE frames.""" run_id = f"run_{uuid.uuid4().hex}" @@ -8754,7 +8849,11 @@ def usage_frame(usage: dict[str, Any]) -> str: } return f"data: {json.dumps(payload, ensure_ascii=False)}\n\n" - security.acquire_run_slot() + if not slot_acquired: + if decision_receipts: + self._acquire_measured_slot() + else: + security.acquire_run_slot() try: if not self._begin_sse() or not self._write_sse( frame({"role": "assistant"}) @@ -8781,6 +8880,7 @@ def usage_frame(usage: dict[str, Any]) -> str: ): return except ToolFallbackStoppedError as exc: + self._decision_failure_reason = "selection_failed" detail = { **_tool_fallback_error_detail(exc), "request_id": current_request_id() or uuid.uuid4().hex, @@ -8797,6 +8897,7 @@ def usage_frame(usage: dict[str, Any]) -> str: if not self._write_sse(frame({}, finish="error")): return except ProviderUpstreamError as exc: + self._decision_failure_reason = "selection_failed" payload = _error_payload( exc.error_code, _provider_upstream_message(exc), @@ -8809,11 +8910,15 @@ def usage_frame(usage: dict[str, Any]) -> str: if not self._write_sse(frame({}, finish="error")): return except Exception: # noqa: BLE001 - headers already sent; surface as a terminal error frame + self._decision_failure_reason = "selection_failed" if not self._write_sse(frame({}, finish="error")): return self._write_sse("data: [DONE]\n\n") finally: - security.release_run_slot() + if slot_acquired or decision_receipts: + self._release_measured_slot() + else: + security.release_run_slot() def _send_security_headers(self) -> None: if getattr(self, "close_connection", False): diff --git a/docs/analytics_spec.md b/docs/analytics_spec.md index ca76f8767..22152524f 100644 --- a/docs/analytics_spec.md +++ b/docs/analytics_spec.md @@ -140,9 +140,94 @@ never present it as an all-request guarantee or encode missing durations as zero Declare the observation window, quantile method, uncertainty method, and workload before comparing policies. A faster failed decision is not a quality improvement. +Implementation tracking: [CO #1110](https://github.com/ContextualWisdomLab/contextual-orchestrator/issues/1110). +For this metric, accepted means authenticated and fully validated, before run-slot +acquisition. A subsequent capacity rejection stays in that denominator, with no +persisted-decision duration. The current nonblocking slot acquisition does not +create a queue-wait observation. Instrument both ordinary callbacks and direct +streaming admission paths; recording only the shared callback wrapper is incomplete. +Separate initial decisions from failover attempts and never substitute a completed +workflow record written after generation for the initial decision acknowledgement. +This specifies the required boundary; it does not claim instrumentation exists. + +Use one admission identity per validated HTTP request, not per execution-slot +acquisition, provider retry, or file replica. Declare separate units for batch +items and non-generation operations. Start the clock before extracting the +policy snapshot or preparing measurement records. A handled streaming error +before selection remains a selection failure even when no exception reaches +the finalizer; a later delivery failure must not rewrite an already acknowledged +initial decision as if its persistence failed. Outcome and initial-decision +statuses are separate observations. These requirements follow the concrete +pre-release review of candidate `01ce9035715fab4ed60e7352caa85512f855e0bb` in +#1110; verification of the repaired implementation remains open. + +### Auxiliary dispatch and task-route decision + +The first upstream call is not necessarily the task-route decision. Automatic +routing can obtain embedding evidence and a model-backed complexity verdict +before choosing task execution. Record these as separate phases. An +`initial_provider_dispatch` receipt is diagnostic evidence, not automatically +`routing_decision_latency_p95`. + +For the headline routing interval, stop after durable acknowledgement of the +initial task-execution route, before its generation starts. Include any earlier +evidence acquisition and triage required to choose that route; report those +component durations separately rather than subtracting them from elapsed time. +Thus “excluding upstream generation” excludes generation after that decision, +not earlier model work required to make it. Cache hits need explicit evidence +and the same endpoint definition; changing endpoint or cache mix cannot establish +a speedup. Later failover decisions remain separate attempts. + +The same embedding helper can also run after task selection to update quality +evidence. Classify each invocation at its own start, using whether selection +has occurred; do not infer its phase from the helper name or from successful +persistence alone. A decision-write failure still follows a selection, but +provides no successful acknowledgement duration. Preserve post-selection +component records separately, without extending or replacing the initial +decision interval. A provider-ready timestamp taken before a diagnostic write +is not a network-send timestamp. Test observers must collect passively and +assert after the request: an assertion raised inside a best-effort provider +hook can change fallback and cache behavior and invalidate the experiment. + +This clarification follows the direct auxiliary calls found at candidate +`07957ee643bf74c5beb13c03827f59331307cc5d` in #1110. Tests must distinguish +cached and uncached triage, embedding evidence, and task execution. Existing +first-dispatch tests do not prove this headline interval; its implementation +and observed-workload baseline remain open. + +### Autonomous experiment targets + +These are engineering acceptance targets selected on 2026-09-09, not measured +results or literature-derived constants. Do not ask the user to choose their +scope. Baseline and candidate must use the same declared population, endpoint +mix, task rubric, model versions, resource budget, and failure accounting. + +| Outcome | Target | Guardrail | +| --- | --- | --- | +| Delivered-correct fraction over all accepted requests | At least +1 percentage point versus baseline, with a 95% confidence interval for the difference wholly above zero. | Independently adjudicated observed outcomes; paired or randomized design declared before evaluation; no silent exclusion of failed delivery. | +| Routing-decision p95 | At most 20 ms and at least 10% lower than baseline, with the 95% interval for the candidate/baseline ratio wholly below 1. | Include selection and durable acknowledgement; preserve workload and failure accounting. This is not the full-page latency SLO. | +| Numerical parameter recovery | No regression in family-wise aligned RMSE under the declared numerical tolerance. | Known-truth unit tests only; never substitute for observed customer outcomes. | + +The end target requires both customer accuracy and decision-latency criteria. +Intermediate changes may advance one while preserving the other's established +baseline; they must not be labelled completion of both. If non-regression cannot +be established, keep the candidate experimental and leave production unchanged. +Use a fresh holdout for confirmation after adaptive experiment selection, or a +predeclared sequential inference procedure; repeated inspection of an ordinary +95% interval is not a stopping rule. Determine sample size from pilot variance, +the +1-point effect target, and declared power before confirmation, retaining +task/model/time clustering. Do not reduce sample size after seeing results. + +Execution and evidence handoff: [autonomous KPI runbook](doctoring/autonomous_kpi_runbook.md). + ## Commercial Due-Diligence KPIs -These metrics support the KRW 2,000,000,000 commercial-readiness review. The +The active goal uses a USD 20,000,000,000 sale-quality ambition. This is an +aspirational quality target, not a measured valuation or a signed transaction. +The KRW 2,000,000,000 references in the legacy metrics below describe the +earlier commercial-readiness review; they must not substitute for the active +goal or be treated as its currency conversion. Accuracy and decision-latency +acceptance still require the observed evidence specified above. The `evidence_type` column is mandatory so measured local evidence is never mixed with proposed production targets. diff --git a/docs/benchmarks/2026-08-11-polytomous-llm-judge.md b/docs/benchmarks/2026-08-11-polytomous-llm-judge.md index a67ae01a7..4a12a6d78 100644 --- a/docs/benchmarks/2026-08-11-polytomous-llm-judge.md +++ b/docs/benchmarks/2026-08-11-polytomous-llm-judge.md @@ -3,6 +3,12 @@ Status: exploratory integration evidence; not a claim that the local judge is unbiased or that this single case is sufficient for IRT estimation. +Evidence classification (2026-09-09 audit): these historical model calls use +constructed release-plan cases. They are not a probability sample of customer +requests and must not supply the observed delivered-correct KPI or parameter +recovery evidence. Tables below are historical reports, not freshly reproduced +raw-artifact verification. + ## Setup The path under test was: @@ -137,7 +143,8 @@ parser rejected the model response; it was never repaired or accepted. | liked | `invalid` / `0.7500; yes` / `0.8333; yes` | `0.5000; no` / `0.0000; no` / `invalid` | | disliked | `0.5000; no` / `0.7500; yes` / `0.8333; yes` | `invalid` / `invalid` / `invalid` | -The good plan parsed in 11/18 calls and was accepted in 5/11 parsed calls; +The good plan parsed in 11/18 calls and was accepted in 5/11 parsed calls +(45.45%, conditional on parsing), or 5/18 of all attempted calls (27.78%); the seven failures were five invalid JSON responses, one out-of-range category, and one non-monotone threshold vector. The unsafe plan parsed in all 18 calls, scored `0.0000` in every case, and was accepted zero times. Direct judging diff --git a/docs/doctoring/autonomous_kpi_runbook.md b/docs/doctoring/autonomous_kpi_runbook.md new file mode 100644 index 000000000..9dcf314df --- /dev/null +++ b/docs/doctoring/autonomous_kpi_runbook.md @@ -0,0 +1,678 @@ +# Autonomous KPI experiment runbook + +## Title-only citation reconciliation, 2026-09-12 + +At parent `14a6a943e99bd1dcf8b65a2780b0c80dbc70edcb`, the Fox and Glas +reference below had no explicit identifier and was absent from the inventory. +Both identifier-discovery tests passed in 2.43s despite that omission. +Adding its verified DOI to this runbook alone produced the expected RED: +one failed, five deselected in 0.72s, naming `10.1007/bf02294839`. +Adding the APA reference and bounded read status to the existing inventory +passed all six citation contracts in 1.04s. Reproduce with the project +environment's `python -m pytest tests/test_paper_contracts.py -q`. +This does not make title-only discovery complete. Normalize manually verified +title-only references to persistent identifiers so the existing guard can +track them thereafter; no new parser or dependency is needed for this repair. + +Publisher metadata and abstract were verified; full-method access and PDF +redistribution permission remain unresolved. The 2025 online migration date +does not replace the original 2001 publication year. Numerical estimation +remains the fast-mlsirm responsibility; no runtime default or KPI gain follows. +New heads invalidate prior hosted acceptance for #1107 and its #1139 successor; +normal restacking, fresh checks and independent review are still required. + +## Baker estimation boundary, 2026-09-09 + +Baker, F. B. (2001). *The basics of item response theory* (2nd ed.). ERIC +Clearinghouse on Assessment and Evaluation. +[Primary PDF](https://files.eric.ed.gov/fulltext/ED458219.pdf). + +Root directly inspected rendered PDF pages 99 and 120 (printed 90 and 111). +Page 90 describes unbounded ability MLEs for all-wrong/all-correct responses +and conditions invariance on one latent trait and a common item metric. +Page 111 defines 2PL item information as `a² P(θ) [1−P(θ)]`; information +depends on ability, not just an item's global ranking. The equations, text, +and table were legible in the directly opened page images. This is a bounded +read, not review of the complete book; empty PDF text extraction cannot prove +that a term is absent. Copyright remains reserved; no PDF is vendored. + +Engineering inference: retain extreme-response and estimator-failure counts in +the numerical recovery denominator. Report recovery RMSE conditional on a valid +estimate alongside that denominator, never silently discard failures. Any +information-based item allocation requires an aligned estimand and calibrated +owner contract before deployment. This does not turn synthetic recovery into +observed customer accuracy or authorize routing defaults. Numerical estimation +belongs to fast-mlsirm; CO retains request/outcome provenance and failure counts. + +## Versioned citation discovery repair, 2026-09-09 + +The citation check at `c226d30d` passed six tests but omitted arXiv HTML URLs +and version-suffixed identifiers. The old word boundary immediately after the +paper number rejected the following `v`, including versioned PDF and abstract +links. Regression checkpoint `c43a7d9214caf54d5276500149c44cea201ecfac` +failed one test with five passing (0.93s). Adding HTML and optional versions +then exposed a second mismatch: the inventory itself contains `1812.00140v4`, +but its identifier extraction also rejected version suffixes (one failed, +five passed in 0.87s). Normalize both sides to the paper identifier while +retaining exact version URLs in the bibliography. + +Reproduce with `.venv/bin/python -m pytest tests/test_paper_contracts.py -q`. +At `9ac931581aae638860833f1b7f79b49cfc9569ff`, all six tests pass in 0.81s. +This checks explicit modern arXiv identifiers and DOI links in tracked Python, +Rust, Markdown and TOML files; it does not establish complete citation discovery +across other formats, legacy arXiv identifiers, title-only citations, full-text +review, redistribution rights or customer KPI improvement. + +## Integrated receipt regression and remaining capacity defect, 2026-09-09 + +Candidate `3b6dd47ebb0f88802bacdd302051d2f03e7d5003` completed the +full local macOS/Python 3.14 regression: **3442 passed, 2 skipped in 771.43s**. +The checkout remained unchanged throughout the run. Its isolated wheel-only +receipt and SSE identity regressions separately passed **33 tests in 14.21s**. +Root independently verified both wheel hashes, disjoint manifests and +outside-checkout imports. These results repair the earlier nine-failure +baseline, but do not prove hosted Linux acceptance, protected merge or release. + +A separate real HTTP unit probe holds the only slot in +`SecurityConfig(max_concurrent_runs=1)` before sending an auto chat streaming +request. At this same installed revision the request calls the mock triage +provider once, then returns 503 with one `capacity_rejected` admission. +The initial expected status of 429 was corrected to the existing 503 contract; +the zero-provider-call assertion still fails. Equivalent Responses-streaming +and nonstreaming-chat controls each make zero calls and pass. The observers +record state only and assert after response completion. + +See [the failing receipt](https://github.com/ContextualWisdomLab/contextual-orchestrator/issues/1110#issuecomment-5598185904) +and [passing sibling controls](https://github.com/ContextualWisdomLab/contextual-orchestrator/issues/1110#issuecomment-5598207533). +Repair chat streaming so its existing execution slot covers classification +before task execution, without double acquisition or release. Test saturated +rejection, capacity-one success, classifier failure and authorized trace +rejection. Start source edits only after the prior full run is terminal; +attribute the next regression to its new head. The initial-decision clock and +failure denominator must survive this change. No customer gain is measured. + +## Publishing secret metadata correction, 2026-09-09 + +The authenticated organization Actions Secrets API succeeded. Both +`CARGO_REGISTRY_TOKEN` and `PIPY_TOKEN` exist with `visibility=all`; the Python +publishing name is spelled `PIPY`, not `PYPI`. A name filter matching only +`PYPI` misses it. No credential values were read. Repository and `production` +environment inventories each returned zero secrets, which does not negate +inherited organization secrets. Earlier permission failures establish unknown +visibility at that time, not missing keys. No permission change was needed. + +Reproduce metadata-only checks with +`gh api orgs/ContextualWisdomLab/actions/secrets/PIPY_TOKEN --jq '{name,visibility,updated_at}'` +and the equivalent `CARGO_REGISTRY_TOKEN` endpoint. This proves registration +and repository visibility, not credential validity or registry ownership. +The current protected-main workflow inventory contains only `nim-benchmark.yml`, +`opencode-hourly-loop.yml`, `provider-catalog-sync.yml`, and `security.yml`; +no package publisher was present there. Canonical release coordination was +notified. Establish a protected, exact-revision package publishing path and +registry acceptance before claiming automatic deployment; do not bypass review +or publish the unmerged measurement candidate to test credential validity. + +## Noema terminal failure attribution, 2026-09-09 + +Two central `.github` runs completed bootstrap but failed their review request: + +| Run / job | Caller observation | Interpretation boundary | +| --- | --- | --- | +| 34316107754 / 102352588433 | 502 after 128.5 s; llama-3.2-11b-vision-instruct | Server artifact records TimeoutError for the same model at 05:56:14.181, just before caller failure at 05:56:14.183; correlation is not request identity. | +| 34315965378 / 102352582904 | 429 after 336.7 s; deepseek-v4-flash-0731 | Terminal rejection is verified; internal candidate history and exhaustion are not. | + +Both callers report one gateway attempt. That is not a count of CO internal +attempts. Artifact `10090380702` (`noema-sidecar-evidence`, first run) contains +stderr and preflight JSON, but no request identifier or source SHA in the +inspected records. Earlier provider failures occurred during preflight; do not +attribute them to the review request. Reproduce retrieval with +`gh run download 34316107754 --repo ContextualWisdomLab/.github --name noema-sidecar-evidence --dir `; +inspect only allowlisted diagnostics, never publish raw credentials or prompts. + +Existing owners are issue #1045, PR #1049 (`e2641c16`, replay safety), +PR #1037 (`4ba6be74`, bounded attempt history), PR #1094 (Noema structured +conduct), and PR #1105 (request correlation). A bare timeout/502 does not prove +the non-idempotent operation was never applied; do not widen retry policy from +this incident. Keep explicit 429 rejection separate. Next verification needs +the deployed sidecar revision and request-correlated reproduction, followed by +owner tests and protected release. See [the evidence receipt](https://github.com/ContextualWisdomLab/contextual-orchestrator/issues/1045#issuecomment-5596669214). +These are failed deliveries in the accuracy denominator, not measured routing +decision latencies. Their elapsed times include work beyond initial selection. + +## Stacked quality-trigger repair + +Lineage correction: existing PR #1066 at +`59a8f4eadfe0e0dcc5ff47cf1acfb80403e241ad` already owns the complete trigger +repair, including Ready/closed admission and PR-only cancellation. The partial +repair below duplicated its base-filter change. Integrate that branch normally, +retain its complete workflow and tests, and consolidate the extra path-filter +and event-permission assertions into `tests/test_repository_security_metadata.py`. +The duplicate `tests/test_stacked_quality_workflow.py` is removed only after those +assertions are preserved. Run the canonical metadata tests plus the NIM workflow +contracts and actionlint. Neither #1066 nor its predecessor #1060 is closed by +this integration; protected delivery is still required. Historical commands and +results below remain attached to their original revisions. + +On 2026-09-09, PR #1108 at `fbb933cbcaa1f1695c6cc305657f450f22b3be4c` +had zero GitHub check runs despite a completed local suite (3,399 passed, +2 skipped). Its base was `autoresearch/20260909-kpi-loop`, excluded by the +repository quality workflow's `pull_request.branches: [main]` filter. +Commit `1a510faa` removes that filter, preserves permissions, and uses the +workflow/repository/PR cancellation key. Central required workflows remain +separate owners; this change cannot provide their approval. + +Reproduce with `.venv/bin/python -m pytest tests/test_stacked_quality_workflow.py -q`. +The old trigger fails its assertion; with the repair, this and the existing NIM +workflow contracts pass (9 tests). `actionlint .github/workflows/security.yml` +has no findings. An initial PyYAML-based test failed collection because that +package is absent; the retained stdlib contract needs no new dependency. +The integrated rollback head `c11df645865062da6c4d1680a285eb5c21a91594` +passed 30 persistence/workflow contracts in 8.71 seconds before push. +Do not assign the earlier full-suite count to this new head. After pushing a +new synchronize event, inspect the live run's head and checked-out merge parents; +no run or a queued run is not a pass. Retain both sides of gap-baseline merge +conflicts so rollback evidence and newer research evidence are not discarded. + +Status: measurement preparation; no measured customer gain. Owner: CO for +request timing and delivered outcomes; fast-mlsirm for numerical estimators. + +## Start and evidence boundaries + +### Local Zotero discovery, 2026-09-09 + +The read-only local API at `http://127.0.0.1:23119/api/users/0/items` +returned HTTP 200. Query exact titles before broad terms: `q=Modeling%20Differences%20Between%20Response%20Times` +returned no matches in the queried personal library; this does not establish +absence from other libraries or external repositories. Quote URL arguments in +zsh because an unquoted `?` is treated as a filename pattern before HTTP runs. +Use `format=json`, a bounded `limit`, and project only bibliographic fields; +do not export the full personal library or private notes. + +A bounded IRT query located item `E4INK5S3`: + +Fox, J.-P., & Glas, C. A. W. (2001). Bayesian estimation of a multilevel IRT +model using Gibbs sampling. *Psychometrika, 66*(2), 271–288. +https://doi.org/10.1007/BF02294839 + +Its bibliographic fields matched the University of Twente repository's record. +The repository linked an accepted manuscript at +`https://research.utwente.nl/files/247180414/Fox2001Bayesian.pdf`, but the +research browser fetch returned HTTP 403. No manuscript was obtained or attached, +and no full-method review or implementation result follows from the abstract. +Next step is lawful full-text access and model/identification review, not +production adoption from bibliographic discovery alone. No Zotero data changed. + +### Completed observation-integrity verification + +PR #1109 code head `4cc0bf2c92181cb5ea175a1f1e1db1c8a85799bc` completed +the three-file local psychometric regression command with **57 passed in +1159.23s**, exit 0 (Python 3.13.14, fast-mlsirm 0.9.1). Process sampling during +the run found Rust CAT/EAP reduction and thread joins on a heavily loaded host; +this elapsed time is not routing-decision latency. +Hosted job [102339210701](https://github.com/ContextualWisdomLab/contextual-orchestrator/actions/runs/34311594936/job/102339210701) +completed full pytest with **3595 passed, 2 skipped in 791.75s**. Its tested +merge `75d08ce905dc3d0c0468420bf4094a4cc7359ab3` has parents +`84a6052369a7bf8b6faae5db475bb68a5ad54a91` and the code head above. +These receipts supersede earlier pending-test observations, not the outstanding +independent GitHub approval, protected-owner integration, or release gates. + +### Multimodal cost evidence boundary + +At CO `632831de9cc2a510af41981711eedc13e38df479`, inspected MMR-Bench v1's +abstract, Section 3.1, and Appendix C.2–C.3; this is a partial read. +The [primary source](https://arxiv.org/html/2601.17814v1) defines a text/image +routing problem. Its normalized cost compares models within a fixed pool; +Appendix C.3 also describes disabled reasoning traces for selected models. +Therefore the reported roughly 33% cost scenario is not a CO invoice saving, +decision-latency p95, or audio/video validation result. CO's model-group +specification cites this paper for evaluation design, not achieved performance. +Any local comparison must fix the candidate and reasoning-policy revisions, +report real prices and latency separately, and retain failed requests. No +production routing or reasoning policy was changed from this partial review. + +### Executable reference inventory check + +At `9a9f1ab2`, run `python -m pytest tests/test_paper_contracts.py -k +explicit_arxiv -q` in a Git checkout with the project test dependencies. +Result: 1 passed, 4 deselected in 16.76s. An in-memory negative check removed +`2601.17814` only from the inventory read; the test failed with that identifier +and its source path as expected. No source file was modified by that check. +The test uses tracked `.py`, `.rs`, `.md`, and `.toml` text and explicit arXiv +identifiers. It detects discovery omissions, not title accuracy, version/license +compliance, evidence quality, full-paper review, or DOI-only references. + +### Expanded-population validity proposal + +Source inspection at CO `479bfe7e096832e1711c4d99b59621a66c3a2f59`: +Kim and Chung (2019), ETS RM-19-07, abstract and printed pages 1–8. +Their method checks item DIF and subgroup score-linking invariance separately; +the report does not establish LLM routing performance. See the +[source and APA reference](../papers/README.md#apa-7th-edition-references). + +Engineering proposal, not an implemented or validated estimator: treat a new +language, task domain, or model revision as an explicitly declared evaluation +population. CO records those evaluation conditions and observed outcomes; +fast-mlsirm owns estimation, alignment, and diagnostic computation. A released +owner contract should return the reference population, anchor-set revision, +identification constraints, supported strata, uncertainty, and diagnostic status. +Do not equate raw accuracy differences with DIF, or a nonsignificant diagnostic +with proven equivalence. Missing support is unknown, not a passing result. + +For acceptance, report held-out correctness and failure denominators by declared +stratum alongside the overall paired accuracy change. Estimate the cost of +retrieving a versioned diagnostic with the same accepted-request-to-durable- +decision clock; do not silently omit it from latency. Offline estimation may +keep the request path small, but requires provenance and an expiry policy whose +operational effect is tested. No new heuristic weights or production default +change follows from this proposal. The next evidence gate is an owner-contract +inventory and an observed-data evaluation design, not implementation copied +into CO. No numerical effect or universal DIF cutoff is claimed. + +### Paper inventory consistency check, 2026-09-09 + +At `b031d3acecc89b29b35fdd769598aa3826615553`, the paper inventory correctly +marked redistribution permission as unverified but its closing paragraph still +claimed the arXiv non-exclusive license authorized redistribution. The +[official license guidance](https://info.arxiv.org/help/license/index.html) +was rechecked: that license grants limited distribution rights to arXiv, not a +general reuse grant. Commit `34bf2f3f5925a84630edfccaf608e06f5e3192ae` removes +the contradictory assurance and places batch references under their own heading. +All five stored PDF hashes still verify; no PDF was added, removed, or republished. +Actual browser inspection of this revision was attempted but the native browser +tool reported the Mac locked and automatic unlock unavailable. Rendering is +**not visually verified**; inspect the headings and license callout after unlock. + +Read AGENTS.md, CLAUDE.md, and the targets in `docs/analytics_spec.md`. +Record head/base, clean or unrelated worktree changes, locked dependencies, +runtime version, dataset identity and permitted use, workload, observation +window, sampling design, resource budget, and failure counts before comparison. +Never replace missing observations with zero or a synthetic oracle score. +Keep experiment results with the tested commit, command, output artifact, +baseline/candidate estimates, uncertainty, and keep/discard decision. + +The reviewed CO measurement owner is PR #1067. At +`84a6052369a7bf8b6faae5db475bb68a5ad54a91`, this command runs focused unit +regressions, not a customer performance benchmark: + +```sh +uv run --frozen pytest tests/test_psychometric_routing.py tests/test_psychometric_benchmark_boundaries.py -q +``` + +Its held-out harness uses known synthetic probabilities and numerical fixtures. +Do not use its output to satisfy the observed-customer accuracy target. A +production interval measuring accepted request through durable decision is +required separately. Current observed-data baseline and that interval's measured +p95 remain unverified; no executable end-to-end customer KPI command is claimed. + +## Long-running numerical tests + +On 2026-09-09, the command above ran in a clean exact-head worktree using +Python 3.14 and installed fast-mlsirm 0.9.1. Process sampling found Rust +`cat_next_item` / EAP CPU reduction, including worker joins. The host reported +10 logical CPUs and approximately 60 one-minute load average. This is evidence +of a busy host, not a measured algorithmic regression or proof of deadlock. +The same execution subsequently completed: **51 passed in 744.99 seconds**, +exit 0. No restart was needed. This duration is host-contended unit execution, +not routing-decision p95 or a baseline suitable for claiming a speedup. For +future runs, reuse the live execution handle, inspect process progress, and do +not restart merely because an observation call yields without output. + +The owner source at `256470c7d1df4910a018841499a74d88b751a774` creates a scoped +CPU worker even for one person in `score_eap_cpu_reduce`. This is an optimization +hypothesis only: match installed artifact to source and measure before changing +it. Do not terminate unrelated host jobs or shrink the fixture to manufacture a +speedup. Use paired/interleaved baseline and candidate measurements with recorded +host conditions; retain unfavorable runs and report interference. + +## Owner unit experiment receipt, 2026-09-09 + +Baseline owner `256470c7d1df4910a018841499a74d88b751a774` completed the +scoring-filtered Cargo command with 41 passed, 1 ignored, 1106 filtered out, +exit 0. Compilation took 27m20s; tests took 0.05s. Neither duration is a +customer latency measurement. Candidate code +`b1709fa1e67a70274126d358506c7e9282fe5605` subsequently completed with exit 0. +A post-completion rerun retained the full local log at +`/tmp/co_candidate_scoring_completed.log`: 42 passed, 1 ignored, 1106 filtered +out, 0 failed, test duration 0.39s. The additional single-person parity test +passed. This is the scoring filter, not the complete crate suite or GPU testing. + +A separately linked unit probe compared three synthetic missingness patterns +across both built libraries. All four output families matched bitwise, with +single-person versus repeated-person equality also checked. Across 300 calls +per version, baseline/candidate median was 569333/180209 ns and nearest-rank +p95 was 7738375/20093625 ns. The worse candidate tail prevents acceptance; +sequential, host-contended unit timings do not prove a causal regression either. +Owner experiment record: `docs/single_person_scoring_experiment.md` at local +commit `e3d9a939` in fast-mlsirm. No release or CO dependency change occurred. + +## Document visual inspection, 2026-09-09 + +Inspected the actual Edge-rendered `docs/analytics_spec.md` at +`dccd37e032f74251e3c02e38046dab68c0049787`, fragment +`autonomous-experiment-targets`, in English document content with Korean browser +chrome. Browser-tab selection timed out, but native app selection of the +already-created CO KPI tab worked. Opened the screenshot directly in the +inspection conversation: all three table columns, wrapped text, caveats, and +the runbook link were legible without overlap; the heading permalink displayed +visible focus. This is a partial desktop document inspection, not admin-product +UI acceptance. Viewport dimensions, durable image export, responsive sizes, +other locales, and error/loading states remain unverified. Do not mark the +full visual-inspection requirement complete from this receipt. + +## State-write atomicity prerequisite (2026-09-09) + +At CO `ab8a7caa6c00a49eede17a03d0897865cfdce9f5`, `run()` calls +`complete()` before saving its workflow record. Its duration includes generation +and cannot represent accepted-request-to-decision latency. Analytics stream +`save()` normally enqueues, whereas `durable=True` reaches the synchronous +commit path. Neither queue acceptance nor final workflow storage is a substitute +for a pre-invocation decision acknowledgement. + +Tracing that common synchronous path exposed an atomicity defect: a keyed +replacement deletes the old row before inserting the new one. An insertion +failure left the deletion in an open transaction, and a subsequent unrelated +save committed it. A real SQLite trigger injecting an insertion failure reproduced +the loss: the previous version disappeared. The new regression failed before +the fix (1 failed, 18 deselected, 6.17s). + +Code commit `d7bba88f3d711883a37effe49ab4f503c4fb8e01` uses the existing +connection's transaction context inside the existing lock. Success commits; +failure rolls back before another writer acquires the lock. No new dependency, +schema, numerical implementation, or production routing default was introduced. +The same project-local Python environment ran `python -m pytest +tests/test_persistence.py -q` from the isolated worktree: 19 passed in 16.38s. +This establishes the tested SQLite failure case, not customer KPI improvement, +full-suite success, protected merge, or production deployment. Next implement +decision timing before provider invocation with explicit failure denominators; +do not relabel existing response-generation timings. + +Follow-up validation: full `python -m pytest -q` at +`877d5112ed470d851afaa2c746b94393cc768ee7` exited 0 with 3,396 passed and +2 skipped in 883.03s. The quiet log does not identify the skip reasons; neither +skip is counted as passed. Test-only commit +`716e012dcb50857000b0fc53c89c6434fdf7e7c2` extends the regression to a real +deferred foreign-key violation at commit, in addition to insertion failure. +The existing transaction implementation is unchanged. At that commit, +`python -m pytest tests/test_persistence.py +tests/test_workflow_run_object_authorization.py tests/test_governance_runtime.py -q` +exited 0: 29 passed in 4.89s. The new test head has not had a full-suite run; +these local results do not establish hosted security checks or independent review. + +## Break release cycles without copying implementation + +Minimum contract → owner RED test → owner implementation → exact-SHA/digest +isolated real integration → protected immutable release → consumer adoption. + +CO can implement its timing/outcome port against test doubles while numerical +owner work proceeds. Doubles prove boundary behavior only. Candidate artifacts +are allowed in isolated CI, never as production dependencies or release proof. +Remove the temporary candidate integration after released-contract adoption; +restore the previous released contract if integration fails. Required reviews, +security checks, and `orchestrator/free` remain enforced throughout. + +## Coverage-gate evidence 2026-09-09 (autoresearch experiment 1: discard) + +Head `74efc7bf` (code+tests identical to `origin/main@414f2297`), coverage +7.15.0, exact issue-#1075 gate block on unmodified code: 134 passed in +143.50s, `nim_benchmark.py` 1205/0/436/0 = 100%, `--fail-under=100` exit 0. +Former gaps 434 / 645 / 671->682 are covered by later-main acceptance tests; +no change needed. Single-file `tests/test_nim_benchmark.py` alone is 93% — +always run the full three-file gate before claiming coverage. Unit evidence +only, not customer accuracy. + +## Shared primary checkout (lesson 2026-09-09) + +The primary checkout may receive commits from a concurrent scheduled session +mid-run (7 docs-only commits observed on the loop branch via reflog; purposes +legible, code+tests untouched). Before and after long runs, record +`git rev-parse HEAD` and `git diff --stat origin/main...HEAD -- +contextual_orchestrator/ tests/`; docs-only drift does not invalidate code +evidence, but any code/test drift does. Prefer isolated worktrees for code +experiments; never rebase or push another session's branch. + +## Autonomous KPI scope, 2026-09-09 (selected without asking) + +Goal: USD 20,000,000,000 sale quality and customer-felt gap closure. +Scope is chosen under `docs/analytics_spec.md`; no KPI-scope question was +asked. Primary product metrics are observed delivered correctness and routing +decision latency, with the acceptance criteria below. `open_pr_count` is a +secondary delivery measure, not evidence of product improvement (lower is better, +`gh pr list --state open --json number | jq length`): 85 at loop start, +87 on recount (concurrent-session growth, not this change). PR 0 only via +merge or verified-successor full-delta inheritance; no force-push and no +close without evidence (user-explicit, no valid delta, malicious change, +or verified complete inheritance only). + +Product acceptance stays observed-only: delivered-correct fraction at +least +1 point with the 95% difference interval wholly above zero, and +routing-decision p95 at most 20 ms with at least 10% reduction and the +95% candidate/baseline ratio interval wholly below 1, each with declared +population, workload, failure denominators, and uncertainty. Synthetic +true-parameter recovery is unit evidence only (family-wise aligned RMSE, +no regression); never substitute it for buyer accuracy. Latency claims +use accepted-request to durable-decision timestamps on one monotonic +clock, including queueing, selection, and persistence; worker durations +that include generation cannot supply this metric. Measured-local +commercial signals (`commercial_readiness_pass_rate`, +`buyer_evidence_completeness`, `security_control_pass_rate`, +`trace_audit_completeness`) may advance; buyer/production warnings stay +warnings until buyer or production evidence arrives. + +## Isolated verification evidence, 2026-09-09 + +Head `0ea2a58d34fb07e15c12dfcfd65f7242b2a45c91`, base +`origin/main`; `contextual_orchestrator/ tests/` diff since `origin/main` +is `benchmark_priors.py` only (17 insertions, 20 deletions, docstrings +and comments bounding the legacy heuristic; runtime unchanged). +`tests/test_model_group.py` plus `tests/test_benchmark_priors.py`: 37 +passed in 25.82s, exit 0, on the primary checkout. + +PR #1108 head `4316be85` verified in isolated worktree +`/tmp/co-verify-1108` (primary checkout untouched): +`tests/test_persistence.py` 20 passed in 32.42s, exit 0. The +`_save_sync` change uses the SQLite connection context under the writer +lock so a failed keyed replacement rolls back instead of leaking its +DELETE into the next commit; the added test covers insert-phase and +deferred-commit-phase failures, closed-transaction state, and reopened +persistence. Focused unit evidence only; no customer KPI, full-suite, +release, or deployment claim. Merge test of loop HEAD `0ea2a58d` into +PR #1108 auto-merged code but conflicted in +`docs/product-technical-gap-baseline.md` (both sides append); restack is +a normal merge with manual docs resolution by the PR owner, never a +force-push. Worktrees removed after verification. + +## Isolated verification evidence, 2026-09-09 (PR #1109) + +Loop HEAD `8839bfc5f3e6608e11faf3c8a2b237d5bd4050a5` is in sync with +`origin/autoresearch/20260909-kpi-loop`; code diff since +`origin/main` remains `benchmark_priors.py` only, while `tests/` +additionally carries the concurrent session's `test_paper_contracts.py` +guard (+24), which this turn does not claim as its evidence. Open-PR recount is 88 +(baseline 85; +1 new draft #1109 on the psychometric stack). + +PR #1109 head `4cc0bf2c` verified in isolated worktree +`/tmp/co-verify-1109` (primary checkout untouched): +`tests/test_psychometric_observation_atomicity.py` 6 passed in 52.15s, +exit 0. The `observe_context_id` change validates the copied vector and +complete dichotomous response row before any retained-state mutation +under the existing lock; no new numerical arithmetic is introduced, so +the Rust-authoritative computation rule is unaffected. Hosted checks at +observation time: CodeQL success; `Tests and package quality` and +`Property and coverage-guided fuzzing` still in progress. PR #1108 head +reports no check runs while `dirty` against the loop branch; PR #1094 +remains `mergeable: true` but `mergeStateStatus: blocked` on main +protection. No merge, readiness flip, or push to another session's +branch was attempted. Unit evidence only; full regression, independent +review, protected merge, and release remain pending. Worktree removal +follows the docs commit. + +## Isolated verification evidence, 2026-09-09 (PR #1109 new head) + +PR #1109 flipped from Draft to Ready during the loop and its head moved +`4cc0bf2c` to `b8d2651d61f1d178971997c3ee49e404580fc2aa`, so the prior +6-pass verification is superseded for the new head. Isolated worktree +`/tmp/co-verify-1109b` (primary checkout untouched): +`tests/test_psychometric_observation_atomicity.py` 19 passed in 13.68s, +exit 0. The new delta replaces `int(value)` with `operator.index(value)` +(`operator` was already imported): fractional rows (`0.7`, `1.7`, +`-0.7`), whole-valued floats (`0.0`, `1.0`), and numeric strings +(`"1"`) are now rejected with `TypeError` instead of silently truncated +to `0`/`1`, while the integer protocol (`int`, `numpy.int64`, bools via +`__index__`) is preserved and covered by +`test_integer_protocol_rows_preserve_binary_values`. Direction is +fail-closed evidence integrity: a fractional row was never a valid +dichotomous observation, so refusing it repairs masking rather than +regressing a contract; the compatibility boundary is documented on the +PR. Hosted checks on the new head: both CodeQL jobs success; `Tests and +package quality` and `Property and coverage-guided fuzzing` still in +progress; no reviews posted. No merge, readiness change, or push to the +owner stack was attempted. Unit evidence only. + +## Re-observation, 2026-09-09 (PR #1109 same head, PR #1108 unknown) + +Loop HEAD `9e08f1448f23a22d10c99f3899021d241910f636` is in sync with +`origin/autoresearch/20260909-kpi-loop`. PR #1109 head is unchanged +(`b8d2651d`), so the 19-pass isolated verification stands without +rerun. Hosted movement since last observation: `Property and +coverage-guided fuzzing` moved from in-progress to success; `Tests and +package quality` remains in progress; still no reviews; state remains +`unstable`, so the merge bar is still unmet and nothing was merged, +flipped, or pushed across branches. PR #1108 reports +`mergeable: null` / `mergeable_state: unknown` (GitHub recomputing the +dirty computation); still not actionable from this loop. Open-PR +recount 88 (baseline 85). + +## Loop-merge review and PR re-observation, 2026-09-09 + +Concurrent session merged the canonical stacked-quality repair into the +loop branch (`d721e04b`, merging `59a8f4ea`). Reviewed rather than +reverted. `security.yml` verdict: compliant, not weakened. The +concurrency group is now exactly +`${{ github.workflow }}-${{ github.repository }}-${{ +github.event.pull_request.number || github.event.schedule || +github.run_id }}` with `cancel-in-progress` only on `pull_request` +events — the demanded `{workflow}-{repository}-{PR}` shape; scheduled +and push runs serialize on schedule/`run_id` and are never cancelled. +The removed `branches: [main]` PR filter expands coverage to stacked-PR +bases instead of weakening main (main PRs still always run). New +`action != 'closed' && draft == false` job guards skip only closed and +Draft PRs, which can never merge; the `ready_for_review` trigger runs +checks on every Draft-to-Ready transition. The deleted +`tests/test_stacked_quality_workflow.py` was consolidated into +`tests/test_repository_security_metadata.py` (updated group/cancel +assertions plus new `test_security_workflow_supports_stacked_pull_requests` +pinning unfiltered triggers, `contents: read`, and no +`pull_request_target`); no assertion was dropped. + +PR #1108 restacked (`4316be85` to `c11df645`, retaining the rollback +delta and activating the stacked checks). Isolated worktree +`/tmp/co-verify-1108c`: `tests/test_persistence.py` 21 passed in +28.10s, exit 0. The `orchestrator.py` fix is unchanged; the delta adds +`durable=True` stream-retention coverage and a new independent-connection +commit-visibility test pinning durable-return-means-committed. Prior +20-pass verification is superseded. Unit evidence only. + +PR #1109 head unchanged (`b8d2651d`): all four hosted groups now green +(both CodeQL jobs, fuzzing, tests) with still no reviews — merge bar +still unmet (independent exact-head approvals missing), so unmerged. +Base remains the owner stack branch; no cross-boundary merge attempted. + +PR #1105 (Ready, `main` base, error-request correlation) is blocked +with 3 `CodeQL compatibility analysis` failures that are +fail-closed pending verdicts, not code defects: job `102342918324` log +shows `DISPATCH_OUTCOME: success` with `VERDICT_STATE: pending`, and the +job's own message states the dispatch workflow will rerun it after +publishing the terminal verdict. Expected to self-heal; re-observe next +turn. No push to any owner branch was made from this loop. + +## Re-observation, 2026-09-09 (PR #1108 loop-merge, PR #1105 unhealed) + +Loop HEAD `0ef13edcefcf562383dc5ef3391cbe46a38635cf` is in sync with +`origin/autoresearch/20260909-kpi-loop` (concurrent docs-only +`analytics_spec.md` dispatch-latency distinction arrived before this +commit; code/test diff since `origin/main` unchanged). PR #1108 head moved `c11df645` +to `129a6650`, which is a normal merge of `origin/loop` into the PR +branch (no force-push): the delta carries the loop's stacked-quality +test consolidation plus this loop's `2996cd3c` docs commit. +`git diff c11df645..129a6650 -- tests/test_persistence.py` is empty, so +the 21-pass isolated verification of the rollback fix stands without +rerun; `contextual_orchestrator/orchestrator.py` is likewise untouched +by the merge. Mergeability reports `null`/`unknown` while GitHub +recomputes after the merge. PR #1109 head unchanged (`b8d2651d`), +still clean with no reviews. PR #1105 head unchanged (`b655fe1b`): +the same 3 `CodeQL compatibility analysis` failures persist with no +self-healing rerun yet observed. Open-PR recount 88 (baseline 85). No +merge, readiness change, or cross-branch push from this loop. + +## DOI discovery visual receipt, 2026-09-09 + +Inspected the actual GitHub-rendered `docs/papers/README.md` at +`ce6e029a63c4126e6c1d0a90f66eef64889a2955`, fragment +`#doi-discovery-register`, in the in-app browser (tab 9, English, +1265 × 712 viewport). The screenshot was opened and directly viewed in +the task, not inferred from the accessibility tree. The heading, limitation +paragraph, and first fifteen complete DOI links were readable, with no +overlap or horizontal clipping in that viewport. The next link continues +below the fold. This receipt does not cover the remaining links, other +viewport sizes, keyboard focus, translated states, or product UI. + +The accompanying discovery contract passed all six tests in 1.36 seconds +at that source revision. It inventories explicit DOI URLs in tracked text; +it does not establish complete paper coverage, correct citation metadata, +full-paper review, reproduction, or customer accuracy improvements. + +## Terminal dispatch-pending is not a live run + +On 2026-09-09, CO PR #1072 head +`730801abfc53d46fb377c1be8b48857d89f6588c` still displayed failed CodeQL +compatibility checks. Direct run API evidence for `33949006276` establishes +`status=completed`, `conclusion=failure`, `run_attempt=1`, last update +`2026-09-05T13:48:59Z`. Job `101298528075` logs record successful dispatch +but `VERDICT_STATE=pending`. That historical pending value does not establish +an active downstream run or a scheduled retry. Reconcile the exact-head verdict +with its canonical publisher before deciding whether a repair or rerun is needed. + +`gh run view` failed while resolving workflow `348317201` (HTTP 404), but +`gh api repos/ContextualWisdomLab/contextual-orchestrator/actions/runs/33949006276` +and `gh api repos/ContextualWisdomLab/contextual-orchestrator/actions/jobs/101298528075/logs` +both succeeded. Use those direct read-only endpoints when workflow resolution +fails; do not infer absent logs or an absent run from that error. Central CI +coordination was notified. No duplicate dispatch, gate weakening, or source fix +was justified by this observation alone. + +## Auxiliary/task interval visual receipt, 2026-09-09 + +Directly viewed the actual GitHub-rendered `docs/analytics_spec.md` at +`0ef13edcefcf562383dc5ef3391cbe46a38635cf`, fragment +`#auxiliary-dispatch-and-task-route-decision`, in the in-app browser at +1265 × 712, English. The complete section was visible and readable without +overlap or horizontal clipping. Its text includes prior evidence acquisition +and triage in the task-route interval, separates first-provider diagnostics, +and forbids changing the cache mix to manufacture improvement. This is a +bounded document inspection, not verification of the implementation, other +viewports/locales, focus interactions, or product UI. + +The earlier PR #1108 re-observation above is historical: its current-head +`129a665016ed1acd79ae12915c905b1020856fcc` Security run `34318012080` +subsequently completed successfully, including 3400 passed and 2 skipped +tests on merge checkout `9516d1d`. That evidence does not replace independent +approval or establish a protected merge. + +## Re-observation, 2026-09-09 (fetch-ref transient, all PRs static) + +Loop HEAD `b5f9a5ac013cc3cd0b0ad443fce6e36d28a51067` is in sync with +`origin/autoresearch/20260909-kpi-loop`. One `git fetch` reported +`30b7b3a1..b5f9a5ac` but refused to update the remote-tracking ref +(transient: a retry updated it, and `merge-base --is-ancestor` plus +`merge --ff-only` confirmed a clean fast-forward with no history +rewrite and no force-push). Lesson: treat a refused ref update as +unresolved sync, never as evidence of a rewrite; retry, verify ancestry, +and fast-forward only. + +PR states all static this turn: #1109 head `b8d2651d`, clean, no +reviews (prior 19-pass verification stands); #1108 head `129a6650`, +mergeability `unknown`, no reviews (prior 21-pass verification stands +on byte-identical fix files); #1105 head `b655fe1b`, still blocked on +the same 3 pending-verdict CodeQL-compat failures with no self-healing +rerun observed. Open-PR recount 88 (baseline 85). No merge, readiness +change, or cross-branch push from this loop. diff --git a/docs/doctoring/batch_request_lineage.md b/docs/doctoring/batch_request_lineage.md new file mode 100644 index 000000000..d8130d517 --- /dev/null +++ b/docs/doctoring/batch_request_lineage.md @@ -0,0 +1,172 @@ +# Deferred batch request lineage + +## RED reproduction, 2026-09-09 + +Base `af8d732e6cfc9c0169ac850f875f42f1db7eecd4`; test commit +`387aa211` on isolated branch `codex/batch-request-lineage-20260909`. +Command: `.venv/bin/python -m pytest tests/test_batch_request_lineage.py -q`. +Terminal result: **1 failed in 3.75s**. Submission returned HTTP 201; +retrieval returned HTTP 200 with two distinct caller custom IDs. Passive backend +observations confirmed distinct trusted submission/retrieval request identities. +After closing and reopening SQLite, the proposed submission/job/item association +was absent: an empty set instead of two links. No runtime implementation changed. + +This test proposes the `batch_request_link` association shape; that shape is not +an existing released contract. It demonstrates the missing durable association, +not a defect in an already-promised schema. The fixture reuses the existing +offline pg-llm-batch client double. No external model was invoked. This is not +remote integration, process-restart batch execution, numerical accuracy, or latency +evidence. The fixture preserves results in test memory across state-store reload; +it does not claim that the default in-memory batch registry survives restart. + +The smallest repair belongs in CO's submission coordinator and durable job +association boundary, with a trusted identity argument from the HTTP adapter. +Preserve multiple submission/job/item associations and keep provider custom IDs +separate. Do not reassign origin when a later HTTP request retrieves outcomes. +Remote execution provenance requires the released pg-llm-batch contract separately. + +Environment: project-local Python 3.14.6, dependencies synchronized with +`uv sync --frozen`. An initial pytest-only environment failed collection because +certifi was absent; this was an environment error, not the RED result. The attempted +`uv sync --frozen --group test` was invalid because test is an optional extra, +not a dependency group; the frozen default synchronization repaired collection. +CodeGraph initialization completed (438 files); no other worktree was modified. + +## Candidate implementation and review repair + +`62239624`: focused lineage, batch backend, and cost-review HTTP tests passed +**42 tests in 12.33s**. Includes existing cross-principal denial coverage. +The initial status/failure contract at `919c945a` failed twice in 1.54s; initial +implementation `9356ac62` passed 39 tests in 14.35s. Follow-up repeated-origin, +repeated-retrieval and no-store cases passed 41 tests at `b565173e` in 11.63s. + +The association is an append-only typed event in the existing state store: +`request_id`, `batch_job_id`, `custom_ids`, and `owner_id`. A single event commits +the complete submission cohort using the existing rollback-safe transaction. +Consumers project one association per custom ID; this replaces the original +RED's proposed per-item event shape. It is not a new relational entity claimed +to satisfy 3NF. Item IDs remain job-scoped and repeated submissions append, +never replace prior origins. No prompt or answer enters the event. + +HTTP 201 means the upstream job was submitted, not that all local state is +durable. `request_link_status` is `durable`, `write_failed`, or `unavailable`. +After remote success, a failed association commit returns the original handle +with `write_failed` and no raw exception. It never automatically resubmits. +No-store/library calls remain supported with unavailable lineage. This response +diagnostic is not reconstructed from remote registry snapshots after restart; +the committed association event is the durable evidence. + +Review found an added second registry assignment could lose the successful +remote handle behind a new exception. Regression `b5baf420` failed once in +0.39s; `62239624` removes that assignment. Existing Valkey writes are separate +`hset` and `expire` calls, not atomic with SQLite. Failure of the original +registry assignment after remote submission remains an unresolved existing +ambiguity; no cross-store transaction, complete recovery, retention/export +window, API reconciliation surface, or remote integration is claimed here. +The event is currently only consumed by the test projection. Full-suite, +package, hosted checks, independent approval, and protected release are pending. + +## Recovery successor (supersedes status-only limitations above) + +Actual HTTP restart regression `d71bcc0a` failed once in 1.96s: an accepted +remote job returned 201 despite a failed registry write, but its rightful owner +received 404 after restart. A different authenticated owner also received 404. +This established unrecoverable work, not just missing status metadata. + +Candidate `853e8609` passes **73 focused tests in 21.17s** across lineage, +batch routing, cost-review HTTP, and registry files. New coverage distinguishes +HSET failure (no handle stored) from expiry failure (HSET partially applied), +and returns the original upstream handle without a second submission. Earlier +`18af2947` expiry tests incorrectly inspected an unprefixed fake-registry key; +`b12981d6` corrected the test and passed ten cases in 6.18s. + +The single durable submission event now includes a prompt-free recovery +descriptor: exact backend name/endpoint alias/endpoint, owner-bound job snapshot, +original token estimates, expected job-scoped item IDs, model/mode/normalized +attribution, and expiry tied to the configured registry retention. This is only +supported for PgLlmBatchBackend, not arbitrary local or embedding backends. +No prompts or credentials are included. Absent source messages remain unavailable; +their contents are never fabricated for token estimates. + +An indexed exact-key lookup permits the original owner to recover a missing +registry handle after SQLite reload. Wrong owner, expired descriptor, malformed +backend object, or changed backend target fail closed. Restored expected item +IDs retain the existing response-identity validator; unexpected result IDs are +rejected. Missing usage remains estimated or unavailable, never relabeled measured. +The malformed-backend test initially returned 500 at `65b41603` (one failure, +five passes in 9.90s); explicit type validation changed that to a concealed 404. + +Request-link status is finalized before the single registry write and survives +Valkey decoding consistently. Registry-write outcome is response-only and is +excluded from dataclass serialization, since HSET may apply before expiry fails. +`stored` means that configured registry operation returned, not proof of process +restart durability for a local dictionary. HTTP 201 still means remote submission; +clients must preserve the handle and must not resubmit solely because local +persistence is incomplete. If both SQLite and the registry fail, automatic recovery +remains unavailable. This is a tested recovery slice, not complete cross-store +atomicity, retention cleanup, production integration, or customer KPI evidence. + +## Independent review repairs and injected-adapter configuration + +At `a6b94855`, the four-file focused suite passes **84 tests in 32.63s**. +Intermediate failures and successful checks remain attributed to their commits: + +| Source | Check | Result | +| --- | --- | --- | +| `e372bc54` | Registry outage, inconsistent item IDs and estimate keys | 3 failed, 3.59s | +| `7fb1a71c` | Lineage file after recovery isolation | 19 passed, 14.94s | +| `83394afa` | Malformed job fields and deployment binding added | 23 passed, 18.26s | +| `f4d036bf` | Healthy coordinator handle, missing backend metadata | 1 failed, 1.37s | +| `6cb530a7` | Four-file suite | 81 passed, 29.90s | +| `42ad396c` | Duplicate persisted item IDs | 1 failed, 1.52s | +| `aaa9b133` | Four-file suite | 82 passed, 31.21s | +| `17cfa611` | Backend registry write fails after remote creation | 1 failed, 0.95s | +| `8ddfeb9f` | Same backend-failure HTTP test | 1 passed, 1.79s | +| `df638d6c` | Healthy active job with expired recovery descriptor | 1 failed, 1.51s | + +The Pg adapter now retains the upstream handle even when its own metadata +registry write fails before returning to the coordinator. The response separates +`backend_registry_persistence_status`, `registry_persistence_status`, +`request_link_status`, and `recovery_status`. The first two describe individual +write outcomes, not restart durability. HSET may apply before expiry fails; +response-only write results are not serialized as claims about their own writes. +Missing durable lineage still remains explicit; no handler repeats submission. + +Recovery tolerates continuing coordinator/backend registry outages. Validated +item metadata is carried on the returned job for that retrieval, without writing +back through the failed registry. The stored envelope's item list, descriptor +items, job count, and estimate keys must agree before downloading. Invalid count +types and null estimates fail closed. A healthy authorized registry job retains +its original refresh-on-read lifecycle even if its separate fixed-deadline +recovery descriptor has expired; missing metadata cannot use expired recovery. + +There is no built-in production Pg adapter constructor in this source tree: +the CLI/default coordinator uses the local backend. Integrators injecting +`PgLlmBatchBackend` configure its optional `recovery_identity` argument with a +stable, non-secret service/deployment/account identifier. Keep this identifier +stable during credential rotation and change it when the service or account +changes. Do not use a credential, infer equivalence from an endpoint alias, or +accept an HTTP caller's value. The default `None` disables durable recovery, +and submission reports `recovery_status=unavailable`; a committed, explicitly +bound Pg descriptor reports `durable_descriptor`. A different or missing binding +cannot recover the old job, even if endpoint aliases are identical. + +The real-HTTP tests use offline injected clients; no deployed integration is +claimed. Simultaneous loss of both durable submission evidence and job registries +still cannot promise recovery. Full suite, clean wheel, hosted review and release +remain pending. Do not stop at a status-only response when recoverable evidence +exists, and do not label a retained remote handle complete recovery by itself. + +## Final focused acceptance before full regression + +Runtime `45066759`: **86 passed in 36.58s** across the four focused files. +The last review exposed inconsistent endpoint binding on healthy metadata: +`c06615ab` failed once in 2.09s. New metadata now stores the exact endpoint and +stable deployment binding before the healthy fast path can use it. A changed +binding, changed endpoint, or missing binding on opted-in metadata cannot bypass +the validated descriptor path. This fixes a contract inconsistency; no cross-service +data leak was established by the offline test. Unbound legacy metadata remains +compatible only with an unbound backend; its missing identity is never described +as validated recovery. The legacy-focused suite passed 85 tests in 35.09s at +`2d68f705`. Independent review accepted freezing this bounded implementation for +full regression and separate installed-artifact verification, not publication. diff --git a/docs/doctoring/decision_receipt_integration.md b/docs/doctoring/decision_receipt_integration.md new file mode 100644 index 000000000..46530d3ce --- /dev/null +++ b/docs/doctoring/decision_receipt_integration.md @@ -0,0 +1,171 @@ +# Initial decision receipt integration candidate + +Status: incomplete implementation for issue #1110; not release evidence. + +RED `85a0b2d2` exposed the missing HTTP measurement option (1 failed, +3.67 s). The working candidate compiles a separate PyO3 extension using the +workspace's existing pyo3 0.29.2 lock resolution. It introduces no new +third-party dependency. The first build failed because a field-level getter +attribute was invalid; explicit read-only getters corrected compilation. +Context7 lookup hit its monthly quota; local compiler/API checks were used. + +The separate Rust receipt owns an Instant clock and state transitions. Python +only associates that receipt with the existing HTTP context and SQLite store; +it computes no durations or statistical estimates. Opt-in requires the native +extension and a configured durable store. The ordinary callback admission and +two direct SSE admissions establish scopes before the nonblocking capacity check. +Initial selection hooks cover _invoke, streaming, generated planning, and +plain/capability proxy candidates. These source hooks are not all validated. + +The initial decision is synchronously committed before the native acknowledgement +timestamp is captured. A separate receipt is exported at scope exit; that export +cannot be its own acknowledgement. A process crash between these writes leaves +an initial decision without a receipt, which must remain in the unfinished +denominator. Persisted successful receipts alone are not an all-request sample. + +Focused verification in the isolated local Python 3.14 environment: 4 tests +passed in 1.49 s. The real HTTP route's dispatch callback queries a separate +SQLite connection (never flush-on-read store.load), verifies the committed +decision, captures its native acknowledgement, and holds answer generation +behind an event. The final acknowledgement must equal that pre-generation value. +Other tests cover native invalid/duplicate acknowledgement, missing-store +startup rejection, write failure, snapshot isolation and secret-safe errors. +Controlled mock provider output is unit/integration evidence, not customer data. + +Remaining before review/merge: direct coordinator/structured synthesis and all +SSE/provider paths need executable coverage; metadata must bind policy revision +and route mode; unsupported and selection/cancellation outcomes need complete +denominator tests; race size/contract/preparation and concurrency need negative +tests; both receipt kinds need an explicit retention/export contract. Validate +actual wheel installation outside the source checkout (the current editable +extension is not package proof). Add the Proposed ADR with the completed seam +inventory, dependency graph and rendering inspection. Do not claim production +instrumentation, p95, accuracy improvement, protected CI, or release. + +## Admission and SSE follow-up + +At candidate descended from `df6eb4a7`, ten focused tests passed in 4.72 s. +These include ordinary HTTP chat, direct chat SSE, direct Responses SSE, +actual race worker context propagation and full candidate-set identity, and +oversized-race rejection before acknowledgement. The first race test fixture +omitted the required group identity and correctly exercised sequential routing; +adding the operator-contract group activated the intended race path. + +An immutable accepted_request row now precedes the capacity check. Export joins +that row with initial_decision and final decision_receipt rather than dropping +unfinished requests. Rows are append-only and not automatically pruned; operators +must retain them for reconciliation and archive after verified export. A storage +outage can prevent the admission itself from being recorded: opt-in then returns +503 before dispatch, states measurement_complete=false and reconciliation_required=true, +and requires external ingress evidence. The local export never claims all-ingress +completeness. This is a deliberate limitation, not a zero-duration observation. + +The existing analytics snapshot carries the joined records only with measurement +enabled and retains its local-runtime labeling. A separate Rust clock remains +the only duration implementation. A trusted request_id argument is the explicit +port for PR #1105; until integrated, identity_source=measurement_scope is honest. +The policy snapshot is hashed; route_mode currently identifies the dispatch +kind/role, not a complete top-level requested-mode contract. + +An isolated noneditable core install plus native wheel, outside the checkout, +passed six tests (one then-current test deselected) in 9.13 s. Both imports were +verified under the isolated environment's site-packages. That build used an +uncommitted candidate and is not final exact-head package evidence. Dependencies +resolved anew there; locked PyO3 remained 0.29.2. Rebuild the final commit before +release. No binary is committed. + +Still unverified: cache-hit and other callback-bypassing accepted-request coverage, +all structured/coordinator/direct-provider paths, selection/cancellation/capacity +HTTP negatives, complete ingress reconciliation, transactional snapshot export, +declared retention window and actual release packaging workflow. No customer KPI +gain is established. The Proposed ADR and full rendering inspection remain due. + +## Integrated request identity and indexed cohort checkpoint + +At `fe2db40e`, receipt, persistence and debug/correlation suites passed together: +54 passed in 9.26 s. Normal merges preserve rollback owner #1108 at `129a6650` +and request identity owner #1105 at `b655fe1b`. The handler now owns one +measurement lifetime across repeated slot acquisitions, and passes its trusted +request identity explicitly. Embedding fallback tests observe one admission and +two backend selections; explicit validated-endpoint admission precedes embedding +candidate ordering. Other endpoints remain labeled first_execution_slot and +must not be assumed to include prior selection work. + +`6e4f87aa` adds a shared per-race invocation identity: replicas deduplicate within +one invocation, while a second race in the same request remains a new attempt. +The regression first failed with one attempt instead of two, then passed. +The earlier route-mode-only deduplication is superseded. + +The state store backfills null keys only for valid measurement JSON identities +inside its startup transaction, never overwrites non-null keys, and adds the +(kind, key, seq) index. Its 2,000-measurement-row unit fixture plus malformed +unrelated row retains all 2,001 rows. EXPLAIN for the actual phase query shows +kind/key index search; this is query-plan evidence, not a measured latency gain. +Export reads a bounded shared admission cohort, exposes sequence/truncation +boundaries and unresolved legacy identities, and never independently prunes +phase rows. Historical storage remains append-only pending an archival policy. + +An exact `01ce9035` native wheel and separately built noneditable core package +passed 10 tests in 4.90 s outside the checkout with isolated imports. That proof +does not cover the later merged runtime: rebuild the final candidate again. + +Outstanding semantic gap: structured triage and ranking-evidence embeddings may +invoke providers before the current task-selection hook. Initial provider +dispatch and final task-route decision must be distinguished; no current receipt +is evidence of the headline routing-decision p95. Cache hits, non-generation +operations, auxiliary dispatch, all preselection SSE failures and cancellation +paths still require complete request-boundary coverage and executable evidence. +No protected merge, production publication, or customer KPI gain is claimed. +# Auxiliary and package acceptance checkpoint (2026-09-09) + +The later frozen `3b6dd47ebb0f88802bacdd302051d2f03e7d5003` full suite passed +3,442 tests with two skips in 771.43 s; its noneditable wheel-only receipt and SSE +identity matrix passed 33 tests in 14.21 s. Independent saturation inspection then +found chat streaming could call triage before acquiring capacity, despite a 503 +response. This was present with measurement disabled too; sibling nonstream chat +and Responses streaming already rejected without a provider call. + +The capacity repair holds one explicit chat-classification lease through either +the direct stream or conducted `_run` path. Request finalization releases the +lease if classification or trace validation exits early. It does not acquire a +second slot, release a slot it never acquired, or depend on measurement being +enabled. Actual HTTP RED covered saturation and early exits (three failures, +one passing control); the expanded enabled/disabled route/conduct/trace/error +matrix plus streaming, disconnect, identity, and trace regression tests passed +93 cases in 33.87 s. Those are local correctness results, not a latency gain. + +At `9707a5e1`, focused receipt/persistence tests passed 41 cases in 8.43 s. +The Rust clock now separates provider-ready diagnostics from initial task-route +acknowledgement. Structured triage, generated planning, and pre-selection evidence +embedding remain inside the task-route interval. Evidence embedding after native +selection is labelled `post_decision_evidence_embedding`, including write-failed +selection; it is not subtracted or presented as preceding routing work. Answer +cache reuse has a distinct terminal outcome with absent provider timing values. +The bounded admission cohort exports auxiliary records with an explicit diagnostic +cap and truncation indicator. Historical retention remains unresolved. + +The initial embedding test asserted inside a best-effort transport callback. +That assertion was swallowed by the existing best-effort path and prevented cache +fill, causing a test-induced warm retry. The corrected spy only collects values; +assertions run after the response. No descriptor invalidation was established. + +Native packaging was rebuilt from the repository root with +`maturin build --locked --release --manifest-path rust/decision_receipt/Cargo.toml +--interpreter .venv/bin/python --out /tmp/co-decision-receipt-wheels`. +Rust 1.97.1, maturin 1.15.0, Python 3.14.6, macOS arm64 produced the native wheel +SHA-256 `e0bf63d790256c6d4eba8598c131d63188a994c899df5124bd9eadf2cc39c568`. +Its five ZIP entries contain only the receipt extension and distribution metadata; +the independently built core wheel has no overlapping files. A separate +noneditable installation outside the checkout passed all 20 receipt tests in +11.83 s using `python -I`, with both import origins under that environment's +site-packages. This proves that local artifact matrix only, not Linux/Python 3.12 +hosted acceptance or a released package. The namespace was retained because the +suspected core-file collision was not observed in either inspected native wheel. + +CI now installs locked native build tooling, builds the extension before the full +suite, validates disjoint wheel ownership before installation, and exercises the +wheel-installed HTTP receipt tests outside the checkout. The existing benchmark +import smoke is preserved. These workflow changes remain pending hosted evidence. +Full endpoint admission validation, cancellation/error-path coverage, deployed +retention/reconciliation, and customer accuracy/latency measurements remain open; +none of the unit or mock-provider evidence establishes a customer KPI gain. diff --git a/docs/doctoring/irt_router_measurement_review.md b/docs/doctoring/irt_router_measurement_review.md index a416c6468..fb87a83fc 100644 --- a/docs/doctoring/irt_router_measurement_review.md +++ b/docs/doctoring/irt_router_measurement_review.md @@ -94,3 +94,110 @@ with a family-specific assertion or fixture for each one. Related: [ADR 0005](../planning/adrs/0005-irt-response-matrix-contract.md). Its matrix-shape validation is necessary boundary checking, not identification or construct-validity evidence. + +## Response-time research extension (2026-09-09) + +Status: proposed research, not a fitted model or production policy. Repository +inspection at `29f417f256f8e50651b49506c042c5ec882ff472` found no reference +matching `van der Linden`, `speed.accuracy`, or `response.time model` in tracked +text. This bounded search does not establish an exhaustive literature inventory. + +van der Linden, W. J. (2007). A hierarchical framework for modeling speed and +accuracy on test items. *Psychometrika, 72*(3), 287–308. +https://doi.org/10.1007/s11336-006-1478-z + +The publisher confirms the final article's bibliographic identity. The separately +available [RR 05-02 report](https://ris.utwente.nl/ws/files/5129699/Linden05hierarchical.pdf) +is an earlier version, not verified identical to the journal article. Its printed +page 6 (PDF page 10) was rendered and visually inspected: equations (8)–(14) +separate response ability from response speed, specify a lognormal time model, +and combine person parameters at a second level. The response/time likelihood +factorization assumes conditional independence given the latent parameters. +The inspected report is linked, not redistributed; redistribution permission +has not been established. + +### CO hypothesis and acceptance boundary + +The following is our proposed application, not a result reported by the paper. +A joint outcome/time model might improve calibration for repeated observed +tasks, but provider generation duration cannot substitute for CO's accepted +request-to-persisted-decision interval. Keep both measurements separate. Provider +load, output length, reasoning settings, retries, and transport failures can +change completion time without changing the intended ability construct. +Do not infer that a faster provider is more capable or add an arbitrary +speed-weighted ability score. + +For an observed-task experiment, compare outcome-only and joint models on the +same predeclared held-out tasks and model revisions, with model/task/time +grouping retained. Report calibration and delivered-correct fraction over all +accepted requests, plus failure and censoring counts. Check residual dependence +and held-out time fit before using the joint model. Fitting and diagnostics +belong in released fast-mlsirm contracts; CO consumes their versioned outputs. + +The decision-latency experiment remains separate: profile selection and durable +write costs under the analytics interval before choosing a runtime optimization. +A joint estimator on the request path could increase that latency. No estimator +is added here: neither observed training/held-out data nor released joint-model +support has been verified. The next experiment must establish that baseline +before claiming improvement, rather than using oracle-score fixtures as buyer +evidence. + +### Conditional-dependence follow-up + +Bolsinova, M., & Tijmstra, J. (2019). Modeling differences between response +times of correct and incorrect responses. *Psychometrika, 84*(4), 1018–1046. +https://doi.org/10.1007/s11336-019-09682-5 + +Read scope on 2026-09-09: publisher abstract and bibliographic information, +not equations, supplementary material, or a reproduced experiment. The +publisher page displays an online date in 2025, but its volume/issue identifies +the article as December 2019; the citation uses the issue year. + +The abstract describes extensions that permit speed-model item parameters to +differ for correct and incorrect responses, including separate latent speeds. +It reports simulation and assessment-data applications, not LLM-routing results. +This gives a concrete alternative to assuming conditional independence in the +joint-model proposal above. + +Engineering inference: compare the independence baseline with this conditional +model on the same held-out design. Correctness is unavailable at routing time: +using the observed outcome to choose a speed component would leak the target. +Any deployable prediction must marginalize unknown outcomes; oracle-conditioned +fit is not a routing KPI. The fitting owner remains fast-mlsirm. No new request-path +estimator or production default is introduced, and full-method review is pending. + +### Response-process identification follow-up + +Bolsinova, M., Tijmstra, J., Molenaar, D., & De Boeck, P. (2017). +Conditional dependence between response time and accuracy: An overview of its +possible sources and directions for distinguishing between them. *Frontiers in +Psychology, 8*, Article 202. https://doi.org/10.3389/fpsyg.2017.00202 + +Read scope: publisher HTML Sections 1–4 on 2026-09-09, with Figure 1 opened +and directly inspected in the actual browser at 1265 × 712, English. Panels +A–E and arrows were readable; the separate caption pane required scrolling. +The initial capture preceded image loading; the second capture confirmed the +diagram. This is a perspective, not a reproduced numerical experiment. The +paper explains why residual time/accuracy association may reflect different +response processes, and why its sign alone does not identify their cause. +Section 4 recommends hypothesis-relevant covariates and model comparisons, +including residual checks. It does not provide a complete estimation algorithm. + +CO engineering proposal at research head `946726408556c3145ee5c996905f5bd5e907645e`: +predeclare model revision, task family, policy revision and configured effort as +available-before-decision covariates. Record realized cache, auxiliary work, +fallback and failure states separately as diagnostics; do not use future +outcomes or realized durations to predict the same request's initial route. +Do not label a latent class “reasoning” merely because its answers took longer. +Compare an outcome-only baseline, a joint independence model and a +covariate-extended model on the same held-out groups before choosing complexity. +Estimation remains a fast-mlsirm responsibility, with a released contract and +predictive checks required before CO adoption. No new estimator is implemented. + +This sharpens the existing KPI experiment rather than changing its targets: +the initial-task decision interval remains distinct from generation duration; +success and failure denominators must retain every request in the declared +cohort. Completeness still requires reconciliation with external ingress. +Candidate #1110 `3b6dd47ebb0f88802bacdd302051d2f03e7d5003` now retains +admitted semantic rejection as selection failure. Its local focused suite passed +83 tests; full-suite and production observation evidence remain separate gates. diff --git a/docs/doctoring/kpi_stack_integration.md b/docs/doctoring/kpi_stack_integration.md new file mode 100644 index 000000000..d6bacd284 --- /dev/null +++ b/docs/doctoring/kpi_stack_integration.md @@ -0,0 +1,135 @@ +# KPI stack integration checkpoint — 2026-09-12 + +Status: Proposed; no main adoption, deployment or KPI improvement claimed. + +## Ownership and retained history + +The numeric acceptance contract belongs to commit +`dccd37e032f74251e3c02e38046dab68c0049787` in PR #1107: observed +delivered-correct fraction improves by at least one percentage point with its +95% difference interval above zero; decision p95 is at most 20 ms and improves +by at least 10%, with the 95% ratio interval below one. The same commit owns +the linked runbook, denominator, holdout and power requirements. These are +targets, not measurements. PR #1103 head `4776a970` is its ancestor. + +Main `012beaacd0631f8cd3391c77744eeb626269b5de` did not contain that ancestry. +The receipt/workflow/batch PR merge commits on the retained research branch +also do not prove main adoption. No deletion from main has been established. + +Ordinary merge `52fd0da99224f0889e8b012667a93940f6a324ee` joins research head +`e30652556320f8c72578443385dd31fad1139ff7` and current main without force or +discarding predecessor delta. Conflicts were confined to AGENTS.md and the +orchestrator/server modules. Both guidance sections were retained. Runtime +resolution preserves current tool suppression, stream candidate failover, +expanded race results, and validated capacity acquisition while adding +request-owned per-attempt measurement. The independent recovery branch at +`726ae2949b026fbcf8567655aaa29cf1c18534ab` remains intact as reconciliation +evidence, not a second production implementation. + +## Verification and resolved test synchronization defect + +Both merged modules parse; whitespace checks pass. Independent read-only +conflict review found no actionable issue within receipt/admission scope. +Focused execution of decision receipt, workflow link, batch lineage, true +streaming and tool-execution fallback suites returned **1 failed, 195 passed +in 13.05s** at merge revision 52fd0da9. The failure is +`test_http_evidence_embedding_cold_and_warm_keep_task_interval`: the second +request has no durable acknowledgement timestamp at the instant sampled. +The synchronization cause and repair are recorded below; an embedding-cache +hit does not mean a response-cache hit. + +The source integration used the existing immutable project test environment +with the separately installed native receipt namespace appended read-only. +It is not installed-core acceptance. Local log: +`/tmp/co-kpi-stack-focused-52fd0da9.log`. + +## Next acceptance + +The failure was independently reproduced by delaying the second final receipt +write. The concurrent export correctly reported acknowledgement unobserved; +the completed database subsequently contained both acknowledged receipts and +both workflows marked cache bypass. HTTP response consumption and listener +shutdown do not join daemon request handlers. Commit `36af4a56` therefore +changes only the test: signal after the original `DecisionMeasurement.close` +returns, then await that event before exporting. All original acknowledgement +assertions remain. The five-suite run then passed **196 tests in 11.50s**. +No runtime acknowledgement is invented or moved ahead of delivery, and this +test-only synchronization does not impose a provider/model timeout. + +At exact head `81ad64cf77a49f7bc2a57f4851a5c3259387ce5d`, the full source +suite terminated successfully: **3,688 passed, 2 skipped in 261.42s**. +The terminal receipt is session `90461`, log `/tmp/co-kpi-stack-full.log`. +It used the same read-only native namespace arrangement described above; +neither installed-core acceptance nor hosted checks are implied. + +Next verify an isolated core/native install. +Retain both #1103 and #1107 until protected review/checks and lineage-preserving +merge are verified. The archived bounded outcome exporter still requires +integration; constants or manually seeded observations are not its successor. + +## Request-level cache aggregation repair — 2026-09-12 + +At published `eeefaca9378e4f70d03aec263fd26be1dccf92a3`, a cache hit +immediately terminated the native request receipt. Repeated cache hits and +selection followed by cache reuse raised an exception; cache followed by +selection silently omitted the first-selection duration. A later failed or +cancelled item could also retain the earlier cache classification. One HTTP +batch contains multiple items but owns one admission clock. + +RED `c4e8343c` reproduced six failures in 0.55s. Runtime repair `0ddb5f1b` +records cache observation without changing native state. At request close, +only an accepted receipt with cache observation and no explicit failure becomes +`cache_hit`. Selection, durable acknowledgement and write-failure terminals keep +their existing meaning. The native state machine is unchanged; catching its +exception alone would leave silent timing loss unresolved. + +Frozen `4bc96045037d04fa7477a1532c75f99f0d7e9898` adds real HTTP local-batch +tests for repeated cache, both mixed orders and cache followed by failure, +plus explicit context cleanup and successor-request isolation. Related suites +passed **82 tests in 3.33s**. Full source regression passed **3,699 tests, +2 skipped in 144.99s**, terminal session `86318`, log +`/tmp/co-kpi-full-cache-0ddb5f1b.log` (the filename names the runtime commit; +the tested head is the frozen revision above). + +Source reproduction uses the root project `.venv/bin/python`, appends the +read-only native namespace at +`/tmp/co-export-native-acceptance-20260912/lib/python3.14/site-packages/contextual_orchestrator` +to the imported package path, then runs `pytest.main(["tests", "-q"])`. +Do not interpret that source/native arrangement as an installed core test. + +Separate installed acceptance used a `git archive` of the frozen revision at +`/tmp/co-cache-wheel-4bc96045.C4cqDg`, Python 3.14.6, all 46 hash-locked +requirements, pytest 9.1.1 and the unchanged native wheel. Core wheel SHA-256: +`3e278e07adf168140b23cde0a3f99be38201dea8ae0abfdc89fcdff2af88514b`. +Native wheel SHA-256: +`8dfee5d228a28733136e25c6006f77006bcba095863a667e0f2a3e71ca8c7c04`. +With `python -I`, working directory `/tmp`, importlib test mode and asserted +core/native `site-packages` origins, `test_decision_cache_aggregation.py`, +`test_decision_receipts.py`, `test_cost_review_server.py` and +`test_workflow_request_link.py` passed **82 tests in 8.93s** (session `18167`). +This verifies measurement correctness, not customer accuracy or latency gains. + +## Central review handoff remains an owner gap + +The coordinating owner task supplied protected-main evidence at +`.github@fb17ef556f94f673234aa557254ae52779e9a7b0`: +[`noema_review_handoff.py`, lines 191–210](https://github.com/ContextualWisdomLab/.github/blob/fb17ef556f94f673234aa557254ae52779e9a7b0/scripts/ci/noema_review_handoff.py#L191-L210) +sends `repository_dispatch` to the target repository. The +[OpenCode handoff, lines 7707–7736](https://github.com/ContextualWisdomLab/.github/blob/fb17ef556f94f673234aa557254ae52779e9a7b0/.github/workflows/opencode-review-dispatch.yml#L7707-L7736) +sets that repository as its destination. The inspected CO `eeefaca` tree has +no `noema-review.yml` receiver. A 204 event-acceptance response is not a Noema +run or independent review. Keep receiver repair in the `.github` owner; +do not copy a workflow into this consumer to claim review coverage. + +At `eeefaca`, Security run `34688379677` completed tests, fuzzing, supply-chain +and CodeQL successfully. That evidence predates the cache repair and does not +approve or validate its new head. Fresh hosted checks, independent review, +protected merge, release and observed KPI acceptance remain separate gates. + +Bounded visual receipt: revision `36ade588`, English, 1265 × 712. The actual +browser preview's cache-repair, installed-verification and final owner-gap +sections were directly opened as three overlapping screenshots. Long hashes +and paths wrapped without horizontal clipping; headings and paragraphs did +not overlap. A stale preview-only revision badge was corrected and its top +view recaptured at `http://127.0.0.1:18771/`. Images remain in task tool output. +This does not cover mobile, other locales, Figma or product interaction states. diff --git a/docs/doctoring/lart_measurement_review.md b/docs/doctoring/lart_measurement_review.md new file mode 100644 index 000000000..b058a7a18 --- /dev/null +++ b/docs/doctoring/lart_measurement_review.md @@ -0,0 +1,258 @@ +# LaRT measurement review — 2026-09-12 + +Status: research intake; no implementation or production adoption. +Integration baseline: `be60791ba85cf9cfa8075b13646fa94b0d8a1031` (PR #1107 +retained research branch). This note does not establish protected-main delivery. + +## Primary evidence and read scope + +Xu, Z., Liu, J., Wang, Y., & Gu, Y. (2026). *Latency-response theory model: +Evaluating large language models via response accuracy and chain-of-thought +length* (Version 4) [Preprint]. arXiv. https://arxiv.org/abs/2512.07019v4 + +Root inspected the abstract, introduction and relevant passages of §§5.1 and +7; independent agent review covered §§3–5 and 7. Neither is a replication or +complete appendix audit. The listed arXiv non-exclusive license does not +establish redistribution permission; no PDF is attached. + +Section 7 defines length as reasoning tokens before the answer, counted with +each model's tokenizer, capped at 10,240. All-wrong models are excluded. +Sections 7.2.2 and 7.2.4 compare subset estimates with full-data fitted +estimates, not known true parameters. Section 5.1 requires informative +loadings for both traits and sign-orientation constraints. These findings do +not establish wall-clock latency reduction or causal benefit from more tokens. +Primary passages: https://arxiv.org/html/2512.07019v4#S7 and +https://arxiv.org/html/2512.07019v4#S5.SS1. + +## Proposed experiment, not a paper result + +Keep estimation in fast-mlsirm and consume its released contract in CO. +First agree an observation contract carrying tokenizer/version, prompt +condition, model family, truncation and missingness. Do not copy an estimator +into the gateway or introduce an uncalibrated token-length routing weight. + +Compare accuracy-only and joint calibration on held-out model families and +items at equal evaluation budgets. Preserve failed outcomes in the target +denominator; report exclusion sensitivity separately. Use observed correctness +and calibration error for real responses. Known-parameter RMSE belongs to +unit recovery checks, not fitted-reference agreement on real data. + +Before implementation, audit joint versus marginal uncertainty and the claimed +optimization conditions. For a positive-definite precision matrix +`[[A, C], [C, B]]`, marginal ability precision is `A - C*C/B`; substituting +`A` requires justification. This is an analytical review question, not a +confirmed defect in the paper or its implementation. + +Measure CO decision-only milliseconds independently under the existing KPI +contract. No length proxy, subset stability statistic or test pass count may +replace that measurement. No production default changes are authorized by +this intake. Remaining gates: complete method audit, owner contract, licensed +data provenance, held-out evaluation and uncertainty verification. + +## Nonlinear dependence: external psychometrics intake + +Bolsinova, M., & Molenaar, D. (2018). Modeling nonlinear conditional dependence +between response time and accuracy. *Frontiers in Psychology, 9*, Article 1525. +https://doi.org/10.3389/fpsyg.2018.01525 + +Read scope on 2026-09-12: publisher HTML abstract, introduction, hierarchical +model, existing conditional-dependence models and opening quadratic-model +section. Equations omitted by HTML extraction, remaining methods, empirical +results and supplements remain unaudited. This is not replication. + +The authors distinguish raw-time median splits from residual log-time effects: +the former can mix higher-level ability/speed association with within-item +dependence. They propose quadratic, multiple-category and nonparametric +approaches, including posterior predictive evaluation of linearity. Opposing +response processes may conceal dependence in a linear summary. + +CO-specific proposal, not the authors' result: before fast-mlsirm supplies a +joint calibration contract, compare residual-dependence diagnostics with an +accuracy-only baseline using held-out model-family/item clusters. Preserve +generation time, queue/network time, reasoning-token count and decision-only +time as different observables. A completed response's time or correctness +cannot select that same request's initial route. Fit preprocessing only on +training data; evaluate frozen policies on subsequent observations. Reject an +uncalibrated monotonic length bonus. Neither human-test results nor this +proposal establishes LLM transfer or the existing accuracy/decision-p95 KPI. + +## Nonparametric diagnostic follow-up, 2026-09-13 + +At research head `b0844bd8`, additional primary-text inspection covered the +two-step procedure and posterior-predictive diagnostic (published pages 5–6) +in the [university-hosted final article](https://pure.uva.nl/ws/files/30670439/fpsyg_09_01525.pdf). +The nonparametric approach treats estimated standardized log-time residuals as +observed covariates. Its linearity check compares an observed deviation statistic +with replicated values under the fitted linear model. This is a model-fit +diagnostic, not a held-out accuracy or routing-latency result. + +CO acceptance implication (our inference): the fast-mlsirm contract must declare +whether residual-estimation uncertainty is propagated or conditioned away. +Fit residual preprocessing inside training partitions; no full-cohort residual +fit may silently enter a held-out policy comparison. Posterior-predictive +replications are diagnostic/unit evidence only under this project's synthetic +data restriction, never the observed customer KPI denominator. Do not implement +a second estimator in CO. Full equations, supplements and transfer remain open. + +The PDF screenshot tool timed out; the actual browser PDF rendered blank. +Consequently equation-level visual inspection is unverified, not passed. + +## Executable marginal-uncertainty check + +Root read §5.2, including equation (7), on 2026-09-12. That equation reports +the ability diagonal of joint posterior precision as inverse variance. For +finite joint Gaussian uncertainty with unknown speed, marginal variance instead +uses the corresponding diagonal of the inverse matrix. Conditional precision +and marginal precision coincide only when the coupling vanishes or in an +appropriate limiting argument. The asymptotic theorem's assumptions and error +order still require a full audit; this check does not refute that theorem. + +Our algebraic unit check below uses fixed known likelihood information, not +simulated customer outcomes or an implementation of LaRT. It checks zero +correlation, both correlation signs, uninformative speed, and finite informative +speed. At ability information 2, speed information 3 and correlation 0.8, +marginal variance is `13/51`, whereas conditional variance is `9/43`. +Neither is a measured CO improvement. Owner implementations must label the +target uncertainty before adopting a formula. + +```rust +fn marginal_variance(ability_information: f64, speed_information: f64, + trait_correlation: f64) -> f64 { + let prior_precision = 1.0 / (1.0 - trait_correlation.powi(2)); + let ability_precision = ability_information + prior_precision; + let speed_precision = speed_information + prior_precision; + let cross_precision = -trait_correlation * prior_precision; + let joint_determinant = ability_precision * speed_precision - cross_precision.powi(2); + assert!(joint_determinant > 0.0); + speed_precision / joint_determinant +} +for trait_correlation in [-0.8, 0.0, 0.8] { + assert!((marginal_variance(2.0, 0.0, trait_correlation) - 1.0 / 3.0).abs() < 1e-12); +} +assert!((marginal_variance(2.0, 3.0, 0.0) - 1.0 / 3.0).abs() < 1e-12); +for trait_correlation in [-0.8, 0.8] { + let observed_variance = marginal_variance(2.0, 3.0, trait_correlation); + assert!((observed_variance - 13.0 / 51.0).abs() < 1e-12); + assert!(observed_variance > 9.0 / 43.0); + assert!(observed_variance < 1.0 / 3.0); +} +``` + +Run `rustdoc --test docs/doctoring/lart_measurement_review.md`. This manual +documentation test is not a hosted owner-estimator conformance test. + +Verification receipt: at `de01e9e1`, the command above passed one Rust +documentation test in 2.89s (terminal process 2866). Root directly inspected +the title/citation area and complete Rust code block in two browser screenshots +at `http://127.0.0.1:18768/lart`, English, 1265 × 712. The inspected text and +code were legible without horizontal clipping or overlap. Middle prose, +other viewport sizes, link destinations and locales were not visually audited. +Screenshots remain in the task output, not published image assets. The shared +local preview has an exporter-oriented browser title/navigation; it is not +the product UI or evidence of published documentation. + +## Additional intake — model/prompt split and theorem boundary + +At official repository commit `8cb9639eb162ff3732df82d4e190e7f902bde19d`, +[data documentation](https://github.com/Toby-X/Latency-Response-Theory-Model/blob/8cb9639eb162ff3732df82d4e190e7f902bde19d/data/README.md) +describes rows as model/prompt combinations, not independent base models. +[Predictive evaluation](https://github.com/Toby-X/Latency-Response-Theory-Model/blob/8cb9639eb162ff3732df82d4e190e7f902bde19d/applications/predictive_power.py) +selects 100 of 128 rows randomly before fitting. Consequently, that split +does not itself establish unseen-base-model or unseen-family generalization. +This is an inference from the code, not a measured leakage magnitude or a +claim that the authors targeted family holdout. CO's proposed experiment must +group verified base-model/prompt siblings and keep family holdout distinct. +The matrices omit raw generations; their availability alone does not establish +complete failed-request denominators, data reuse permission, or CO latency. + +Root read Appendix D.1; independent review also checked PDF pages 32–34. +Assumption 2 requires both component bounds to be negative whenever the joint +trait differs from truth. At `theta = theta_true` and `tau != tau_true`, the +accuracy log-ratio is identically zero, contradicting its strictly negative +bound. The symmetric speed-axis case also fails. Separate suprema over the +two-dimensional ball complement preserve this problem. This is a printed +quantifier inconsistency, not evidence that the intended normality conclusion +or empirical results are false. A possible repair is componentwise +nonpositivity with joint uniform separation; Lemma 4 must also be revisited. +No corrected theorem has been proved here. Primary locations: +[Appendix D.1](https://arxiv.org/html/2512.07019v4#A4.SS1) and +[PDF pages 32–34](https://arxiv.org/pdf/2512.07019v4#page=32). + +Owner acceptance implication: do not treat the printed assumption as a +verified precondition for production uncertainty claims. Keep the existing +finite-matrix unit check separate from an asymptotic proof and from held-out +calibration. This intake changes no estimator or routing default; dataset +licensing/provenance audit and final rendering of this addition are pending. + +### Pinned matrix identity audit + +On 2026-09-12 root parsed the complete public combined correctness CSV at the +upstream commit above, without running upstream code or fitting an estimator. +Raw SHA-256: `231ea6d561e63229747de56b3c8aa8456a1ae16442945758f1ac4944df89b244`. +There are 128 rows and 100 columns, but only 40 unique column labels; 30 labels +repeat. No ragged rows, duplicate full row IDs, or nonbinary cells were found. +Column position must therefore be preserved until benchmark-qualified item +identity is established; a label-keyed join could merge distinct questions. + +Reproducing only the published `RandomState(42).choice(128, 100, replace=False)` +split with NumPy 2.5.2, and stripping only explicit `_one_shot`/`_zero_shot` +suffixes for this diagnostic, gives 65 base identifiers. Of 28 held-out rows, +21 have a base identifier in training (21 shared identifiers). This verifies +overlap under that naming rule, not family ancestry or its predictive impact. +It is a read-only dataset audit, not an autoresearch treatment or KPI gain. +No response content was sent to a model; no dataset was committed. + +The existing finite-uncertainty Rust documentation test passed at `a73737e4` +(session `30596`, 1 test, 1.30s). It does not test this CSV audit or prove a +corrected asymptotic theorem. + +Visual follow-up: source `85fabb16` was rendered at +`http://127.0.0.1:18774/lart`, English, 1265 × 712. Root directly opened two +overlapping browser captures covering the complete additional intake and +pinned-matrix audit through its final paragraph. Text, identifiers and links +were readable without observed clipping or overlap. This supersedes the +earlier pending-render statement for these additions only. Captures remain +in the task output; other viewports/locales, link destinations and product UI +were not audited. Dataset rights and all statistical/deployment limits remain. + +### Existing installed owner gate + +The installed `fast-mlsirm` 0.9.1 distribution already exposes +`validate_group_partition`. Its installed `model_validation.py` SHA-256 is +`7441501eb8c9ee2fb79a8c5e8fddff4a9be9385331e5f0909c4e023eb63bd7e9`. +Root verified the site-packages origin with Python isolated mode from `/tmp`. +Passing the pinned matrix's suffix-derived group IDs and published seeded +training/evaluation fold IDs to this existing function raised the expected +cross-fold-group `ValueError` (terminal command `221880`, exit 0 because the +diagnostic explicitly required rejection). Input bytes were SHA-256 checked +before parsing. No owner source imports, external estimator execution, runtime +changes or new validation abstraction were needed. + +Reuse this released identity gate for the proposed experiment. It validates +declared group boundaries, not the scientific correctness of those declarations, +family ancestry, label validity, calibration, or data rights. A manually verified +identity map remains a prerequisite; accepting a repaired partition will not +constitute successful model estimation or KPI improvement. + +Verification receipt: at `1ffd5d44`, root directly opened a browser screenshot +of this complete owner-gate section at `http://127.0.0.1:18774/lart`, English, +1265 × 712. The text and complete digest wrapped legibly without observed +clipping or overlap. The capture remains in the task output; other viewports, +locales and product UI remain uninspected. The existing Rust documentation +test also passed (1 test, 1.61s; terminal command `c2e46a`); that algebra check +is separate from the installed partition-rejection evidence above. + +### Preprocessing evidence successor + +The four preprocessing follow-ups formerly ending at `b4887c28` are preserved +in [Draft PR #1139](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1139), +exact head `0024522146803627b8741470ebefaf79ffa4a310`. Before this normal revert, +the research head was verified as its ancestor and the successor diff contained +only additions to this note and AGENTS/CLAUDE; runtime, tests, Rust and Gap +contents were identical. This removes duplicate ownership, not the research +finding or its unresolved acceptance gates. Read the successor for all retained +cell comparisons, encoding provenance, chosen owner repair and bounded tests. +[Upstream PR #2](https://github.com/Toby-X/Latency-Response-Theory-Model/pull/2) +remains Draft at `e5c82a91918a26fab469efd9f04d5152088b5c84`; its hosted run +requires maintainer approval. Full estimator execution, data rights, held-out +accuracy and CO decision latency remain unverified. No deployment is claimed. diff --git a/docs/doctoring/measured-routing-evidence.md b/docs/doctoring/measured-routing-evidence.md index 1ccdef35c..eb20b3af2 100644 --- a/docs/doctoring/measured-routing-evidence.md +++ b/docs/doctoring/measured-routing-evidence.md @@ -32,14 +32,152 @@ fails closed to conducted orchestration when its reply violates the exact | Implementation boundary | Evidence-informed reason | Acceptance evidence | | --- | --- | --- | | EWMA with gain 1/8 for latency and throughput | Jacobson's congestion-avoidance estimator is the canonical low-pass filter for volatile network measurements; it needs no tuning window. | Exact-arithmetic tests reproduce hand-computed EWMA values. | -| Laplace rule of succession as stability prior | The uniform Beta(1,1) posterior mean is the minimum-assumption estimate of a Bernoulli accept probability (Gelman et al., 2013). | Stability tests assert alpha/(alpha+beta) exactly. | +| Laplace rule of succession as stability prior | Beta(1,1) is an explicit uniform prior for Bernoulli probability, not a uniquely assumption-free choice (Gelman et al., 2013). Calibration and sensitivity to the prior require separate evidence. | Stability tests assert alpha/(alpha+beta) exactly; arithmetic correctness is not calibration evidence. | | Cosine similarity over declared metadata documents | Dense retrieval established query-document cosine ordering without keyword overlap (Karpukhin et al., 2020). Affinity uses operator-declared descriptors only. | Deterministic mock-embedding tests verify cosine ordering and zero-vector guards. | -| Strict JSON triage verdict | LLM judges are reliable only under constrained output schemas; Zheng et al. (2023) show judge agreement collapses without structure. Fail-closed preserves verification guarantees. | Parser tests reject seven malformed-reply classes and cache verdicts by content hash. | +| Strict JSON triage verdict | The schema is a CO boundary-validation decision. Zheng et al. (2023) examine judge agreement and biases; their citation does not establish that this schema makes a verdict correct. | Parser tests reject seven malformed-reply classes and cache verdicts by content hash; these checks do not measure human agreement. | | Real-time judging before returning answers | RouteLLM/FrugalGPT motivate quality-aware routing between models (Ong et al., 2024; Chen et al., 2023); here quality is measured per deployment instead of trained offline. | Judge-driven failover tests prove rejection routes to the next candidate within budget while updating both ledgers. | -| Multi-layer simple-structure measurement (fast-mlsirm) | Judged quality is modeled per member rather than pooled, avoiding atomistic fallacy across heterogeneous providers (Jeon et al., 2021). | Quality-ledger reports expose per-member posteriors consumed by `_measured_member_order`. | +| Per-member quality ledger; latent-interaction research candidate | Jeon et al. (2021) model latent item–respondent interactions. Separate per-member Beta posteriors do not implement that model, establish multilevel validity, or justify group-level inference from individual results. | Quality-ledger reports expose per-member posteriors consumed by `_measured_member_order`; interaction recovery and cross-group validity remain separate acceptance work. | + +### Citation correction, 2026-09-09 + +Audit source: `279f7e03`, before this correction. The Jeon et al. entry previously +used an incorrect title and DOI. The authors' [arXiv record, version 3](https://arxiv.org/abs/2007.08719v3) +identifies the latent-space paper and links its published DOI. This corrects the +reference, not evidence that CO implements the cited estimator. The former +claims that separate ledgers prevent atomistic fallacy and that JSON structure +establishes judge reliability are withdrawn. No runtime behavior or score +formula changes in this correction. Current calibration, sampling design, and +group/time validity must be measured before interpreting ledger scores as +psychometric accuracy. + +### Hybrid LLM attribution correction, 2026-09-09 + +Reviewed the authors' [version-one text](https://arxiv.org/html/2404.14618v1), +especially Sections 2–3. Model choice based on predicted quality gaps does not +establish sync/batch transport choice. Removed that attribution from the paper +index; no runtime policy changed. In psychometric work, request urgency and +batchability must not become unvalidated proxies for item difficulty. A policy +combining these signals needs separately identified constructs and observed +outcome validation. Existing `test_paper_contracts.py` checks role/trace behavior +and citation strings, not faithful paper reproduction or empirical validity. + +Section 4.4/Table 2 reports router mean latency 0.036 ± 0.002 seconds over +200 randomly chosen queries; the uncertainty is one standard error, not a +95% interval or p95. The paper explicitly excludes GPT-3.5 API timing because +network, queueing, and inference cannot be separated there. This cannot serve +as CO's 20 ms durable-decision p95 baseline. Section 4.5 selects thresholds on +500 validation examples and evaluates them on a separate test set: retain that +selection/evaluation separation, but do not adopt 500 as a universal sample-size +requirement. CO must choose its sample size from the declared effect, variance, +power, and clustering design. These are interpretation boundaries, not a new +measured performance result or a reproduction of the paper. ## APA 7 references +### Multilevel follow-up source (2026-09-09) + +The local Zotero API returned a matching bibliographic record for Jin et al. +(2022); the authors' [arXiv version 5](https://arxiv.org/abs/1810.07876v5) +independently confirms the title and author list. The abstract distinguishes +differences among individual schools from differences between school systems. +This is a relevant methodological lead for preserving task/model/group levels, +not evidence that CO's per-member Beta ledger implements a multilevel model. +At that checkpoint, full model review and gateway suitability remained open. No participant +data, attachments, private notes, or new Zotero records were copied or created. + +Follow-up at source `8352d155`: read the [version-5 HTML](https://arxiv.org/html/1810.07876v5), +Sections 2–4.2 and the conclusion; supplements and implementation remain unverified. +Section 3.2 constructs networks from products of binary responses and explicitly +rejects treating both derived network families as conditionally independent. +Section 3.4's displayed posterior uses the item-network family. Translation, +rotation and reflection require equivalence-aware interpretation; Procrustes +matching does not identify substantive axis meanings. Section 4.2 evaluates +in-sample adjacency predictions using posterior means, not held-out request +correctness or full posterior predictive checks. Section 5 notes potential +computational advantages of simpler models. The paper does not establish CO's +accuracy or decision-latency KPI. Dataset reuse requires permission. + +Proposed CO applicability check: retain repeated-task and model-revision identity +before defining groups; do not force crossed observations into school nesting. +The numerical owner must validate a common outcome estimand, group-held-out +calibration and uncertainty before publishing a consumer contract. Keep fitting +outside the request path unless measured latency supports otherwise. No routing +default or estimator was changed by this reading. + +Visual evidence, 2026-09-09: opened the version-5 PDF in the actual browser +at page 7 of 24, 100% zoom, 1280 × 720 viewport, English document. Directly +inspected the screenshot: equation (2), its interpretation, the joint-model +warning and identifiability paragraphs were legible, with no visible overlap +or clipped equation in that viewport. Other pages and supplements were not +visually audited; this does not establish a complete PDF inspection. No PDF +was committed because redistribution permission remains unverified. + +Jin, I. H., Jeon, M., Schweinberger, M., Yun, J., & Lin, L. (2022). +Multilevel network item response modelling for discovering differences between +innovation and regular school systems in Korea. *Journal of the Royal +Statistical Society: Series C (Applied Statistics), 71*(5), 1225–1244. +https://doi.org/10.1111/rssc.12569 + +### LSIRM identification and version discrepancy (2026-09-09) + +Read scope: the authors' [arXiv v2 HTML](https://arxiv.org/html/2007.08719v2), +sections 2.1, 2.3.2, 3.2, and 3.3; not the full paper or verified final journal +text. At CO `4e078c5152d7aa021986ffb53735edb6891e1cea`, this extends the +existing Jeon et al. citation without changing an estimator or routing policy. +Section 3.2 identifies translation, reflection, and rotation invariance, uses +Procrustes post-processing, and cautions that interpretation concerns relative +distances rather than absolute positions. + +Section 3.3 needs version-level reconciliation before implementation: its +displayed mixture assigns the slab to delta=1, but the following prose selects +Rasch when P(delta=0) is below 0.5 and then describes estimation using delta=1 +counts. These event descriptions do not agree. It also describes a spike near +zero for log(gamma); that is not a point mass at gamma=0. This is a discrepancy +in the inspected preprint rendering, not a demonstrated defect in the final +article, supplementary code, or fast-mlsirm. The publisher DOI fetch failed and +the NSF PDF fetch timed out; neither failure establishes source absence. + +Engineering consequence: fast-mlsirm owns reconciliation against the final PDF +and supplementary model-selection implementation before publishing such a +contract. CO must not copy this threshold, infer axis meanings, or treat +posterior model selection as held-out customer accuracy. Keep aligned recovery, +observed prediction, and decision latency as separate acceptance measures. + +Follow-up: a direct download of the [NSF repository copy](https://par.nsf.gov/servlets/purl/10229440) +succeeded after the browser fetch failures. The 26-page PDF identifies Springer +as creator and carries the journal DOI; SHA-256 is +`587e6967678a639dec5e9824b75baad92e58a042a017c34a837ead45828960a3`. +PDF page 12 was rendered at 1188 × 1600 and directly visually inspected. +The mixture/event discrepancy is also printed there, so it is not solely an +arXiv HTML conversion artifact. This archived Springer-formatted copy is not +yet byte-compared with the current publisher version or supplementary code. +It specifies the spike as N(-3, 1), unlike the generic near-zero description; +do not implement a prior from that description alone. No PDF is vendored. +Local inspection artifacts: `/tmp/lsirm-10229440.pdf` and +`/tmp/lsirm-model-selection.png`; these temporary paths are not release assets. + +### Existing references + +Kang, I., & Jeon, M. (2025). Multidimensional latent space item response models: +A note on the relativity of conditional dependence. *Psychometrika, 90*(2), +799–826. https://doi.org/10.1017/psy.2025.5 + +Read receipt (2026-09-09): publisher abstract and model introduction, plus +[supplement S1, pages 1–3](https://static.cambridge.org/content/id/urn%3Acambridge.org%3Aid%3Aarticle%3AS0033312325000055/resource/name/S0033312325000055sup001.pdf). +The paper extends Rasch-based latent-space models with between-item dimensions +and discrimination. Conditional dependence is relative to what the base model +explains. Engineering inference: CO comparison evidence should retain base +formulation and dimensional structure; residual-dependence values from different +models are not automatically interchangeable quality scores. + +Direct visual inspection of supplement page 2 (1082 × 1400) confirmed a +declaration of `log_gamma` followed by `lambda = exp(log_lambda)`; S1's extracted +pages 1–3 contain no declaration of `log_lambda`. This is a published-example +name mismatch, not a compiler-run result or evidence against the reported +empirical findings. No Stan execution or Rust port was performed. Reconcile +the executable source before reproduction; fast-mlsirm issue #1713 remains +the candidate-specification owner, not permission to add estimation to CO. + Chen, L., Zaharia, M., & Zou, J. (2023). *FrugalGPT: How to use large language models while reducing cost and improving performance*. arXiv. https://arxiv.org/abs/2305.05176 @@ -51,9 +189,15 @@ Jacobson, V. (1988). Congestion avoidance and control. *ACM SIGCOMM Computer Communication Review, 18*(4), 314–329. https://doi.org/10.1145/52325.52356 -Jeon, M., Jin, I. H., Schweinberger, M., & Baugh, S. (2021). Estimating -parameters for unidimensional multidimensional logistic item response -models. *Psychometrika*. https://doi.org/10.1007/s11336-021-09783-y +Ding, D., Mallick, A., Wang, C., Sim, R., Mukherjee, S., Ruhle, V., +Lakshmanan, L. V. S., & Awadallah, A. H. (2024). *Hybrid LLM: Cost-efficient +and quality-aware query routing*. arXiv. +https://doi.org/10.48550/arXiv.2404.14618 + +Jeon, M., Jin, I. H., Schweinberger, M., & Baugh, S. (2021). Mapping unobserved +item–respondent interactions: A latent space item response model with interaction +map. *Psychometrika, 86*(2), 378–403. +https://doi.org/10.1007/s11336-021-09762-5 Karpukhin, V., Oguz, B., Min, S., Lewis, P., Wu, L., Edunov, S., Chen, D., & Yih, W.-t. (2020). Dense passage retrieval for open-domain question diff --git a/docs/doctoring/nim-benchmark-evidence-grade.md b/docs/doctoring/nim-benchmark-evidence-grade.md index add6a2b1b..a1a26e4cd 100644 --- a/docs/doctoring/nim-benchmark-evidence-grade.md +++ b/docs/doctoring/nim-benchmark-evidence-grade.md @@ -214,6 +214,31 @@ succeed: No earlier head, local-only result, queued check, or stale approval is accepted as release evidence. +## Synthetic classification verification, 2026-09-12 + +Frozen head `07b95c9ab8e1c7b2f0031c39ea45349314079c34` passed the full source +suite: **3,688 passed, 2 skipped, 170.85s**, process 12368, exit 0. +Log: `/tmp/co-kpi-full-07b95c9a.log`. Reproduction from that checkout: + +```sh +/Users/seonghobae/Documents/ChatGPT/contextual-orchestrator/.venv/bin/python -c 'import contextual_orchestrator; contextual_orchestrator.__path__.append("/tmp/co-export-native-acceptance-20260912/lib/python3.14/site-packages/contextual_orchestrator"); import pytest; raise SystemExit(pytest.main(["tests", "-q"]))' +``` + +This reused existing dependencies read-only and is source integration evidence. +Separately, root process 40384 completed **126 passed, 18.44s** against the exact +head's installed wheel, with `python -I` from `/tmp` and the benchmark import +asserted under installed site-packages. Isolated package directory: +`/tmp/co-diagnostic-wheel-07b95c9a.EWQ86S`; core wheel SHA-256: +`24a3528f273a42e17700ea26173af4f3e2820bd6de65ac56ab304a00a627a036`. +Runtime dependencies used hash-locked requirements; test tooling used pytest 9.1.1. + +Root process 56391 also generated a dry-run artifact and checked consistent +classification in JSON evaluation, run provenance and Markdown. The root reviewer +directly inspected the generated summary's top screenshot in an actual browser, +English at 1265 × 712. This is a scoped visual check, not full-document or mobile +inspection. These results validate synthetic evidence labeling, not customer +accuracy, observed decision latency, production promotion, or a protected release. + ## References Autio, C., Schwartz, R., Dunietz, J., Jain, S., Stanley, M., Tabassi, E., Hall, diff --git a/docs/doctoring/noema_gateway_failure_20260909.md b/docs/doctoring/noema_gateway_failure_20260909.md new file mode 100644 index 000000000..5cb16d165 --- /dev/null +++ b/docs/doctoring/noema_gateway_failure_20260909.md @@ -0,0 +1,146 @@ +# Noema gateway failure: evidence provenance + +Status: investigation; no runtime repair or KPI improvement claimed. + +## Verified source + +- Consumer PR: contextual-orchestrator #1103, head + `4776a970ed8bdef3406684aef84952740b476d88`. +- Workflow run: `34195571142`; first Noema job: `101967720557`. +- The job reports HTTP 502 after 926.0 seconds at + `2026-09-08T07:43:21Z`, with `provider_connection_error` in the sidecar. +- Matching artifact: `10045693660`, created `2026-09-08T07:43:22Z`, digest + `sha256:d3e1ca380217c7e8f5c4ae80e862b6f9333deb9e64e9368a8f02bb11270a3ff9`. +- Run artifacts also contain the same name, `noema-sidecar-evidence`, under + IDs `10046985455` and `10048993704`, created at 08:21:36Z and 09:16:08Z. + Name-only download selected later evidence; use artifact ID when attributing + a failure to a specific attempt. + +## Observations and limits + +The matching sidecar records repeated approximately 90-second `TimeoutError` +outcomes for `deepseek-ai/deepseek-v4-flash-0731`. It records `circuit_opened` +at 07:39:13.828 and another attempt for the same agent at 07:39:13.911. +It also records other model attempts and circuit resets. These logs do not +contain sufficient request correlation to establish whether adjacent attempts +belong to the same request or whether an already-admitted call crossed the +circuit transition. A circuit bypass is therefore a hypothesis, not a finding. + +The job bootstrap explicitly reports gateway source revision +`414f22973658c4ddc3d4320fcf7acd9b4e8ba991` at 07:20:03Z. This differs from +the reviewed PR head. At that source revision, `orchestrator.py:1696` declares +the transport constructor default `timeout: int = 90`; line 2256 selects +`self.timeout` when the request timeout argument is `None`. This is a concrete +candidate explanation for the observed durations, but the caller construction +and any bootstrap overrides still need verification. The bootstrap also reports +five of five provider secrets present; this run does not support a missing-key +diagnosis. + +Next investigation: resolve the sidecar constructor and effective timeout +configuration, then trace admission, probes, retries, and fallback in +the deployed source revision. Reproduce any confirmed violation before changing the canonical +gateway. The 926-second failure duration is not routing-decision p95 or buyer +accuracy evidence. Preserve the failed request in the operational denominator. + +## Existing repair ownership + +### Stack-validation owner dependency + +At inspection, PR #1066 (`ci(security): materialize stacked-PR validation on +trusted branch`) remains open at `59a8f4eadfe0e0dcc5ff47cf1acfb80403e241ad`. +Its [Noema job 101609961568](https://github.com/ContextualWisdomLab/contextual-orchestrator/actions/runs/34073907313/job/101609961568) +reports HTTP 502 after 1806.8s at 2026-09-07T05:06:50Z, with one caller +attempt and a reported served model. This does not count internal provider +attempts or establish a particular timeout mechanism. +Its [Strix job 101610071110](https://github.com/ContextualWisdomLab/contextual-orchestrator/actions/runs/34073907351/job/101610071110) +reports failure after 6819s using `orchestrator/free`, explicitly saying that +authoritative vulnerability analysis could not complete. The message lists +several possible backend causes; it does not distinguish them. Neither failed +review is a vulnerability-free verdict, and neither duration is decision p95. +Keep this owner dependency open while diagnosing the gateway and verdict paths; +do not bypass stacked-PR review or interpret ordinary security-job success as +completion of these model-backed reviews. No rerun was dispatched in this audit. + +PR [#1053](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1053) +already owns removal of the implicit model timeout. On inspection its head is +`76c047585f54fcbe940fe168412f51627d3f79dd`, base `main`, and it remains Draft. +Its description contains older head and test claims; those claims must not be +transferred to the current revision. Continue that PR's implementation and +review rather than creating a duplicate timeout fix. Its documented remaining +scope includes model-specific administrator policy propagation, protected +merge, release, consumer adoption, and runtime recovery. + +The three failed compatibility jobs returned by #1053's check rollup +(`101701439950`, `101701439874`, `101701439897`, run `34098501126`) +all end with `DISPATCH_OUTCOME=success`, `VERDICT_STATE=pending` and an +explicit failure awaiting an authenticated terminal verdict. Their log tails +do not report a source vulnerability. This evidence identifies a verdict +delivery investigation at the central workflow owner; it does not establish +that a scan succeeded, or authorize bypassing these checks. + +Reading the complete Python step narrows this further: its successful step +only GETs the live PR and commit statuses, validates the head, and writes +`verdict=pending`. It contains no dispatch POST. Consequently +`DISPATCH_OUTCOME=success` is step completion, not a dispatch receipt. The +terminal message "CodeQL scan dispatched" overstates the evidence. The +central dispatch-run listing for 2026-09-07 10:40–11:10Z returned 13 runs, +none for #1053; this bounded search is not proof that no dispatch ever existed. +Investigate the separate dispatch trigger and correct the misleading message +at the central workflow owner. + +Central owner inventory identifies open `.github` PR #2044 at +`3720dd853fe399fd453e093a90944b2b0e78a8e6` for versioned head envelopes, +#2040 at `6706c231ab06a3c91c43fdb5b989cfcd79fff593` for target-app wake +credentials, and #2051 at `a34dc5af8363a86531d70e51983ad336b9f57096` for +sibling-shard rerun races. These are candidate repair lanes, not established +causes of #1053. Match dispatch receipts and wake-job logs before adopting one. +An open owner PR does not provide a released consumer contract. + +The paginated commit-status endpoint for #1053 head `76c04758` returned no +`codeql-dispatch/*` contexts on inspection. The Python required-job log queries +those contexts, but does not print a dispatch-run identifier. Thus the required +job's successful status-read step cannot locate or prove completion of the scan. +Follow-up must locate the dispatch using repository, PR, language, and full +head SHA; it must not fabricate a status to satisfy the consumer. + +The separate coordinator job `101717808225` ran later, at 12:33:14Z, and +POSTed the full #1053 payload. This locates central run `34122498232`, whose +title includes repository, PR #1053, and full head `76c04758`. Validation +succeeded; all three language jobs failed. Python job `101756437515` reports +HTTP 403 `Resource not accessible by integration` while publishing the +dispatch status with both `target-app-token` and `github-token`, then fails +closed without waking the required job. This establishes a concrete status +publication permission failure. It does not prove scanner success or that +either credential is absent. Inspect app permission and installation scope at +the canonical owner before changing the publisher or repeating the scan. + +Current metadata from `GET /orgs/ContextualWisdomLab/installations` shows +`opencode-agent` installation `141441800` covers all repositories but grants +only `statuses: read` and `actions: read`. `GET /apps/opencode-agent` reports +the same registered permissions and owner `anomalyco`. Thus repository +selection expansion cannot supply the missing write permissions, and there +is no evidence of a pending write-permission upgrade to accept. The external +app owner must change the registration, or the central workflow must adopt +an explicitly governed publisher identity with the required permissions and +matching verdict authentication. Do not substitute an unrelated installed +app merely because it has write access. This metadata is current evidence, +not a historical grant snapshot for the failed execution. + +Sources: [job log](https://github.com/ContextualWisdomLab/contextual-orchestrator/actions/runs/34195571142/job/101967720557), +[matching artifact](https://github.com/ContextualWisdomLab/contextual-orchestrator/actions/runs/34195571142/artifacts/10045693660). + +Additional reproducibility sources: + +- [Central dispatch run](https://github.com/ContextualWisdomLab/.github/actions/runs/34122498232) + and [Python publication failure](https://github.com/ContextualWisdomLab/.github/actions/runs/34122498232/job/101756437515). +- Required compatibility jobs: [Python](https://github.com/ContextualWisdomLab/contextual-orchestrator/actions/runs/34098501126/job/101701439950), + [Actions](https://github.com/ContextualWisdomLab/contextual-orchestrator/actions/runs/34098501126/job/101701439874), + and [JavaScript](https://github.com/ContextualWisdomLab/contextual-orchestrator/actions/runs/34098501126/job/101701439897). +- [Separate dispatch coordinator](https://github.com/ContextualWisdomLab/contextual-orchestrator/actions/runs/34098501126/job/101717808225). +- Permission metadata rechecked at `2026-09-09T01:59:29Z` using + `GET /apps/opencode-agent` and + `GET /orgs/ContextualWisdomLab/installations`, selecting installation + `141441800`. Both still report `statuses: read` and `actions: read`; + installation scope remains `all`. These endpoints require appropriate + authentication; their responses are current metadata, not execution-time + permission evidence. No credential values are recorded. diff --git a/docs/doctoring/reasoning-effort-profile.md b/docs/doctoring/reasoning-effort-profile.md index f476e9f8a..f439895c0 100644 --- a/docs/doctoring/reasoning-effort-profile.md +++ b/docs/doctoring/reasoning-effort-profile.md @@ -35,8 +35,91 @@ uv run ruff check . ## Research basis (APA 7th) +### Fugu read scope and implementation boundary, 2026-09-09 + +The June 22 report is pinned to author-repository revision +`1397abb416e4b774003a09b689ea120e0da02262`; PDF SHA-256 +`00a0e5065551c80c12a019018e18d8365cc3da229303f1765aa49fdf22876ce2`. +Read scope: opening abstract, pages 3–10, and Appendix A/page 26; page 5/Figure 2 was rendered and +directly inspected, with legible labels, arrows, and caption. No full-report +review or reproduction is claimed; no PDF is vendored. +Page 8 was also directly inspected as a 1132×1600 render: the two reward +conditions and equations 6–7 are legible without clipping. This verifies the +source reading, not CO's rendered documentation or a numerical reproduction. + +Section 3.1 uses a trained lightweight head over backbone hidden states for +worker selection without autoregressive decision text. Unlike TRINITY, this +Fugu variant does not assign roles. Section 3.1.2 derives soft training targets +from repeated worker outcomes on tasks with reference solutions. These are +not CO's deterministic effort-rank fixture or evidence validating its role +catalog. Figure 1's non-Fugu scores are provider-reported, not a common CO +evaluation cohort. + +Section 3.1.3 optimizes repeated end-to-end task completion under a fixed turn +budget, not isolated turn accuracy. Section 3.2.1 describes Fugu-Ultra's different +training reward: malformed workflows receive 0, parseable incorrect workflows +0.5, and correct final outputs 1. That shaped reward is not a correctness rate. +For CO experiments, retain separate format validity, final correctness, and +training reward fields; never count the intermediate reward as half a correct +customer answer. Repeated runs and turns share a task/harness context and must +not inflate the independent sample count. This is an evaluation-design inference, +not a reproduced result or authorization to train on customer transcripts. + +CO source mapping at `204e306c046797b812589b9b66062296beaa1c8d`: +`orchestrator.py::_score_config` averages a caller-supplied quality function; +`evolve_orchestration` caches each configuration's evaluation once. Neither +implements Fugu's trained selection head or replicated trajectory fitness. +The deterministic fixtures in `tests/test_optimizer.py` and +`tests/test_evolve_optimizer.py` establish ranking/accounting behavior, not +empirical worker superiority. Existing `tests/test_batch_optimizer.py` covers +the real `batch_route` rejection of incomplete provider results while retaining +other items' spend. A suspicious `zip` in the scorer alone therefore does not +prove that this provider path silently shrinks the evaluation denominator. + +Sections 3.2.2–3.2.3 distinguish current-workflow access-list isolation from +cross-workflow shared tool history, and retain the emitting agent's identity +when routing tool responses. CO ablations must therefore declare memory scope +and tool-response ownership, not just role labels. Shared history can correlate +agents' errors; independent-looking agent counts are not independent evidence. +Section 4.1.1 matches maximum reasoning effort against frontier workers, which +does not establish equal total tokens, cost, or latency. The remaining main +evaluation sections have not yet been audited. These observations +do not authorize changing CO's memory disclosure or free-pool policy. + +Appendix A confirms heterogeneous benchmark protocols and externally reported +baselines, rather than one common end-to-end experiment. It specifies five +baseline retries for LiveCodeBench Pro timeout/token exhaustion, pass@4 for +banking, and benchmark-specific harness/dependency repairs. CO comparison +receipts must pin these protocol choices, preserve attempts and exhaustion +counts, and distinguish multi-attempt metrics from single-request correctness. +The appendix's two-hour limit for one long-context benchmark is an evaluation +configuration, not evidence for a universal production model timeout. No +benchmark rerun, retry parity verification, or claimed score gain follows from +this source reading. + +Independent read-only audit additionally covered pages 11–19; root directly +cross-checked pages 13–14 and viewed Figure 6/Table 2 at 1132×1600. The table, +legend, axes, and distinction between the mean and best-run curve were legible. +The AutoResearch experiment runs 123 sequential experiments for each of three +independent seeds, with a shared H100 task-compute budget. Thus the independent +trajectory count is three per system, not 369 independent final results. +Table 2 reports mean best validation BPB and standard deviation: Ultra +0.9774 ± 0.0019 versus Model C 0.9781 ± 0.0011. These are not confidence +intervals or an untouched-test result; this subsection supplies no significance +test. The 0.0007 mean difference alone cannot establish a general improvement. +CO must report trajectory-level uncertainty and reserve independent confirmation +data after adaptive selection. Shared task-compute budgets also do not prove +matched orchestration tokens, calls, or provider spend. No aggregate table is +being converted into synthetic customer observations. + +Engineering inference: compare decision-only selection with generative triage +under the same observed cohort, correctness guardrail, and resource accounting. +Do not claim Fugu's results for a black-box API router without its trained +representation/head, or treat temperature as reasoning effort. This motivates +an experiment, not a default change or a new unvalidated estimator. + Sakana AI. (2026). *Sakana Fugu technical report*. -https://github.com/SakanaAI/fugu/blob/main/Fugu_technical_report.pdf +https://github.com/SakanaAI/fugu/blob/1397abb416e4b774003a09b689ea120e0da02262/Fugu_technical_report.pdf Xu, J., Sun, Q., Schwendeman, P., Nielsen, S., Cetin, E., & Tang, Y. (2025). *Trinity: An evolved LLM coordinator* (arXiv:2512.04695). diff --git a/docs/doctoring/review_phase_transport_comparison.md b/docs/doctoring/review_phase_transport_comparison.md new file mode 100644 index 000000000..9844f63da --- /dev/null +++ b/docs/doctoring/review_phase_transport_comparison.md @@ -0,0 +1,146 @@ +# Free review transport: deployed-source comparison + +Recorded 2026-09-12. This is a bounded diagnostic record, not a deployment +receipt, production validation, or a new routing-policy decision. + +## Incident and evidence boundary + +The coordinator inspected GitHub Actions run `34688188671`, job +`103539568718`, for `.github` PR #2052. The job reported HTTP 502 with +`provider_connection_error`. Its actual job log, lines 980–981, records +vendoring and installing dependencies for contextual-orchestrator revision +`414f22973658c4ddc3d4320fcf7acd9b4e8ba991` at 10:34:23–10:34:26. +The trusted workflow revision was +`fb17ef556f94f673234aa557254ae52779e9a7b0`. + +The supplied local evidence directory was +`/tmp/cwl-noema-evidence.8M4Tel/`. Its sidecar stderr records repeated Flash +and Pro attempts, circuit opening/reset/clearing, and a final approximately +90-second TimeoutError. Preflight reports 24 candidates, 16 probed, five +ready, two deferred, and nine rejected. This disproves the hypothesis that +only one internal attempt occurred. Readiness is not proof that every ready +candidate satisfied every later role/request constraint. + +The source pin is established by the job log. A runtime image/package digest +and request-correlated terminal role are not established. The terminal call +cannot be identified as final synthesis solely from these logs. Missing +provider usage remains unavailable, not zero. + +## Frozen unit comparison + +Tests were committed **before execution** in +`8065ada18b5e0c785446f2e081b26d5cf60d0aca`, based on +`50e1b0d0a7eddc0f866ab69162d6cf098efe693f`. The test is +`tests/test_passthrough_provider_failover.py::test_free_review_phase_timeout_preserves_eligible_sibling`. +It reuses the free-tagged two-candidate fixture pattern, executes actual +conduct stages, passively records planner/stage calls, and injects an +immediate unit TimeoutError. There is no sleep or provider request. +The unavailable calibration resolver remains fail-closed; a test double is +not a production judge approval or a real review-provider integration. + +| Source | Generated-planner injection | Final structured-synthesis injection | +| --- | --- | --- | +| `50e1b0d0` with test commit `8065ada1` | PASS: planner never entered | PASS: primary then eligible free sibling | +| `414f2297` | PASS: planner never entered | FAIL: `ProviderUpstreamError`, sibling not reached | + +Current-source pytest session `72922` completed with **2 passed, 67 +deselected in 1.50s**. These are passing current regression tests, not RED +results. Both versions route `FREE_MODEL` to fixed planning even when the +generated-planning policy is configured: old source lines 6504–6506, current +source lines 7231–7234. Therefore the armed planner exception never executes; +this test cannot establish generated-planner recovery. + +For the old-source comparison, a detached worktree at +`/tmp/co-review-phase-deployed-20260912` preserved the source unchanged. +Whole-module pytest collection failed because that old revision lacks the +unrelated newer `_is_ambiguous_passthrough_transport_failure` import used by +other tests. The fallback harness parsed the frozen committed test module +with stdlib `ast`, selected only `SequencedProxyClient` and the test function, +and executed those unchanged definitions with their explicit dependencies. +It invoked both parameter cases with `pytest.MonkeyPatch.context()`. +The printed imported source was +`/private/tmp/co-review-phase-deployed-20260912/contextual_orchestrator/__init__.py`. +The harness caught and printed failures for comparison, so its zero shell +exit status must **not** be read as a passing old-source test run. + +Old-source final synthesis reaches `send_synthesis`, then raises the +classified exception at `orchestrator.py:4848`. That branch advances on size +or selected stale-model failures but not this retryable transport failure. +Current code advances eligible virtual synthesis candidates on +`classified.retryable`. This is a reproduced unit-level source difference, +not proof of the incident's terminal phase or sole cause. + +## Existing repair and next action + +The repair already exists in commit +`1c61eff2da012382255bf8b4e1aa6dd0d6dd05ca`: +“fix(gateway): fail over structured 502 onto the next free worker.” +The coordinator independently confirmed [PR #1094](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1094) +MERGED at `2026-09-09T23:24:35Z`, head prefix `73338d21`, merge +`9334dc91aaf853b758077e983517a822b6b21edb`. +Local ancestry places the repair in recorded `origin/main` +`012beaacd0631f8cd3391c77744eeb626269b5de`. + +No duplicate runtime fix is warranted by this comparison. Release/sidecar +adoption should be checked against the existing repaired owner revision, +with immutable artifact identity and correlated execution evidence before +claiming recovery. The coordinator found no listed GitHub releases; this +does not prove that no registry package exists. + +No paid fallback, consumer retry, timeout/default change, eligibility +relaxation, push, or PR mutation was performed. HTTP integration, live +provider behavior, final rendered-document visual inspection, and incident +causal certainty remain unverified by this bounded exercise. + +Additional coordinator checks: `gh release list --limit 5` for this repository +returned no entries; local `git tag --contains 1c61eff2` returned none (local +tag inventory only); the public PyPI endpoint +`https://pypi.org/pypi/contextual-orchestrator/json` returned HTTP 404 in a +separate request. These checks do not rule out alternate distribution names +or private registries. No immutable released adoption has been verified. + +## Exact protected-merge adoption check + +The minimal candidate `9334dc91aaf853b758077e983517a822b6b21edb` was checked +in detached `/tmp/co-review-phase-merge-20260912`. The unchanged selected +test definitions described above passed both cases. Its own +`tests/test_review_gateway.py` passed **13 tests in 0.81s**. The ancestry +check against `012beaacd0631f8cd3391c77744eeb626269b5de` returned exit 0. +The generated-planner case still means no planner invocation, not recovery. + +The central launcher was read at `.github` PR #2052 head +`68daf0f61d2afc0ebf68aa260713b2e481f5112d`, path +`scripts/ci/contextual_orchestrator_review_sidecar.sh`. It installs only the +hash-locked dependencies, then imports the selected CO source. The old and +candidate `requirements.lock` files have identical SHA-256 +`c80752a4c6bbbc1bc9b0cb2b938831693a88dfdca7d1130bb4c00e2f9fe21345`. +The changed project dependency URL does not participate in this install path. + +The exact import statement and unchanged startup heredoc were executed with +`PYTHONPATH=/tmp/co-review-phase-merge-20260912:/private/tmp/cwl-pr2052-current`, +the isolated interpreter +`/tmp/co-cache-wheel-4bc96045.C4cqDg/installed/bin/python`, and working +directory `/tmp`. The imported source path was printed and verified. The +startup body SHA-256 was +`3dc3b5bffb95f90ba9ef1db1dbe93003def994e4d063637420ed9bbe0645b178`. +Session `83034` exited 0 with `IMPORT_AND_STARTUP_CONTRACT_PASS`. +This exercised real loopback HTTP rejection above the body limit, acceptance +above 64 KiB, exact tool-description byte preservation, and connection/server +cleanup. Provider responses were test doubles; no credentials or provider +requests were used. This did not reinstall dependencies or execute the full +provisioning shell. It proves this bounded startup contract, not hosted CI, +registry release, live discovery, or recovery of the original incident. + +### Rendered-document inspection + +Source `49362265` was rendered at `http://127.0.0.1:18773/transport` in the +actual in-app browser, English, 1265 × 712 viewport. Four successive viewport +captures were directly opened: introduction/evidence boundary, source +comparison table, existing repair/limitations, and exact-merge startup proof. +Table cells, long hashes/paths, paragraph spacing, contrast and vertical +scrolling were legible without observed clipping or overlap. The captures +are retained in this task's visual tool results, not as repository image +files. This supersedes only the earlier uninspected-document statement; +mobile, other locales, application UI states and live deployment remain +outside this bounded inspection. No Figma artifact was supplied for this +diagnostic document. diff --git a/docs/doctoring/stacked_review_dispatch_checkpoint.md b/docs/doctoring/stacked_review_dispatch_checkpoint.md new file mode 100644 index 000000000..356eda22b --- /dev/null +++ b/docs/doctoring/stacked_review_dispatch_checkpoint.md @@ -0,0 +1,32 @@ +# Stacked review dispatch checkpoint — 2026-09-12 + +## Verified cause and preserved policy + +PR #1138 at `3419abf11bc7f67dc4d854b347f1dbe2d1f517dc` first had +Security and Quality run `34687370076` skipped while Draft. Its checked-in +`.github/workflows/security.yml` explicitly excludes Draft PRs from all three +jobs and includes `ready_for_review` among its pull-request events. +After marking the PR ready, run `34687504508` was created for the same head +and was authoritatively queued at this checkpoint. This verifies local event +delivery; Draft filtering is intentional, not a defect candidate. No gate, +ruleset, required approval or model-routing policy was changed. + +## Current-head evidence and remaining boundary + +PR #1107 at `ff590dce735f6f9378ded1e8ac2160cd1077e4db` has Security and +Quality run `34687545746`, with jobs `103537072059`, `103537071997` and +`103537072142` queued. Earlier `de01e9e1` evidence is historical only. +The queried current-head rollup contained no central Required OpenCode, +Noema or Strix CheckRun entries. This is not success and does not establish +missing credentials, failed event delivery or a central execution failure. +The canonical workflow owner was asked to trace central admission/dispatch; +the consumer must not copy review workflows or weaken branch protections. + +Continue observing these existing runs; queued jobs are not restart requests. +Keep local source tests, installed-wheel checks, hosted checks, independent +approval, protected-main adoption and release as separate evidence classes. + +Reproduce read-only with `gh pr view 1107 --json headRefOid,statusCheckRollup` +and `gh run view 34687504508 --json headSha,status,conclusion,event`, using +`--repo ContextualWisdomLab/contextual-orchestrator`. Re-fetch heads before +interpreting results; this checkpoint is not a permanently current status. diff --git a/docs/doctoring/workflow_request_link.md b/docs/doctoring/workflow_request_link.md new file mode 100644 index 000000000..66b0cedd5 --- /dev/null +++ b/docs/doctoring/workflow_request_link.md @@ -0,0 +1,63 @@ +# Durable workflow origin identity + +## Full-suite environment repair + +Full collection at `4cf7feafd554fbbd65dfc3b790f1081623b0d05a` failed with +exit 2 after 5.41 seconds: the shared wheel-smoke environment lacked Hypothesis. +No tests completed in that run. The remedy is a separate project-local environment, +not a runtime change or an installation into the shared historical environment: + +```sh +uv sync --locked --extra api --extra db --extra queue --group dev +.venv/bin/python -c 'import contextual_orchestrator; contextual_orchestrator.__path__.append("/tmp/co-receipt-wheel-install-20260909/lib/python3.14/site-packages/contextual_orchestrator"); import pytest; raise SystemExit(pytest.main(["-q"]))' +``` + +Setup completed with CPython 3.14.6 and the existing lock, including Hypothesis +6.165.10. The reused base-native artifact is `_decision_receipt.abi3.so` under +that explicitly named site-packages directory, SHA-256 +`ddac17f8c6a25e52a9233bd9e75f5ca3c754eb24641caa58113148df8ef60b4d`. +Native source is unchanged from `c7345670`; this is source-integration evidence, +not an installed successor-wheel or release result. Do not modify the shared +native environment. Full-suite results remain pending until its handle terminates. + +The accepted-request ledger and completed workflow records previously had no +durable join. At base c7345670, real HTTP route, conduct and streamed route +requests could not match their persisted outcome to the trusted request ID. + +The shared workflow replacement seam now records the trusted HTTP identity on +first creation and preserves it on replacement. Explicit state restoration +retains stored identity instead of attributing history to the loading context. +Non-HTTP runs remain valid without an ID. Cache hits create separate outcome +records with their existing cache-hit classification; the earlier run's origin +is unchanged. This does not make reused output a new provider execution. + +The regression also exposed that stream_route never saved its completed run. +It now uses the same synchronous workflow store path as run/conduct. A failed +save emits the existing terminal SSE error, retains the admission and does not +rewrite an earlier initial-decision acknowledgement. As in run/conduct, the +in-memory run and budget update precede storage: memory is not durable proof, +and no atomic memory/database transaction is claimed. Consumers must reconcile +against persisted records rather than count the in-memory run as stored. + +Reproduction prerequisite: the unchanged native extension built at base +`c7345670e08f029ad3aa5dd1133037bb4b451d9b` is installed in the existing isolated +Python 3.14 environment below. From `/tmp/co-outcome-request-link-20260909`, run: + +```sh +/tmp/co-receipt-wheel-install-20260909/bin/python -c 'import contextual_orchestrator; contextual_orchestrator.__path__.append("/tmp/co-receipt-wheel-install-20260909/lib/python3.14/site-packages/contextual_orchestrator"); import pytest; raise SystemExit(pytest.main(["tests/test_workflow_request_link.py", "tests/test_persistence.py", "tests/test_stream_error_identity.py", "-q"]))' +``` + +This explicitly loads successor Python source and the unchanged base native +extension. The namespace append is test-only; it does not install the successor +or establish wheel/release acceptance. Preserve that environment unchanged. +The three-file command passed 34 tests in 9.36 seconds. A final linkage-only +run after adding the in-memory failure-state assertion passed 10 in 5.43 seconds. +The independent original HTTP probe also confirmed one admission joins one +durably stored route outcome. Initial four cases failed; subsequent HTTP enabled/disabled, +cache, reload, persistence and SSE regressions passed. Tests use controlled +provider output and establish identity, not independently adjudicated accuracy. +The request may map to multiple workflow outcomes; it is not a one-to-one join +contract for every API. Detached background execution without propagated HTTP +context remains unlinked, rather than guessed from the current thread. Explicit +batch-parent/item linkage and crash recovery after an in-memory-only update +remain outside this bounded repair. diff --git a/docs/library_research.md b/docs/library_research.md index 3cee44389..9fc271f2b 100644 --- a/docs/library_research.md +++ b/docs/library_research.md @@ -72,8 +72,8 @@ reasoning effort. | Area | Researched | Decision | Skipped | |---|---|---|---| -| Profile object | Existing `OrchestrationPolicy` dataclass; OpenAI `reasoning_effort` enum; Anthropic thinking-token budget | New stdlib module `reasoning_effort_profile.py` with a frozen dataclass, fail-closed parser, role catalog, and snapshot hash. No production-default change until the RMSE gate passes. | New dependency, provider SDK, treating temperature as an effort proxy, a second policy factory. | -| Ablation | Fugu latency-vs-quality frontier; TRINITY role split; Conductor steps/access lists; Baker (2001) IRT true-θ RMSE | Deterministic offline θ̂ = (1−λ)θ with RMSE(θ̂, θ). Access-list scope and recursion depth change λ. Record quality, budget, estimated tokens used, and `measurement_status=estimated`. Persist the same snapshot on `run` / `stream_route` / `batch_route`. | Live NVIDIA NIM calls in this slice (issue #86 evidence plane); changing `OrchestrationPolicy` defaults. | +| Profile object | Existing `OrchestrationPolicy` dataclass; OpenAI `reasoning_effort` enum; Anthropic thinking-token budget | New stdlib module `reasoning_effort_profile.py` with a frozen dataclass, fail-closed parser, role catalog, and snapshot hash. Synthetic recovery scores do not authorize production-default changes. | New dependency, provider SDK, treating temperature as an effort proxy, a second policy factory. | +| Ablation | Fugu latency-vs-quality frontier; TRINITY role split; Conductor steps/access lists; Baker (2001) as IRT background, not a verified source for this recovery criterion | Deterministic offline θ̂ = (1−λ)θ with RMSE(θ̂, θ). Access-list scope and recursion depth impose changes in λ; this is a unit fixture, not an estimated causal effect of effort. Record recovery error, budget, estimated tokens used, and `measurement_status=estimated`. Persist the same snapshot on `run` / `stream_route` / `batch_route`. | Live NVIDIA NIM calls in this slice (issue #86 evidence plane); changing `OrchestrationPolicy` defaults. | | Doctoring | Sakana Fugu (2026); Xu et al. (2025) TRINITY arXiv:2512.04695; Nielsen et al. (2025) Conductor arXiv:2512.04388 | APA 7th citations in `docs/architecture.md` and `docs/papers/README.md`. PDFs are not vendored when redistribution is unclear. | Training a learned coordinator. | Buyer next action: call `default_role_effort_catalog()` / `run_equal_budget_ablation()` diff --git a/docs/papers/README.md b/docs/papers/README.md index 6680d87b7..8d4d8c81a 100644 --- a/docs/papers/README.md +++ b/docs/papers/README.md @@ -2,8 +2,80 @@ These papers ground the design of the LLM **cost review** ledger and the **sync-vs-batch / upstream** routing added in `feat/cost-review-and-batch-routing`. -All three are arXiv preprints distributed under licenses that permit -redistribution; each is cited below with its arXiv identifier. +References below include arXiv preprints and other publications. Redistribution +must be verified for the exact version of each PDF: arXiv's perpetual, +non-exclusive license grants distribution rights to arXiv, not a blanket +permission for this repository to redistribute it. See the +[arXiv license guidance](https://info.arxiv.org/help/license/index.html). +Version-page licenses are recorded below. Additional redistribution permission, +where required, and the contents of release archives remain unverified. + +## Stored PDF version inventory + +First-page text inspection on 2026-09-09 identifies the following versions. +This establishes document identity, not redistribution permission or a complete +research review. License verification must use these versions, not whichever +version the unversioned abstract page currently serves. + +| Stored paper | Embedded arXiv version | Embedded date | +| --- | --- | --- | +| FrugalGPT | 2305.05176v1 | 2023-05-09 | +| RouteLLM | 2406.18665v4 | 2025-02-23 | +| Hybrid LLM | 2404.14618v1 | 2024-04-22 | +| HELM | 2211.09110v2 | 2023-10-01 | +| The Art, Science, and Engineering of Fuzzing: A Survey | 1812.00140v4 | 2019-04-08 | + +Version-specific arXiv abstract pages were checked on 2026-09-09: + +| Version page | Declared license | Release implication | +| --- | --- | --- | +| [2305.05176v1](https://arxiv.org/abs/2305.05176v1) | arXiv nonexclusive-distrib/1.0 | Additional redistribution basis not established. | +| [2406.18665v4](https://arxiv.org/abs/2406.18665v4) | arXiv nonexclusive-distrib/1.0 | Additional redistribution basis not established. | +| [2404.14618v1](https://arxiv.org/abs/2404.14618v1) | CC BY-NC-ND 4.0 | Do not assume commercial redistribution rights. | +| [2211.09110v2](https://arxiv.org/abs/2211.09110v2) | CC BY 4.0 | Preserve attribution and license requirements. | +| [1812.00140v4](https://arxiv.org/abs/1812.00140v4) | arXiv nonexclusive-distrib/1.0 | Additional redistribution basis not established. | + +This checks the declared version-page licenses. The local bytes were hashed on +2026-09-09 in [stored_pdf_sha256.txt](stored_pdf_sha256.txt); from the repository +root run `shasum -a 256 -c docs/papers/stored_pdf_sha256.txt` to verify all five. +These hashes identify the stored copies only: comparison with publisher bytes, +redistribution permission, and release-package inclusion remain separate checks. +Existing files are retained during the audit. + +At source commit `15b8f52b`, `uv build` produced both package formats and +archive-member inspection found no `.pdf` members in either: + +- `contextual_orchestrator-0.2.0-py3-none-any.whl`, SHA-256 + `9091060d05ee07fa52e48918195c9e14388d6d9e85182b7919fcdf3f5db6e152`. +- `contextual_orchestrator-0.2.0.tar.gz`, SHA-256 + `75747f6a0a841046bdad44f5edc408f81839fa8c53b09efa2c2e5710c4c6d180`. + +These are local build artifacts, not published-package evidence. Git source +archives include tracked PDFs and require a separate redistribution decision. + +## Response-process validity + +Xu, Z., Liu, J., Wang, Y., & Gu, Y. (2026). *Latency-response theory model: +Evaluating large language models via response accuracy and chain-of-thought +length* (Version 4) [Preprint]. arXiv. https://arxiv.org/abs/2512.07019v4 + +Citation only; no PDF vendored. The version page declares the arXiv perpetual +non-exclusive license, not repository redistribution permission. See the +[measurement review and read scope](../doctoring/lart_measurement_review.md). +Experimental intake only: no production estimator or measured KPI improvement. + +Bolsinova, M., Tijmstra, J., Molenaar, D., & De Boeck, P. (2017). +Conditional dependence between response time and accuracy: An overview of its +possible sources and directions for distinguishing between them. *Frontiers in +Psychology, 8*, Article 202. https://doi.org/10.3389/fpsyg.2017.00202 + +[Unmodified publisher PDF](bolsinova_2017_conditional_dependence.pdf), downloaded +2026-09-09 from the [publisher](https://www.frontiersin.org/journals/psychology/articles/10.3389/fpsyg.2017.00202/pdf). +Copyright © 2017 Bolsinova, Tijmstra, Molenaar and De Boeck; +[CC BY 4.0](https://creativecommons.org/licenses/by/4.0/), verified in the +publisher copyright statement. No endorsement is implied. SHA-256: +`3391e166d9c136cec8cdfdcc235e0e06b418a00525aac3005de97940ae9452a5`. +See the [read scope and proposed experiment](../doctoring/irt_router_measurement_review.md#response-process-identification-follow-up). ## Cost optimisation @@ -13,8 +85,9 @@ redistribution; each is cited below with its arXiv identifier. Motivates the **configurable price table + per-request cost accounting** and cost-optimising model selection: cost varies by orders of magnitude across providers/models, so a gateway should price each request and route to the - cheapest capable upstream. Distributed under arXiv's non-exclusive license to - distribute (arXiv perpetual, non-exclusive license 1.0). + cheapest capable upstream. Version-page license: arXiv non-exclusive; + additional redistribution basis unverified. + source: https://arxiv.org/abs/2305.05176. ## Query routing (which upstream / which tier) @@ -30,16 +103,19 @@ redistribution; each is cited below with its arXiv identifier. `routellm-routing-2406.18665.pdf` Grounds the **routing decision** layer (`RoutingPolicy` + cost-aware upstream selection): route strong/weak model choices to hit a cost/quality target. - arXiv preprint; distributed under the arXiv non-exclusive distribution license. + Version-page license: arXiv non-exclusive; additional redistribution basis + unverified. Source: https://arxiv.org/abs/2406.18665. - **Hybrid LLM: Cost-Efficient and Quality-Aware Query Routing** — Dujian Ding, Ankur Mallick, Chi Wang, Robert Sim, Subhabrata Mukherjee, Victor Rühle, Laks V. S. Lakshmanan, Ahmed Hassan Awadallah. arXiv:2404.14618 (ICLR 2024). `hybrid-llm-query-routing-2404.14618.pdf` - Grounds **latency-tolerant vs interactive routing** and the sync/batch split: - route easy/bulk queries to the cheaper path, keep hard/interactive queries on - the responsive path. Distributed under the arXiv non-exclusive license / - CC BY as marked on arXiv. + Supports selecting between small and large models using predicted response + quality differences and a configurable quality target. It does not establish + this repository's sync/batch transport policy or equate bulk requests with + easy items. Those are separate product decisions requiring their own evidence. + Version-page license: CC BY-NC-ND 4.0; + source: https://arxiv.org/abs/2404.14618. ## Role reasoning-effort profiles @@ -49,9 +125,10 @@ one replayable compute snapshot. PDFs are cited rather than vendored when redistribution is unclear. - Sakana AI. (2026). *Sakana Fugu Technical Report*. - https://github.com/SakanaAI/fugu/blob/main/Fugu_technical_report.pdf - Grounds the latency-quality frontier: route is the low-compute path, - conduct is the high-quality path. Do not proxy that split with temperature. + https://github.com/SakanaAI/fugu/blob/1397abb416e4b774003a09b689ea120e0da02262/Fugu_technical_report.pdf + See the [bounded method review](../doctoring/reasoning-effort-profile.md#fugu-read-scope-and-implementation-boundary-2026-09-09). + CO's route/conduct analogy is an engineering interpretation, not an + implementation or reproduction of the trained Fugu variants. - Xu, J., Sun, Q., Schwendeman, P., Nielsen, S., Cetin, E., & Tang, Y. (2025). *Trinity: An evolved LLM coordinator* (arXiv:2512.04695). https://arxiv.org/abs/2512.04695 @@ -65,8 +142,19 @@ redistribution is unclear. - Baker, F. B. (2001). *The basics of item response theory* (2nd ed.). ERIC Clearinghouse on Assessment and Evaluation. https://eric.ed.gov/?id=ED458219 - Grounds RMSE(θ̂, θ) as the accuracy metric. The ablation must emit θ̂ - and compare it to known true parameters; a rank constant is not an estimate. + Background for IRT parameter estimation and information, not a verified + source for this project's choice of RMSE as its recovery criterion. The + ablation must emit θ̂ and compare it to known true parameters; a rank + constant is not an estimate. That engineering criterion measures synthetic + parameter recovery, not observed task correctness. + Read scope on 2026-09-09: ERIC bibliographic record and the directly viewed + copyright page and publisher's note (PDF pages 4 and 6 of 187), not the + full estimation chapters. Page 4 states copyright 2001, ERIC Clearinghouse + on Assessment and Evaluation, with all rights reserved. + [ERIC-hosted original](https://files.eric.ed.gov/fulltext/ED458219.pdf) + is linked only; redistribution permission has not been established. + Subsequent bounded inspection of printed pages 90 and 111 is recorded in + [the estimation-boundary runbook](../doctoring/autonomous_kpi_runbook.md#baker-estimation-boundary-2026-09-09). `run_equal_budget_ablation` emits estimated, synthetic true-parameter evidence for unit-test recovery checks. It does not establish buyer accuracy or authorize @@ -79,6 +167,17 @@ verified observed-task evidence and the protected release process. ## Evaluation methodology (NIM cost-quality benchmark) +- Chiang, W.-L., Zheng, L., Sheng, Y., Angelopoulos, A. N., Li, T., Li, D., + Zhang, H., Zhu, B., Jordan, M. I., Gonzalez, J. E., & Stoica, I. (2024). + *Chatbot Arena: An open platform for evaluating LLMs by human preference*. + arXiv. https://doi.org/10.48550/arXiv.2403.04132 + Cited by `contextual_orchestrator/benchmark_priors.py`. Version 1 metadata + and HTML sections 3–4 were inspected on 2026-09-09. Pairwise human preference + is distinct from absolute task correctness. The repository's equal-weight + composite of normalized Arena and Quality Index scores is not validated by + this citation; its historical snapshot values also need archived provenance. + Citation only; no PDF copied. This is not a completed full-paper review. + - **Holistic Evaluation of Language Models (HELM)** — Percy Liang, Rishi Bommasani, Tony Lee, et al. arXiv:2211.09110, 2022 (TMLR 2023). `helm-holistic-evaluation-2211.09110.pdf` @@ -87,13 +186,59 @@ verified observed-task evidence and the protected release process. latency, cost) instead of a single leaderboard number; report incompleteness honestly (skipped/unsupported/rate-limited cells stay machine-readable rather than silently dropped); and standardize conditions across compared systems - (same tasks, scorers, caps, and budgets). Distributed under the arXiv -non-exclusive license / CC BY as marked on arXiv. - -## Batch execution / load balancing + (same tasks, scorers, caps, and budgets). Version-page license: CC BY 4.0; + source: https://arxiv.org/abs/2211.09110. ## Psychometrics beyond orchestration +### Multilevel ability and measurement error + +Fox, J.-P., & Glas, C. A. W. (2001). Bayesian estimation of a multilevel IRT +model using Gibbs sampling. *Psychometrika, 66*(2), 271–288. +https://doi.org/10.1007/BF02294839 + +The publisher abstract describes joint estimation of a two-parameter normal +ogive measurement model and a two-level regression on latent ability using +Gibbs sampling. Research lead: evaluate measurement error and clustering +together rather than treating observed scores as error-free abilities. +This is an engineering proposal, not an implemented CO estimator. +Read status: abstract only; full method, identification assumptions, +implementation and customer accuracy/latency gains remain unverified. +The original publication year is 2001, not the publisher's 2025 online date. +No PDF is attached: full-text access and redistribution permission have not +been established. See the [discovery receipt](../doctoring/autonomous_kpi_runbook.md#local-zotero-discovery-2026-09-09). + +### Cross-document reference index + +At `fcf0047c37706975d9bff1ab4b95c54a2a383f54`, a tracked-text census found +six explicit arXiv identifiers referenced elsewhere but absent from this index. +Their existing evidence records remain canonical; these links do not imply +that the papers were fully reviewed, their claims reproduced, or PDFs licensed. + +| Identifier | Existing evidence and implementation discussion | +| --- | --- | +| 1810.07876 | [Measured routing evidence](../doctoring/measured-routing-evidence.md) | +| 2007.08719 | [Measured routing evidence](../doctoring/measured-routing-evidence.md) | +| 2306.05685 | [Measured routing evidence](../doctoring/measured-routing-evidence.md) | +| 2506.22316 | [Polytomous judge benchmark](../benchmarks/2026-08-11-polytomous-llm-judge.md) and [judge calibration ADR](../planning/adrs/0006-polytomous-llm-judge-bias-calibration.md) | +| 2110.15150 | [Purpose-limited protection ADR](../planning/adrs/0028-purpose-limited-pii-protection.md) | +| 2601.17814 | [Model-group specification](../model-group-product-technical-spec.md) and [free-pool admission research](../research/review-gateway-free-pool-admission.md) | + +Scope: explicit arXiv URL, colon, and DOI-style identifiers in tracked Python, +Rust, Markdown, and TOML files. This is a discovery census, not a complete +bibliography: DOI-only sources, author/year citations, other formats, and +references inside PDFs need separate reconciliation. The table includes +non-psychometric references to avoid hiding cross-cutting dependencies. + +### Interpretation constraints + +Additional DOI-only lead: Kang and Jeon (2025), +[Multidimensional latent space item response models](https://doi.org/10.1017/psy.2025.5). +The [bounded read and reproduction receipt](../doctoring/measured-routing-evidence.md) +records the abstract/model scope and a directly viewed supplementary-code name +mismatch. This extends discovery beyond the arXiv census; it does not establish +a completed reproduction or a released fast-mlsirm implementation. + These sources constrain what a routing score may mean; they are not routing algorithms. They add validity, identification, and fair-comparison checks: @@ -107,6 +252,13 @@ algorithms. They add validity, identification, and fair-comparison checks: - Reise, Ainsworth, and Haviland (2005) separate fit, parameter interpretation, and practical measurement. Report family-wise fit and uncertainty separately from route accuracy and latency, and fail closed when identification is absent. +- Kim and Chung (2019) examine expanded test use using item DIF and subgroup + linking invariance. Their methods distinguish item-level behavior from + score-level comparability. Read scope: abstract and printed pages 1–8, not a + complete review of results. The [implementation proposal](../doctoring/autonomous_kpi_runbook.md#expanded-population-validity-proposal) + is an engineering inference, not a demonstrated LLM-routing result. + +## Batch execution / load balancing The external `pg-llm-batch` service carries its own grounding papers, including PagedAttention / vLLM (2309.06180) and DeepSpeed-FastGen (2401.08671), which @@ -114,9 +266,9 @@ motivate throughput-oriented **batched** inference and the load-balancing that makes the latency-tolerant batch route economical. Those sources are referenced but not vendored here so this repository remains one deployable control plane. -> Citations are provided for scholarly attribution. Redistribution here relies -> on the arXiv non-exclusive distribution license each author granted; no -> GPL/AGPL-licensed material is vendored anywhere in this repository. +> Citations provide attribution, not redistribution permission. Apply the +> version-specific inventory above before including PDFs in a release; the +> arXiv non-exclusive license alone does not authorize this repository's reuse. ## APA 7th edition references @@ -151,3 +303,76 @@ Reise, S. P., Ainsworth, A. T., & Haviland, M. G. (2005). Item response theory: Fundamentals, applications, and promise in psychological research. *Current Opinion in Psychiatry, 18*(5), 611–616. https://doi.org/10.1097/01.yco.0000170421.57227.9b + +Kim, S., & Chung, S. (2019). *Psychometric evidence to assess expanded test +use* (Research Memorandum No. RM-19-07). Educational Testing Service. +https://www.ets.org/Media/Research/pdf/RM-19-07.pdf + +The report states all rights reserved. Link and citation only; no PDF vendored. +## Conditional-independence diagnostics + +van der Linden, W. J., & Glas, C. A. W. (2010). Statistical tests of +conditional independence between responses and/or response times on test +items. *Psychometrika, 75*(1), 120–139. +https://doi.org/10.1007/s11336-009-9129-9 + +Read status: publisher metadata and abstract verified; original PDF text on +printed pages 120–123 inspected. The model distinguishes latent speed from +observed response time and develops conditional-independence diagnostics. +This is not a complete methods review or a reproduced implementation. +Publisher HTML was visually inspected, but PDF rendering was blank; equations +and tables remain visually unverified. The publisher identifies CC BY-NC 2.0; +cite and link only, with no commercial PDF redistribution assumed. + +Engineering inference: numerical adoption belongs to fast-mlsirm and requires +identified person/item roles and dependence diagnostics. CO's routing-decision +clock cannot be substituted for a person's item response time. This source +does not demonstrate an LLM accuracy or latency gain. + +## DOI discovery register + +The links below are explicit citations found in tracked source and documentation, +not newly reviewed papers or verified implementation evidence. Some identify +standards rather than papers. Existing citation discussions remain authoritative; +this register prevents DOI-only sources from escaping the discovery inventory. +Case and sentence-final punctuation are normalized by the inventory test. + +- [DOI 10.1007/s11336-006-1478-z](https://doi.org/10.1007/s11336-006-1478-z) +- [Bolsinova and Tijmstra (2019)](https://doi.org/10.1007/s11336-019-09682-5) — [bounded abstract review and outcome-leakage constraint](../doctoring/irt_router_measurement_review.md#conditional-dependence-follow-up). +- [Bolsinova and Molenaar (2018)](https://doi.org/10.3389/fpsyg.2018.01525) — [bounded nonlinear-dependence intake and proposed leakage-safe evaluation](../doctoring/lart_measurement_review.md#nonlinear-dependence-external-psychometrics-intake); no LLM transfer or replication claim. +- [DOI 10.1007/s11336-021-09762-5](https://doi.org/10.1007/s11336-021-09762-5) +- [DOI 10.1017/psy.2025.5](https://doi.org/10.1017/psy.2025.5) +- [DOI 10.1037/0003-066X.50.9.741](https://doi.org/10.1037/0003-066X.50.9.741) +- [DOI 10.1093/biomet/39.3-4.324](https://doi.org/10.1093/biomet/39.3-4.324) +- [DOI 10.1097/01.yco.0000170421.57227.9b](https://doi.org/10.1097/01.yco.0000170421.57227.9b) +- [DOI 10.1109/IAS.2007.29](https://doi.org/10.1109/IAS.2007.29) +- [DOI 10.1111/rssc.12569](https://doi.org/10.1111/rssc.12569) +- [DOI 10.1145/2043556.2043566](https://doi.org/10.1145/2043556.2043566) +- [DOI 10.1145/2080.357392](https://doi.org/10.1145/2080.357392) +- [DOI 10.1145/2408776.2408794](https://doi.org/10.1145/2408776.2408794) +- [DOI 10.1145/362384.362685](https://doi.org/10.1145/362384.362685) +- [DOI 10.1145/38713.38742](https://doi.org/10.1145/38713.38742) +- [DOI 10.1145/52325.52356](https://doi.org/10.1145/52325.52356) +- [DOI 10.1287/opre.2016.1582](https://doi.org/10.1287/opre.2016.1582) +- [RFC 4193](https://doi.org/10.17487/RFC4193) +- [RFC 6265](https://doi.org/10.17487/RFC6265) +- [RFC 6598](https://doi.org/10.17487/RFC6598) +- [RFC 9110](https://doi.org/10.17487/RFC9110) +- [DOI 10.18653/v1/2020.emnlp-main.550](https://doi.org/10.18653/v1/2020.emnlp-main.550) +- [DOI 10.18653/v1/2025.acl-long.761](https://doi.org/10.18653/v1/2025.acl-long.761) +- [arXiv 2211.09110 DOI](https://doi.org/10.48550/arXiv.2211.09110) +- [arXiv 2305.05176 DOI](https://doi.org/10.48550/arXiv.2305.05176) +- [arXiv 2403.04132 DOI](https://doi.org/10.48550/arXiv.2403.04132) +- [arXiv 2404.14618 DOI](https://doi.org/10.48550/arXiv.2404.14618) +- [arXiv 2406.18665 DOI](https://doi.org/10.48550/arXiv.2406.18665) +- [arXiv 2512.04388 DOI](https://doi.org/10.48550/arXiv.2512.04388) +- [arXiv 2512.04695 DOI](https://doi.org/10.48550/arXiv.2512.04695) +- [arXiv 2601.17814 DOI](https://doi.org/10.48550/arXiv.2601.17814) +- [NIST AI 100-1](https://doi.org/10.6028/NIST.AI.100-1) +- [NIST AI 600-1](https://doi.org/10.6028/NIST.AI.600-1) +- [NIST SP 800-204](https://doi.org/10.6028/NIST.SP.800-204) +- [NIST SP 800-207](https://doi.org/10.6028/NIST.SP.800-207) +- [NIST SP 800-53r5](https://doi.org/10.6028/NIST.SP.800-53r5) +- [NIST SP 800-57pt1r5](https://doi.org/10.6028/NIST.SP.800-57pt1r5) +- [NIST SP 800-63b](https://doi.org/10.6028/NIST.SP.800-63b) +- [NIST SP 800-92](https://doi.org/10.6028/NIST.SP.800-92) diff --git a/docs/papers/bolsinova_2017_conditional_dependence.pdf b/docs/papers/bolsinova_2017_conditional_dependence.pdf new file mode 100644 index 000000000..cdb4aa0ae Binary files /dev/null and b/docs/papers/bolsinova_2017_conditional_dependence.pdf differ diff --git a/docs/papers/stored_pdf_sha256.txt b/docs/papers/stored_pdf_sha256.txt new file mode 100644 index 000000000..e1b4b193c --- /dev/null +++ b/docs/papers/stored_pdf_sha256.txt @@ -0,0 +1,5 @@ +035ae8b90333dad8b7817fc8f55e7c4cbca435368c5c1a4dbf7bba9e5db87473 docs/papers/frugalgpt-cost-2305.05176.pdf +b683da872e369882f0b8ebcb7423058e1317aa9571f8187ccd1c6148e3ac3992 docs/papers/fuzzing-art-science-engineering-manes-2019.pdf +30970bda8958a1a8adf6a0a82feef9728331840288b7d2b49368bac87fd25f18 docs/papers/helm-holistic-evaluation-2211.09110.pdf +c0ebc21073112b8f8e51c7ea279eeac4d5b62be88c69a5cd2bccc2d58ec28cbb docs/papers/hybrid-llm-query-routing-2404.14618.pdf +c9bc9c8171cab95bb3832cde8767c6b5e0925cd62930e51ddbd60d7cb2616741 docs/papers/routellm-routing-2406.18665.pdf diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 7fdbb9e40..ebd2cfa56 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1,5 +1,566 @@ # Contextual Orchestrator: Product & Technical Gap Baseline +## 2026-09-13 request-decision export candidate + +[PR #1158](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1158) +implements request-scoped milliseconds on export owner #1138 at base +`1881ef06ed90ee72eb7209434db366c851cd68dc`. Source +`45cc666f9fd52aedf6484b345f30857d7f9d72bf` also repairs admission metadata +incorrectly supplying missing final acknowledgement or selection evidence. +This preserves the intent of #1125 without claiming its per-step trace contract +fully inherited; neither predecessor is closed. No new timer or routing policy +is introduced. Focused strict contracts passed 69 tests; default full regression +passed 3,764 with 2 skipped. Isolated noneditable macOS ARM64/Python 3.12 wheels +passed the same 69 strict contracts. These are controlled test results, not an +observed correctness cohort or decision-latency improvement. + +Candidate head `0a2626867c0baa6a95ad40f3f00e40008359cca2` has a real +[manual Security run](https://github.com/ContextualWisdomLab/contextual-orchestrator/actions/runs/34708859235) +whose three jobs were observed running, not completed. Draft PR automatic jobs +were skipped by their explicit Draft condition; that is not secret absence. +Protected review, merge-result verification, publication, ingress-denominator +reconciliation and observed KPI comparison remain open. Actual GitHub +screenshots covered the two changed production files and both changed test +files at 1265 x 712 English without observed clipping or overlap; this is not +the complete product UI/locale matrix. The default full-test log's isolated +request-log fragment remains unresolved: strict standalone telemetry tests +passed 50 in 3.26s without reproducing it. Detailed receipts remain in the +[candidate runbook](https://github.com/ContextualWisdomLab/contextual-orchestrator/blob/0a2626867c0baa6a95ad40f3f00e40008359cca2/docs/doctoring/request_outcome_export_validation.md). + +## 2026-09-13 residual diagnostic acceptance gap + +Research follow-up `e9d51ac7` identifies an owner-contract requirement: declare +whether uncertainty in estimated response-time residuals is propagated or held +fixed, and fit preprocessing within training partitions. Posterior-predictive +model checks do not establish observed accuracy or decision latency. The +[primary-method intake](doctoring/lart_measurement_review.md#nonparametric-diagnostic-follow-up-2026-09-13) +records the read scope and failed PDF rendering; no estimator or production +policy is changed. Owner implementation and observed evaluation remain open. + +## 2026-09-12 Title-only psychometric citation gap + +Research parent `14a6a943` cited Fox and Glas (2001) without an identifier, +so passing identifier checks missed its absence from the paper inventory. +The verified DOI and APA entry now connect that lead to the existing guard. +The [reproduction record](doctoring/autonomous_kpi_runbook.md#title-only-citation-reconciliation-2026-09-12) +records RED and six passing citation contracts. This closes one known discovery +omission, not complete paper coverage. Full-method review, owner implementation, +observed accuracy/latency improvement and protected delivery remain unverified. + +## 2026-09-12 Nonlinear response-process evidence gap + +Follow-up classification repair: RED `c8b358ad` reproduced a dry-run report +being labelled for production-candidate review (1 failed, 1.87s). Runtime +`383b4a1e` marks assembled dry-run reports `synthetic_diagnostic_only` and +`benchmark_smoke_only`, preserving live sufficiency logic and the null routing +recommendation. Both benchmark test suites passed: 126 tests in 8.65s, +session 70083, exit 0. Independent read-only tracing found the misleading +classification in published JSON/Markdown but no inspected deployment consumer +that uses it to activate policy. This repairs evidence classification, not a +demonstrated promotion bypass or measured accuracy/latency gain. Full suite, +installed artifacts, rendered output and hosted review remain unverified for +this repair; earlier research-branch runtime acceptance does not cover it. + +Evaluation readiness probe at `4359401f` ran the documented NIM command with +`--dry-run --pricing-scenario examples/nim_pricing_scenario.json` and isolated +output `/tmp/co-kpi-readiness-20260912.r1sYeS` (session 75417, exit 0). +It completed 589 simulated requests, with 30 paired tasks and no production +recommendation. All policy scores were zero and reported latency was the +fixed 1 ms test value: neither is an observed KPI baseline. The generated +report labels `decision_use=production_candidate_review` despite dry-run mode; +admission semantics require audit before this label can support any decision. +The readiness command proves harness execution only. No provider egress, +observed cohort, independent adjudication, initial-decision p95, or production +promotion was verified by this probe. + +Research baseline `60ee94c2e941d2883aca623867138798e2ba9bc9`, PR #1107: +[external psychometrics intake](doctoring/lart_measurement_review.md#nonlinear-dependence-external-psychometrics-intake) +adds a diagnostic alternative to monotonic token-length assumptions. Proposed +owner work belongs in fast-mlsirm; CO consumes a released calibration contract. +Keep post-response observations out of the same request's initial routing +features, and separate decision latency from generation and queueing time. +No estimator, production default, observed KPI, or protected-main delivery is +established by this literature addition. Remaining work includes full method +audit, lawful observed data, held-out diagnostic comparison and frozen-policy +accuracy/decision-latency measurement. Rendering of this addition is unverified. + +## 2026-09-09 Optimizer score-domain repair finding + +Independent exact-source probing at +`204e306c046797b812589b9b66062296beaa1c8d` found that the shared +`_score_config` boundary accepts nonfinite and out-of-range callback scores +despite its public `[0,1]` contract. In both serial and batch mock paths, +NaN, infinity, and 1.1 could recommend an invalid candidate over a valid 0.9 +candidate. The 0.44-second probe used AST-extracted functions, not installed +package or real-provider execution. Subsequent public optimize/evolve RED +at `98153df6` produced 20 failed invalid-score cases and 20 passed compatibility +cases. Shared repair `db700768` rejects each invalid score before aggregation. +Frozen review checkpoint `22246762e3ce0b7e8624d457d8905835565d6a5a` +passed 86 focused tests in 3.04 seconds, including invalid observations whose +mean is valid. Preserve valid +fractional scores; reject invalid evidence rather than clamping or omitting it. +This protects recommendation integrity, not a measured customer accuracy gain. + +Isolated wheel execution `22123` passed 48 public score-domain tests in +0.93 seconds outside the checkout, using Python `-I`. Root independently +confirmed the installed import and wheel SHA-256 +`8dea451f3722dc91b3f4e9c10bfc9b55ab3372e39a2f49aeb5a027d67fce28f3`. +The installed environment resolves declared dependencies; the separate live +full suite `78368` uses the frozen project lock. Those environments are not +claimed identical. Full-suite, hosted acceptance, protected merge, and release +remain pending. The guard does not claim early provider-call cancellation or +recovery of spend already incurred before score validation. + +The separate provider-truncation hypothesis was rejected: `batch_route` returns +the ordered input cardinality or raises, and existing missing/content tests +preserve incurred spend. Merged PR #961 owns that earlier provider repair; +a short-list test double alone would not establish an actual provider defect. + +## 2026-09-09 Batch recovery PR delivery + +[PR #1115](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1115) +is Ready/Open at `73404f89d7a90f7a6b98caf514af859b02a375dd`, based on +#1113 `af8d732e6cfc9c0169ac850f875f42f1db7eecd4`, delivered without force. +Full-suite execution `82637` completed with **3,491 passed, 2 skipped in +757.78 seconds**, supplementing the separate **69 installed-package tests**. +Root verified the live head/base and empty review inventory. Hosted Security +run **34333545448** completed all three jobs successfully on tested merge +`138fa7aca0554d5695137a1cfa6b74a48b26715d`. Linux/Python 3.12.14 hosted +full suite: **3,491 passed, 2 skipped in 785.64 seconds**; benchmark/docstring +checks: **134 passed in 10.85 seconds**; installed wheel checks: **40 passed +in 15.85 seconds**. Hosted core wheel SHA-256: +`4c34fdc911270ab07297fdd7bbf782e8ee4368547e879f288cf9c3805ed3798e`. +Current PR head/base still match the above revisions, Ready/Open with no reviews. +No independent approval, +protected merge, release, remote-provider integration, or observed KPI gain is +established. The untracked local native extension was not committed. + +## 2026-09-09 Request-outcome export gap + +[Issue #1114](https://github.com/ContextualWisdomLab/contextual-orchestrator/issues/1114) +records an installed-wheel HTTP probe at frozen `73404f89`: batch submission +returned 201 with durable lineage and one committed association, while the +200 analytics snapshot omitted the submitted batch job ID. Execution `83174` +completed successfully as an observation probe, not a passing assertion that +the proposed export exists. Its script is +`/tmp/co-batch-export-probe.s0ytig/probe_export.py` on the validation host. +The correct `measurement_complete=false` remains unchanged. + +Independent source review confirms associations currently support internal +recovery, not an operator outcome-link export. The next delta is a bounded, +prompt-free, purpose-authorized join preserving many-to-one links, cache +provenance and unmatched/failed admissions. Existing admissions are not +owner-filtered; do not expose a global join through owner-scoped replay or +infer ownership from request IDs. Recovery descriptors and private payloads +must stay excluded. This operational capability is needed before collecting +the requested observed KPI cohort; it does not supply adjudicated outcomes. + +Successor `codex/request-outcome-export-20260909`, based on `73404f89`, +established actual HTTP RED at `3e14dfbbb87e8cc22a98ddd9e6ebb8ed576c0021`: +**1 failed in 5.37 seconds**, terminal execution `24161`. After successful +workflow/batch requests, an invalid admission, and SQLite restart, the proposed +admin export returned 404; an inference principal was denied with 401. +This is not installed-successor evidence. Design review found keyed workflow +replacement deletes prior versions, so a high-water query alone cannot promise +historical reproducibility. The successor must retain the necessary prompt-free +association revisions in the same journal transaction or otherwise prove the +claimed snapshot semantics; missing legacy history cannot be fabricated. + +Successor checkpoint `f0304d7bab409823ab17f8b0d3f08f69701441d0` now has +**68 focused tests passed in 23.08 seconds** (execution `55267`) and a clean +bounded independent source review of persistence/privacy. Root review found a +separate HTTP parsing gap: default query parsing discards empty values before +unknown/duplicate validation. Blank-query RED/fix is pending after the live +frozen full-suite execution `50159`; no source or environment mutation during +that execution is authorized by this receipt. Proposed ADR 0131 and the +runbook record the service-wide admin boundary; no independent purpose-claim +verification is supplied by the existing external verifier. + +Direct browser inspection of the temporary local runbook first viewport +(`1265×712`, English) found its heading and PRD text legible. The ADR preview +(`1129×1022`, English) exposed a renderer defect: YAML frontmatter became +merged prose/list content. This is being repaired in the temporary renderer, +not hidden by changing the source ADR. Whole-document and rendered UML +acceptance remain pending. Neither inspection establishes deployed UI quality. + +## 2026-09-09 Installed batch recovery validation + +Frozen `73404f89d7a90f7a6b98caf514af859b02a375dd` produced separately built +core and native wheels. Isolated installed-package execution `31999` completed +**69 batch-lineage and decision-receipt tests in 44.46 seconds**, exit 0, +on macOS arm64/Python 3.14.6. Root independently verified outside-checkout +imports resolve under `/private/tmp/co-batch-installed-73404f89.cX4GMN/venv` +for both the core package and native extension. No editable source import is +used in this receipt. Core SHA-256: +`d51ea2844064a5f5674791c6ef7789a0a277d1eae43fe49d506f4ccaa120a18c`; +native SHA-256: +`fb0a88ff477f422d496720d551f051a06172c848d2a6bc95d7caaa5e95bc87ea`. + +At this installed-package checkpoint the separate full-suite execution `82637` +was live; it subsequently passed as recorded in the delivery section above. +Installed focused success alone does +not prove full-suite or hosted success, real remote integration, review +approval, protected merge, publishing, or customer KPI improvement. + +## 2026-09-09 Hosted outcome-link acceptance receipt + +PR #1113 head `af8d732e6cfc9c0169ac850f875f42f1db7eecd4`, based on +`c7345670e08f029ad3aa5dd1133037bb4b451d9b`, completed repository Security +run **34329594602** successfully. Its tested merge was +`e127f7a0aef94949a4a8f3eb16371e155f370fb1`, not a protected-main merge. +Linux CPython **3.12.14** tests/package job **102394807283** reports +**3,462 passed, 2 skipped in 756.10 seconds**, benchmark/public-docstring +checks **134 passed in 10.00 seconds**, and installed-wheel checks +**40 passed in 15.72 seconds**. The built core wheel SHA-256 is +`e705cec46453123eae92b0c7979b6bde13a479afb5e021978c2defe849e995d8`. +CodeQL/supply-chain/SBOM job **102394807100** and fuzz job **102394807264** +also completed successfully. Evidence: [terminal run and job logs](https://github.com/ContextualWisdomLab/contextual-orchestrator/actions/runs/34329594602). + +The current-head review inventory remains empty. These hosted results supplement +the separately recorded local installed-wheel request/outcome tests; they do not +replace independent approval, protected merge, publisher acceptance, or measured +customer accuracy/latency. The unpushed batch recovery successor is a different +revision and does not inherit this test result. + +## 2026-09-09 Batch restart recovery checkpoint + +The isolated batch successor, based on #1113, retains the failed restart +experiment `d71bcc0a`: **1 failed in 1.96 seconds**. Remote submission succeeded, +but after registry failure and restart the authenticated owner received 404; +another owner correctly received 404 without a remote download. Runtime +`853e860946d2d55a06d824e042057806c3ad79d8` then passed **73 focused tests in +21.17 seconds** (terminal execution `86930`). Documentation checkpoint +`b4abdd660b0ee44fd1a5b7a8b8c7a7ad5d1e1f92` remains unpushed. + +The candidate reuses an indexed durable submission event for an owner-bound, +expiring, prompt-free recovery descriptor. Focused cases cover target mismatch, +malformed descriptors, unexpected result IDs, missing usage, and no resubmission. +This is not acceptance: independent review and additional RED cases must check +registry reads/writes that remain unavailable during recovery, and consistency +between submission-envelope item IDs and restored descriptor IDs. Full-suite, +installed-package, hosted, protected-merge, and release evidence remain absent +for this candidate. Simultaneous durable-store and registry failure cannot be +reported as recoverable. No observed customer KPI improvement is established. + +Follow-up checkpoint `e372bc542f8fbec8d047a9affae15c018698d48a` +reproduced **3 failures in 3.59 seconds**: continuing registry outage, inconsistent +submission/item identities, and inconsistent estimate keys. At +`7fb1a71ce1b4c14d1ba29e12501ba22fb589cca9`, **19 tests passed in 14.94 seconds** +(terminal execution `37121`). Recovery metadata now travels with the authorized +request instead of requiring another registry write. Independent source review +found no additional silent usage/model-attribution defect in this diff: absent +prompt estimates do not fall through to estimating an empty reconstructed prompt. +That review does not cover all existing attribution behavior. Remaining checks +include coordinator-registry hits with missing backend metadata, malformed job +field types, and a changed deployment using the same backend alias. A backend +alias or API path alone cannot establish service/account identity. + +At `aaa9b133`, the focused suite reports **82 passed in 31.21 seconds** +(terminal execution `6240`), and independent source review clears the preceding +typed-identity and partial-registry findings within its inspected scope. Root +end-to-end review nevertheless identified two remaining paths before full-suite +acceptance: the Pg adapter writes its own registry after remote acceptance but +before returning the handle to the coordinator, and healthy registry reads +refresh retention whereas recovery descriptors use a fixed expiry. Reproduce +backend-registry submission failure without losing the accepted handle; also +ensure an expired recovery descriptor does not invalidate otherwise healthy, +authorized, complete registry state. Expired recovery with missing registry +state must remain denied. These findings supersede any bounded recommendation +to freeze the candidate for full-suite verification. + +Those paths were reproduced independently: backend submission checkpoint +`17cfa611` failed once in **0.95 seconds**, and its `8ddfeb9f` repair passed +the focused case in **1.79 seconds**. Healthy-expired checkpoint `df638d6c` +failed once in **1.51 seconds**; `a6b94855` then passed **84 tests in 32.63 +seconds**. Review found its healthy fast path skipped the new deployment +binding. Checkpoint `08a660dc` reproduced that regression (**1 failed in 1.41 +seconds**); `cd38d9c4811deb6e9fde9c0c11a78869d9f39dcf` passed **85 tests in +34.48 seconds** after persisting and checking the binding. Full-suite acceptance +is still deferred: healthy metadata must also preserve the endpoint comparison +already required by descriptor recovery. This is a demonstrated contract +inconsistency, not demonstrated cross-service data disclosure. Legacy unbound +jobs must have an explicit compatibility test; they cannot count as validated +deployment-bound recovery. + +Endpoint checkpoint `c06615ab` reproduced the healthy-path mismatch (**1 failed +in 2.09 seconds**). Runtime `450667593285679c92d1d0a698f35eadb2b2c879` +passed **86 focused tests in 36.58 seconds** (terminal execution `63908`). +Independent read-only review confirms endpoint equality now applies to new +bound metadata; missing endpoints remain compatible only for explicitly unbound +legacy records. Documentation head `73404f89d7a90f7a6b98caf514af859b02a375dd` +is frozen for full-suite and separate installed-package verification. Neither +has a completed result yet; this checkpoint is not hosted or released evidence. + +Visual receipt: the GitHub-rendered document at +`aee00ac9da1e7f17ddfaec4ad3ebbafc06dee01f` was opened in the actual browser, +and its screenshot directly inspected at **1265 × 712**, English. The title, +recovery heading, first paragraph, and full revision strings were readable +without overlap or horizontal clipping in that viewport. The screenshot is +inline in the validation task. Lower sections and other viewports/locales were +not inspected; this does not constitute product UI acceptance. +## 2026-09-09 State persistence integrity prerequisite + +PR [#1108](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1108) +repairs a reproduced failed-replacement data-loss case in the common SQLite +state writer. Code `d7bba88f3d711883a37effe49ab4f503c4fb8e01` rolls back failed +writes under the existing lock. Test follow-up `f1abe1e3` checks that no +transaction remains immediately after failure and that both the previous and +unrelated subsequent records survive reopening the database. The persistence +suite passed 19 tests in 9.28s; this is not a latency or customer-accuracy result. + +Full local suite at `877d5112ed470d851afaa2c746b94393cc768ee7`: 3,396 passed, +2 skipped, exit 0 (883.03s). Test-only follow-up +`716e012dcb50857000b0fc53c89c6434fdf7e7c2` covers a deferred commit failure +with real SQLite constraints; persistence, workflow authorization, and governance +tests pass together (29 passed, 4.89s). Full-suite evidence remains attached to +the earlier head, not silently reassigned to the new regression. + +At the earlier PR head `aa674187b0341c7852f85c27fb696aec21f1a799`, GitHub +reported zero check runs and two success statuses whose descriptions explicitly +said reviews were skipped (Draft; expired trial/no credits). Those statuses do +not establish review approval or security validation. Keep protected merge and +release pending actual exact-head evidence. The root cause and reproduction are +in the [canonical runbook](doctoring/autonomous_kpi_runbook.md). + +## 2026-09-09 Stacked quality-trigger repair + +Correction: PR #1066 at `59a8f4eadfe0e0dcc5ff47cf1acfb80403e241ad` already +owns this repair and its Ready/closed admission checks. The partial local repair +described below missed that lineage. Its full branch was integrated at `d721e04b` +without force, with the extra filter/permission assertions consolidated into +the owner's `tests/test_repository_security_metadata.py`. The duplicate test +file is removed after preserving those assertions. #1066 and #1060 stay open; +integration is not protected delivery. The #1108 hosted run +`34316962950` at `c11df645` completed with **1 failed, 3,399 passed, 2 skipped +in 737.97 seconds**. The single failure was the existing metadata assertion +for the old concurrency key, omitted by the partial repair. Fuzz and +CodeQL/supply-chain/SBOM jobs passed. This was a contract-update omission, +not a flaky test. Full #1066 inheritance repairs that assertion. + +The non-force integrated #1108 head `129a665016ed1acd79ae12915c905b1020856fcc` +passed **42 metadata, NIM workflow, and persistence tests in 4.19 seconds**; +actionlint and diff-check also passed. New hosted run `34318012080` was queued +at the latest observation. Neither the historical full-suite result nor focused +success proves current-head hosted completion, independent approval, or release. + +At `035b58c252cd4f4a79e712d028e8265264326c94`, the repository-owned +Security and Quality workflow filters pull requests to `main`. PR #1108 targets +another PR branch, so its zero check-run count is consistent with this trigger +exclusion, not a successful Security run. The repair removes the base filter +without changing job permissions or switching to privileged `pull_request_target`. +It also keys cancellation by workflow, repository, and PR number. Central review +and security ownership is unchanged; this does not repair or replace their gates. + +The regression assertion failed on the old filter. After repair, the new contract +and existing benchmark workflow contracts passed (9 tests), and actionlint emitted +no findings. An initial test collection failed because PyYAML is not installed; +the test instead uses the existing stdlib text-contract pattern, with actionlint +checking YAML syntax. No dependency was added. A new synchronize event must still +demonstrate hosted execution on the actual stacked merge revision. Trigger syntax +and local tests alone are not that execution evidence. GitHub documents that PR +branch filters match the [target branch](https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/trigger-a-workflow). + +## 2026-09-09 Integrated rollback regression receipt + +PR #1108 at `fbb933cbcaa1f1695c6cc305657f450f22b3be4c` includes the +non-force base integration and transaction rollback repair. Its clean, pinned +worktree completed the full local suite: **3,399 passed, 2 skipped in 823.46 +seconds**, exit 0. The focused persistence, workflow-object authorization, and +governance checks separately passed **30 tests in 10.06 seconds**. The full log +is `/tmp/co-rollback-integrated-fbb933cb.log` on the validation host; it is a +local receipt, not a hosted artifact or release attestation. At the subsequent +GitHub check, this exact head had **zero check runs and zero reviews**. Status +contexts alone do not satisfy the Security job or independent review gates; +protected merge and release remain unverified. + +The analytics specification at `ddf087da136dbb5f2110aba6db20dd9bd384db7c` +was opened in the actual browser and its screenshots directly inspected at +1265 × 712, English locale. The opening context and autonomous-target table +were readable; the table's three columns and three rows had no overlap or +horizontal clipping in that view. Images are inline in the validation task, +linked by [the PR inspection receipt](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1107#issuecomment-5596505419). +This is bounded document evidence, not responsive, multilingual, Figma, or +product-interaction verification. No customer accuracy or latency gain follows +from either receipt. + +## 2026-09-09 Decision-latency durable acknowledgement gap + +[Issue #1110](https://github.com/ContextualWisdomLab/contextual-orchestrator/issues/1110) +tracks the accepted-request-to-durable-decision interval required by +[the analytics specification](analytics_spec.md). At +`b2c09930a8d335952e4f4d8de5371460cb14fcd6`, `route_once` still times the upstream +invocation, not that interval. `conduct` persists a completed workflow after +generation. Neither is a measured initial-decision receipt. + +The existing state store has synchronous `durable=True` writes and best-effort +queued stream writes. Reuse that owner; a successful queue operation, a later +flush-on-read, or an audit append without configured storage is not a durable +acknowledgement. Three retention tests now explicitly exercise the synchronous +path. An additional test prevents background draining and verifies committed +records through an independent SQLite connection before any store read. +All 19 persistence tests passed in 7.19 seconds. An in-memory mutation forcing +asynchronous writes fails the new assertion. This proves commit visibility in +the tested SQLite configuration, not power-loss survival or customer latency. + +Next implementation must connect validated admission, selection completion and +successful commit acknowledgement within one monotonic clock domain; distinguish +initial/failover decisions and retain failed/unfinished requests in accounting. +PR #1108 owns transaction rollback repair and remains open; its valid delta must +be retained during integration. No measured p95 or accuracy improvement is yet +established, and no production routing default changes follow from these tests. + +## 2026-09-09 LSIRM model-selection source reconciliation + +At source `1b31166024a20b5ab5be6547a0cfd95a7099a7f1`, direct visual inspection +of the NSF-hosted Springer-formatted LSIRM PDF confirmed the same mixture/event +description discrepancy previously found in arXiv v2. The +[version-specific receipt](doctoring/measured-routing-evidence.md#lsirm-identification-and-version-discrepancy-2026-09-09) +records the PDF hash and inspected page. Before publishing a model-selection +contract, fast-mlsirm must reconcile the decision event and prior parameterization +against the current publisher copy and supplementary implementation. CO must +not copy the ambiguous threshold into routing or count this investigation as +an accuracy improvement. No current fast-mlsirm implementation defect has been +demonstrated by this source comparison. + +## 2026-09-09 Expanded-population validity research gap + +At source `479bfe7e096832e1711c4d99b59621a66c3a2f59`, the research inventory +was extended with ETS RM-19-07 and a bounded read receipt. The +[proposed evaluation design](doctoring/autonomous_kpi_runbook.md#expanded-population-validity-proposal) +separates item DIF, scale linking, and observed routing accuracy. CO owns +evaluation context and outcome collection; fast-mlsirm remains the numerical +owner. Released diagnostic-contract availability, observed-data support by +language/domain/model revision, and any accuracy or latency gain remain +unverified. This is a research proposal, not a production gate already shipped. + +## 2026-09-09 Benchmark prior interpretation gap + +At `ef374defdf4037a829d4b4d6d61c5a8b314e8c05`, +`contextual_orchestrator/benchmark_priors.py` labels an equal-weight composite +of median/MAD-normalized Arena and Quality Index scores as a measured success +probability. Inspection of the cited Chatbot Arena v1 metadata and sections 3–4 +does not justify that cross-instrument calibration. The docstrings now identify +the legacy heuristic and correct the paper's author list; runtime behavior is +unchanged. Outstanding work: authenticate archived rating snapshots, define +the target outcome and model-version mapping, fit/calibrate against observed +held-out tasks in the numerical owner, and validate accuracy/decision latency +before protected adoption. A bounded score and constant prior mass do not prove +measurement validity. No customer KPI gain is claimed. + +## 2026-09-09 Paper inventory consistency + +`34bf2f3f5925a84630edfccaf608e06f5e3192ae` removes a stale blanket +redistribution assurance that contradicted the version-specific license audit. +All five stored PDF fingerprints verify, but publisher-byte equivalence and +additional redistribution rights remain unverified. Browser inspection of this +revision could not proceed because the Mac was locked; no visual pass is claimed. +See the [runbook evidence](doctoring/autonomous_kpi_runbook.md). + +## 2026-09-09 Numerical candidate and rendered KPI evidence + +The [autonomous KPI runbook](doctoring/autonomous_kpi_runbook.md) records a +completed owner baseline scoring-filtered run (41 passed, 1 ignored), finite-case +bitwise cross-version parity, and an unfavorable candidate unit p95. The candidate +remains experimental: candidate scoring-filtered tests now pass (42 passed, +1 ignored), but full-suite tests, observed customer outcomes, durable +decision timing, protected release, and consumer adoption are not established. +The exact-revision KPI table received a partial desktop screenshot inspection; +this does not complete responsive, multilingual, or product UI verification. + +The live REST PR inventory still includes correction PR #1043 at +`05fe7178c12f37206458e256e468f498e9a5ce20`. Its proposed correction supersedes +the historical blanket `_invoke` deadline recommendation below in intent, but +is not merged evidence. Current policy remains default-null model timeouts; +do not implement the historical overall-cap recommendation. Keep #1043's delta +in its owner PR rather than duplicating its patch in this research branch. + +## 2026-09-09 Autoresearch loop: KPI baseline, #1075 closure evidence, main-protection blocker + +Loop branch `autoresearch/20260909-kpi-loop` (code+tests identical to +`origin/main@414f2297`; branch diff is docs-only). KPI baseline: **85 open +PRs** (`gh pr list --state open`, lower-is-better toward 0 via merge or +verified-successor full-delta inheritance only; no force-push, no close +without evidence). + +- **Issue #1075 (nim_benchmark 100% branch coverage): gap already closed.** + The issue's exact gate block passes on unmodified code: 134 passed, + `nim_benchmark.py 1205 stmts / 436 branches / 0 miss / 0 partial = 100%` + (coverage 7.15.0, `--fail-under=100` exit 0), no `pragma: no cover` in the + module. Former gaps `434`, `645`, branch `671->682` are covered by + `EqualBudgetModelClient` + `_BudgetDelegate` cases (including `usage=None`) + in `tests/test_nim_benchmark_release_acceptance.py` that landed after filing + (e.g. `7ba5fefc`, `a080297d`). Evidence comment posted on #1075; issue left + open for owner close. No code change (experiment 1: discard, unit evidence). +- **Issue #1079 (main branch protection): owner-action blocker confirmed live.** + Protection still requires stale `Hypothesis property tests`, + `Atheris coverage-guided`, `CodeQL analysis`, `Python supply chain` + (API-verified); every post-#1054 PR to `main` is unmergeable via protection + (e.g. #1094 `MERGEABLE` but `BLOCKED`). Not bypassed; documented for owner. +- **PR #1094 RCA:** `opencode-review` failure is reviewer-verdict wait, not a + code defect. The wait does not stop independent work. +- **Mergeable path:** non-main-base stacked PRs (e.g. #1104→#1101: + `MERGEABLE`/`CLEAN`, core checks green) but Draft — no readiness flip + attempted (owner process boundary). + +## 2026-09-09 Autonomous KPI targets and verified unit execution + +`docs/analytics_spec.md` now sets an observed delivered-correct improvement +target of at least one percentage point and decision p95 at most 20 ms with +at least 10% reduction, subject to stated uncertainty and non-regression gates. +These are selected targets, not results. The +[single experiment runbook](doctoring/autonomous_kpi_runbook.md) records owner +boundaries, isolated contract bootstrapping, and reproducible commands. +AGENTS.md and CLAUDE.md link to it; the existing hourly automation was updated +without adding a duplicate. PR #1067 exact head +`84a6052369a7bf8b6faae5db475bb68a5ad54a91` passed all 51 focused routing and +benchmark-boundary tests in 744.99 seconds on a heavily loaded host. No customer +accuracy, decision p95, full-suite, protected merge, or deployment follows from +that unit result. + +## 2026-09-09 Existing research attribution repaired + +Audit of `279f7e03` found an incorrect Jeon et al. title/DOI in +[measured routing evidence](doctoring/measured-routing-evidence.md), plus claims +that per-member ledgers prevent atomistic fallacy and JSON structure establishes +judge reliability. The corrected citation identifies the latent-space +item–respondent interaction model; the mapping now separates ledger arithmetic, +schema validation, and unverified psychometric validity. This removes unsupported +research justification without changing runtime policy. A calibrated observed +task evaluation and released owner diagnostics remain required. + +## 2026-09-09 Response-time research and measurement gap + +The [psychometric review](doctoring/irt_router_measurement_review.md#response-time-research-extension-2026-09-09) +now traces van der Linden's hierarchical speed/accuracy framework to a visually +inspected earlier report and the final publication's DOI. It records a proposed +joint-model experiment, not evidence that fast responses imply high ability. +CO decision overhead and provider completion duration require separate clocks +and denominators. Observed-data availability, released owner support, baseline +accuracy, and routing-decision p95 remain unverified; no production default +change or measured improvement is justified by this literature extension. + +## 2026-09-09 Review gateway failure and existing timeout repair + +PR #1103 at `4776a970ed8bdef3406684aef84952740b476d88` has a terminal +Noema 502 after 926 seconds. Its gateway bootstrap used CO `414f2297`, +whose transport default is 90 seconds. Artifact attribution and the remaining +causal uncertainty are recorded in +[the incident investigation](doctoring/noema_gateway_failure_20260909.md). +The existing repair owner is PR #1053 at +`76c047585f54fcbe940fe168412f51627d3f79dd`, still Draft, with failures in all +three CodeQL compatibility analyses on inspection. Older test claims in its +description do not validate this head. Next work is repair and verification in +#1053, followed by release and consumer adoption; no runtime recovery or buyer +accuracy/decision-latency improvement is established by these observations. + +The linked central scan run `34122498232` reached status publication, where +Python job `101756437515` received HTTP 403 with both configured credential +paths. Current `opencode-agent` installation `141441800` covers all repos but +has only `statuses: read` and `actions: read`; the app registration owned by +`anomalyco` also requests only those read permissions. Central publisher +identity/permission repair is required before validating the consumer gates. +No missing-secret claim or successful-scan claim follows from this evidence. + ## 2026-09-08 Psychometrics evidence boundary: research trace added PR #1103 head `19448d95` adds APA 7 references and implementation constraints @@ -2853,3 +3414,548 @@ shows this is now occasional, not the dominant failure mode (most is an overall deadline on `_invoke`'s candidate/retry loop, not another timeout increase on the sidecar's client side — deferred rather than rushed into this heavily-tested core file without dedicated validation. + +## 2026-09-09 Autoresearch loop: autonomous KPI scope, PR #1108 verification, hourly-prompt hardening + +PRD/Goal adjustment: KPI scope was selected autonomously under +`docs/analytics_spec.md` without asking (see the runbook scope entry). +Loop metric `open_pr_count` is 87 on recount (baseline 85; growth from +concurrent sessions). PR 0 only via merge or verified-successor +full-delta inheritance; single-writer deltas are integrated, never +discarded; no force-push; close only on user instruction, no valid +delta, malicious change, or verified complete inheritance. + +- **PR #1108 (fix(persistence): roll back failed state replacements):** + valid minimal root-cause fix. `_save_sync` now runs under the writer + lock plus the SQLite connection context so a failed keyed replacement + rolls back instead of leaking its DELETE into a later unrelated commit. + Isolated-worktree evidence at head `4316be85`: + `tests/test_persistence.py` 20 passed in 32.42s, exit 0 (insert-phase + and deferred-commit-phase failures, closed-transaction checks, + reopen persistence). Unit evidence only. The PR is `dirty` against + loop HEAD `0ea2a58d` because both sides appended to this baseline + file; code auto-merges. Action: owner restacks with a normal merge + and manual docs resolution; this loop does not push to that branch. +- **Current HEAD `0ea2a58d` (`benchmark_priors.py` calibration bound):** + docstrings/comments only in effect; `tests/test_model_group.py` plus + `tests/test_benchmark_priors.py` 37 passed in 25.82s, exit 0. No + runtime, routing-default, or numerical-formula change; no customer KPI + claim. +- **Actions concurrency (reviewed, no change):** `security.yml` groups by + `local-quality`-repository-event-PR/schedule/ref with same-group + cancel only, so distinct PRs stay independent and pushes/schedules + serialize on ref/schedule; the hourly loop uses its own + `opencode-hourly-loop` group with `cancel-in-progress: false` and never + cancels merge/release/deploy/migration. Renaming groups without an ADR + would churn CI for no functional gain; left as is. +- **Hourly prompt:** `.github/opencode/ hourly-loop-prompt.md` now records + the shared-checkout, live-handle, synthetic-vs-observed, and PR-0 + rules so the next scheduled pass inherits them without re-derivation. + Follow-up: keep #1079 (main-protection stale job names) with the + owner; keep #1075 closure with the owner; re-observe #1108 after its + restack and hosted checks. + +## 2026-09-09 Autoresearch loop: PR #1109 atomicity review, no merge, prompt stacking rule + +KPI reaffirmation (no scope question asked): `open_pr_count` 88 +(baseline 85). #1109 is a new draft on the psychometric stack +(`codex/psychometric-kpi-successor` base); #1108 is still `dirty` +against the loop branch; #1094 is still protection-blocked. No PR met +the merge bar this turn (terminal-success checks plus resolved threads +plus independent exact-head approvals), so no merge, readiness flip, or +cross-session push was attempted. + +- **PR #1109 (fix(psychometrics): preserve evidence when observations + are rejected):** read-only review plus isolated verification. The + reorder validates before mutating retained vectors, order, and + revision under the existing lock; valid-input behavior is preserved + and no new Python-side numerical arithmetic is added. Isolated + evidence at head `4cc0bf2c`: + `tests/test_psychometric_observation_atomicity.py` 6 passed in + 52.15s, exit 0. Hosted checks: CodeQL success; tests and fuzzing still + in progress at observation time. Unit evidence only; full regression, + independent review, protected merge, and release remain pending. + Action: leave the draft with its owner stack; re-observe after hosted + checks complete. +- **Hourly prompt (this hour):** added the single-writer stacking rule + (integrate deltas, normal-merge restack only, never flip another + session's Draft) and the PRD/TRD case-preservation rule alongside the + existing fail-closed ordering guidance. + +## 2026-09-09 Autoresearch loop: PR #1109 Ready flip and integer-index hardening, still unmerged + +KPI reaffirmation (no scope question asked): `open_pr_count` 88 +(baseline 85). #1109 is now Ready (`draft: false`, `mergeable: true`, +`mergeable_state: unstable`); #1108 is confirmed `dirty` again; #1094 +remains protection-blocked. No PR met the merge bar (terminal-success +checks plus resolved threads plus independent exact-head approvals), so +no merge, readiness change, or cross-session push was attempted. + +- **PR #1109 new head `b8d2651d`:** the owner hardened validation from + `int(value)` to `operator.index(value)`, rejecting fractional rows, + whole-valued floats, and numeric strings that truncation previously + masked as valid dichotomous data, while keeping the integer protocol + including `numpy.int64`. Isolated evidence: 19 passed in 13.68s, exit + 0. Hosted checks: both CodeQL jobs success; tests and fuzzing still in + progress; no reviews yet. Unit evidence only; full regression, + independent review, protected merge into the owner stack, and release + remain pending. Action: re-observe after hosted checks and first + review; do not merge across the stack boundary from this loop. +- **Hourly prompt (this hour):** queue-exhausted continuation now + explicitly names gap development plus ContextualWisdomLab repository + and connector linkage under responsibility boundaries, so scheduled + passes do not idle after the PR list drains. + +## 2026-09-09 Autoresearch loop: PR #1109 fuzzing green, tests pending, failure-never-idles rule + +KPI reaffirmation (no scope question asked): `open_pr_count` 88 +(baseline 85). #1109 head unchanged (`b8d2651d`): fuzzing success is +new since last turn, tests still in progress, no reviews, still +`unstable` — the prior 19-pass isolated verification stands and no +merge was attempted. #1108 mergeability is `unknown` (recomputing); +#1094 remains protection-blocked. + +- **Hourly prompt (this hour):** a failing check never idles the loop — + fix and rerun owned failures immediately while continuing safe + independent work, and codify manual workarounds with log-grounded RCA + for PYTHONPATH, Actions, and execution errors. + +## 2026-09-09 Autoresearch loop: stacked-quality merge adopted, #1108 restack verified, #1105 pending-verdict diagnosed + +KPI reaffirmation (no scope question asked): `open_pr_count` 88 +(baseline 85). No PR met the merge bar, so no merge, readiness change, +or cross-session push was attempted. + +- **Loop merge `d721e04b` (adopted, reviewed):** the stacked-quality + repair now on this branch is compliant — exact + `{workflow}-{repository}-{PR}` concurrency with same-group PR-only + cancellation, expanded stacked-PR coverage, Draft/closed-only skips, + and test consolidation without dropped assertions (see runbook for + the clause-level verdict). Action: none; keep. +- **PR #1108 restacked head `c11df645`:** isolated evidence 21 passed + in 28.10s, exit 0 (prior 20-pass run superseded). Mergeability still + recomputing. Action: re-observe; owner restacks with normal merges. +- **PR #1109 head `b8d2651d`:** all hosted checks green, still no + reviews — awaiting independent approval on the owner stack. Action: + re-observe. +- **PR #1105 (Ready, `main` base):** 3 CodeQL-compat failures are + pending-verdict fail-closed (`DISPATCH_OUTCOME: success`, + `VERDICT_STATE: pending`, self-rerun promised), not code defects. + Action: re-observe next turn for self-heal; owner owns any real fix. +- **Hourly prompt (this hour):** never make a full foundation or mutual + official release a precondition — cut owner/consumer cycles with a + minimal contract, port, or ACL and complete independently verifiable + functionality first. + +## 2026-09-09 Autoresearch loop: #1108 loop-merge absorbed, #1105 still unhealed + +KPI reaffirmation (no scope question asked): `open_pr_count` 88 +(baseline 85). No PR met the merge bar, so no merge, readiness change, +or cross-session push was attempted. + +- **PR #1108 head `129a6650`:** owner merged the loop branch with a + normal merge (no force). The fix files are byte-identical to the + verified head, so the 21-pass evidence stands. Mergeability + recomputing. Action: re-observe for clean state, then hosted checks. +- **PR #1109 head `b8d2651d`:** still clean, still no reviews. Action: + await independent approval; re-observe. +- **PR #1105 head `b655fe1b`:** same 3 pending-verdict failures, no + self-healing rerun observed yet. Action: re-observe; owner owns any + real fix. +- **Hourly prompt (this hour):** wrong closes are recovered through + reopen or successor and never left closed (close only on the four + evidenced conditions). + +## 2026-09-09 Autoresearch loop: all PRs static, fetch transient absorbed + +KPI reaffirmation (no scope question asked): `open_pr_count` 88 +(baseline 85). No PR met the merge bar, so no merge, readiness change, +or cross-session push was attempted. + +- **PR #1109 / #1108 / #1105:** all heads unchanged; prior isolated + verifications stand (19-pass and 21-pass). #1109 clean without + reviews; #1108 mergeability recomputing without reviews; #1105 still + blocked on the same 3 pending-verdict failures. Action: re-observe + all three next turn. +- **Sync incident:** one fetch refused the remote-tracking ref update; + retry plus ancestry check plus fast-forward-only resolved it with no + rewrite. Lesson recorded in the runbook: never infer a rewrite from a + refused ref update. +- **Hourly prompt (this hour):** record merge and delete rationale + before committing; remove self-modifying or source-fix workflows + whose purpose is done. + +### PR #1108 terminal repair evidence, 2026-09-09 + +At head `129a665016ed1acd79ae12915c905b1020856fcc`, base +`2996cd3c360444b792d499f3b09a783abdd830c2`, hosted run +[34318012080](https://github.com/ContextualWisdomLab/contextual-orchestrator/actions/runs/34318012080) +completed successfully. Checkout log records merge `9516d1d` of those revisions; +tests job `102358241912` reports **3400 passed, 2 skipped, 734.73 seconds**, plus +134 package/docstring checks passed in 10.61 seconds. All four actual check runs +were successful. This supersedes the earlier metadata-assertion failure for +current-head CI only; it does not erase that failure or establish a protected +release. The Ready PR still has no reviews. Next gate: independent review and +protected stack integration, preserving the canonical #1066 workflow delta. + +Research source `7734e89c` adds the bounded Bolsinova–Tijmstra response-time +follow-up and prohibits outcome leakage in the proposed joint-model comparison. +The DOI discovery check caught its missing inventory entry (one failing test); +after linking the source, all six paper contracts passed in 6.23 seconds. +No observed-task accuracy or decision-latency improvement has been measured. + +### Decision-measurement pre-release review, 2026-09-09 + +Candidate `01ce9035715fab4ed60e7352caa85512f855e0bb` for +[issue #1110](https://github.com/ContextualWisdomLab/contextual-orchestrator/issues/1110#issuecomment-5597096146) +is not release-ready. Read-only call-site review found that server `_run` +is called inside embedding retry loops and file-replica deletion. Admission +inside that helper counts attempts as requests and starts some clocks after +selection. Repair: one validated request-owned admission, explicit endpoint and +measurement unit, separate attempts, and a first-failure/second-success regression +that retains exactly one admission. File operations and evaluation batches must +not silently become individual generation samples. + +The automatic proxy also acknowledged selection before file binding and effort +configuration could reject the request. Move acknowledgement after those checks, +immediately before transport, and prove rejection produces neither dispatch nor +a committed decision. The existing persistence store does not bound the three +new receipt kinds; all-record export is unbounded in memory. Cohort-aware export +and explicit retention remain required. A successful isolated package test does +not resolve these semantic findings. The assigned implementation agent is repairing +them; no routing default or release was changed. + +Integration checkpoint `07957ee643bf74c5beb13c03827f59331307cc5d` normally +merges #1108 head `129a6650` into the measurement branch. The implementation +agent reports terminal receipt/persistence verification: **34 passed in 5.88 +seconds**; this is focused local evidence, not full CI or package acceptance. +The checkpoint still needs per-invocation race identity, indexed phase lookup, +trusted HTTP identity integration, and remaining endpoint coverage. + +A root-run in-memory SQLite plan comparison used the checkpoint's table and +index definitions: the JSON-filtered phase query searches only by `kind`; +the proposed `(kind, key, seq)` index with `key IN (...)` searches by both +`kind` and `key`. Both plans use a temporary ordering B-tree. This validates +the proposed lookup shape only, not a measured customer latency improvement. +Keep historical records, validate migration identity, and test rollback before +adopting the index/backfill. The implementation agent owns that change. + +At committed candidate `05b512effe0045340224e5e0408ae984f5784d1e`, an +independent read-only review found that a successful answer-cache return bypasses +selection hooks and can finalize as `unfinished`. The required HTTP regression +uses two identical authenticated requests: two admissions, one provider dispatch, +and a distinct cache-hit terminal outcome with absent provider-selection duration. +Keep the cache hit in the accepted denominator. This is tracked in +[the existing owner issue](https://github.com/ContextualWisdomLab/contextual-orchestrator/issues/1110#issuecomment-5597507862), +not a separate implementation branch. + +The in-progress auxiliary/task repair has a reported cold/warm triage HTTP result +of one passed test in 8.72 seconds, but it ran on an uncommitted delta over that +candidate. It is development feedback, not exact-head acceptance. Root inspection +also found that the new auxiliary records were not yet included in the bounded +receipt export. Export the same admission cohort's component evidence and test +it before claiming component reporting. A provider-ready timestamp preceding a +diagnostic write is not evidence of the actual network-send instant. Generated +planning, evidence embedding, and answer-cache outcomes remain explicit coverage +items; no customer KPI gain or release is established by this checkpoint. + +### Native packaging and release owner, 2026-09-09 + +Root independently inspected the native wheel built from candidate `9707a5e1`: +SHA-256 `e0bf63d790256c6d4eba8598c131d63188a994c899df5124bd9eadf2cc39c568`. +Its five entries contain only the extension and distribution metadata/SBOM, not +core Python sources. The hypothesized source-file collision was not observed; +retain the existing namespace and verify both manifests rather than rename +without evidence. Clean-checkout CI still needs native build/install and +outside-checkout core/native acceptance. Local ABI success is not Linux or +hosted-CI proof. + +The existing canonical release owner is +[organization issue #1552](https://github.com/ContextualWisdomLab/.github/issues/1552), +verified open. At organization main `7fd571dbcdbae6acf29d8f4ee704d7ba6297e4db`, +the inspected `exact-artifact-sbom-attestation.yml` reusable component attests +artifacts; it does not publish packages. CO owns its build adapter and package +acceptance; generic release eligibility, immutable tagging, and idempotent +delivery remain with that owner. The verified organization publishing secret +names are `PIPY_TOKEN` and `CARGO_REGISTRY_TOKEN`, both visible to all repositories. +Registration is not credential-validity or registry-ownership proof. Complete +the minimal owner contract and CO adapter independently, then integrate exact +revisions; do not require the whole foundation or publish an unmerged candidate. + +### Streaming admission reproduction, 2026-09-09 + +An independent installed-package HTTP probe revises the initial source-only +hypothesis: invalid empty `user` and array-valued `routing` each returned 400, +with zero provider calls and zero accepted records. Earlier shared validation +already rejects these inputs; later duplicate validators do not prove unsafe +spend. Retain these cases as guards, not failing regressions. + +The valid auto-streaming control returned 200, invoked the observed chat client +once, and retained one admission. That provider call had no active measurement +scope, proving its triage work preceded the acceptance clock. Installed +`server.py` SHA-256 +`a28fd4aaafb3852315d0b69541ab12235c6cdcd6c75b5ceff7d8f1c5ff08e08a` +matches candidate `bbe7eae1a24a95e17b5933ca75cc6b2598f896e4`. +The spy only recorded state, with assertions after the response. This is a +mock-provider unit reproduction over real HTTP, not customer latency evidence. +Repair the valid-stream timing boundary without bypassing existing validation; +see [the reproduction receipt](https://github.com/ContextualWisdomLab/contextual-orchestrator/issues/1110#issuecomment-5597865731). +The full-suite checkout remained unchanged during this independent probe. + +Full regression at `bbe7eae1a24a95e17b5933ca75cc6b2598f896e4` terminated +with **9 failed, 3420 passed, 2 skipped in 794.44 seconds** on local +macOS/Python 3.14. Eight failures exposed compatibility with lightweight +handlers when measurement was disabled; one workflow contract required the +existing standalone benchmark import spelling. Repair +`0b10b553ab916f341ddfbb5c1fc6989c23e0f293` keeps the unmeasured acquisition, +release, and disconnect paths independent of measurement state and restores +the explicit import. The failed-file plus receipt regression passed **55 tests +in 10.92 seconds**. A transient indentation error during repair caused collection +failure and was corrected before that run; it does not replace the original RCA. +Full regression on the repaired final head and hosted Linux acceptance remain +required, along with the valid-stream timing and typed-error accounting repairs. + +Integrated candidate `3b6dd47ebb0f88802bacdd302051d2f03e7d5003` preserves +#1105 trusted SSE identities and includes valid-stream triage in admission time. +An actual HTTP rejection after authorized triage reproduced `unfinished`; the +shared error adapter now classifies pending admitted failures without replacing +acknowledged, capacity, cancellation or write-failure outcomes. Focused local +verification: **83 passed in 35.72s**. Independently installed exact-head wheels: +**33 passed in 14.21s**, with disjoint package manifests and imports verified +outside the checkout. Full regression is still pending; no hosted, deployed or +customer-KPI success follows from these local receipts. + +The [response-process follow-up](doctoring/irt_router_measurement_review.md#response-process-identification-follow-up) +adds a lawfully redistributable 2017 perspective and a predecision-covariate +comparison proposal. Its source figure was inspected in the actual browser. +Observed-data calibration and an immutable owner estimation contract remain +unverified; latency correlation cannot substitute for those acceptance gates. + +Full regression for `3b6dd47ebb0f88802bacdd302051d2f03e7d5003` is now +terminal: **3442 passed, 2 skipped in 771.43s** on macOS/Python 3.14. +This supersedes the pending observation above, not the separate hosted/release +gates. An independent installed-wheel HTTP probe exposed pre-capacity triage +in auto chat streaming; Responses streaming and nonstreaming chat passed the +same saturated-slot controls. Preserve the green regression as historical +evidence and repair the uncovered case, as specified in +[the runbook](doctoring/autonomous_kpi_runbook.md#integrated-receipt-regression-and-remaining-capacity-defect-2026-09-09). + +Next accuracy gap, independently reproduced against installed candidate +`c7345670e08f029ad3aa5dd1133037bb4b451d9b`: a successful real HTTP route +request retains one admission receipt and one workflow result, but neither +record exposes an explicit durable link to the other's identity. The receipt +has `request_id`; the workflow has `workflow_run_id` and `owner_id` only. +Field-level equality assertions fail; this mock-provider probe does not claim +observed customer accuracy. A separately stacked successor must bind trusted +request identity to outcomes without treating cache reuse as a new execution +or discarding requests that fail before producing a workflow. + +The bounded tracked-data audit at `53a9266a639361651064d7748fa74b396dd493ef` +found no qualifying observed-accuracy cohort. The NIM manifest contains authored +tasks, historical benchmark reports contain aggregates, and Noema incidents +select failed deliveries rather than a complete request window. These cannot +supply the existing customer KPI. Require permitted-use provenance, a complete +bounded ingress window, independent outcome adjudication and exact model/policy +revisions before the first observed baseline. The linkage successor prepares +that measurement; it does not itself establish its correctness or improvement. + +Capacity repair `c7345670e08f029ad3aa5dd1133037bb4b451d9b` now has terminal +local full-suite evidence: **3452 passed, 2 skipped in 753.15s** (macOS, +Python 3.14; execution 39067). Its isolated installed-wheel receipt and SSE +identity slice passed **43 tests in 17.88s**. Independent saturated-capacity +HTTP probes confirm zero classifier/provider calls with measurement enabled +and disabled. These results do not establish hosted checks, protected merge, +publication or customer accuracy. + +The separate linkage successor `4cf7feafd554fbbd65dfc3b790f1081623b0d05a` +retains focused passing evidence, but its first full-suite attempt (28355) +terminated during collection: missing `hypothesis`, exit 2 after 5.41s. +This is an incomplete test environment, not a passing full regression. Repair +the successor's isolated test dependencies without changing the base candidate's +installed-wheel environment, then rerun against a frozen documented checkpoint. + +The capacity candidate is now [PR #1112](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1112), +head `c7345670e08f029ad3aa5dd1133037bb4b451d9b`, stacked on #1108 at +`129a665016ed1acd79ae12915c905b1020856fcc`. Security run +[34327884508](https://github.com/ContextualWisdomLab/contextual-orchestrator/actions/runs/34327884508) +was admitted with three queued jobs. The review list is empty. CodeRabbit's +SUCCESS status is explicitly a +[skipped review](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1112#issuecomment-5598554823) +because its automatic reviews exclude non-default target branches; it is not +approval. Canonical review coordination received this finding. No merge or +deployment follows from the page's “Able to merge” indicator. + +Bounded visual inspection: opened the actual PR in the browser and directly +viewed its 1265 × 712 English screenshot. The title wraps onto two lines; +branch labels, Scope heading and opening provenance paragraph remain readable +without overlap in the inspected viewport. Lower sections require scrolling +and were not visually audited. This is a PR-document inspection, not product +UI, responsive, locale or Figma acceptance. + +The linkage successor at `af8d732e6cfc9c0169ac850f875f42f1db7eecd4` +completed its frozen local full suite: **3462 passed, 2 skipped in 822.68s**, +exit 0 (execution 81304). This supersedes the collection-only failure above: +a separate locked project environment supplied the missing test dependency, +without changing the base candidate's wheel environment. The run used successor +Python source and the unchanged base native extension, so dedicated successor +wheel acceptance remains separate and in progress. See the successor's +`docs/doctoring/workflow_request_link.md` for exact reproduction and failure +history. No observed accuracy baseline, protected merge or release is established. + +Research follow-up [now records](doctoring/measured-routing-evidence.md#multilevel-follow-up-source-2026-09-09) +the read scope and proposed applicability conditions for Jin et al. (2022), +including independent review and direct inspection of PDF page 7. The remaining +work is an owner-validated observation/estimand contract and dependence-aware +held-out evaluation, not production adoption based on a literature citation. + +Linkage delivery checkpoint: [PR #1113](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1113) +now stacks frozen `af8d732e6cfc9c0169ac850f875f42f1db7eecd4` on #1112. +Dedicated noneditable wheel acceptance completed **53 tests in 26.39s**; +root independently verified disjoint archive members, both installed import +origins and the original real-HTTP request/outcome join probe. The PR preserves +artifact hashes and the missing-Setuptools and macOS path-alias probe failures. +Hosted run [34329594602](https://github.com/ContextualWisdomLab/contextual-orchestrator/actions/runs/34329594602) +has three queued jobs, not acceptance. Review, protected delivery and batch +submission-to-item lineage remain open. + +Hosted #1112 checkpoint: Security run 34327884508 is now terminal **SUCCESS** +across all three jobs. Job 102389289972 checked out merge +`aca87f45839f7d03956cc4e37f9646ff2dadbbab` (head `c7345670`, base `129a6650`) +on Linux/Python 3.12.14: **3452 passed, 2 skipped in 760.25s**, then 134 +benchmark/docstring checks and 40 installed-wheel checks passed. This supplies +hosted-platform evidence previously pending; it does not establish independent +approval, protected-main merge, registry publication or customer KPI improvement. + +Next batch gap: isolated test `387aa2111142b13b327f7065226c0f22c305b872` +reproduced successful HTTP submission/retrieval of two items but no durable +submission-request association after SQLite reopen (one failed in 3.75s). +Candidate `369ea1e34dd0f4b3da9672ceae068656af6a69a8` passed 65 focused +checks in 16.51s after repairing association persistence, redundant registry +writes and response-only registry-write diagnostics. This is not full or +installed-package acceptance and no batch PR has been submitted. + +Independent review still found a recovery gap: a remotely accepted job can +return its handle after registry-write failure, yet later retrieval is unavailable. +Recovery must use an owner-bound, expiring backend descriptor and exact item +identities; a naked remote handle must never bypass ownership. Preserve absent +usage and distinguish remote acceptance, association commit, registry persistence +and actual recoverability. A status-only response does not complete this gap. +The new work remains isolated from the tested #1112/#1113 candidates. + +### Research-stack integration checkpoint — 2026-09-12 + +The earlier stacked-PR receipts above are historical evidence, not proof that +their changes reached protected main. Main `012beaacd0631f8cd3391c77744eeb626269b5de` +lacked the retained research ancestry; no deletion from main was established. +Ordinary merge `52fd0da99224f0889e8b012667a93940f6a324ee` preserves that +ancestry and current main behavior. The test synchronization repair at +`36af4a56` waits for actual receipt finalization, without inventing response +delivery or moving its timestamp. + +Frozen integration `81ad64cf77a49f7bc2a57f4851a5c3259387ce5d` completed +**3,688 passed, 2 skipped in 261.42s**, terminal session 90461. This is source +testing with a separately installed native namespace, not installed-core, +hosted, release or observed-customer acceptance. The accuracy and latency +targets remain unmeasured; the next gap is protected delivery of complete +request/outcome evidence and independent held-out evaluation. +See [the integration runbook](doctoring/kpi_stack_integration.md) for lineage, +the original failure, its reproduction and remaining acceptance work. + +Visual inspection: the runbook at `0fd408aa` was rendered at +`http://127.0.0.1:18766/` in a real browser, English, 1265 × 712. Two screenshots +were directly opened in the task: the top and the verification/acceptance +section after scrolling. Text, revision identifiers and headings were readable +without horizontal clipping or overlap in those views. The viewport edge cut +off continuing vertical content normally; the final paragraph was not inspected. +This is a local document preview, not Figma, product UI, mobile or locale +acceptance; screenshots remain in the task tool output, not repository assets. + +Research intake: [LaRT measurement review](doctoring/lart_measurement_review.md) +separates token-length evidence from wall-clock decision latency and +full-data fitted references from known true parameters. The proposed +fast-mlsirm-owned calibration experiment remains unimplemented and unmeasured; +it does not change routing defaults or close the observed-outcome gap. + +Cache aggregation follow-up: frozen `4bc96045037d04fa7477a1532c75f99f0d7e9898` +repairs repeated/mixed cache items losing initial-decision timing or masking +failure within one HTTP admission. Full source: **3,699 passed, 2 skipped**; +separate installed core/native: **82 passed**. These are correctness receipts, +not observed accuracy or latency improvement. See the +[single integration runbook](doctoring/kpi_stack_integration.md#request-level-cache-aggregation-repair--2026-09-12) +for RED evidence, hashes and reproduction. Central Noema dispatch acceptance +still lacks receiver/run proof; its exact owner evidence is recorded there. +Do not substitute a successful event submission or old-head Security result +for current-head independent review, protected delivery or release. + +### Existing gateway repair not adopted by the review sidecar — 2026-09-12 + +The [transport comparison receipt](doctoring/review_phase_transport_comparison.md) +connects central run `34688188671` / job `103539568718` to source pin +`414f22973658c4ddc3d4320fcf7acd9b4e8ba991`. That source fails the eligible-free +final-synthesis transport regression. Existing protected merge +`9334dc91aaf853b758077e983517a822b6b21edb` passes the same selected regression, +13 bootstrap tests, and the exact central `68daf0f` import/startup contract. +The dependency lock is byte-identical. This is an adoption gap for the central +workflow owner, not justification for duplicate CO fallback code or consumer +retries. The original incident's terminal role remains uncorrelated; do not +claim this defect is its sole cause. + +Next evidence: central protected pin adoption, current-head mandatory reviews +and checks, then a real successful review with immutable runtime identity. +Until then this gap stays open. Local loopback/test-double success does not +establish provider recovery, release, observed accuracy, or decision-latency +improvement. Test commit `8065ada1` remains preserved on its diagnostic branch; +this research-PR update imports documentation only and does not claim that +test is already part of its own CI suite. + +The historical overall-deadline recommendation above is superseded by the +user's model-specific timeout policy: no common application/agent/gateway +deadline by default. Provider termination, explicit user cancellation and +configured per-model administrative timeout must remain distinguishable. + +### Retrospective calibration intake: split identity remains open + +The [pinned LaRT matrix audit](doctoring/lart_measurement_review.md#pinned-matrix-identity-audit) +records actual public-data identity checks, not an estimator result. The +published row split places 21 of 28 evaluation rows alongside a training row +with the same suffix-derived base-model identifier. Its 100 item columns have +only 40 distinct labels. These observations rule out treating that split as +independent base-model generalization or joining items by raw labels alone. + +The next owner experiment must freeze benchmark-qualified item identities, +reviewed base-model groups and separate family lineage, then fit only on +training observations. Compare paired held-out Brier score/log loss at equal +observation budgets; record excluded populations and unavailable generation +failures explicitly. Released fast-mlsirm contracts remain the estimator +boundary. Public matrices do not supply CO decision timestamps or known true +latent parameters, so neither decision-p95 nor true-parameter RMSE can be +claimed from this retrospective experiment. Data rights, grouping verification, +owner implementation and observed gains remain open; no route default changes. + +Preprocessing follow-ups now live in [Draft successor PR #1139](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1139) +at `0024522146803627b8741470ebefaf79ffa4a310`, preserving the complete prior +delta before this normal document revert. The [handoff](doctoring/lart_measurement_review.md#preprocessing-evidence-successor) +keeps the upstream Draft/maintainer-approval boundary and unverified estimator, +rights, accuracy and latency gates visible. This is not gap closure or release. + +## 2026-09-13 constant-only KPI PR repair findings + +Exact-source review found that [PR #1125](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1125) +at `dbfcc0c17177af0339f1b326f3584db866d8a943` only declares +`ROUTE_DECISION_LATENCY_FIELD`; its test checks the constant's existence/value, +not a measured trace interval. [PR #1126](https://github.com/ContextualWisdomLab/contextual-orchestrator/pull/1126) +at `2ecc4a03ffe30707d9ce69ae3448e9f83a50e312` does the same for +`REQUEST_OUTCOME_ASSOCIATIONS_FIELD`, without exercising an authorized export. +Both tests describe runtime behavior they do not actually assert. Passing them +cannot establish decision latency, outcome linkage, or customer KPI acceptance. + +Keep both Draft PRs open as repair findings. Reconcile their proposed public +field names with the canonical request measurement/export owner before any +successor claims complete inheritance. Existing receipt timing is nanoseconds +and request-scoped; do not copy it onto per-step traces or reinterpret total +generation latency as decision time. Acceptance needs real HTTP requests, +durable acknowledgement and missing-value checks, plus authorized one-to-many +request/outcome joins retaining unfinished requests and excluding other owners. +PR #1138 is an export candidate, not proof that either legacy field contract +has already been adopted. No predecessor closure or production change follows +from this audit. diff --git a/mise.toml b/mise.toml new file mode 100644 index 000000000..19305ad64 --- /dev/null +++ b/mise.toml @@ -0,0 +1,2 @@ +[tools] +rust = "1.97.1" diff --git a/pyproject.toml b/pyproject.toml index 6760da4dc..ace50c59d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ queue = [ ] [dependency-groups] +native-build = ["maturin==1.15.0"] dev = [ "hypothesis>=6.100", "pytest>=8.0", diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 000000000..725551148 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "1.97.1" +profile = "minimal" diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 96d3c6a1d..58fc95de5 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -49,6 +49,13 @@ dependencies = [ "serde_core", ] +[[package]] +name = "contextual-decision-receipt" +version = "0.1.0" +dependencies = [ + "pyo3", +] + [[package]] name = "contextual-token-packer" version = "0.1.0" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index b7d83efbb..c69753fbb 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -1,3 +1,3 @@ [workspace] -members = ["token_counter"] +members = ["token_counter", "decision_receipt"] resolver = "2" diff --git a/rust/decision_receipt/Cargo.toml b/rust/decision_receipt/Cargo.toml new file mode 100644 index 000000000..8e1c1d19a --- /dev/null +++ b/rust/decision_receipt/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "contextual-decision-receipt" +version = "0.1.0" +edition = "2021" + +[dependencies] +pyo3 = { version = "0.29", features = ["abi3-py310", "auto-initialize"] } + +[lib] +name = "_decision_receipt" +crate-type = ["cdylib"] diff --git a/rust/decision_receipt/pyproject.toml b/rust/decision_receipt/pyproject.toml new file mode 100644 index 000000000..6c498c112 --- /dev/null +++ b/rust/decision_receipt/pyproject.toml @@ -0,0 +1,13 @@ +[build-system] +requires = ["maturin==1.15.0"] +build-backend = "maturin" + +[project] +name = "contextual-decision-receipt" +version = "0.1.0" +requires-python = ">=3.10" + +[tool.maturin] +module-name = "contextual_orchestrator._decision_receipt" +python-source = "../.." +features = ["pyo3/extension-module"] diff --git a/rust/decision_receipt/src/lib.rs b/rust/decision_receipt/src/lib.rs new file mode 100644 index 000000000..5630c1dfc --- /dev/null +++ b/rust/decision_receipt/src/lib.rs @@ -0,0 +1,117 @@ +//! One-clock initial-decision measurement; storage acknowledgement is caller-owned. +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use std::time::Instant; + +/// Request-local monotonic state with no caller-supplied clock values. +#[pyclass] +pub struct DecisionReceipt { + accepted_at: Instant, + #[pyo3(get)] + status: String, + #[pyo3(get)] + selection_elapsed_ns: Option, + #[pyo3(get)] + durable_ack_elapsed_ns: Option, + #[pyo3(get)] + first_provider_elapsed_ns: Option, +} + +impl DecisionReceipt { + fn elapsed_ns(&self) -> PyResult { + self.accepted_at + .elapsed() + .as_nanos() + .try_into() + .map_err(|_| PyValueError::new_err("decision elapsed time overflow")) + } +} + +#[pymethods] +impl DecisionReceipt { + /// Start at validated acceptance, before capacity admission. + #[new] + fn accepted() -> Self { + Self { + accepted_at: Instant::now(), + status: "accepted".into(), + selection_elapsed_ns: None, + durable_ack_elapsed_ns: None, + first_provider_elapsed_ns: None, + } + } + + /// Mark the first provider-ready boundary without advancing task selection. + fn record_provider_dispatch(&mut self) -> PyResult { + if let Some(elapsed) = self.first_provider_elapsed_ns { + return Ok(elapsed); + } + let elapsed = self.elapsed_ns()?; + self.first_provider_elapsed_ns = Some(elapsed); + Ok(elapsed) + } + + /// Timestamp an auxiliary phase in the same acceptance clock domain. + fn current_elapsed_ns(&self) -> PyResult { + self.elapsed_ns() + } + + /// Complete answer reuse without inventing a provider selection or acknowledgement. + fn record_cache_hit(&mut self) -> PyResult<()> { + if self.status != "accepted" { + return Err(PyValueError::new_err("cache hit requires accepted state")); + } + self.status = "cache_hit".into(); + Ok(()) + } + + /// Record the initial selection without allowing later attempts to replace it. + fn record_selection(&mut self) -> PyResult<()> { + if self.status != "accepted" { + return Err(PyValueError::new_err("selection requires accepted state")); + } + let elapsed = self.elapsed_ns()?; + self.selection_elapsed_ns = Some(elapsed); + self.status = "selected".into(); + Ok(()) + } + + /// Call only after a synchronous durable decision write returns successfully. + fn record_durable_ack(&mut self) -> PyResult<()> { + if self.status != "selected" { + return Err(PyValueError::new_err( + "acknowledgement requires selected state", + )); + } + let elapsed = self.elapsed_ns()?; + self.durable_ack_elapsed_ns = Some(elapsed); + self.status = "acknowledged".into(); + Ok(()) + } + + /// Preserve a denominator-only terminal observation without a success duration. + fn record_failure(&mut self, reason: &str) -> PyResult<()> { + if !matches!(self.status.as_str(), "accepted" | "selected") { + return Err(PyValueError::new_err("observation is already terminal")); + } + if !matches!( + reason, + "capacity_rejected" + | "selection_failed" + | "write_failed" + | "cancelled" + | "unfinished" + | "store_unavailable" + ) { + return Err(PyValueError::new_err("unknown failure reason")); + } + self.status = reason.into(); + Ok(()) + } +} + +/// Export the receipt separately from token counting. +#[pymodule] +fn _decision_receipt(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::() +} diff --git a/scripts/verify_decision_wheel_manifest.py b/scripts/verify_decision_wheel_manifest.py new file mode 100644 index 000000000..461dd6379 --- /dev/null +++ b/scripts/verify_decision_wheel_manifest.py @@ -0,0 +1,23 @@ +"""Reject native/core wheel ownership overlap before isolated installation.""" + +import sys +import zipfile + + +def verify_wheels(core_path, native_path): + """Require a native-only extension distribution disjoint from the core wheel.""" + with zipfile.ZipFile(core_path) as core_archive, zipfile.ZipFile(native_path) as native_archive: + core_files = set(core_archive.namelist()) + native_files = set(native_archive.namelist()) + if core_files & native_files: + raise ValueError("core and native wheel files overlap") + payload = {name for name in native_files if ".dist-info/" not in name} + if len(payload) != 1 or not all( + name.startswith("contextual_orchestrator/_decision_receipt.") + and name.endswith((".so", ".pyd")) for name in payload + ): + raise ValueError("native wheel must contain only the receipt extension") + + +if __name__ == "__main__": + verify_wheels(*sys.argv[1:]) diff --git a/tests/test_batch_request_lineage.py b/tests/test_batch_request_lineage.py new file mode 100644 index 000000000..eaeef59ca --- /dev/null +++ b/tests/test_batch_request_lineage.py @@ -0,0 +1,363 @@ +"""Deferred batch outcomes retain their original trusted HTTP admission.""" + +import copy +import threading +import pytest + +from contextual_orchestrator import CostRoutingCoordinator, ModelAgent, TaskOrchestrator +from contextual_orchestrator.batch_routing import PgLlmBatchBackend +from contextual_orchestrator.server import SecurityConfig, build_server +from contextual_orchestrator.telemetry import current_request_id +from test_batch_routing import _FakeBatchApiClient +from test_cost_review_server import _request + + +@pytest.mark.parametrize("write_failure", [False, True]) +@pytest.mark.parametrize("registry_failure", [None, "hset", "expire"]) +def test_http_batch_origin_survives_distinct_retrieval_and_reload(tmp_path, monkeypatch, write_failure, registry_failure): + """One submission joins two item outcomes without trusting their custom IDs.""" + class ObservedBatchClient(_FakeBatchApiClient): + """Reuse the existing offline provider contract with passive identity capture.""" + + async def create_batch_job(self, *args, **kwargs): + self.submission_request_id = current_request_id() + return await super().create_batch_job(*args, **kwargs) + + async def download_results(self, *args, **kwargs): + self.retrieval_request_id = current_request_id() + payload = await super().download_results(*args, **kwargs) + second_result = copy.deepcopy(payload["responses"][0]) + second_result["custom_id"] = "b" + payload["responses"].append(second_result) + return payload + + state_path = tmp_path / "state.db" + agents = [ModelAgent("worker_one", "mock/worker")] + orchestrator = TaskOrchestrator(agents, state_db=state_path) + if write_failure: + original_save = orchestrator._store.save + + def reject_link(kind, *args, **kwargs): + if kind == "batch_request_link": + raise RuntimeError("private-store-secret") + return original_save(kind, *args, **kwargs) + + monkeypatch.setattr(orchestrator._store, "save", reject_link) + batch_client = ObservedBatchClient() + coordinator = CostRoutingCoordinator( + orchestrator, batch_backend=PgLlmBatchBackend(batch_client) + ) + if registry_failure: + from contextual_orchestrator.batch_job_registry import ValkeyJsonMapping + from contextual_orchestrator.batch_routing import BatchJob + from test_batch_job_registry import FakeValkeyClient + + class RejectingClient(FakeValkeyClient): + """Distinguish no registry write from partial HSET-before-expiry.""" + + def hset(self, *args, **kwargs): + if registry_failure == "hset": + raise RuntimeError("private-registry-secret") + return super().hset(*args, **kwargs) + + def expire(self, *args, **kwargs): + raise RuntimeError("private-registry-secret") + + registry_client = RejectingClient() + coordinator._batch_jobs = ValkeyJsonMapping( + registry_client, "jobs", decode=lambda raw: BatchJob(**raw) + ) + server = build_server(orchestrator, port=0, coordinator=coordinator, + security=SecurityConfig(auth_token="unit-token")) + worker_thread = threading.Thread(target=server.serve_forever, daemon=True) + worker_thread.start() + base_url = f"http://127.0.0.1:{server.server_address[1]}" + try: + status, submitted = _request("POST", f"{base_url}/api/v1/batch_routing_jobs", + "unit-token", {"requests": [ + {"custom_id": item_id, "model": "mock/worker", "mode": "route", + "messages": [{"role": "user", "content": "Offline contract fixture."}]} + for item_id in ("a", "b") + ]}) + assert status == 201, submitted + assert submitted["request_link_status"] == ("write_failed" if write_failure else "durable") + assert "private-store-secret" not in str(submitted) + assert submitted["recovery_status"] == "unavailable" + assert submitted["registry_persistence_status"] == ("write_failed" if registry_failure else "stored") + assert "private-registry-secret" not in str(submitted) + if registry_failure: + assert submitted["job_id"] == "batch-789" + assert batch_client.calls.count("create_batch_job") == 1 + stored_handles = registry_client.hashes.get("batch_job_registry:jobs", {}) + assert bool(stored_handles) == (registry_failure == "expire") + denied_status, _ = _request( + "POST", f"{base_url}/api/v1/batch_routing_jobs/{submitted['job_id']}/results", + "other-owner-token", + ) + assert denied_status in {401, 403} + assert "download_results" not in batch_client.calls + links = orchestrator._store.load("batch_request_link") + assert len(links) == (0 if write_failure else 1) + if links: + assert links[0]["request_id"] == batch_client.submission_request_id + return + status, retrieved = _request( + "POST", f"{base_url}/api/v1/batch_routing_jobs/{submitted['job_id']}/results", + "unit-token", + ) + assert status == 200, retrieved + assert {item["custom_id"] for item in retrieved["results"]} == {"a", "b"} + assert batch_client.submission_request_id + assert batch_client.retrieval_request_id + assert batch_client.submission_request_id != batch_client.retrieval_request_id + assert batch_client.submission_request_id not in {"a", "b"} + repeated_status, repeated = _request( + "POST", f"{base_url}/api/v1/batch_routing_jobs/{submitted['job_id']}/results", + "unit-token", + ) + assert repeated_status == 200 + assert {item["custom_id"] for item in repeated["results"]} == {"a", "b"} + assert batch_client.calls.count("create_batch_job") == 1 + assert coordinator._batch_jobs[submitted["job_id"]].job_id == submitted["job_id"] + finally: + server.shutdown() + worker_thread.join() + server.server_close() + orchestrator.close() + + restored = TaskOrchestrator(agents, state_db=state_path) + try: + # Proposed CO-owned association contract: one row per submission/job/item, + # not a replacement of provider custom_id or one origin per eventual job. + links = restored._store.load("batch_request_link") + actual_links = { + (row["request_id"], row["batch_job_id"], custom_id) + for row in links for custom_id in row["custom_ids"] + } + if write_failure: + assert not actual_links + return + assert actual_links == { + (batch_client.submission_request_id, submitted["job_id"], item["custom_id"]) + for item in retrieved["results"] + } + finally: + restored.close() + + +def test_batch_submission_links_keep_job_scoped_item_ids(tmp_path): + """Repeated item identifiers across submissions retain every origin association.""" + from contextual_orchestrator.batch_routing import BatchRequest + + agents = [ModelAgent("worker_one", "mock/worker")] + orchestrator = TaskOrchestrator(agents, state_db=tmp_path / "state.db") + coordinator = CostRoutingCoordinator( + orchestrator, batch_backend=PgLlmBatchBackend(_FakeBatchApiClient()) + ) + try: + for request_id in ("trusted_origin_one", "trusted_origin_two"): + coordinator.submit_batch([ + BatchRequest(messages=[{"role": "user", "content": "Fixture"}], custom_id="a") + ], owner_id="owner_one", request_id=request_id) + links = orchestrator._store.load("batch_request_link") + assert {row["request_id"] for row in links} == {"trusted_origin_one", "trusted_origin_two"} + assert [row["custom_ids"] for row in links] == [["a"], ["a"]] + finally: + orchestrator.close() + + +def test_library_batch_without_state_store_keeps_legacy_submission(): + """Standalone calls explicitly report unavailable durable request lineage.""" + from contextual_orchestrator.batch_routing import BatchRequest + + orchestrator = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")]) + coordinator = CostRoutingCoordinator( + orchestrator, batch_backend=PgLlmBatchBackend(_FakeBatchApiClient()) + ) + try: + job = coordinator.submit_batch([BatchRequest( + messages=[{"role": "user", "content": "Fixture"}], custom_id="a" + )]) + assert job.request_link_status == "unavailable" + assert job.job_id == "batch-789" + # Legacy metadata without a deployment binding remains usable only + # while the injected backend also has no recovery identity configured. + coordinator.batch_backend._jobs[job.job_id].pop("recovery_identity") + assert coordinator.poll_batch(job.job_id)["is_complete"] is True + finally: + orchestrator.close() + + +def test_batch_link_does_not_rewrite_submitted_registry_handle(tmp_path): + """Lineage status must not add another failure-prone registry assignment.""" + from contextual_orchestrator.batch_routing import BatchRequest + + class SingleWriteRegistry(dict): + """Reject a redundant second remote-registry assignment.""" + + def __setitem__(self, key, value): + if key in self: + raise RuntimeError("second registry write failed") + super().__setitem__(key, value) + + orchestrator = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")], + state_db=tmp_path / "state.db") + coordinator = CostRoutingCoordinator( + orchestrator, batch_backend=PgLlmBatchBackend(_FakeBatchApiClient()) + ) + coordinator._batch_jobs = SingleWriteRegistry() + try: + job = coordinator.submit_batch([BatchRequest( + messages=[{"role": "user", "content": "Fixture"}], custom_id="a" + )], request_id="trusted_origin_one") + assert job.request_link_status == "durable" + finally: + orchestrator.close() + + +def test_valkey_job_snapshot_does_not_prove_lineage_commit(tmp_path): + """Decoded registry status is non-authoritative; committed events supply proof.""" + from contextual_orchestrator.batch_job_registry import ValkeyJsonMapping + from contextual_orchestrator.batch_routing import BatchJob, BatchRequest + from test_batch_job_registry import FakeValkeyClient + + orchestrator = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")], + state_db=tmp_path / "state.db") + coordinator = CostRoutingCoordinator( + orchestrator, batch_backend=PgLlmBatchBackend(_FakeBatchApiClient()) + ) + coordinator._batch_jobs = ValkeyJsonMapping( + FakeValkeyClient(), "jobs", decode=lambda raw: BatchJob(**raw) + ) + try: + response_job = coordinator.submit_batch([BatchRequest( + messages=[{"role": "user", "content": "Fixture"}], custom_id="a" + )], owner_id="owner_one", request_id="trusted_origin_one") + decoded_job = coordinator._batch_jobs[response_job.job_id] + assert decoded_job is not response_job + assert decoded_job.request_link_status == "durable" + assert decoded_job.registry_persistence_status == "unavailable" + assert response_job.registry_persistence_status == "stored" + assert decoded_job.owner_id == "owner_one" + assert response_job.request_link_status == "durable" + assert orchestrator._store.load("batch_request_link")[0]["request_id"] == "trusted_origin_one" + finally: + orchestrator.close() + + +@pytest.mark.parametrize("recovery_case", ["valid", "expired", "malformed", "backend_mismatch", "unexpected_item", "missing_usage", "registry_outage", "item_mismatch", "estimate_mismatch", "null_estimates", "boolean_count", "deployment_mismatch", "missing_identity", "coordinator_hit", "duplicate_ids", "backend_write_outage", "healthy_expired", "healthy_deployment_mismatch", "healthy_endpoint_mismatch"]) +def test_http_batch_failed_registry_recovers_authorized_job_after_restart(tmp_path, recovery_case): + """SQLite recovery binds the original owner without another remote submission.""" + class MissingRegistry(dict): + """Lose the registry assignment, retaining only committed SQLite evidence.""" + + def __setitem__(self, key, value): + raise RuntimeError("registry unavailable") + + state_path = tmp_path / "state.db" + agents = [ModelAgent("worker_one", "mock/worker")] + security = SecurityConfig(bearer_verifier=lambda token, scope: + token in {"owner-one", "owner-two"}) + clients = [] + for restarted in (False, True): + orchestrator = TaskOrchestrator(agents, state_db=state_path) + class RecoveryClient(_FakeBatchApiClient): + """Return controlled result variants without changing routing execution.""" + + async def download_results(self, *args, **kwargs): + result = await super().download_results(*args, **kwargs) + if recovery_case == "unexpected_item": + result["responses"][0]["custom_id"] = "unsubmitted-item" + if recovery_case == "missing_usage": + result["responses"][0]["response"]["body"].pop("usage") + return result + + client = RecoveryClient() + clients.append(client) + coordinator = CostRoutingCoordinator(orchestrator, + batch_backend=PgLlmBatchBackend(client, endpoint_alias=( + "changed-endpoint" if restarted and recovery_case == "backend_mismatch" + else "original-endpoint"), endpoint=( + "/v1/completions" if restarted and recovery_case == "healthy_endpoint_mismatch" + else "/v1/chat/completions"), recovery_identity=( + None if restarted and recovery_case == "missing_identity" else + "different-deployment" if restarted and recovery_case in {"deployment_mismatch", "healthy_deployment_mismatch"} + else "unit-deployment-account"))) + if not restarted: + if recovery_case == "backend_write_outage": + coordinator.batch_backend._jobs = MissingRegistry() + else: + coordinator._batch_jobs = MissingRegistry() + elif recovery_case in {"coordinator_hit", "healthy_expired", "healthy_deployment_mismatch", "healthy_endpoint_mismatch"}: + from contextual_orchestrator.batch_routing import BatchJob + coordinator._batch_jobs[submitted["job_id"]] = BatchJob(**record["recovery_descriptor"]["job"]) + if recovery_case in {"healthy_expired", "healthy_deployment_mismatch", "healthy_endpoint_mismatch"}: + coordinator.batch_backend._jobs = retained_backend_metadata + elif recovery_case == "registry_outage": + class UnavailableRegistry(MissingRegistry): + """All reads and writes remain unavailable during recovery.""" + + def get(self, *args, **kwargs): + raise RuntimeError("registry still unavailable") + + coordinator._batch_jobs = UnavailableRegistry() + coordinator.batch_backend._jobs = UnavailableRegistry() + server = build_server(orchestrator, port=0, coordinator=coordinator, security=security) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + base_url = f"http://127.0.0.1:{server.server_address[1]}" + try: + if not restarted: + status, submitted = _request("POST", f"{base_url}/api/v1/batch_routing_jobs", + "owner-one", {"requests": [{"custom_id": "a", "model": "mock/worker", + "messages": [{"role": "user", "content": "Never persist this prompt."}]}]}) + assert status == 201 + assert submitted["registry_persistence_status"] == ( + "stored" if recovery_case == "backend_write_outage" else "write_failed") + if recovery_case == "backend_write_outage": + assert submitted["backend_registry_persistence_status"] == "write_failed" + assert submitted["recovery_status"] == "durable_descriptor" + record = orchestrator._store.load("batch_request_link")[0] + assert "Never persist this prompt." not in str(record) + retained_backend_metadata = coordinator.batch_backend._jobs + if recovery_case in {"expired", "healthy_expired"}: + record["recovery_descriptor"]["expires_at"] = 0 + if recovery_case == "malformed": + record["recovery_descriptor"]["backend"] = [] + if recovery_case == "item_mismatch": + record["custom_ids"] = ["different-original-item"] + if recovery_case == "estimate_mismatch": + record["recovery_descriptor"]["job"]["prompt_token_estimates"] = {"different-item": 99} + if recovery_case == "null_estimates": + record["recovery_descriptor"]["job"]["prompt_token_estimates"] = None + if recovery_case == "boolean_count": + record["recovery_descriptor"]["job"]["request_count"] = True + if recovery_case == "duplicate_ids": + record["custom_ids"] = ["a", "a"] + if recovery_case in {"expired", "healthy_expired", "malformed", "item_mismatch", "estimate_mismatch", "null_estimates", "boolean_count", "duplicate_ids"}: + orchestrator._store.save("batch_request_link", submitted["job_id"], record, durable=True) + continue + result_url = f"{base_url}/api/v1/batch_routing_jobs/{submitted['job_id']}/results" + denied_status, _ = _request("POST", result_url, "owner-two") + assert denied_status == 404 + assert "download_results" not in client.calls + status, retrieved = _request("POST", result_url, "owner-one") + if recovery_case in {"expired", "malformed", "backend_mismatch", "item_mismatch", "estimate_mismatch", "null_estimates", "boolean_count", "deployment_mismatch", "missing_identity", "duplicate_ids", "healthy_deployment_mismatch", "healthy_endpoint_mismatch"}: + assert status == 404, retrieved + assert "download_results" not in client.calls + continue + if recovery_case == "unexpected_item": + assert status != 200 + continue + assert status == 200, retrieved + poll_status, _ = _request("GET", result_url.removesuffix("/results"), "owner-one") + assert poll_status == 200 + assert retrieved["results"][0]["custom_id"] == "a" + if recovery_case == "missing_usage": + assert retrieved["results"][0]["measurement_status"] != "measured" + assert sum(item.calls.count("create_batch_job") for item in clients) == 1 + finally: + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() diff --git a/tests/test_decision_cache_aggregation.py b/tests/test_decision_cache_aggregation.py new file mode 100644 index 000000000..a17602a5b --- /dev/null +++ b/tests/test_decision_cache_aggregation.py @@ -0,0 +1,151 @@ +"""Answer reuse must not terminate a request containing later batch work.""" + +import pytest +import http.client +import json +import threading + +from contextual_orchestrator.decision_receipts import ( + DecisionMeasurement, + record_answer_cache_hit, +) + + +class RecordingStore: + """Keep receipt writes observable without a provider or storage dependency.""" + + def __init__(self): + self.records = [] + + def save(self, record_kind, record_key, payload, *, durable=False): + self.records.append((record_kind, dict(payload))) + + +@pytest.mark.parametrize("operations", ["cache_cache", "cache_select", "select_cache"]) +def test_request_cache_observations_preserve_first_selection(operations): + """Mixed batch order cannot erase selection or reject an otherwise valid item.""" + store = RecordingStore() + measurement = DecisionMeasurement(store) + try: + for operation in operations.split("_"): + if operation == "cache": + record_answer_cache_hit() + else: + measurement.select(["worker_one"], "route") + snapshot = measurement.snapshot() + if "select" in operations: + assert snapshot["status"] == "acknowledged" + assert snapshot["selection_elapsed_ns"] is not None + assert snapshot["durable_ack_elapsed_ns"] is not None + else: + assert snapshot["status"] == "accepted" + finally: + measurement.close() + final_snapshot = store.records[-1][1] + assert final_snapshot["status"] == ( + "acknowledged" if "select" in operations else "cache_hit" + ) + if "select" not in operations: + assert final_snapshot["selection_elapsed_ns"] is None + assert final_snapshot["durable_ack_elapsed_ns"] is None + + +@pytest.mark.parametrize("failure_reason", ["selection_failed", "cancelled"]) +def test_cached_item_does_not_hide_later_request_failure(failure_reason): + """Failure after an answer-cache item remains a failed admitted request.""" + store = RecordingStore() + measurement = DecisionMeasurement(store) + record_answer_cache_hit() + measurement.close(failure_reason) + assert store.records[-1][1]["status"] == failure_reason + + +def test_cache_observation_preserves_existing_write_failure(): + """A later cached item cannot overwrite a durable selection-write failure.""" + store = RecordingStore() + measurement = DecisionMeasurement(store) + try: + measurement.receipt.record_failure("write_failed") + record_answer_cache_hit() + finally: + measurement.close() + assert store.records[-1][1]["status"] == "write_failed" + + +def test_cache_observation_context_is_cleared_between_requests(): + """A completed request cannot leak cache-only classification into its successor.""" + from contextual_orchestrator.decision_receipts import _CURRENT_DECISION + + first_store = RecordingStore() + first_measurement = DecisionMeasurement(first_store) + first_measurement.close() + assert _CURRENT_DECISION.get() is None + record_answer_cache_hit() + assert first_measurement._answer_cache_observed is False + second_store = RecordingStore() + second_measurement = DecisionMeasurement(second_store) + try: + assert second_measurement._answer_cache_observed is False + record_answer_cache_hit() + finally: + second_measurement.close() + assert _CURRENT_DECISION.get() is None + assert first_store.records[-1][1]["status"] == "unfinished" + assert second_store.records[-1][1]["status"] == "cache_hit" + + +@pytest.mark.parametrize("item_order", [ + ["cached", "cached"], ["cached", "fresh"], ["fresh", "cached"], + ["cached", "failure"], +]) +def test_http_local_batch_cache_aggregation(tmp_path, monkeypatch, item_order): + """An actual admitted batch retains cache-only, mixed, and failed outcomes.""" + from contextual_orchestrator import ModelAgent, TaskOrchestrator + from contextual_orchestrator.decision_receipts import export_decision_receipts + from contextual_orchestrator.server import SecurityConfig, build_server + + orchestrator = TaskOrchestrator( + [ModelAgent("worker_one", "mock/worker")], + state_db=tmp_path / "state.db", cache_ttl=60, + ) + orchestrator.complete([{"role": "user", "content": "cached"}], mode="route") + original_dispatch = orchestrator._dispatch + def dispatch(messages, *args, **kwargs): + if messages[-1]["content"] == "failure": + raise ValueError("unit batch failure before selection") + return original_dispatch(messages, *args, **kwargs) + monkeypatch.setattr(orchestrator, "_dispatch", dispatch) + finished = threading.Event() + original_close = DecisionMeasurement.close + def close_and_signal(measurement, reason="unfinished"): + original_close(measurement, reason) + finished.set() + monkeypatch.setattr(DecisionMeasurement, "close", close_and_signal) + server = build_server(orchestrator, port=0, decision_receipts=True, + security=SecurityConfig(auth_token="test-token")) + server_worker = threading.Thread(target=server.serve_forever, daemon=True) + server_worker.start() + try: + connection = http.client.HTTPConnection(*server.server_address) + connection.request("POST", "/api/v1/batch_routing_jobs", json.dumps({ + "requests": [{"messages": [{"role": "user", "content": item}], + "mode": "route"} for item in item_order], + }), {"Authorization": "Bearer test-token", "Content-Type": "application/json"}) + response = connection.getresponse() + response.read() + connection.close() + assert finished.wait(5), "request receipt did not finalize" + assert response.status == (400 if "failure" in item_order else 201) + observations = export_decision_receipts(orchestrator._store)["observations"] + assert len(observations) == 1 + observation = observations[0] + expected_status = ("selection_failed" if "failure" in item_order else + "acknowledged" if "fresh" in item_order else "cache_hit") + assert observation["status"] == expected_status + assert (observation["durable_ack_elapsed_ns"] is not None) == ("fresh" in item_order) + assert (observation["selection_elapsed_ns"] is not None) == ("fresh" in item_order) + finally: + server.shutdown() + server_worker.join() + server.server_close() + orchestrator.close() diff --git a/tests/test_decision_receipts.py b/tests/test_decision_receipts.py new file mode 100644 index 000000000..bc1873474 --- /dev/null +++ b/tests/test_decision_receipts.py @@ -0,0 +1,795 @@ +"""Initial decisions must be acknowledged before any answer is generated.""" + +import http.client +import json +import threading +import sqlite3 +import pytest + +from contextual_orchestrator import ModelAgent, TaskOrchestrator +from contextual_orchestrator.server import build_server, SecurityConfig + + +@pytest.mark.parametrize("scenario", ["saturated", "success", "conduct_success", "classifier_error", "trace_rejection"]) +@pytest.mark.parametrize("measurement_enabled", [False, True]) +def test_chat_stream_classification_owns_one_capacity_lease(tmp_path, monkeypatch, scenario, measurement_enabled): + """Classification and task execution share capacity and release it exactly once.""" + from contextual_orchestrator.server import RequestError + orchestrator = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")], + state_db=tmp_path / "state.db") + security = SecurityConfig(auth_token="test-token", max_concurrent_runs=1) + original_acquire, original_release = security.acquire_run_slot, security.release_run_slot + acquired, released, calls = [], [], [] + def acquire(): + original_acquire() + acquired.append(True) + def release(): + released.append(True) + original_release() + monkeypatch.setattr(security, "acquire_run_slot", acquire) + monkeypatch.setattr(security, "release_run_slot", release) + original_chat = orchestrator.client.chat + def chat(agent, messages, **kwargs): + calls.append(True) + if messages[0]["content"] == orchestrator.TRIAGE_SYSTEM_PROMPT: + return json.dumps({"workflow_required": scenario == "conduct_success"}) + return original_chat(agent, messages, **kwargs) + monkeypatch.setattr(orchestrator.client, "chat", chat) + if scenario == "classifier_error": + def reject_classifier(*args, **kwargs): + raise ValueError("unit classifier rejection") + monkeypatch.setattr(orchestrator, "would_route", reject_classifier) + server = build_server(orchestrator, port=0, decision_receipts=measurement_enabled, security=security) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + if scenario == "saturated": + original_acquire() + connection = http.client.HTTPConnection(*server.server_address) + try: + body = {"model": "orchestrator/auto", "mode": "auto", "stream": True, + "messages": [{"role": "user", "content": "capacity question"}]} + if scenario == "trace_rejection": + body["include_orchestration_trace"] = True + connection.request("POST", "/v1/chat/completions", json.dumps(body), + {"Content-Type": "application/json", "Authorization": "Bearer test-token"}) + response = connection.getresponse() + response.read() + assert response.status == {"saturated": 503, "success": 200, "conduct_success": 200, + "classifier_error": 400, "trace_rejection": 400}[scenario] + connection.close() + server.shutdown() + if scenario == "saturated": + assert calls == [] + assert acquired == released == [] + else: + assert len(acquired) == len(released) == 1 + original_acquire() + with pytest.raises(RequestError): + original_acquire() + original_release() + finally: + if scenario == "saturated": + original_release() + connection.close() + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() + + +@pytest.mark.parametrize("invalid_field", [None, "routing", "attribution", "user", "metadata", "implicit_trace", "explicit_trace", "authorized_trace"]) +def test_http_auto_stream_admits_before_triage(tmp_path, monkeypatch, invalid_field): + """Auto stream classification must share the eventual task's admission clock.""" + from contextual_orchestrator.decision_receipts import _CURRENT_DECISION, export_decision_receipts + orchestrator = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")], + state_db=tmp_path / "state.db") + snapshots = [] + original_chat = orchestrator.client.chat + def observed_chat(agent, messages, **kwargs): + current = _CURRENT_DECISION.get() + snapshots.append(current.snapshot() if current else None) + if messages[0]["content"] == orchestrator.TRIAGE_SYSTEM_PROMPT: + return '{"workflow_required": false}' + return original_chat(agent, messages, **kwargs) + monkeypatch.setattr(orchestrator.client, "chat", observed_chat) + security = SecurityConfig(auth_token="test-token") + if invalid_field in ("implicit_trace", "explicit_trace"): + security = SecurityConfig(bearer_verifier=lambda token, scope: token == "test-token" and scope == "inference", + expose_trace_by_default=True) + server = build_server(orchestrator, port=0, decision_receipts=True, security=security) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + connection = http.client.HTTPConnection(*server.server_address) + try: + request_body = { + "model": "orchestrator/auto", "mode": "auto", "stream": True, + "messages": [{"role": "user", "content": "streamed question"}], + } + if invalid_field in ("routing", "attribution", "user", "metadata"): + request_body[invalid_field] = "" if invalid_field == "user" else [] + if invalid_field in ("explicit_trace", "authorized_trace"): + request_body["include_orchestration_trace"] = True + connection.request("POST", "/v1/chat/completions", json.dumps(request_body), + {"Content-Type": "application/json", "Authorization": "Bearer test-token"}) + response = connection.getresponse() + response.read() + expected_status = 401 if invalid_field == "explicit_trace" else (400 if invalid_field in ("routing", "attribution", "user", "metadata", "authorized_trace") else 200) + assert response.status == expected_status + connection.close() + server.shutdown() + if invalid_field == "authorized_trace": + observation, = export_decision_receipts(orchestrator._store)["observations"] + assert observation["status"] == "selection_failed" + assert observation["durable_ack_elapsed_ns"] is None + assert snapshots[0]["request_id"] == observation["request_id"] + return + if expected_status != 200: + assert snapshots == [] + assert orchestrator._store.load("accepted_request") == [] + return + assert snapshots and all(snapshot is not None for snapshot in snapshots) + observation, = export_decision_receipts(orchestrator._store)["observations"] + assert observation["first_provider_phase"] == "structured_triage" + assert observation["durable_ack_elapsed_ns"] is not None + assert snapshots[0]["request_id"] == observation["request_id"] + finally: + connection.close() + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() + + +@pytest.mark.parametrize("endpoint", ["/v1/chat/completions", "/v1/responses"]) +def test_http_typed_stream_failure_before_selection_is_retained(tmp_path, monkeypatch, endpoint): + """Handled provider failures before task selection retain a failure denominator.""" + from contextual_orchestrator.provider_errors import ProviderUpstreamError + from contextual_orchestrator.decision_receipts import export_decision_receipts + orchestrator = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")], + state_db=tmp_path / "state.db") + def rejected(*args, **kwargs): + raise ProviderUpstreamError(agent_id="worker_one", model="mock/worker", + error_code="rate_limit_exceeded", message="unit rejection", + client_status=429, provider_status=429, retryable=True) + monkeypatch.setattr(orchestrator, "stream_route", rejected) + monkeypatch.setattr(orchestrator, "conduct", rejected) + if endpoint == "/v1/responses": + monkeypatch.setattr(orchestrator, "would_route", lambda *args, **kwargs: False) + server = build_server(orchestrator, port=0, decision_receipts=True, + security=SecurityConfig(auth_token="test-token")) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + connection = http.client.HTTPConnection(*server.server_address) + try: + request_body = {"model": "orchestrator/auto", "mode": "route", "stream": True} + if endpoint == "/v1/responses": + request_body["input"] = "question" + request_body.pop("mode") + else: + request_body["messages"] = [{"role": "user", "content": "question"}] + connection.request("POST", endpoint, json.dumps(request_body), + {"Content-Type": "application/json", "Authorization": "Bearer test-token"}) + response = connection.getresponse() + payload = response.read() + assert response.status == 200 + assert b"rate_limit_exceeded" in payload + connection.close() + server.shutdown() + observation, = export_decision_receipts(orchestrator._store)["observations"] + assert observation["status"] == "selection_failed" + assert observation["durable_ack_elapsed_ns"] is None + finally: + connection.close() + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() + + +def test_http_evidence_embedding_cold_and_warm_keep_task_interval(tmp_path, monkeypatch): + """Routing evidence calls occur only cold and remain before task acknowledgement.""" + from contextual_orchestrator.decision_receipts import DecisionMeasurement, _CURRENT_DECISION, export_decision_receipts + receipt_closed = threading.Event() + original_close = DecisionMeasurement.close + + def close_and_signal(measurement, reason="unfinished"): + """Observe real durable finalization without treating response bytes as a join.""" + original_close(measurement, reason) + receipt_closed.set() + + monkeypatch.setattr(DecisionMeasurement, "close", close_and_signal) + orchestrator = TaskOrchestrator([ + ModelAgent("worker_one", "mock/worker", tags=("writing",)), + ModelAgent("embedding_one", "mock-embedding", tags=("embedding",)), + ], state_db=tmp_path / "state.db") + original_embed = orchestrator.client.embed + embedding_snapshots = [] + embedding_inputs = [] + def observed_embed(*args, **kwargs): + embedding_inputs.extend(args[1]) + snapshot = _CURRENT_DECISION.get().snapshot() + embedding_snapshots.append(snapshot) + return original_embed(*args, **kwargs) + monkeypatch.setattr(orchestrator.client, "embed", observed_embed) + server = build_server(orchestrator, port=0, decision_receipts=True, + security=SecurityConfig(auth_token="test-token")) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + try: + counts = [] + for _ in range(2): + receipt_closed.clear() + connection = http.client.HTTPConnection(*server.server_address) + connection.request("POST", "/v1/chat/completions", json.dumps({ + "model": "orchestrator/auto", "mode": "route", + "messages": [{"role": "user", "content": "same evidence request"}], + }), {"Content-Type": "application/json", "Authorization": "Bearer test-token", + "x-cache-bypass": "true"}) + response = connection.getresponse() + response.read() + assert response.status == 200 + connection.close() + assert receipt_closed.wait(5), "request measurement did not finalize" + counts.append(len(embedding_snapshots)) + server.shutdown() + assert counts[0] > 0 + assert counts[1] == counts[0] + assert embedding_inputs.count("same evidence request") == 1 + assert all(snapshot["first_provider_phase"] == "routing_evidence_embedding" + for snapshot in embedding_snapshots) + cold, warm = export_decision_receipts(orchestrator._store)["observations"] + assert cold["auxiliary_dispatches"] + assert all(row["finished_elapsed_ns"] <= cold["selection_elapsed_ns"] + for row in cold["auxiliary_dispatches"] if row["phase"] == "routing_evidence_embedding") + assert any(row["phase"] == "post_decision_evidence_embedding" + for row in cold["auxiliary_dispatches"]) + assert warm["auxiliary_dispatches"] == [] + assert cold["durable_ack_elapsed_ns"] is not None + assert warm["durable_ack_elapsed_ns"] is not None + finally: + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() + + +def test_generated_planner_is_auxiliary_until_worker_selection(tmp_path, monkeypatch): + """A planning provider does not freeze the task-execution route interval.""" + from contextual_orchestrator.decision_receipts import DecisionMeasurement + orchestrator = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")], + state_db=tmp_path / "state.db") + measurement = DecisionMeasurement(orchestrator._store) + def plan_reply(agent, messages, **kwargs): + assert measurement.snapshot()["durable_ack_elapsed_ns"] is None + return json.dumps({"steps": [{"id": 0, "role": "worker", "agent_id": agent.id, + "subtask": "work", "access": []}, + {"id": 1, "role": "synthesizer", "agent_id": agent.id, + "subtask": "answer", "access": [0]}]}) + monkeypatch.setattr(orchestrator.client, "chat", plan_reply) + try: + orchestrator._plan_generated("answer") + assert measurement.snapshot()["status"] == "accepted" + assert measurement.snapshot()["first_provider_phase"] == "generated_planner" + assert not orchestrator._store.load("initial_decision") + finally: + measurement.close() + orchestrator.close() + + +@pytest.mark.parametrize("diagnostic_count", [16, 17]) +def test_diagnostic_window_cap_preserves_shared_admissions(tmp_path, diagnostic_count): + """One noisy request cannot remove another admitted request from the export.""" + from contextual_orchestrator.orchestrator import _StateStore + from contextual_orchestrator.decision_receipts import export_decision_receipts + store = _StateStore(tmp_path / "state.db") + try: + for request_id in ("outside_cohort", "quiet_request", "noisy_request"): + store.save("accepted_request", None, {"request_id": request_id}, durable=True) + store.save("auxiliary_dispatch", "outside_cohort", + {"request_id": "outside_cohort"}, durable=True) + for index in range(diagnostic_count): + store.save("auxiliary_dispatch", "noisy_request", + {"request_id": "noisy_request", "test_index": index}, durable=True) + exported = export_decision_receipts(store, limit=2) + assert exported["window"]["diagnostic_limit"] == 16 + assert exported["window"]["diagnostic_truncated"] is (diagnostic_count > 16) + quiet, noisy = exported["observations"] + assert quiet["request_id"] == "quiet_request" + assert quiet["status"] == "unfinished" + assert quiet["auxiliary_dispatches"] == [] + assert noisy["request_id"] == "noisy_request" + assert len(noisy["auxiliary_dispatches"]) == 16 + assert all(row["request_id"] == "noisy_request" for row in noisy["auxiliary_dispatches"]) + finally: + store.close() + + +def test_http_answer_cache_keeps_admission_without_provider_duration(tmp_path, monkeypatch): + """Answer reuse has its own terminal outcome, never a copied provider timing.""" + from contextual_orchestrator.decision_receipts import export_decision_receipts + orchestrator = TaskOrchestrator( + [ModelAgent("worker_one", "mock/worker")], + state_db=tmp_path / "state.db", cache_ttl=60, + ) + original_chat = orchestrator.client.chat + provider_calls = [] + def counted_chat(*args, **kwargs): + provider_calls.append(True) + return original_chat(*args, **kwargs) + monkeypatch.setattr(orchestrator.client, "chat", counted_chat) + server = build_server(orchestrator, port=0, decision_receipts=True, + security=SecurityConfig(auth_token="test-token")) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + answers = [] + try: + for _ in range(2): + connection = http.client.HTTPConnection(*server.server_address) + connection.request("POST", "/v1/chat/completions", json.dumps({ + "model": "orchestrator/auto", "mode": "route", + "messages": [{"role": "user", "content": "same answer"}], + }), {"Content-Type": "application/json", "Authorization": "Bearer test-token"}) + response = connection.getresponse() + answers.append(json.loads(response.read())["choices"][0]["message"]["content"]) + assert response.status == 200 + connection.close() + server.shutdown() + observations = export_decision_receipts(orchestrator._store)["observations"] + assert len(observations) == 2 + assert observations[0]["status"] == "acknowledged" + assert observations[1]["status"] == "cache_hit" + assert observations[1]["selection_elapsed_ns"] is None + assert observations[1]["durable_ack_elapsed_ns"] is None + assert observations[1]["first_provider_elapsed_ns"] is None + assert len(orchestrator._store.load("provider_dispatch")) == 1 + assert len(provider_calls) == 1 + assert answers[0] == answers[1] + finally: + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() + + +@pytest.mark.parametrize("endpoint,stream", [ + ("/v1/chat/completions", False), ("/v1/chat/completions", True), + ("/v1/responses", True), +]) +def test_http_route_persists_initial_decision(tmp_path, monkeypatch, endpoint, stream): + """A real HTTP route retains one native-clock receipt before completion.""" + orchestrator = TaskOrchestrator( + [ModelAgent("worker_one", "mock/worker")], state_db=tmp_path / "state.db" + ) + dispatch_records = [] + dispatch_ready = threading.Event() + generation_allowed = threading.Event() + dispatched_snapshots = [] + + def inspect_committed_decision(original_call, *args, **kwargs): + with sqlite3.connect(tmp_path / "state.db") as independent: + rows = independent.execute( + "SELECT payload FROM orchestration_records WHERE kind = 'initial_decision'" + ).fetchall() + assert independent.execute( + "SELECT COUNT(*) FROM orchestration_records WHERE kind = 'accepted_request'" + ).fetchone()[0] == 1 + assert len(rows) == 1 + dispatch_records.extend(rows) + from contextual_orchestrator.decision_receipts import _CURRENT_DECISION + dispatched_snapshots.append(_CURRENT_DECISION.get().snapshot()) + dispatch_ready.set() + assert generation_allowed.wait(10) + return original_call(*args, **kwargs) + + for method_name in ("chat", "stream_chat"): + original_call = getattr(orchestrator.client, method_name) + monkeypatch.setattr(orchestrator.client, method_name, + lambda *args, _call=original_call, **kwargs: + inspect_committed_decision(_call, *args, **kwargs)) + server = build_server(orchestrator, port=0, decision_receipts=True, + security=SecurityConfig(auth_token="test-token")) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + connection = http.client.HTTPConnection(*server.server_address) + try: + body = {"model": "orchestrator/auto", "mode": "route", "stream": stream, + "messages": [{"role": "user", "content": "hello"}]} + if endpoint == "/v1/responses": + body = {"model": "orchestrator/auto", "stream": True, "input": "hello"} + connection.request( + "POST", endpoint, json.dumps(body), + {"Content-Type": "application/json", "Authorization": "Bearer test-token"}, + ) + assert dispatch_ready.wait(10) + assert dispatched_snapshots[0]["status"] == "acknowledged" + assert dispatched_snapshots[0]["durable_ack_elapsed_ns"] is not None + # Hold generation after the ack: the final interval must remain exactly + # the pre-generation native value, irrespective of the hold duration. + assert not generation_allowed.is_set() + generation_allowed.set() + response = connection.getresponse() + response.read() + assert response.status == 200 + assert dispatch_records + records = orchestrator._store.load("decision_receipt") + assert len(records) == 1 + assert records[0]["status"] == "acknowledged" + assert records[0]["selected_agent_ids"] == ["worker_one"] + assert records[0]["selection_elapsed_ns"] <= records[0]["durable_ack_elapsed_ns"] + assert records[0]["durable_ack_elapsed_ns"] == dispatched_snapshots[0]["durable_ack_elapsed_ns"] + assert len(orchestrator._store.load("initial_decision")) == 1 + finally: + generation_allowed.set() + connection.close() + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() + + +def test_measurement_requires_durable_store(): + """Opt-in cannot silently lose every missing-store denominator observation.""" + orchestrator = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")]) + with pytest.raises(ValueError, match="durable state store"): + build_server(orchestrator, port=0, decision_receipts=True) + orchestrator.close() + + +def test_native_rejects_out_of_order_and_duplicate_ack(): + """Native transitions never manufacture missing elapsed times.""" + from contextual_orchestrator._decision_receipt import DecisionReceipt + + receipt = DecisionReceipt() + with pytest.raises(ValueError): + receipt.record_durable_ack() + assert receipt.status == "accepted" + assert receipt.durable_ack_elapsed_ns is None + receipt.record_selection() + receipt.record_durable_ack() + before = receipt.durable_ack_elapsed_ns + with pytest.raises(ValueError): + receipt.record_durable_ack() + assert receipt.durable_ack_elapsed_ns == before + + +def test_write_failure_has_no_ack_and_does_not_log_exception_contents(caplog): + """Failed commits cannot become successful timing samples or leak detail.""" + from contextual_orchestrator.decision_receipts import DecisionMeasurement + + auxiliary_records = [] + class FailedStore: + def save(self, *args, **kwargs): + if args[0] == "initial_decision": + raise RuntimeError("secret-canary-never-log") + if args[0] == "auxiliary_dispatch": + auxiliary_records.append(args[2]) + + measurement = DecisionMeasurement(FailedStore()) + try: + measurement.select(["worker_one"], "route") + snapshot = measurement.snapshot() + assert snapshot["status"] == "write_failed" + assert snapshot["durable_ack_elapsed_ns"] is None + snapshot["selected_agent_ids"].append("untrusted_mutation") + assert measurement.snapshot()["selected_agent_ids"] == ["worker_one"] + with measurement.auxiliary_call(["embedding_one"], "routing_evidence_embedding"): + pass + assert auxiliary_records[0]["phase"] == "post_decision_evidence_embedding" + finally: + measurement.close() + assert "secret-canary-never-log" not in caplog.text + assert "error_type=RuntimeError" in caplog.text + + +def test_admission_write_failure_rejects_before_dispatch_and_recovers(tmp_path, monkeypatch): + """A failed ingress receipt returns safe 503 and does not poison the next context.""" + orchestrator = TaskOrchestrator( + [ModelAgent("worker_one", "mock/worker")], state_db=tmp_path / "state.db" + ) + original_save = orchestrator._store.save + original_chat = orchestrator.client.chat + failed_once = [] + dispatched = [] + + def fail_first_admission(kind, *args, **kwargs): + if kind == "accepted_request" and not failed_once: + failed_once.append(True) + raise RuntimeError("never-disclose-storage-secret") + return original_save(kind, *args, **kwargs) + + def record_dispatch(*args, **kwargs): + dispatched.append(True) + return original_chat(*args, **kwargs) + + monkeypatch.setattr(orchestrator._store, "save", fail_first_admission) + monkeypatch.setattr(orchestrator.client, "chat", record_dispatch) + server = build_server(orchestrator, port=0, decision_receipts=True, + security=SecurityConfig(auth_token="test-token")) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + connection = http.client.HTTPConnection(*server.server_address) + try: + for expected_status in (503, 200): + connection.request("POST", "/v1/chat/completions", json.dumps({ + "model": "orchestrator/auto", "mode": "route", + "messages": [{"role": "user", "content": "hello"}], + }), {"Content-Type": "application/json", "Authorization": "Bearer test-token"}) + response = connection.getresponse() + payload = response.read().decode() + assert response.status == expected_status + assert "never-disclose-storage-secret" not in payload + if expected_status == 503: + assert not dispatched + assert '"measurement_complete": false' in payload + assert len(orchestrator._store.load("accepted_request")) == 1 + assert len(orchestrator._store.load("decision_receipt")) == 1 + finally: + connection.close() + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() + + +def test_export_retains_accepted_request_without_finalization(tmp_path): + """Crash-like missing finalization stays an unfinished denominator row.""" + from contextual_orchestrator.decision_receipts import export_decision_receipts + from contextual_orchestrator.orchestrator import _StateStore + + store = _StateStore(tmp_path / "state.db") + try: + store.save("accepted_request", None, { + "request_id": "accepted_only", "status": "accepted", + "selection_elapsed_ns": None, "durable_ack_elapsed_ns": None, + }, durable=True) + exported = export_decision_receipts(store) + assert exported["measurement_complete"] is False + assert exported["reconciliation_required"] is True + assert exported["observations"][0]["request_id"] == "accepted_only" + assert exported["observations"][0]["status"] == "unfinished" + assert exported["observations"][0]["durable_ack_elapsed_ns"] is None + finally: + store.close() + + +def test_export_window_keeps_unfinished_selected_cohort(tmp_path): + """A bounded admission cohort never silently drops its unfinished member.""" + from contextual_orchestrator.decision_receipts import export_decision_receipts + from contextual_orchestrator.orchestrator import _StateStore + + store = _StateStore(tmp_path / "state.db") + try: + for request_id in ("older_request", "unfinished_request", "newest_request"): + store.save("accepted_request", None, {"request_id": request_id, "status": "accepted"}, durable=True) + exported = export_decision_receipts(store, limit=2) + assert exported["window"]["truncated"] is True + assert [row["request_id"] for row in exported["observations"]] == ["unfinished_request", "newest_request"] + assert all(row["status"] == "unfinished" for row in exported["observations"]) + assert len(store.load("accepted_request")) == 3 # No retention deletion. + finally: + store.close() + + +def test_failed_nested_admission_restores_same_thread_context(tmp_path): + """A constructor failure restores its prior token before leaving the thread.""" + from contextual_orchestrator.decision_receipts import DecisionMeasurement, _CURRENT_DECISION + from contextual_orchestrator.orchestrator import _StateStore + + class FailedStore: + def save(self, *args, **kwargs): + raise RuntimeError("unavailable") + + store = _StateStore(tmp_path / "state.db") + outer = DecisionMeasurement(store) + try: + with pytest.raises(RuntimeError, match="could not be persisted"): + DecisionMeasurement(FailedStore()) + assert _CURRENT_DECISION.get() is outer + finally: + outer.close() + store.close() + assert _CURRENT_DECISION.get() is None + + +@pytest.mark.parametrize("invalid_capacity", [False, True]) +def test_race_receipt_retains_candidate_set_not_winner(tmp_path, monkeypatch, invalid_capacity): + """Real race workers share one clock; rejected races cannot acknowledge selection.""" + from contextual_orchestrator.decision_receipts import DecisionMeasurement + from contextual_orchestrator.orchestrator import MAX_LOCAL_CONCURRENCY + + contract = { + "contract_id": "test_contract", "model_revision": "test_revision", + "reasoning_effort_profile": "worker_medium", "capability_set": ["text"], + "structured_output_contract": "openai_response_v1", "accuracy_class": "full_precision", + "data_residency_policy": "test_region", "retention_policy": "zero_retention", + "context_limit": 128000, "pricing_evidence_id": "test_price_evidence", + "hedge_eligible": True, "cancellation_supported": False, + "execution_policy": "immediate_race", + } + agents = [ModelAgent(f"worker_{index}", "mock/worker", group_name="shared_group", + endpoint_equivalence=contract) + for index in range(2)] + orchestrator = TaskOrchestrator(agents, state_db=tmp_path / "state.db") + measurement = DecisionMeasurement(orchestrator._store) + try: + if invalid_capacity: + monkeypatch.setattr(orchestrator, "_equivalent_race_members", + lambda *args, **kwargs: [agents[0]] * (MAX_LOCAL_CONCURRENCY + 1)) + with pytest.raises(ValueError, match="concurrency capacity"): + orchestrator._invoke(agents[0], [{"role": "user", "content": "hello"}], + text="hello", role="worker") + assert measurement.receipt.status == "accepted" + assert not orchestrator._store.load("initial_decision") + else: + for _ in range(2): + orchestrator._invoke(agents[0], [{"role": "user", "content": "hello"}], + text="hello", role="worker") + snapshot = measurement.snapshot() + assert snapshot["status"] == "acknowledged" + assert snapshot["selection_attempt_count"] == 2 + assert set(snapshot["selected_agent_ids"]) == {"worker_0", "worker_1"} + assert len(orchestrator._store.load("initial_decision")) == 1 + finally: + measurement.close() + orchestrator.close() + + +def test_embedding_failover_has_one_admission_and_two_selection_attempts(tmp_path, monkeypatch): + """Two backend submissions remain children of one validated HTTP request.""" + from contextual_orchestrator.cost_router import CostRoutingCoordinator + + agents = [ModelAgent(f"embedding_{index}", f"mock-embedding-{index}", tags=("embedding",)) + for index in range(2)] + orchestrator = TaskOrchestrator(agents, state_db=tmp_path / "state.db") + class FixtureTokenCounter: + def count_text(self, text, model): + assert text == "hello" + return 1 + + coordinator = CostRoutingCoordinator(orchestrator, embedding_token_counter=FixtureTokenCounter()) + backend = coordinator.embedding_batch_backend + original_submit = backend.submit + submissions = [] + + def fail_first_submission(*args, **kwargs): + submissions.append(True) + if len(submissions) == 1: + raise RuntimeError("controlled first member failure") + return original_submit(*args, **kwargs) + + monkeypatch.setattr(backend, "submit", fail_first_submission) + server = build_server(orchestrator, port=0, coordinator=coordinator, + decision_receipts=True, security=SecurityConfig(auth_token="test-token")) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + connection = http.client.HTTPConnection(*server.server_address) + try: + connection.request("POST", "/v1/embeddings", json.dumps({"input": "hello"}), + {"Content-Type": "application/json", "Authorization": "Bearer test-token"}) + response = connection.getresponse() + response.read() + assert response.status == 200 + connection.close() + server.shutdown() + server.server_close() + assert len(submissions) == 2 + assert len(orchestrator._store.load("accepted_request")) == 1 + receipts = orchestrator._store.load("decision_receipt") + assert len(receipts) == 1 + assert receipts[0]["selection_attempt_count"] == 2 + assert receipts[0]["admission_boundary"] == "validated_endpoint" + assert len(orchestrator._store.load("selection_attempt")) == 1 + finally: + connection.close() + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() + + +def test_legacy_identity_backfill_and_indexed_window(tmp_path): + """Legacy keys migrate without touching unrelated data or scanning every phase.""" + from contextual_orchestrator.orchestrator import _StateStore + + database = tmp_path / "state.db" + store = _StateStore(database) + with store._conn: + store._conn.executemany( + "INSERT INTO orchestration_records(kind, key, payload) VALUES (?, NULL, ?)", + [(kind, json.dumps({"request_id": f"request_{index}", "status": "accepted"})) + for index in range(1000) for kind in ("accepted_request", "initial_decision")] + + [("unrelated_legacy", "not valid JSON")], + ) + store.close() + store = _StateStore(database) + traced = [] + try: + assert store._conn.execute("SELECT COUNT(*) FROM orchestration_records").fetchone()[0] == 2001 + assert store._conn.execute( + "SELECT key FROM orchestration_records WHERE kind = 'unrelated_legacy'" + ).fetchone()[0] is None + store._conn.set_trace_callback(traced.append) + cohort = store.load_decision_window(2) + store._conn.set_trace_callback(None) + assert len(cohort["accepted"]) == len(cohort["decisions"]) == 2 + phase_query = next(query for query in traced if query.startswith("SELECT kind, key, payload")) + plan = store._conn.execute("EXPLAIN QUERY PLAN " + phase_query).fetchall() + assert any("orchestration_records_kind_key_seq" in row[3] + and "kind=? AND key=?" in row[3] for row in plan) + assert cohort["window"]["truncated"] is True + finally: + store.close() + + +def test_http_cold_and_cached_triage_keep_task_ack_after_auxiliary_work(tmp_path, monkeypatch): + """Cold triage is diagnostic only; warm triage still measures the task decision.""" + from contextual_orchestrator.decision_receipts import _CURRENT_DECISION + + orchestrator = TaskOrchestrator( + [ModelAgent("worker_one", "mock/worker", tags=("writing",))], + state_db=tmp_path / "state.db", + ) + original_chat = orchestrator.client.chat + auxiliary_ready = threading.Event() + auxiliary_release = threading.Event() + auxiliary_snapshots = [] + task_snapshots = [] + + def controlled_chat(agent, messages, **kwargs): + measurement = _CURRENT_DECISION.get() + if messages[0]["content"] == orchestrator.TRIAGE_SYSTEM_PROMPT: + auxiliary_snapshots.append(measurement.snapshot()) + auxiliary_ready.set() + assert auxiliary_release.wait(10) + return '{"workflow_required": false}' + task_snapshots.append(measurement.snapshot()) + return original_chat(agent, messages, **kwargs) + + monkeypatch.setattr(orchestrator.client, "chat", controlled_chat) + server = build_server(orchestrator, port=0, decision_receipts=True, + security=SecurityConfig(auth_token="test-token")) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + try: + for request_index in range(2): + connection = http.client.HTTPConnection(*server.server_address) + connection.request("POST", "/v1/chat/completions", json.dumps({ + "model": "orchestrator/auto", "mode": "auto", + "messages": [{"role": "user", "content": "same question"}], + }), {"Content-Type": "application/json", "Authorization": "Bearer test-token", + "x-cache-bypass": "true"}) + if request_index == 0: + assert auxiliary_ready.wait(10) + snapshot = auxiliary_snapshots[0] + assert snapshot["durable_ack_elapsed_ns"] is None + assert snapshot.get("first_provider_elapsed_ns") is not None + assert snapshot.get("first_provider_phase") == "structured_triage" + auxiliary_release.set() + response = connection.getresponse() + response.read() + assert response.status == 200 + connection.close() + server.shutdown() + server.server_close() + assert len(auxiliary_snapshots) == 1 + assert len(task_snapshots) == 2 + cold, warm = task_snapshots + assert cold["first_provider_elapsed_ns"] < cold["selection_elapsed_ns"] <= cold["durable_ack_elapsed_ns"] + assert warm["first_provider_phase"] != "structured_triage" + assert warm["durable_ack_elapsed_ns"] is not None + from contextual_orchestrator.decision_receipts import export_decision_receipts + exported = export_decision_receipts(orchestrator._store) + cold_export, warm_export = exported["observations"] + assert cold_export["first_provider_boundary"] == "provider_ready_before_diagnostic_commit" + assert len(cold_export["auxiliary_dispatches"]) == 1 + auxiliary = cold_export["auxiliary_dispatches"][0] + assert auxiliary["phase"] == "structured_triage" + assert auxiliary["outcome"] == "completed" + assert auxiliary["finished_elapsed_ns"] <= cold_export["selection_elapsed_ns"] + assert warm_export["auxiliary_dispatches"] == [] + finally: + auxiliary_release.set() + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() diff --git a/tests/test_decision_wheel_manifest.py b/tests/test_decision_wheel_manifest.py new file mode 100644 index 000000000..5921ce11e --- /dev/null +++ b/tests/test_decision_wheel_manifest.py @@ -0,0 +1,31 @@ +"""Wheel ownership is validated before installation can mask package omissions.""" + +from pathlib import Path +import runpy +import zipfile + +import pytest + +verify_wheels = runpy.run_path( + str(Path(__file__).resolve().parents[1] / "scripts" / "verify_decision_wheel_manifest.py") +)["verify_wheels"] + + +@pytest.mark.parametrize("native_payload,expected_error", [ + ("contextual_orchestrator/_decision_receipt.abi3.so", None), + ("contextual_orchestrator/__init__.py", "overlap"), + ("unexpected_package/module.py", "only the receipt extension"), +]) +def test_native_wheel_ownership(tmp_path, native_payload, expected_error): + """Reject overlapping or unexpected native payloads; accept disjoint bindings.""" + core_path, native_path = tmp_path / "core.whl", tmp_path / "native.whl" + with zipfile.ZipFile(core_path, "w") as archive: + archive.writestr("contextual_orchestrator/__init__.py", "") + with zipfile.ZipFile(native_path, "w") as archive: + archive.writestr(native_payload, "unit fixture") + archive.writestr("native.dist-info/METADATA", "") + if expected_error: + with pytest.raises(ValueError, match=expected_error): + verify_wheels(core_path, native_path) + else: + verify_wheels(core_path, native_path) diff --git a/tests/test_nim_benchmark_release_acceptance.py b/tests/test_nim_benchmark_release_acceptance.py index 59ecde9ff..110f7d89c 100644 --- a/tests/test_nim_benchmark_release_acceptance.py +++ b/tests/test_nim_benchmark_release_acceptance.py @@ -345,7 +345,7 @@ def test_video_probe_fixture_is_one_decodable_frame_with_stable_hash() -> None: def test_smoke_manifest_cannot_authorize_production_routing(tmp_path: Path) -> None: - """The smoke manifest produces review evidence, not an automatic decision.""" + """A simulated run cannot label its cells as production-candidate evidence.""" report = nb.run_benchmark( "dry_run", TASK_MANIFEST_PATH, @@ -356,8 +356,8 @@ def test_smoke_manifest_cannot_authorize_production_routing(tmp_path: Path) -> N ) evaluation = report["evaluation"] - assert evaluation["evidence_status"] == "evidence_review_required" - assert evaluation["decision_use"] == "production_candidate_review" + assert evaluation["evidence_status"] == "synthetic_diagnostic_only" + assert evaluation["decision_use"] == "benchmark_smoke_only" assert evaluation["minimum_paired_task_count"] == 30 assert evaluation["required_completion_fraction"] == 0.9 assert evaluation["routing_recommendation"] is None diff --git a/tests/test_paper_contracts.py b/tests/test_paper_contracts.py index a87d620dc..79c3e09d7 100644 --- a/tests/test_paper_contracts.py +++ b/tests/test_paper_contracts.py @@ -1,6 +1,8 @@ from __future__ import annotations from pathlib import Path +import re +import subprocess import sys sys.path.insert(0, str(Path(__file__).resolve().parents[1])) @@ -10,6 +12,57 @@ ROOT_DIR = Path(__file__).resolve().parents[1] +def test_explicit_arxiv_references_have_inventory_entries() -> None: + """Keep tracked-text paper discovery complete without claiming paper review.""" + reference_pattern = re.compile( + r"(?:arxiv\.org/(?:abs|pdf|html)/|arxiv[:.])(\d{4}\.\d{4,5})(?:v\d+)?\b", + re.IGNORECASE, + ) + for citation_text in ( + "https://arxiv.org/abs/2601.17814", + "https://arxiv.org/pdf/2601.17814v1.pdf", + "https://arxiv.org/html/2601.17814v1", + "arXiv:2601.17814v2", + ): + assert reference_pattern.findall(citation_text) == ["2601.17814"] + tracked_paths = subprocess.check_output( + ["git", "ls-files", "-z"], cwd=ROOT_DIR, text=True + ).split("\0") + inventory = (ROOT_DIR / "docs/papers/README.md").read_text(encoding="utf-8") + inventoried_ids = set(re.findall(r"\b(\d{4}\.\d{4,5})(?:v\d+)?\b", inventory, re.I)) + missing_references = {} + for relative_path in tracked_paths: + source_path = ROOT_DIR / relative_path + if source_path.suffix not in {".py", ".rs", ".md", ".toml"}: + continue + references = set(reference_pattern.findall(source_path.read_text(encoding="utf-8"))) + if missing_ids := references - inventoried_ids: + missing_references[relative_path] = sorted(missing_ids) + assert not missing_references, missing_references + + +def test_explicit_doi_links_have_inventory_entries() -> None: + """Index DOI-only citations too; discovery does not establish source review.""" + reference_pattern = re.compile(r"https?://(?:dx\.)?doi\.org/([^\s<>\"`)]+)", re.I) + inventory = (ROOT_DIR / "docs/papers/README.md").read_text(encoding="utf-8") + inventoried_ids = {value.rstrip(".,;").lower() for value in reference_pattern.findall(inventory)} + tracked_paths = subprocess.check_output( + ["git", "ls-files", "-z"], cwd=ROOT_DIR, text=True + ).split("\0") + missing_references = {} + for relative_path in tracked_paths: + source_path = ROOT_DIR / relative_path + if source_path.suffix not in {".py", ".rs", ".md", ".toml"}: + continue + references = { + value.rstrip(".,;").lower() + for value in reference_pattern.findall(source_path.read_text(encoding="utf-8")) + } + if missing_ids := references - inventoried_ids: + missing_references[relative_path] = sorted(missing_ids) + assert not missing_references, missing_references + + class RecordingClient: def __init__(self) -> None: self.calls = [] diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 923c026df..02e0a7912 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -16,6 +16,8 @@ import threading import time +import pytest + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 @@ -121,6 +123,49 @@ def test_store_upserts_keyed_records_and_appends_streams() -> None: store.close() +@pytest.mark.parametrize("failure_phase", ["insert", "commit"]) +def test_failed_keyed_save_preserves_previous_committed_record(failure_phase: str) -> None: + """A failed replacement must not leak its deletion into the next commit.""" + with tempfile.TemporaryDirectory() as directory: + store = _StateStore(os.path.join(directory, "state.db")) + try: + store._conn.execute("PRAGMA foreign_keys = ON") + store.save("workflow_run", "run_existing", {"version": 1}) + if failure_phase == "insert": + store._conn.execute( + "CREATE TRIGGER reject_replacement BEFORE INSERT ON orchestration_records " + "WHEN NEW.payload = '{\"version\": 2}' " + "BEGIN SELECT RAISE(FAIL, 'injected write failure'); END" + ) + else: + # Both writes succeed; the deferred constraint fails only at commit. + store._conn.execute( + "CREATE TABLE linked_record (record_seq INTEGER REFERENCES " + "orchestration_records(seq) DEFERRABLE INITIALLY DEFERRED)" + ) + store._conn.execute( + "INSERT INTO linked_record SELECT seq FROM orchestration_records" + ) + store._conn.commit() + try: + store.save("workflow_run", "run_existing", {"version": 2}) + except sqlite3.IntegrityError: + pass + else: + raise AssertionError("the injected write failure did not occur") + assert not store._conn.in_transaction + store.save("workflow_run", "run_other", {"version": 3}) + assert store.load("workflow_run") == [{"version": 1}, {"version": 3}] + assert not store._conn.in_transaction + finally: + store.close() + reopened = _StateStore(os.path.join(directory, "state.db")) + try: + assert reopened.load("workflow_run") == [{"version": 1}, {"version": 3}] + finally: + reopened.close() + + def test_store_treats_kind_key_and_limit_as_sql_parameters() -> None: with tempfile.TemporaryDirectory() as directory: store = _StateStore(os.path.join(directory, "s.db")) @@ -294,7 +339,7 @@ def test_durable_audit_retention_is_bounded() -> None: store = _StateStore(os.path.join(directory, "s.db")) limit = store._STREAM_LIMITS["audit"] for index in range(limit + 3): - store.save("audit", None, {"index": index}) + store.save("audit", None, {"index": index}, durable=True) assert len(store.load("audit")) == limit assert store.load("audit", 1) == [{"index": limit + 2}] @@ -306,7 +351,7 @@ def test_durable_authorization_retention_is_bounded() -> None: store = _StateStore(os.path.join(directory, "s.db")) limit = store._STREAM_LIMITS["authorization"] for index in range(limit + 3): - store.save("authorization", None, {"index": index}) + store.save("authorization", None, {"index": index}, durable=True) assert len(store.load("authorization")) == limit assert store.load("authorization", 1) == [{"index": limit + 2}] @@ -319,13 +364,32 @@ def test_durable_analytics_retention_is_bounded() -> None: assert store._STREAM_LIMITS["analytics"] == 256 limit = 256 for index in range(limit + 3): - store.save("analytics", None, {"index": index}) + store.save("analytics", None, {"index": index}, durable=True) assert len(store.load("analytics")) == limit assert store.load("analytics", 1) == [{"index": limit + 2}] store.close() +def test_durable_stream_return_is_visible_to_an_independent_connection() -> None: + """A durable return must acknowledge commit, not enqueue or flush-on-read.""" + with tempfile.TemporaryDirectory() as directory: + database_path = os.path.join(directory, "s.db") + store = _StateStore(database_path) + try: + # Prevent the asynchronous worker from making a queued write look durable. + with store._stream_condition, sqlite3.connect(database_path) as reader: + for stream_kind in store._STREAM_LIMITS: + store.save(stream_kind, None, {"committed": True}, durable=True) + rows = reader.execute( + "SELECT payload FROM orchestration_records WHERE kind = ?", + (stream_kind,), + ).fetchall() + assert rows == [('{"committed": true}',)] + finally: + store.close() + + def test_authorization_stream_persists_separately_from_audit() -> None: with tempfile.TemporaryDirectory() as directory: db = os.path.join(directory, "state.db") diff --git a/tests/test_repository_security_metadata.py b/tests/test_repository_security_metadata.py index 6633a1b72..7f213ea27 100644 --- a/tests/test_repository_security_metadata.py +++ b/tests/test_repository_security_metadata.py @@ -90,12 +90,16 @@ def test_security_workflow_covers_core_repository_security_process(): def test_security_workflow_supports_stacked_pull_requests(): + """Keep stacked validation unfiltered without escalating event permissions.""" workflow_text = read_text(".github/workflows/security.yml") pull_request_trigger = workflow_text.split(" pull_request:\n", 1)[1].split( " schedule:\n", 1 )[0] - assert "branches:" not in pull_request_trigger + for filter_name in ("branches", "branches-ignore", "paths", "paths-ignore"): + assert f"{filter_name}:" not in pull_request_trigger + assert "permissions:\n contents: read\n" in workflow_text + assert "pull_request_target:" not in workflow_text def test_dependabot_tracks_actions_and_python_dependencies(): @@ -214,7 +218,10 @@ def test_unit_workflow_uses_the_project_lock_for_git_runtime_dependencies(): assert re.search(r"@[0-9a-f]{40}(?:\s+#|$)", setup_uv_line) assert "# v" in setup_uv_line assert 'version: "0.12.5"' in workflow_text - assert "uv run --locked --extra api --extra db --extra queue --group dev python -m pytest -q" in workflow_text + locked_sync = "uv sync --locked --extra api --extra db --extra queue --group dev --group native-build" + native_build = "uv run --no-sync maturin develop --locked --release --features pyo3/extension-module" + full_tests = "uv run --no-sync python -m pytest -q" + assert workflow_text.index(locked_sync) < workflow_text.index(native_build) < workflow_text.index(full_tests) def test_local_full_suite_installs_runtime_and_test_lockfiles(): diff --git a/tests/test_workflow_request_link.py b/tests/test_workflow_request_link.py new file mode 100644 index 000000000..3f35cb053 --- /dev/null +++ b/tests/test_workflow_request_link.py @@ -0,0 +1,151 @@ +"""Trusted HTTP request identity survives durable workflow persistence.""" + +import http.client +import json +import threading + +import pytest + +from contextual_orchestrator import ModelAgent, TaskOrchestrator +from contextual_orchestrator.server import SecurityConfig, build_server +from contextual_orchestrator.telemetry import current_request_id, request_identity + + +@pytest.mark.parametrize("mode,stream", [("route", False), ("conduct", False), ("route", True), ("write_failure", True)]) +@pytest.mark.parametrize("measurement_enabled", [False, True]) +def test_http_workflow_retains_origin_request(tmp_path, monkeypatch, mode, stream, measurement_enabled): + """Provider-observed identity joins the stored run without caller identity trust.""" + orchestrator = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")], + state_db=tmp_path / "state.db") + observed_ids = [] + if mode == "write_failure": + original_save = orchestrator._store.save + + def fail_workflow_write(kind, *args, **kwargs): + if kind == "workflow_run": + raise RuntimeError("private-storage-failure") + return original_save(kind, *args, **kwargs) + + monkeypatch.setattr(orchestrator._store, "save", fail_workflow_write) + for method_name in ("chat", "stream_chat"): + original_method = getattr(orchestrator.client, method_name) + + def observe(*args, _method=original_method, **kwargs): + observed_ids.append(current_request_id()) + return _method(*args, **kwargs) + + monkeypatch.setattr(orchestrator.client, method_name, observe) + server = build_server(orchestrator, port=0, decision_receipts=measurement_enabled, + security=SecurityConfig(auth_token="unit-token")) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + connection = http.client.HTTPConnection(*server.server_address, timeout=20) + try: + connection.request("POST", "/v1/chat/completions", json.dumps({ + "model": "orchestrator/auto", "mode": "route" if mode == "write_failure" else mode, "stream": stream, + "messages": [{"role": "user", "content": "Explain addition briefly."}], + }), {"Content-Type": "application/json", "Authorization": "Bearer unit-token", + "X-Request-ID": "untrusted-client-id"}) + response = connection.getresponse() + response_payload = response.read().decode() + connection.close() + server.shutdown() + server.server_close() + assert response.status == 200 + stored_runs = orchestrator._store.load("workflow_run") + if mode == "write_failure": + assert not stored_runs + assert len(orchestrator._workflow_runs) == 1 + assert '"finish_reason": "error"' in response_payload + assert "private-storage-failure" not in response_payload + if measurement_enabled: + from contextual_orchestrator.decision_receipts import export_decision_receipts + receipts = export_decision_receipts(orchestrator._store)["observations"] + assert len(receipts) == 1 + assert receipts[0]["status"] == "acknowledged" + return + assert len(stored_runs) == 1 + assert observed_ids and len(set(observed_ids)) == 1 + assert observed_ids[0] and observed_ids[0] != "untrusted-client-id" + assert stored_runs[0]["request_id"] == observed_ids[0] + if measurement_enabled: + from contextual_orchestrator.decision_receipts import export_decision_receipts + receipts = export_decision_receipts(orchestrator._store)["observations"] + assert len(receipts) == 1 + assert receipts[0]["request_id"] == stored_runs[0]["request_id"] + run_id = stored_runs[0]["workflow_run_id"] + orchestrator.close() + restored = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")], + state_db=tmp_path / "state.db") + try: + assert restored.get_workflow_run(run_id)["request_id"] == observed_ids[0] + finally: + restored.close() + finally: + connection.close() + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() + + +def test_workflow_update_preserves_original_request_identity(): + """Later request contexts cannot claim an existing run or a non-HTTP run.""" + orchestrator = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")]) + try: + for origin_http in (False, True): + run_id = f"run_{origin_http}" + record = {"workflow_run_id": run_id, "trace": []} + if origin_http: + with request_identity() as origin_id: + orchestrator._replace_workflow_run(record) + else: + origin_id = None + orchestrator._replace_workflow_run(record) + with request_identity(): + replacement = {"workflow_run_id": run_id, "trace": []} + orchestrator._replace_workflow_run(replacement) + assert replacement.get("request_id") == origin_id + finally: + orchestrator.close() + + +def test_http_cache_hit_keeps_distinct_outcome_identity(tmp_path): + """A reused answer creates a cache-hit outcome, not a reassigned execution.""" + from contextual_orchestrator.decision_receipts import export_decision_receipts + + orchestrator = TaskOrchestrator([ModelAgent("worker_one", "mock/worker")], + state_db=tmp_path / "state.db", cache_ttl=60) + server = build_server(orchestrator, port=0, decision_receipts=True, + security=SecurityConfig(auth_token="unit-token")) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + try: + for _ in range(2): + connection = http.client.HTTPConnection(*server.server_address, timeout=20) + try: + connection.request("POST", "/v1/chat/completions", json.dumps({ + "model": "orchestrator/auto", "mode": "route", + "messages": [{"role": "user", "content": "Explain addition briefly."}], + }), {"Content-Type": "application/json", "Authorization": "Bearer unit-token"}) + response = connection.getresponse() + response.read() + assert response.status == 200 + finally: + connection.close() + server.shutdown() + server.server_close() + stored_runs = orchestrator._store.load("workflow_run") + receipts = export_decision_receipts(orchestrator._store)["observations"] + assert [record["cache_status"] for record in stored_runs] == ["miss", "hit"] + assert len({record["workflow_run_id"] for record in stored_runs}) == 2 + assert len({record["request_id"] for record in stored_runs}) == 2 + assert {record["request_id"] for record in stored_runs} == { + receipt["request_id"] for receipt in receipts + } + assert sorted(receipt["status"] for receipt in receipts) == ["acknowledged", "cache_hit"] + finally: + server.shutdown() + worker.join() + server.server_close() + orchestrator.close() diff --git a/uv.lock b/uv.lock index 6e9687c5a..47d34571f 100644 --- a/uv.lock +++ b/uv.lock @@ -424,6 +424,9 @@ dev = [ { name = "hypothesis" }, { name = "pytest" }, ] +native-build = [ + { name = "maturin" }, +] [package.metadata] requires-dist = [ @@ -450,6 +453,7 @@ dev = [ { name = "hypothesis", specifier = ">=6.100" }, { name = "pytest", specifier = ">=8.0" }, ] +native-build = [{ name = "maturin", specifier = "==1.15.0" }] [[package]] name = "cryptography" @@ -886,6 +890,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, ] +[[package]] +name = "maturin" +version = "1.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b9/c8/22e5e21b2679c9bce6415ca578034ca2cc9316be0642ae21e051a2d5198c/maturin-1.15.0.tar.gz", hash = "sha256:94b26cc8e8aba61a5f2099715fe640e18c5f678e9a500408b38761263954228a", size = 385504, upload-time = "2026-08-24T12:11:22.665Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/69/5c01b461044eb1f45ddcce006706eb88110c793cdb11c7ae0b5e08492e94/maturin-1.15.0-py3-none-linux_armv6l.whl", hash = "sha256:6bf6dc62e22d4dcfd5a51244ff0d58975fa4979c48209fe84159617648956d82", size = 10206220, upload-time = "2026-08-24T12:10:53.327Z" }, + { url = "https://files.pythonhosted.org/packages/eb/1f/2b431554e11687cdb1077e0cdadcc118c53f611086b3af00c8545a67c6a5/maturin-1.15.0-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:cd35772633f489841132bc8e71d6fc7f842df30b9c05cd5cdf1ee1ddcb744cc7", size = 19416513, upload-time = "2026-08-24T12:10:56.126Z" }, + { url = "https://files.pythonhosted.org/packages/51/36/e23a21cb34a648b711036b9b2fe1d4f3f4ee24f8db54215d73f1a9a3a3ec/maturin-1.15.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:c40b4eae7bf5ef1f4b1af8d623fe4105016f93578fb15b764e741d08ec3b92dd", size = 10014962, upload-time = "2026-08-24T12:10:58.486Z" }, + { url = "https://files.pythonhosted.org/packages/8c/80/33b15cb2d8f30f12c807955e8f2fd775027692904e30ec0784744ce8cd83/maturin-1.15.0-py3-none-manylinux_2_12_i686.manylinux2010_i686.musllinux_1_1_i686.whl", hash = "sha256:7eb066372f541f8eb4909c79c5d9bd0b9e8125980bdf1ec9e8aba23c6c8d6c55", size = 10196223, upload-time = "2026-08-24T12:11:00.696Z" }, + { url = "https://files.pythonhosted.org/packages/fe/91/b495e19e2f5c503b540452b2039115e7b2363867e8c5ad4179eb752fa92c/maturin-1.15.0-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl", hash = "sha256:653020a63525bb224e5ab0adf02e17a2e08bc86dbea7fc1399c9a56d7529b99e", size = 10541186, upload-time = "2026-08-24T12:11:02.857Z" }, + { url = "https://files.pythonhosted.org/packages/5d/2b/2abff58037188d852b124871b1f0d720e1c2bfb3d4f1b03d87c52cd66488/maturin-1.15.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:0ebf9767892725083138e671c34482c660317a2f3d6a29fc0e0f34e9d8c99136", size = 10083468, upload-time = "2026-08-24T12:11:05.012Z" }, + { url = "https://files.pythonhosted.org/packages/3f/07/b7e9f8be99a6627849e81ac7b6694876bce8f50a92995fe17e3cf2610f0a/maturin-1.15.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:7ab7eebffd7b8debca2265985de4eaeb332141276d24b9560b5ad484d4b3add1", size = 10047786, upload-time = "2026-08-24T12:11:07.042Z" }, + { url = "https://files.pythonhosted.org/packages/ca/ab/167e3cb7accee11b507dbe53e0e87aeccb376d44ae66284c96ee4df3a9fd/maturin-1.15.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.musllinux_1_1_ppc64le.whl", hash = "sha256:126e12e618b4db42f68c779a56d41f82a390145ba36ac3f621d057eb34f5ad9d", size = 13315332, upload-time = "2026-08-24T12:11:09.433Z" }, + { url = "https://files.pythonhosted.org/packages/14/4d/801379f646cbc6b00998e5289b0630a886be3a4ee4c75b6bc9b87478a7f1/maturin-1.15.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4f9d33e6c3f9615c8caceecbbbd440f8eb25a3ddeb687077682cd5eca2e9ae15", size = 10807183, upload-time = "2026-08-24T12:11:11.73Z" }, + { url = "https://files.pythonhosted.org/packages/89/27/2e612e1cbd1580e9e94d4722c227b5180dca27b32b955a34b79918aa1292/maturin-1.15.0-py3-none-manylinux_2_31_riscv64.musllinux_1_1_riscv64.whl", hash = "sha256:bf29beddd0c6708f112db51d5275fc28b28b9e9c9c5faae387eaef662918b176", size = 10413274, upload-time = "2026-08-24T12:11:14.04Z" }, + { url = "https://files.pythonhosted.org/packages/70/d8/202a7b4d75a51f20f84ec9ce3b7345b12b822207072164e2b1c6ef665125/maturin-1.15.0-py3-none-win32.whl", hash = "sha256:da649988be98e87e009e51b1bf0d301b6a301bc0cecbdd60d40d8ba60748d1ca", size = 8928744, upload-time = "2026-08-24T12:11:16.269Z" }, + { url = "https://files.pythonhosted.org/packages/40/dc/4e90da594986ba78dd3bc8a5921ecdcdb11085b22b02a412caab3b225601/maturin-1.15.0-py3-none-win_amd64.whl", hash = "sha256:552c2be4afd43fe8d5c9f3ec8d4c4756d973b8dcbe94c14084390301f50243e1", size = 10335085, upload-time = "2026-08-24T12:11:18.326Z" }, + { url = "https://files.pythonhosted.org/packages/8b/10/15d4314edf130955edf2dc237aa393a8a7c10f2b9b57b89fa2f61f915659/maturin-1.15.0-py3-none-win_arm64.whl", hash = "sha256:c7dc0c66c78d3debdd9c5aa807e861fbcbf07f3505d34b125df74c03986b0f48", size = 9713795, upload-time = "2026-08-24T12:11:20.83Z" }, +] + [[package]] name = "numpy" version = "2.5.2"