From 35a428cafd8c246515d087912611d2a0c092e2d5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 17:25:59 +0900 Subject: [PATCH 01/78] fix(gateway): remove implicit model request timeout --- CHANGELOG.md | 3 +++ contextual_orchestrator/cost_router.py | 5 ++++- contextual_orchestrator/endpoint_race.py | 12 ++++++++---- contextual_orchestrator/orchestrator.py | 10 +++++----- contextual_orchestrator/server.py | 19 +++++++++++++------ tests/test_orchestrator_client_boundaries.py | 16 ++++++++++++++++ .../test_provider_embedding_batch_backend.py | 14 ++++++++++++++ 7 files changed, 63 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f0b268c9..fbcc6e4a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,9 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ### Fixed +- Model, Agent, gateway, and structured-output repair requests now default to + no application timeout. Explicit probe, discovery, benchmark, and operator + limits remain bounded. - Workflow workers now preserve the caller message array exactly once, while the added envelope carries only the subtask and Conductor-style prior-step access list instead of duplicating the task or source attachments. diff --git a/contextual_orchestrator/cost_router.py b/contextual_orchestrator/cost_router.py index 751c9e366..f56934beb 100644 --- a/contextual_orchestrator/cost_router.py +++ b/contextual_orchestrator/cost_router.py @@ -232,7 +232,10 @@ def _run_embedding_shard( def _provider_embedding_backend(self) -> ProviderEmbeddingBatchBackend: client = getattr(self.orchestrator, "client", None) - client_timeout = float(getattr(client, "timeout", 0)) + configured_timeout = getattr(client, "timeout", None) + client_timeout = ( + float(configured_timeout) if configured_timeout is not None else 0.0 + ) return ProviderEmbeddingBatchBackend( self._run_provider_embeddings, job_registry=self.job_registry, diff --git a/contextual_orchestrator/endpoint_race.py b/contextual_orchestrator/endpoint_race.py index 7d6b5c682..6f4b531f5 100644 --- a/contextual_orchestrator/endpoint_race.py +++ b/contextual_orchestrator/endpoint_race.py @@ -79,7 +79,7 @@ def race_first_valid( attempts: list[EndpointAttempt[T]], *, validate: Callable[[T], bool], - deadline_seconds: float, + deadline_seconds: float | None, max_concurrency: int, on_attempt_complete: Callable[[str, T | None, BaseException | None], None] | None = None, ) -> RaceOutcome[T]: @@ -94,7 +94,7 @@ def race_first_valid( raise ValueError("immediate_race requires concurrency capacity of at least two") if max_concurrency < len(attempts): raise ValueError("immediate_race capacity must cover every declared endpoint") - if deadline_seconds <= 0: + if deadline_seconds is not None and deadline_seconds <= 0: raise ValueError("deadline_seconds must be positive") contract = attempts[0].contract if any(attempt.contract != contract for attempt in attempts[1:]): @@ -128,8 +128,12 @@ def execute(attempt: EndpointAttempt[T]) -> T: last_error: BaseException | None = None try: while pending: - remaining = deadline_seconds - (time.monotonic() - started) - if remaining <= 0: + remaining = ( + None + if deadline_seconds is None + else deadline_seconds - (time.monotonic() - started) + ) + if remaining is not None and remaining <= 0: raise TimeoutError("equivalent endpoint race exceeded its deadline") done, pending = wait(pending, timeout=remaining, return_when=FIRST_COMPLETED) if not done: diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 9d168b3b3..12dae62bc 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -1030,7 +1030,7 @@ def _local_provider_state(base_url: str) -> _LocalProviderState: def _local_provider_slot( agent: ModelAgent, capacity: int, - timeout: float, + timeout: float | None, ): """Bound local requests and serialize model switches on a shared endpoint.""" if not _is_local_provider_url(agent.base_url): @@ -1038,7 +1038,7 @@ def _local_provider_slot( return state = _local_provider_state(agent.base_url) - deadline = time.monotonic() + max(float(timeout), 0.0) + deadline = None if timeout is None else time.monotonic() + max(float(timeout), 0.0) with state.condition: while True: if state.active == 0: @@ -1051,8 +1051,8 @@ def _local_provider_slot( state.active += 1 break - remaining = deadline - time.monotonic() - if remaining <= 0: + remaining = None if deadline is None else deadline - time.monotonic() + if remaining is not None and remaining <= 0: raise TimeoutError("local provider endpoint is busy past its request deadline") state.condition.wait(remaining) @@ -1693,7 +1693,7 @@ class ModelClient: def __init__( self, - timeout: int = 90, + timeout: float | None = None, max_output_tokens: int = 2048, max_retries: int = 2, local_max_retries: int = 0, diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index eb6a77519..7c6060dfd 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -7286,25 +7286,32 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di if not attribution.get("service"): attribution["service"] = "embeddings_api" started_at = time.perf_counter() - embedding_deadline = time.monotonic() + float( - orchestrator.client.timeout + configured_timeout = orchestrator.client.timeout + embedding_deadline = ( + None + if configured_timeout is None + else time.monotonic() + float(configured_timeout) ) document = None last_embedding_error: Exception | None = None for embedding_agent in embedding_agents: - remaining_timeout = embedding_deadline - time.monotonic() - if remaining_timeout <= 0: + remaining_timeout = ( + None + if embedding_deadline is None + else embedding_deadline - time.monotonic() + ) + if remaining_timeout is not None and remaining_timeout <= 0: break attempt_started_at = time.perf_counter() try: - document = self._run(lambda agent=embedding_agent: coordinator.complete_embeddings_batch( + document = self._run(lambda agent=embedding_agent, wait_timeout=remaining_timeout: coordinator.complete_embeddings_batch( inputs, model=agent.model, attribution=attribution, metadata={"actor_scope": "inference", "endpoint_alias": "embeddings"}, zdr_only=zdr_only, agent_id=agent.id, - wait_timeout=remaining_timeout, + wait_timeout=wait_timeout, owner_id=security.principal_id(self.headers), )) except Exception as exc: # noqa: BLE001 - measured member failover diff --git a/tests/test_orchestrator_client_boundaries.py b/tests/test_orchestrator_client_boundaries.py index 9bb489b41..eb227554f 100644 --- a/tests/test_orchestrator_client_boundaries.py +++ b/tests/test_orchestrator_client_boundaries.py @@ -236,6 +236,22 @@ def test_batch_results_must_be_a_mapping() -> None: # -- local provider slot concurrency ------------------------------------------ +def test_default_model_timeout_is_unbounded() -> None: + """Model and repair requests inherit no application wall-clock cap.""" + assert ModelClient().timeout is None + + +def test_local_slot_accepts_unbounded_waits() -> None: + """The local-agent coordinator preserves the shared unbounded default.""" + agent = ModelAgent( + id="unbounded_slot_agent", + model="unbounded-slot-model", + base_url="local://127.0.0.1:59343/v1", + ) + with _local_provider_slot(agent, 1, None): + pass + + def test_slot_shrinks_capacity_for_same_model_and_resets_when_empty() -> None: """Concurrent same-model holders shrink capacity; last release resets.""" url = "local://127.0.0.1:59341/v1" diff --git a/tests/test_provider_embedding_batch_backend.py b/tests/test_provider_embedding_batch_backend.py index 0eb661fbb..201211c19 100644 --- a/tests/test_provider_embedding_batch_backend.py +++ b/tests/test_provider_embedding_batch_backend.py @@ -46,6 +46,20 @@ def count_text(self, text, model): return len(text.split()) +def test_default_client_keeps_batch_lifecycle_separate_from_model_timeout() -> None: + """A null model timeout does not break the existing batch-retention boundary.""" + coordinator = CostRoutingCoordinator( + TaskOrchestrator([], allow_empty_agents=True), + embedding_token_counter=_SyntheticExactCounter(), + ) + + backend = coordinator._provider_embedding_backend() + + assert backend._execution_timeout_seconds == 604_800 + assert backend._claim_lease_seconds is None + backend.close() + + def test_unknown_tokenizer_uses_authoritative_provider_usage() -> None: """A byte-safe request completes only after the provider supplies exact usage.""" agent = ModelAgent( From 284447fcee372437370ca86b5f6760c47f044dd2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 4 Sep 2026 19:09:28 +0900 Subject: [PATCH 02/78] fix(embeddings): preserve unbounded completion wait Signed-off-by: Seongho Bae --- contextual_orchestrator/batch_routing.py | 4 +- contextual_orchestrator/cost_router.py | 21 +++++--- .../test_provider_embedding_batch_backend.py | 54 +++++++++++++++++-- 3 files changed, 66 insertions(+), 13 deletions(-) diff --git a/contextual_orchestrator/batch_routing.py b/contextual_orchestrator/batch_routing.py index 5650e441d..27e2c1c2d 100644 --- a/contextual_orchestrator/batch_routing.py +++ b/contextual_orchestrator/batch_routing.py @@ -1111,8 +1111,8 @@ def _publish_terminal( self._errors[job_id] = error self._states[job_id] = status - def wait(self, job: BatchJob, *, timeout: float) -> Dict[str, Any]: - """Wait within the caller's explicit deadline for a terminal state.""" + def wait(self, job: BatchJob, *, timeout: float | None) -> Dict[str, Any]: + """Wait for a terminal state, bounded only when the caller sets a deadline.""" event = self._terminal_events.get(job.job_id) if event is not None: event.wait(timeout=timeout) diff --git a/contextual_orchestrator/cost_router.py b/contextual_orchestrator/cost_router.py index f56934beb..cb584c1b3 100644 --- a/contextual_orchestrator/cost_router.py +++ b/contextual_orchestrator/cost_router.py @@ -59,6 +59,7 @@ _DEFAULT_EMBEDDING_MAX_TOKENS_PER_REQUEST = 280_000 _DEFAULT_EMBEDDING_MAX_CHARS_PER_PART = 240_000 _DEFAULT_EMBEDDING_MAX_INPUTS_PER_REQUEST = 1 +_DEFAULT_PROVIDER_EMBEDDING_CLAIM_LEASE_SECONDS = 30.0 _BATCH_LEDGER_SETTLEMENT_TIMEOUT_SECONDS = 1.0 _EMBEDDING_UNIT_RE = re.compile(r"\S+\s*|\s+", re.UNICODE) @@ -243,7 +244,11 @@ def _provider_embedding_backend(self) -> ProviderEmbeddingBatchBackend: claim_lease_seconds=( client_timeout if self.job_registry.durable and client_timeout > 0 - else None + else ( + _DEFAULT_PROVIDER_EMBEDDING_CLAIM_LEASE_SECONDS + if self.job_registry.durable + else None + ) ), execution_timeout_seconds=client_timeout if client_timeout > 0 else None, ) @@ -1837,9 +1842,9 @@ def complete_embeddings_batch( ) -> Dict[str, Any]: """Submit an embeddings batch and return its document (one round-trip). - Local backends complete immediately. Callers that require a synchronous - provider result pass ``wait_timeout``; a timed-out queued job is - cancelled so the synchronous surface does not leave orphaned work. + Local backends complete immediately. ``wait_timeout=None`` waits without + an application deadline; a timed-out queued job is cancelled only when + the caller supplied a finite deadline. """ job = self.submit_embeddings_batch( inputs, @@ -1851,9 +1856,13 @@ def complete_embeddings_batch( owner_id=owner_id, ) backend = self._embedding_backend_for(job) - if wait_timeout is not None and hasattr(backend, "wait"): + if hasattr(backend, "wait"): status = backend.wait(job, timeout=wait_timeout) - if not status.get("is_complete") and hasattr(backend, "cancel"): + if ( + wait_timeout is not None + and not status.get("is_complete") + and hasattr(backend, "cancel") + ): backend.cancel(job, reason="synchronous request deadline elapsed") return self.embeddings_batch_document(job.job_id, owner_id=owner_id) diff --git a/tests/test_provider_embedding_batch_backend.py b/tests/test_provider_embedding_batch_backend.py index 201211c19..37a7311d4 100644 --- a/tests/test_provider_embedding_batch_backend.py +++ b/tests/test_provider_embedding_batch_backend.py @@ -5,11 +5,7 @@ import pytest -from contextual_orchestrator.batch_routing import ( - EmbeddingBatchRequest, - ProviderEmbeddingBatchBackend, -) -from contextual_orchestrator.batch_job_registry import JobRegistryFactory +import contextual_orchestrator.cost_router as cost_router_module from contextual_orchestrator import ( CostRoutingCoordinator, InMemoryConfigStore, @@ -18,6 +14,11 @@ PriceEntry, TaskOrchestrator, ) +from contextual_orchestrator.batch_job_registry import JobRegistryFactory +from contextual_orchestrator.batch_routing import ( + EmbeddingBatchRequest, + ProviderEmbeddingBatchBackend, +) from contextual_orchestrator.orchestrator import ModelClient from contextual_orchestrator.provider_errors import ProviderUpstreamError from contextual_orchestrator.server import SecurityConfig, build_server @@ -60,6 +61,49 @@ def test_default_client_keeps_batch_lifecycle_separate_from_model_timeout() -> N backend.close() +def test_durable_claim_lease_does_not_depend_on_model_timeout(monkeypatch) -> None: + """A null model timeout still supplies the durable registry a positive lease.""" + coordinator = CostRoutingCoordinator( + TaskOrchestrator([], allow_empty_agents=True), + embedding_token_counter=_SyntheticExactCounter(), + ) + coordinator.job_registry._client = object() + captured = {} + + def capture_backend(*_args, **kwargs): + captured.update(kwargs) + return object() + + monkeypatch.setattr(cost_router_module, "ProviderEmbeddingBatchBackend", capture_backend) + + coordinator._provider_embedding_backend() + + assert captured["claim_lease_seconds"] == 30.0 + assert captured["execution_timeout_seconds"] is None + + +def test_unbounded_synchronous_embedding_waits_for_provider_completion() -> None: + """No application deadline means wait, rather than return an unfinished document.""" + def delayed_runner(requests): + time.sleep(0.05) + return [[1.0] for _request in requests], len(requests) + + backend = ProviderEmbeddingBatchBackend(delayed_runner) + coordinator = CostRoutingCoordinator( + TaskOrchestrator([], allow_empty_agents=True), + embedding_batch_backend=backend, + embedding_token_counter=_SyntheticExactCounter(), + ) + + document = coordinator.complete_embeddings_batch( + ["delayed provider input"], model="synthetic-model" + ) + + assert document["status"] == "completed" + assert document["embeddings"][0]["embedding"] == [1.0] + backend.close() + + def test_unknown_tokenizer_uses_authoritative_provider_usage() -> None: """A byte-safe request completes only after the provider supplies exact usage.""" agent = ModelAgent( From 35465a5097587882926b6c48475182f7e8feff39 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:08:29 +0900 Subject: [PATCH 03/78] test(gateway): require durable model timeout policy Signed-off-by: Seongho Bae --- tests/test_model_timeout_policy.py | 50 ++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 tests/test_model_timeout_policy.py diff --git a/tests/test_model_timeout_policy.py b/tests/test_model_timeout_policy.py new file mode 100644 index 000000000..7cf9c079e --- /dev/null +++ b/tests/test_model_timeout_policy.py @@ -0,0 +1,50 @@ +"""Administrator-owned model timeout policy must survive configuration changes.""" + +from pathlib import Path + +import pytest + +from contextual_orchestrator import ModelAgent, TaskOrchestrator + + +def test_model_timeout_policy_defaults_to_null() -> None: + """An ordinary model has no administrator-imposed execution limit.""" + model_agent = ModelAgent("timeout_agent", "example-model") + assert model_agent.to_config()["model_timeout_seconds"] is None + + +def test_model_timeout_policy_survives_restart_and_rediscovery(tmp_path: Path) -> None: + """An explicit limit belongs to its model and remains until explicitly cleared.""" + model_agent = ModelAgent("timeout_agent", "example-model") + other_agent = ModelAgent("other_agent", "other-model") + database_path = str(tmp_path / "agent-pool.db") + orchestrator = TaskOrchestrator([model_agent, other_agent], agents_db=database_path) + updated = orchestrator.patch_agent( + "default", model_agent.id, {"model_timeout_seconds": 7200.5} + ) + assert updated["model_timeout_seconds"] == 7200.5 + assert orchestrator._agent(other_agent.id).to_config()["model_timeout_seconds"] is None + + restored = TaskOrchestrator([model_agent, other_agent], agents_db=database_path) + assert restored._agent(model_agent.id).to_config()["model_timeout_seconds"] == 7200.5 + restored.sync_discovered_agents([model_agent]) + updated = restored.patch_agent("default", model_agent.id, {"priority": 2}) + assert updated["model_timeout_seconds"] == 7200.5 + cleared = restored.patch_agent( + "default", model_agent.id, {"model_timeout_seconds": None} + ) + assert cleared["model_timeout_seconds"] is None + restarted = TaskOrchestrator([model_agent, other_agent], agents_db=database_path) + assert restarted._agent(model_agent.id).to_config()["model_timeout_seconds"] is None + + +@pytest.mark.parametrize("invalid_limit", [True, False, 0, -1, "90", float("nan"), float("inf")]) +def test_model_timeout_policy_rejects_invalid_patch(invalid_limit: object) -> None: + """Invalid administrator limits must be rejected without changing the model.""" + model_agent = ModelAgent("timeout_agent", "example-model") + orchestrator = TaskOrchestrator([model_agent]) + with pytest.raises((TypeError, ValueError)): + orchestrator.patch_agent( + "default", model_agent.id, {"model_timeout_seconds": invalid_limit} + ) + assert orchestrator._agent(model_agent.id) == model_agent From c276fcec0c48b6b33d72d0e95a0c07916769c863 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:11:50 +0900 Subject: [PATCH 04/78] feat(gateway): persist administrator model timeout policy Signed-off-by: Seongho Bae --- contextual_orchestrator/orchestrator.py | 35 ++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 98289fd12..8ca9af08b 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -597,6 +597,8 @@ class ModelAgent: endpoint_equivalence: dict[str, Any] | None = None # Provider-declared support for the Chat Completions terminal usage frame. stream_usage_supported: bool = False + # Administrator-owned execution policy; runtime admission is a separate gate. + model_timeout_seconds: float | None = None def __post_init__(self) -> None: require_object_name(self.id, "agent.id") @@ -622,6 +624,10 @@ def __post_init__(self) -> None: raise TypeError("reasoning_effort_supported must be true, false, or null") if type(self.stream_usage_supported) is not bool: raise TypeError("stream_usage_supported must be a boolean") + if self.model_timeout_seconds is not None: + value = self.model_timeout_seconds + if type(value) not in (int, float) or not 0 < value <= 1.7976931348623157e308: + raise ValueError("model_timeout_seconds must be finite positive seconds or null") if self.endpoint_equivalence is not None: contract = EndpointEquivalenceContract(**self.endpoint_equivalence) object.__setattr__(self, "endpoint_equivalence", dict(contract.__dict__)) @@ -647,6 +653,7 @@ def to_config(self) -> dict[str, Any]: "reasoning_effort_supported": self.reasoning_effort_supported, "endpoint_equivalence": self.endpoint_equivalence, "stream_usage_supported": self.stream_usage_supported, + "model_timeout_seconds": self.model_timeout_seconds, } @property @@ -682,6 +689,7 @@ def from_dict(cls, value: dict[str, Any]) -> "ModelAgent": # pragma: no cover reasoning_effort_supported=value.get("reasoning_effort_supported"), endpoint_equivalence=value.get("endpoint_equivalence"), stream_usage_supported=value.get("stream_usage_supported", False), + model_timeout_seconds=value.get("model_timeout_seconds"), ) @@ -3157,6 +3165,7 @@ class _AgentPoolStore: "context_window", "reasoning_effort_supported", "stream_usage_supported", + "model_timeout_seconds", } ) @@ -3196,6 +3205,8 @@ def _create_normalized_schema(cls, conn: sqlite3.Connection) -> None: context_window INTEGER, reasoning_effort_supported INTEGER, stream_usage_supported INTEGER NOT NULL DEFAULT 0, + model_timeout_seconds REAL CHECK (model_timeout_seconds IS NULL OR + (model_timeout_seconds > 0 AND model_timeout_seconds <= 1.7976931348623157e308)), CONSTRAINT agent_pool_disabled_flag_check CHECK (disabled IN (0, 1)), CONSTRAINT agent_pool_max_output_tokens_check CHECK ( @@ -3255,8 +3266,9 @@ def _insert_agent(cls, conn: sqlite3.Connection, agent: "ModelAgent") -> None: INSERT INTO agent_pool ( agent_id, model_name, base_url, api_key_env, credential_key, priority, disabled, provider_name, local_credential_key, auth_scheme, - max_output_tokens, context_window, reasoning_effort_supported, stream_usage_supported - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + max_output_tokens, context_window, reasoning_effort_supported, stream_usage_supported, + model_timeout_seconds + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( config["id"], @@ -3273,6 +3285,7 @@ def _insert_agent(cls, conn: sqlite3.Connection, agent: "ModelAgent") -> None: config["context_window"], config["reasoning_effort_supported"], int(config["stream_usage_supported"]), + config["model_timeout_seconds"], ), ) conn.executemany( @@ -3344,6 +3357,13 @@ def _initialize_schema(cls, conn: sqlite3.Connection) -> None: "CHECK (stream_usage_supported IN (0, 1))" ) columns.add("stream_usage_supported") + if "model_timeout_seconds" not in columns: + conn.execute( + "ALTER TABLE agent_pool ADD COLUMN model_timeout_seconds REAL " + "CHECK (model_timeout_seconds IS NULL OR " + "(model_timeout_seconds > 0 AND model_timeout_seconds <= 1.7976931348623157e308))" + ) + columns.add("model_timeout_seconds") if not cls._AGENT_COLUMNS.issubset(columns): missing = ", ".join(sorted(cls._AGENT_COLUMNS - columns)) raise RuntimeError(f"unsupported agent_pool schema; missing columns: {missing}") @@ -3439,7 +3459,8 @@ def save(self, agent: "ModelAgent") -> None: priority = ?, disabled = ?, provider_name = ?, local_credential_key = ?, auth_scheme = ?, max_output_tokens = ?, context_window = ?, - reasoning_effort_supported = ?, stream_usage_supported = ? + reasoning_effort_supported = ?, stream_usage_supported = ?, + model_timeout_seconds = ? WHERE agent_id = ? """, ( @@ -3456,6 +3477,7 @@ def save(self, agent: "ModelAgent") -> None: config["context_window"], config["reasoning_effort_supported"], int(config["stream_usage_supported"]), + config["model_timeout_seconds"], agent.id, ), ) @@ -3552,7 +3574,7 @@ def load_all(self) -> list["ModelAgent"]: SELECT agent_id, model_name, base_url, api_key_env, credential_key, priority, disabled, provider_name, local_credential_key, auth_scheme, max_output_tokens, context_window, - reasoning_effort_supported, stream_usage_supported + reasoning_effort_supported, stream_usage_supported, model_timeout_seconds FROM agent_pool ORDER BY agent_id """ ).fetchall() @@ -3618,6 +3640,7 @@ def load_all(self) -> list["ModelAgent"]: context_window=row[11], reasoning_effort_supported=(None if row[12] is None else bool(row[12])), stream_usage_supported=bool(row[13]), + model_timeout_seconds=row[14], group_name=group_by_agent.get(row[0], ""), endpoint_equivalence=contract_by_agent.get(row[0]), ) @@ -6023,6 +6046,8 @@ def patch_agent(self, agent_pool_id: str, worker_agent_id: str, patch: dict[str, patched = replace(patched, max_output_tokens=patch["max_output_tokens"]) if "context_window" in patch: patched = replace(patched, context_window=patch["context_window"]) + if "model_timeout_seconds" in patch: + patched = replace(patched, model_timeout_seconds=patch["model_timeout_seconds"]) if "endpoint_equivalence" in patch: value = patch["endpoint_equivalence"] if value is not None and not isinstance(value, dict): @@ -6220,6 +6245,7 @@ def sync_discovered_agents(self, discovered_agents: list[ModelAgent]) -> dict[st agent = replace( agent, group_name=updated_candidates[index].group_name, + model_timeout_seconds=updated_candidates[index].model_timeout_seconds, ) updated_candidates[index] = agent updated.append(agent.id) @@ -8427,6 +8453,7 @@ def _agent_to_admin_payload(self, agent: ModelAgent) -> dict[str, Any]: "max_output_tokens": agent.max_output_tokens, "context_window": agent.context_window, "stream_usage_supported": agent.stream_usage_supported, + "model_timeout_seconds": agent.model_timeout_seconds, "group_name": agent.group_name, "group_routing": self._group_router.member_report(agent.id) if agent.group_name else None, } From 94c6856c5471ca741f33814977df1401414d62c1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:14:08 +0900 Subject: [PATCH 05/78] test(gateway): cover timeout migration and numeric storage Signed-off-by: Seongho Bae --- tests/test_model_timeout_policy.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_model_timeout_policy.py b/tests/test_model_timeout_policy.py index 7cf9c079e..284d555c7 100644 --- a/tests/test_model_timeout_policy.py +++ b/tests/test_model_timeout_policy.py @@ -1,6 +1,7 @@ """Administrator-owned model timeout policy must survive configuration changes.""" from pathlib import Path +import sqlite3 import pytest @@ -13,6 +14,27 @@ def test_model_timeout_policy_defaults_to_null() -> None: assert model_agent.to_config()["model_timeout_seconds"] is None +def test_model_timeout_policy_accepts_large_finite_seconds(tmp_path: Path) -> None: + """Valid seconds must not accidentally use SQLite's signed integer binding.""" + model_agent = ModelAgent("timeout_agent", "example-model", model_timeout_seconds=2**63) + database_path = str(tmp_path / "agent-pool.db") + TaskOrchestrator([model_agent], agents_db=database_path) + restored = TaskOrchestrator([model_agent], agents_db=database_path) + assert restored._agent(model_agent.id).model_timeout_seconds == float(2**63) + + +def test_model_timeout_policy_migrates_existing_pool(tmp_path: Path) -> None: + """An older normalized pool gains a null policy without losing its models.""" + model_agent = ModelAgent("timeout_agent", "example-model") + database_path = str(tmp_path / "agent-pool.db") + TaskOrchestrator([model_agent], agents_db=database_path) + with sqlite3.connect(database_path) as connection: + connection.execute("ALTER TABLE agent_pool DROP COLUMN model_timeout_seconds") + restored = TaskOrchestrator([model_agent], agents_db=database_path) + assert restored._agent(model_agent.id) == model_agent + assert restored._agent(model_agent.id).model_timeout_seconds is None + + def test_model_timeout_policy_survives_restart_and_rediscovery(tmp_path: Path) -> None: """An explicit limit belongs to its model and remains until explicitly cleared.""" model_agent = ModelAgent("timeout_agent", "example-model") From dac678c1d3c4013f8242203dc044a43d81c3a2dc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:15:04 +0900 Subject: [PATCH 06/78] test(gateway): prove persisted timeout rows rather than seed replay Signed-off-by: Seongho Bae --- tests/test_model_timeout_policy.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/test_model_timeout_policy.py b/tests/test_model_timeout_policy.py index 284d555c7..0b30cf8cc 100644 --- a/tests/test_model_timeout_policy.py +++ b/tests/test_model_timeout_policy.py @@ -16,9 +16,14 @@ def test_model_timeout_policy_defaults_to_null() -> None: def test_model_timeout_policy_accepts_large_finite_seconds(tmp_path: Path) -> None: """Valid seconds must not accidentally use SQLite's signed integer binding.""" - model_agent = ModelAgent("timeout_agent", "example-model", model_timeout_seconds=2**63) + model_agent = ModelAgent("timeout_agent", "example-model") database_path = str(tmp_path / "agent-pool.db") - TaskOrchestrator([model_agent], agents_db=database_path) + orchestrator = TaskOrchestrator([model_agent], agents_db=database_path) + orchestrator.patch_agent("default", model_agent.id, {"model_timeout_seconds": 2**63}) + with sqlite3.connect(database_path) as connection: + assert connection.execute( + "SELECT model_timeout_seconds FROM agent_pool WHERE agent_id = ?", (model_agent.id,) + ).fetchone() == (float(2**63),) restored = TaskOrchestrator([model_agent], agents_db=database_path) assert restored._agent(model_agent.id).model_timeout_seconds == float(2**63) @@ -27,11 +32,12 @@ def test_model_timeout_policy_migrates_existing_pool(tmp_path: Path) -> None: """An older normalized pool gains a null policy without losing its models.""" model_agent = ModelAgent("timeout_agent", "example-model") database_path = str(tmp_path / "agent-pool.db") - TaskOrchestrator([model_agent], agents_db=database_path) + orchestrator = TaskOrchestrator([model_agent], agents_db=database_path) + orchestrator.patch_agent("default", model_agent.id, {"priority": 7}) with sqlite3.connect(database_path) as connection: connection.execute("ALTER TABLE agent_pool DROP COLUMN model_timeout_seconds") restored = TaskOrchestrator([model_agent], agents_db=database_path) - assert restored._agent(model_agent.id) == model_agent + assert restored._agent(model_agent.id).priority == 7 assert restored._agent(model_agent.id).model_timeout_seconds is None From 439da2e585e11dfbd24911984ee1b82b29f8094d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:15:26 +0900 Subject: [PATCH 07/78] fix(gateway): normalize timeout seconds before durable binding Signed-off-by: Seongho Bae --- contextual_orchestrator/orchestrator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 8ca9af08b..eb484b4ad 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -628,6 +628,7 @@ def __post_init__(self) -> None: value = self.model_timeout_seconds if type(value) not in (int, float) or not 0 < value <= 1.7976931348623157e308: raise ValueError("model_timeout_seconds must be finite positive seconds or null") + object.__setattr__(self, "model_timeout_seconds", float(value)) if self.endpoint_equivalence is not None: contract = EndpointEquivalenceContract(**self.endpoint_equivalence) object.__setattr__(self, "endpoint_equivalence", dict(contract.__dict__)) From 6236e982bfc431de61ca9b30d39158985e2a50f5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:17:40 +0900 Subject: [PATCH 08/78] test(gateway): reject unaudited model timeout mutation Signed-off-by: Seongho Bae --- tests/test_model_timeout_policy.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_model_timeout_policy.py b/tests/test_model_timeout_policy.py index 0b30cf8cc..731be3629 100644 --- a/tests/test_model_timeout_policy.py +++ b/tests/test_model_timeout_policy.py @@ -76,3 +76,25 @@ def test_model_timeout_policy_rejects_invalid_patch(invalid_limit: object) -> No "default", model_agent.id, {"model_timeout_seconds": invalid_limit} ) assert orchestrator._agent(model_agent.id) == model_agent + + +def test_model_timeout_policy_audit_failure_does_not_apply( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A rejected policy update must not leave a new durable or serving limit.""" + model_agent = ModelAgent("timeout_agent", "example-model") + database_path = str(tmp_path / "agent-pool.db") + orchestrator = TaskOrchestrator([model_agent], agents_db=database_path) + + def reject_audit(*args: object, **kwargs: object) -> None: + """Simulate unavailable audit storage before reporting update success.""" + raise OSError("audit storage unavailable") + + monkeypatch.setattr(orchestrator, "_append_audit_event", reject_audit) + with pytest.raises(OSError, match="audit storage unavailable"): + orchestrator.patch_agent("default", model_agent.id, {"model_timeout_seconds": 7200}) + restored = TaskOrchestrator([model_agent], agents_db=database_path) + assert ( + orchestrator._agent(model_agent.id).model_timeout_seconds, + restored._agent(model_agent.id).model_timeout_seconds, + ) == (None, None) From 32b01993721f9bb3c83002ce544b11ddeb5ea394 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:19:07 +0900 Subject: [PATCH 09/78] docs(gateway): track timeout policy and audit acceptance gaps Signed-off-by: Seongho Bae --- .../model-timeout-policy-evidence.md | 86 +++++++++++++++++++ docs/product-technical-gap-baseline.md | 20 +++++ 2 files changed, 106 insertions(+) create mode 100644 docs/doctoring/model-timeout-policy-evidence.md diff --git a/docs/doctoring/model-timeout-policy-evidence.md b/docs/doctoring/model-timeout-policy-evidence.md new file mode 100644 index 000000000..a334ba9e5 --- /dev/null +++ b/docs/doctoring/model-timeout-policy-evidence.md @@ -0,0 +1,86 @@ +# Model timeout policy: evidence and unfinished acceptance + +Observation date: 2026-09-06. Status: local implementation, not released. + +## Buyer requirement + +An ordinary model call has no implicit application-wide execution limit. +Administrators can eventually set, inspect, clear and restore an explicit +model-specific limit in seconds, with validated input, precedence, inheritance +and an auditable policy revision. Cancellation, provider termination and +administrator timeout must remain distinguishable. A termination reason is +not automatically an incorrect-answer observation for psychometric fitting. + +## Verified scope + +PR #1053 at `661ce8db75460c9f5752ba1493aad026e01f5316` removes the implicit +client timeout and preserves null through existing waiting boundaries. Its +full suite completed with 3400 passed, 2 skipped, exit 0, in 918.02 seconds. +Clean start/end revisions matched; parsed JUnit had 3402 cases and no errors +or failures. This is software regression evidence, not buyer response latency +or a deployed recovery claim. Real Edge inspection covered the PR body only. + +Local `439da2e585e11dfbd24911984ee1b82b29f8094d` adds durable configuration +using the existing normalized agent store, not a second settings service. +Eleven policy cases plus twenty existing pool cases passed in 10.23 seconds. +The tests exercise null defaults, invalid input, actual stored rows, migration, +restart, omission, clearing and rediscovery preservation. A large integer +binding failure was first reproduced at `dac678c1`, then corrected by +normalizing validated seconds to floating point. This does not establish that +every representable value is executable by a future transport clock. + +The first related run at `c276fcec` had 28 passes and one existing HTTP test +timeout. Its isolated rerun passed, but the timeout cause is unconfirmed. Both +results are retained. The initial migration/large-value checks at `94c6856c` +could replay seeds without proving persisted rows; their pass is not counted +as durable-storage evidence. + +## Confirmed unresolved audit failure + +At `6236e982`, the committed policy suite reports 1 failed and 11 passed in +4.72 seconds. Injecting an audit-storage exception during a 7200-second update +raises an error to the caller, yet both the serving candidate and a fresh +instance read 7200.0 rather than the previous null. This is a direct internal +configuration-path reproduction, not an externally admitted HTTP exploit: +HTTP create/PATCH allowlists still reject the new field. + +The current pool save commits before the general audit append, which uses a +separate state-store transaction. Moving the audit earlier cannot prove atomic +success, and blindly restoring a prior value could overwrite a concurrent +update. The next implementation must make policy revision/history and the +configuration change atomic at their owning store, with failure injection and +concurrent-update checks. General telemetry must not be mistaken for the +authoritative policy history. + +## Remaining delivery gates + +- Implement atomic change/history and restore with authenticated actor evidence. +- Resolve request snapshot, precedence, inheritance and in-flight update rules. +- Bind actual execution to the released canonical Rust runtime contract; + do not add a Python timer clone or consume an unreleased owner branch. +- Distinguish model response waiting from DNS, connection, pool and body-safety + budgets. Preserve destination validation and DNS pinning. +- Verify streams, tools, local queues, embedding and endpoint races, including + cancellation classification and resource cleanup. +- Only then admit HTTP writes and expose administrator controls; perform actual + visual inspection and authenticated end-to-end tests. +- Re-run exact-head checks and independent reviews before protected merge, + release, consumer pinning and deployed-version verification. + +EgressWeave's inspected main `bd0339bf43cf5041e861bac86a84cb6e7e32637e` +documents finite phase ceilings that replace null values. That contract is not +silently compatible with unrestricted model response waiting. GitHub release +and tag queries were empty; this says nothing about every other registry. +Existing security PRs #220 and #210 are preserved; their historical review +comments do not establish a current writer lease. + +Local raw evidence is retained under `/tmp/co-uptime-path.T7v9Rj/`, including +the original failures and JUnit reports. These temporary paths are not public +release artifacts. The local policy delta remains unpushed pending the gates +above; the remote PR's completed full-suite result applies only to `661ce8db`. + +## Source reference + +ContextualWisdomLab. (n.d.). *Finite outbound request-timeout boundaries* +(revision bd0339bf43cf5041e861bac86a84cb6e7e32637e). EgressWeave. +https://github.com/ContextualWisdomLab/EgressWeave/blob/bd0339bf43cf5041e861bac86a84cb6e7e32637e/docs/research/request-timeout-boundaries.md diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index d145a0b1d..564d25103 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1,5 +1,25 @@ # Contextual Orchestrator: Product & Technical Gap Baseline +## 2026-09-06 model-specific timeout policy: local, not delivered + +PR #1053's remote `661ce8db` has a completed 3400-pass/2-skip regression suite +for removing the implicit client timeout. The subsequent local durable-policy +implementation at `439da2e5` has 31 related passes, but does not yet enforce +model-specific execution limits. HTTP write admission and UI controls remain +closed for the new setting. Neither result proves deployment or buyer latency. + +A new failure-injection test at `6236e982` demonstrates an unresolved audit +atomicity gap: the update reports an audit error while both memory and a +restarted instance retain the new 7200-second policy. Policy change/history +must commit atomically before administrator write admission. Restore, +precedence, cancellation semantics, released Rust runtime integration and +actual administrator visual/E2E evidence also remain open. + +See [the evidence record](doctoring/model-timeout-policy-evidence.md) for exact +revisions, retained failures, corrected test-evidence limitations, owner +boundaries and the full remaining acceptance gates. Local configuration work +must not be represented as released enforcement or psychometric accuracy gain. + ## 2026-09-01 Autonomous Commercialization Loop: PR #970 Merge, Token Accounting & Cost Gateway Harmonization Observation time: 2026-09-01 Asia/Seoul. From 197e8880ee02811373f6f17e873f472cbc5dc3b2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:21:59 +0900 Subject: [PATCH 10/78] test(gateway): require transactional timeout history Signed-off-by: Seongho Bae --- tests/test_model_timeout_policy.py | 41 +++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/tests/test_model_timeout_policy.py b/tests/test_model_timeout_policy.py index 731be3629..aafc49a45 100644 --- a/tests/test_model_timeout_policy.py +++ b/tests/test_model_timeout_policy.py @@ -78,23 +78,46 @@ def test_model_timeout_policy_rejects_invalid_patch(invalid_limit: object) -> No assert orchestrator._agent(model_agent.id) == model_agent -def test_model_timeout_policy_audit_failure_does_not_apply( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: +def test_model_timeout_policy_audit_failure_does_not_apply(tmp_path: Path) -> None: """A rejected policy update must not leave a new durable or serving limit.""" model_agent = ModelAgent("timeout_agent", "example-model") database_path = str(tmp_path / "agent-pool.db") orchestrator = TaskOrchestrator([model_agent], agents_db=database_path) - def reject_audit(*args: object, **kwargs: object) -> None: - """Simulate unavailable audit storage before reporting update success.""" - raise OSError("audit storage unavailable") - - monkeypatch.setattr(orchestrator, "_append_audit_event", reject_audit) - with pytest.raises(OSError, match="audit storage unavailable"): + with sqlite3.connect(database_path) as connection: + connection.execute( + "CREATE TRIGGER reject_timeout_history BEFORE INSERT ON model_timeout_history " + "BEGIN SELECT RAISE(ABORT, 'audit storage unavailable'); END" + ) + with pytest.raises(sqlite3.IntegrityError, match="audit storage unavailable"): orchestrator.patch_agent("default", model_agent.id, {"model_timeout_seconds": 7200}) restored = TaskOrchestrator([model_agent], agents_db=database_path) assert ( orchestrator._agent(model_agent.id).model_timeout_seconds, restored._agent(model_agent.id).model_timeout_seconds, ) == (None, None) + with sqlite3.connect(database_path) as connection: + assert connection.execute("SELECT COUNT(*) FROM model_timeout_history").fetchone() == (0,) + + +def test_model_timeout_policy_requires_durable_store() -> None: + """Administrator policy cannot silently fall back to volatile memory.""" + model_agent = ModelAgent("timeout_agent", "example-model") + orchestrator = TaskOrchestrator([model_agent]) + with pytest.raises(ValueError, match="durable"): + orchestrator.patch_agent("default", model_agent.id, {"model_timeout_seconds": 7200}) + assert orchestrator._agent(model_agent.id).model_timeout_seconds is None + + +def test_model_timeout_policy_records_atomic_history(tmp_path: Path) -> None: + """Committed revisions retain their old and new limits in order.""" + model_agent = ModelAgent("timeout_agent", "example-model") + database_path = str(tmp_path / "agent-pool.db") + orchestrator = TaskOrchestrator([model_agent], agents_db=database_path) + for limit in (7200, None): + orchestrator.patch_agent("default", model_agent.id, {"model_timeout_seconds": limit}) + with sqlite3.connect(database_path) as connection: + assert connection.execute( + "SELECT previous_seconds, timeout_seconds FROM model_timeout_history " + "WHERE agent_id = ? ORDER BY policy_revision", (model_agent.id,) + ).fetchall() == [(None, 7200.0), (7200.0, None)] From 4e839ce1dc8db4c32e10234053a150c0cb8c1cc4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:23:13 +0900 Subject: [PATCH 11/78] fix(gateway): commit timeout policy and history atomically Signed-off-by: Seongho Bae --- contextual_orchestrator/orchestrator.py | 47 ++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index eb484b4ad..600948df2 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -3406,6 +3406,13 @@ def __init__(self, path: str) -> None: "contract_id TEXT NOT NULL REFERENCES endpoint_equivalence_contract(contract_id) ON DELETE RESTRICT)" ) self._migrate_legacy_groups(conn) + conn.execute( + "CREATE TABLE IF NOT EXISTS model_timeout_history (" + "policy_revision INTEGER PRIMARY KEY AUTOINCREMENT, " + "agent_id TEXT NOT NULL REFERENCES agent_pool(agent_id), " + "previous_seconds REAL, timeout_seconds REAL, " + "created_at REAL NOT NULL)" + ) conn.commit() except Exception: conn.rollback() @@ -3447,11 +3454,27 @@ def _migrate_legacy_groups(conn: sqlite3.Connection) -> None: ) conn.execute("DROP TABLE agent_pool_legacy_payloads") - def save(self, agent: "ModelAgent") -> None: + def save(self, agent: "ModelAgent", *, timeout_previous: "ModelAgent | None" = None) -> None: """Persist one normalized model-agent definition.""" with self._lock: conn = self._connect(self._path) try: + if timeout_previous is not None: + conn.execute("BEGIN IMMEDIATE") + row = conn.execute( + "SELECT model_timeout_seconds FROM agent_pool WHERE agent_id = ?", + (agent.id,), + ).fetchone() + if row is not None: + if row[0] != timeout_previous.model_timeout_seconds: + raise ValueError("model timeout policy changed; reload before updating") + conn.execute( + "UPDATE agent_pool SET model_timeout_seconds = ? WHERE agent_id = ?", + (agent.model_timeout_seconds, agent.id), + ) + self._append_timeout_history(conn, timeout_previous, agent) + conn.commit() + return config = agent.to_config() conn.execute( """ @@ -3561,10 +3584,23 @@ def save(self, agent: "ModelAgent") -> None: "INSERT INTO endpoint_equivalence_member (agent_id, contract_id) VALUES (?, ?)", (agent.id, contract.contract_id), ) + if timeout_previous is not None: + self._append_timeout_history(conn, timeout_previous, agent) conn.commit() finally: conn.close() + @staticmethod + def _append_timeout_history( + conn: sqlite3.Connection, previous: "ModelAgent", updated: "ModelAgent" + ) -> None: + """Write the policy change using the same uncommitted configuration transaction.""" + conn.execute( + "INSERT INTO model_timeout_history " + "(agent_id, previous_seconds, timeout_seconds, created_at) VALUES (?, ?, ?, ?)", + (updated.id, previous.model_timeout_seconds, updated.model_timeout_seconds, time.time()), + ) + def load_all(self) -> list["ModelAgent"]: """Load every persisted model-agent definition.""" with self._lock: @@ -6064,6 +6100,15 @@ def patch_agent(self, agent_pool_id: str, worker_agent_id: str, patch: dict[str, if not updated_agents: raise ValueError("cannot disable the last enabled agent") self._require_role_effort_pool(updated_candidates) + if "model_timeout_seconds" in patch: + if set(patch) != {"model_timeout_seconds"}: + raise ValueError("model timeout policy must be updated separately") + if self._pool_store is None: + raise ValueError("model timeout policy requires a durable agent store") + self._pool_store.save(patched, timeout_previous=current) + self.candidates = updated_candidates + self.agents = updated_agents + return self._agent_to_admin_payload(patched) if self._pool_store is not None: self._pool_store.save(patched) self.candidates = updated_candidates From c3879439cd4a0547ff06e7cbe8561cce787e3a1f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:24:02 +0900 Subject: [PATCH 12/78] test(gateway): cover timeout rollback and stale policy writers Signed-off-by: Seongho Bae --- tests/test_model_timeout_policy.py | 42 +++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/tests/test_model_timeout_policy.py b/tests/test_model_timeout_policy.py index aafc49a45..2b69f806d 100644 --- a/tests/test_model_timeout_policy.py +++ b/tests/test_model_timeout_policy.py @@ -78,11 +78,16 @@ def test_model_timeout_policy_rejects_invalid_patch(invalid_limit: object) -> No assert orchestrator._agent(model_agent.id) == model_agent -def test_model_timeout_policy_audit_failure_does_not_apply(tmp_path: Path) -> None: +@pytest.mark.parametrize("previous_limit", [None, 3600.0]) +def test_model_timeout_policy_audit_failure_does_not_apply( + tmp_path: Path, previous_limit: float | None +) -> None: """A rejected policy update must not leave a new durable or serving limit.""" model_agent = ModelAgent("timeout_agent", "example-model") database_path = str(tmp_path / "agent-pool.db") orchestrator = TaskOrchestrator([model_agent], agents_db=database_path) + if previous_limit is not None: + orchestrator.patch_agent("default", model_agent.id, {"model_timeout_seconds": previous_limit}) with sqlite3.connect(database_path) as connection: connection.execute( @@ -95,9 +100,11 @@ def test_model_timeout_policy_audit_failure_does_not_apply(tmp_path: Path) -> No assert ( orchestrator._agent(model_agent.id).model_timeout_seconds, restored._agent(model_agent.id).model_timeout_seconds, - ) == (None, None) + ) == (previous_limit, previous_limit) with sqlite3.connect(database_path) as connection: - assert connection.execute("SELECT COUNT(*) FROM model_timeout_history").fetchone() == (0,) + assert connection.execute("SELECT COUNT(*) FROM model_timeout_history").fetchone() == ( + int(previous_limit is not None), + ) def test_model_timeout_policy_requires_durable_store() -> None: @@ -121,3 +128,32 @@ def test_model_timeout_policy_records_atomic_history(tmp_path: Path) -> None: "SELECT previous_seconds, timeout_seconds FROM model_timeout_history " "WHERE agent_id = ? ORDER BY policy_revision", (model_agent.id,) ).fetchall() == [(None, 7200.0), (7200.0, None)] + + +def test_model_timeout_policy_rejects_stale_writer(tmp_path: Path) -> None: + """A stale model view cannot overwrite a different committed timeout value.""" + model_agent = ModelAgent("timeout_agent", "example-model") + database_path = str(tmp_path / "agent-pool.db") + first = TaskOrchestrator([model_agent], agents_db=database_path) + stale = TaskOrchestrator([model_agent], agents_db=database_path) + first.patch_agent("default", model_agent.id, {"model_timeout_seconds": 7200}) + with pytest.raises(ValueError, match="reload"): + stale.patch_agent("default", model_agent.id, {"model_timeout_seconds": 3600}) + restored = TaskOrchestrator([model_agent], agents_db=database_path) + assert restored._agent(model_agent.id).model_timeout_seconds == 7200 + assert stale._agent(model_agent.id).model_timeout_seconds is None + with sqlite3.connect(database_path) as connection: + assert connection.execute("SELECT COUNT(*) FROM model_timeout_history").fetchone() == (1,) + + +def test_model_timeout_policy_preserves_other_stored_attributes(tmp_path: Path) -> None: + """A timeout-only write does not replay an old copy of unrelated attributes.""" + model_agent = ModelAgent("timeout_agent", "example-model") + database_path = str(tmp_path / "agent-pool.db") + first = TaskOrchestrator([model_agent], agents_db=database_path) + stale = TaskOrchestrator([model_agent], agents_db=database_path) + first.patch_agent("default", model_agent.id, {"priority": 9}) + stale.patch_agent("default", model_agent.id, {"model_timeout_seconds": 7200}) + restored = TaskOrchestrator([model_agent], agents_db=database_path) + assert restored._agent(model_agent.id).priority == 9 + assert restored._agent(model_agent.id).model_timeout_seconds == 7200 From cb24fa9f6e3a613d6a6f249f17cc88e29ca00c58 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:24:51 +0900 Subject: [PATCH 13/78] docs(gateway): record transactional timeout evidence and remaining gates Signed-off-by: Seongho Bae --- .../model-timeout-policy-evidence.md | 26 ++++++++++++++++--- docs/product-technical-gap-baseline.md | 6 ++++- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/docs/doctoring/model-timeout-policy-evidence.md b/docs/doctoring/model-timeout-policy-evidence.md index a334ba9e5..8d4f29ecd 100644 --- a/docs/doctoring/model-timeout-policy-evidence.md +++ b/docs/doctoring/model-timeout-policy-evidence.md @@ -35,7 +35,7 @@ results are retained. The initial migration/large-value checks at `94c6856c` could replay seeds without proving persisted rows; their pass is not counted as durable-storage evidence. -## Confirmed unresolved audit failure +## Reproduced audit failure and local transactional repair At `6236e982`, the committed policy suite reports 1 failed and 11 passed in 4.72 seconds. Injecting an audit-storage exception during a 7200-second update @@ -44,7 +44,7 @@ instance read 7200.0 rather than the previous null. This is a direct internal configuration-path reproduction, not an externally admitted HTTP exploit: HTTP create/PATCH allowlists still reject the new field. -The current pool save commits before the general audit append, which uses a +The reproduced pool save committed before the general audit append, which uses a separate state-store transaction. Moving the audit earlier cannot prove atomic success, and blindly restoring a prior value could overwrite a concurrent update. The next implementation must make policy revision/history and the @@ -52,9 +52,29 @@ configuration change atomic at their owning store, with failure injection and concurrent-update checks. General telemetry must not be mistaken for the authoritative policy history. +Local `4e839ce1` now commits timeout history and the configuration change on +the same pool connection, then publishes the in-memory candidate. Existing +rows receive a timeout-only update, preserving unrelated stored attributes. +Changing policy requires a durable store and a separate timeout-only patch. +The generic audit stream no longer owns this policy transaction. A database +trigger that rejects history insertion rolls back the associated policy write, +including initial-row creation. The failure test now targets that actual +transaction rather than the former generic audit callback; the original +failure remains preserved in the earlier commit and JUnit. + +At `c3879439cd4a0547ff06e7cbe8561cce787e3a1f`, 37 related cases passed in +2.65 seconds, exit 0. They cover rejection on history failure for both missing +and existing rows, ordered old/new values, durable-store requirements, a +stale writer with a different committed timeout and preservation of other +stored model attributes. This is not complete concurrent-policy correctness: +value-based conflict detection does not detect an ABA change, and serving +snapshots across processes do not yet carry a policy revision. Authenticated +actor evidence, revision-based restore and complete concurrency/commit-failure +injection remain required before exposing administrator policy writes. + ## Remaining delivery gates -- Implement atomic change/history and restore with authenticated actor evidence. +- Complete revision-based change/history and restore with authenticated actor evidence. - Resolve request snapshot, precedence, inheritance and in-flight update rules. - Bind actual execution to the released canonical Rust runtime contract; do not add a Python timer clone or consume an unreleased owner branch. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 564d25103..848381c50 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -11,7 +11,11 @@ closed for the new setting. Neither result proves deployment or buyer latency. A new failure-injection test at `6236e982` demonstrates an unresolved audit atomicity gap: the update reports an audit error while both memory and a restarted instance retain the new 7200-second policy. Policy change/history -must commit atomically before administrator write admission. Restore, +was subsequently moved into one local pool transaction at `4e839ce1`; +`c3879439` has 37 related passes, including history-insertion rollback for +new/existing rows and rejection of a stale different-value writer. This does +not yet cover ABA revisions, authenticated actor evidence or cross-process +serving snapshots. Restore, precedence, cancellation semantics, released Rust runtime integration and actual administrator visual/E2E evidence also remain open. From ef76ade9aed5a24a4f580d60375a1f11f1bc9d59 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:26:06 +0900 Subject: [PATCH 14/78] test(gateway): reject stale timeout policy after value restoration Signed-off-by: Seongho Bae --- tests/test_model_timeout_policy.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/test_model_timeout_policy.py b/tests/test_model_timeout_policy.py index 2b69f806d..7d3913f9d 100644 --- a/tests/test_model_timeout_policy.py +++ b/tests/test_model_timeout_policy.py @@ -157,3 +157,19 @@ def test_model_timeout_policy_preserves_other_stored_attributes(tmp_path: Path) restored = TaskOrchestrator([model_agent], agents_db=database_path) assert restored._agent(model_agent.id).priority == 9 assert restored._agent(model_agent.id).model_timeout_seconds == 7200 + + +def test_model_timeout_policy_rejects_aba_writer(tmp_path: Path) -> None: + """Returning to null must not make an older policy snapshot current again.""" + model_agent = ModelAgent("timeout_agent", "example-model") + database_path = str(tmp_path / "agent-pool.db") + first = TaskOrchestrator([model_agent], agents_db=database_path) + stale = TaskOrchestrator([model_agent], agents_db=database_path) + for limit in (7200, None): + first.patch_agent("default", model_agent.id, {"model_timeout_seconds": limit}) + with pytest.raises(ValueError, match="reload"): + stale.patch_agent("default", model_agent.id, {"model_timeout_seconds": 3600}) + restored = TaskOrchestrator([model_agent], agents_db=database_path) + assert restored._agent(model_agent.id).model_timeout_seconds is None + with sqlite3.connect(database_path) as connection: + assert connection.execute("SELECT COUNT(*) FROM model_timeout_history").fetchone() == (2,) From d911a38e1907a79c521b6ced058244497a252cef Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:26:41 +0900 Subject: [PATCH 15/78] fix(gateway): compare durable timeout policy revisions Signed-off-by: Seongho Bae --- contextual_orchestrator/orchestrator.py | 36 ++++++++++++++++++++----- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 600948df2..35ed0346c 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -599,6 +599,7 @@ class ModelAgent: stream_usage_supported: bool = False # Administrator-owned execution policy; runtime admission is a separate gate. model_timeout_seconds: float | None = None + model_timeout_revision: int = 0 def __post_init__(self) -> None: require_object_name(self.id, "agent.id") @@ -629,6 +630,8 @@ def __post_init__(self) -> None: if type(value) not in (int, float) or not 0 < value <= 1.7976931348623157e308: raise ValueError("model_timeout_seconds must be finite positive seconds or null") object.__setattr__(self, "model_timeout_seconds", float(value)) + if type(self.model_timeout_revision) is not int or self.model_timeout_revision < 0: + raise ValueError("model_timeout_revision must be a non-negative integer") if self.endpoint_equivalence is not None: contract = EndpointEquivalenceContract(**self.endpoint_equivalence) object.__setattr__(self, "endpoint_equivalence", dict(contract.__dict__)) @@ -655,6 +658,7 @@ def to_config(self) -> dict[str, Any]: "endpoint_equivalence": self.endpoint_equivalence, "stream_usage_supported": self.stream_usage_supported, "model_timeout_seconds": self.model_timeout_seconds, + "model_timeout_revision": self.model_timeout_revision, } @property @@ -691,6 +695,7 @@ def from_dict(cls, value: dict[str, Any]) -> "ModelAgent": # pragma: no cover endpoint_equivalence=value.get("endpoint_equivalence"), stream_usage_supported=value.get("stream_usage_supported", False), model_timeout_seconds=value.get("model_timeout_seconds"), + model_timeout_revision=value.get("model_timeout_revision", 0), ) @@ -3454,13 +3459,19 @@ def _migrate_legacy_groups(conn: sqlite3.Connection) -> None: ) conn.execute("DROP TABLE agent_pool_legacy_payloads") - def save(self, agent: "ModelAgent", *, timeout_previous: "ModelAgent | None" = None) -> None: + def save(self, agent: "ModelAgent", *, timeout_previous: "ModelAgent | None" = None) -> int | None: """Persist one normalized model-agent definition.""" with self._lock: conn = self._connect(self._path) try: if timeout_previous is not None: conn.execute("BEGIN IMMEDIATE") + revision = conn.execute( + "SELECT COALESCE(MAX(policy_revision), 0) FROM model_timeout_history WHERE agent_id = ?", + (agent.id,), + ).fetchone()[0] + if revision != timeout_previous.model_timeout_revision: + raise ValueError("model timeout policy changed; reload before updating") row = conn.execute( "SELECT model_timeout_seconds FROM agent_pool WHERE agent_id = ?", (agent.id,), @@ -3472,9 +3483,9 @@ def save(self, agent: "ModelAgent", *, timeout_previous: "ModelAgent | None" = N "UPDATE agent_pool SET model_timeout_seconds = ? WHERE agent_id = ?", (agent.model_timeout_seconds, agent.id), ) - self._append_timeout_history(conn, timeout_previous, agent) + revision = self._append_timeout_history(conn, timeout_previous, agent) conn.commit() - return + return revision config = agent.to_config() conn.execute( """ @@ -3585,21 +3596,23 @@ def save(self, agent: "ModelAgent", *, timeout_previous: "ModelAgent | None" = N (agent.id, contract.contract_id), ) if timeout_previous is not None: - self._append_timeout_history(conn, timeout_previous, agent) + revision = self._append_timeout_history(conn, timeout_previous, agent) conn.commit() + return revision if timeout_previous is not None else None finally: conn.close() @staticmethod def _append_timeout_history( conn: sqlite3.Connection, previous: "ModelAgent", updated: "ModelAgent" - ) -> None: + ) -> int: """Write the policy change using the same uncommitted configuration transaction.""" - conn.execute( + cursor = conn.execute( "INSERT INTO model_timeout_history " "(agent_id, previous_seconds, timeout_seconds, created_at) VALUES (?, ?, ?, ?)", (updated.id, previous.model_timeout_seconds, updated.model_timeout_seconds, time.time()), ) + return int(cursor.lastrowid) def load_all(self) -> list["ModelAgent"]: """Load every persisted model-agent definition.""" @@ -3626,6 +3639,9 @@ def load_all(self) -> list["ModelAgent"]: groups = conn.execute( "SELECT agent_id, group_name FROM model_group_member ORDER BY agent_id" ).fetchall() + timeout_revisions = dict(conn.execute( + "SELECT agent_id, MAX(policy_revision) FROM model_timeout_history GROUP BY agent_id" + ).fetchall()) contracts = conn.execute( "SELECT endpoint_equivalence_member.agent_id, endpoint_equivalence_contract.* " "FROM endpoint_equivalence_member JOIN endpoint_equivalence_contract USING (contract_id)" @@ -3678,6 +3694,7 @@ def load_all(self) -> list["ModelAgent"]: reasoning_effort_supported=(None if row[12] is None else bool(row[12])), stream_usage_supported=bool(row[13]), model_timeout_seconds=row[14], + model_timeout_revision=timeout_revisions.get(row[0], 0), group_name=group_by_agent.get(row[0], ""), endpoint_equivalence=contract_by_agent.get(row[0]), ) @@ -6105,7 +6122,10 @@ def patch_agent(self, agent_pool_id: str, worker_agent_id: str, patch: dict[str, raise ValueError("model timeout policy must be updated separately") if self._pool_store is None: raise ValueError("model timeout policy requires a durable agent store") - self._pool_store.save(patched, timeout_previous=current) + revision = self._pool_store.save(patched, timeout_previous=current) + patched = replace(patched, model_timeout_revision=revision) + updated_candidates = [patched if agent.id == worker_agent_id else agent for agent in self.candidates] + updated_agents = [agent for agent in updated_candidates if not agent.disabled] self.candidates = updated_candidates self.agents = updated_agents return self._agent_to_admin_payload(patched) @@ -6292,6 +6312,7 @@ def sync_discovered_agents(self, discovered_agents: list[ModelAgent]) -> dict[st agent, group_name=updated_candidates[index].group_name, model_timeout_seconds=updated_candidates[index].model_timeout_seconds, + model_timeout_revision=updated_candidates[index].model_timeout_revision, ) updated_candidates[index] = agent updated.append(agent.id) @@ -8500,6 +8521,7 @@ def _agent_to_admin_payload(self, agent: ModelAgent) -> dict[str, Any]: "context_window": agent.context_window, "stream_usage_supported": agent.stream_usage_supported, "model_timeout_seconds": agent.model_timeout_seconds, + "model_timeout_revision": agent.model_timeout_revision, "group_name": agent.group_name, "group_routing": self._group_router.member_report(agent.id) if agent.group_name else None, } From bceaeb2310aa1f220efdedef4841bf328a3579cb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:27:39 +0900 Subject: [PATCH 16/78] test(gateway): reject mixed timeout value and revision snapshots Signed-off-by: Seongho Bae --- tests/test_model_timeout_policy.py | 34 ++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/test_model_timeout_policy.py b/tests/test_model_timeout_policy.py index 7d3913f9d..9affffdf9 100644 --- a/tests/test_model_timeout_policy.py +++ b/tests/test_model_timeout_policy.py @@ -173,3 +173,37 @@ def test_model_timeout_policy_rejects_aba_writer(tmp_path: Path) -> None: assert restored._agent(model_agent.id).model_timeout_seconds is None with sqlite3.connect(database_path) as connection: assert connection.execute("SELECT COUNT(*) FROM model_timeout_history").fetchone() == (2,) + + +def test_model_timeout_policy_loads_value_and_revision_together( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A concurrent commit cannot attach a new revision to an older value.""" + model_agent = ModelAgent("timeout_agent", "example-model") + database_path = str(tmp_path / "agent-pool.db") + writer = TaskOrchestrator([model_agent], agents_db=database_path) + writer.patch_agent("default", model_agent.id, {"model_timeout_seconds": 3600}) + with sqlite3.connect(database_path) as connection: + connection.execute("PRAGMA journal_mode=WAL") + reader = TaskOrchestrator([model_agent], agents_db=database_path) + original_connect = reader._pool_store._connect + updates = [] + + def connect_with_interleaved_write(database: str) -> sqlite3.Connection: + """Commit a second version after value selection but before history selection.""" + connection = original_connect(database) + + def on_query(statement: str) -> None: + """Interleave one independently committed writer without sleeping.""" + if "SELECT agent_id, MAX(policy_revision)" in statement and not updates: + updates.append(writer.patch_agent( + "default", model_agent.id, {"model_timeout_seconds": 7200} + )) + + connection.set_trace_callback(on_query) + return connection + + monkeypatch.setattr(reader._pool_store, "_connect", connect_with_interleaved_write) + loaded = reader._pool_store.load_all()[0] + assert len(updates) == 1 + assert (loaded.model_timeout_seconds, loaded.model_timeout_revision) == (3600, 1) From e5e9c96fdab3255058c491721701b61a24764b1d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:28:03 +0900 Subject: [PATCH 17/78] fix(gateway): read model configuration from one database snapshot Signed-off-by: Seongho Bae --- contextual_orchestrator/orchestrator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 35ed0346c..e47f77f6e 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -3619,6 +3619,7 @@ def load_all(self) -> list["ModelAgent"]: with self._lock: conn = self._connect(self._path) try: + conn.execute("BEGIN") rows = conn.execute( """ SELECT agent_id, model_name, base_url, api_key_env, credential_key, From 57653076d06aa0ab2997533b3d95d08a0d7e33c8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:28:56 +0900 Subject: [PATCH 18/78] docs(gateway): record timeout revision and snapshot proofs Signed-off-by: Seongho Bae --- docs/doctoring/model-timeout-policy-evidence.md | 16 ++++++++++++++++ docs/product-technical-gap-baseline.md | 7 +++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/docs/doctoring/model-timeout-policy-evidence.md b/docs/doctoring/model-timeout-policy-evidence.md index 8d4f29ecd..9233e8330 100644 --- a/docs/doctoring/model-timeout-policy-evidence.md +++ b/docs/doctoring/model-timeout-policy-evidence.md @@ -72,6 +72,22 @@ snapshots across processes do not yet carry a policy revision. Authenticated actor evidence, revision-based restore and complete concurrency/commit-failure injection remain required before exposing administrator policy writes. +The ABA limitation was then directly reproduced at `ef76ade9` (1 failed, +17 passed, 0.50 seconds). `d911a38e` reuses each model's latest history sequence +as its policy revision and compares it inside the write transaction, rejecting +a stale snapshot even when the value returned to null. Related tests passed +38/38 in 4.30 seconds. Revision allocation remains owned by committed history. + +At `bceaeb23`, a deterministic interleaved WAL writer showed that loading model +values and history in separate reads could attach revision 2 to the old +3600-second value (1 failed, 18 passed, 6.88 seconds). `e5e9c96f` starts a read +transaction before selecting model rows, so values, relations and revisions +come from the same database snapshot. The same interleaving now returns +3600 seconds with revision 1; 39 related tests passed in 4.43 seconds, exit 0. +This proves that interleaving, not all distributed serving coherence. Policy +restore, authenticated actor attribution, complete concurrent-write failure +coverage and actual runtime enforcement are still unfinished and unshipped. + ## Remaining delivery gates - Complete revision-based change/history and restore with authenticated actor evidence. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 848381c50..636652c4f 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -14,8 +14,11 @@ restarted instance retain the new 7200-second policy. Policy change/history was subsequently moved into one local pool transaction at `4e839ce1`; `c3879439` has 37 related passes, including history-insertion rollback for new/existing rows and rejection of a stale different-value writer. This does -not yet cover ABA revisions, authenticated actor evidence or cross-process -serving snapshots. Restore, +not yet cover authenticated actor evidence or cross-process serving refresh. +Subsequent `d911a38e` adds history-revision comparisons for ABA conflicts; +`e5e9c96f` reads values and revisions in one database snapshot. Their committed +RED cases reproduced stale-value acceptance and mixed revisions respectively; +the latter head has 39 related passes. Restore, precedence, cancellation semantics, released Rust runtime integration and actual administrator visual/E2E evidence also remain open. From cbab94c8734bf263103874be0c7dc2383057cec1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:33:39 +0900 Subject: [PATCH 19/78] test(gateway): retain authenticated principal in timeout history Signed-off-by: Seongho Bae --- tests/test_model_timeout_policy.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/test_model_timeout_policy.py b/tests/test_model_timeout_policy.py index 9affffdf9..a765a5c51 100644 --- a/tests/test_model_timeout_policy.py +++ b/tests/test_model_timeout_policy.py @@ -207,3 +207,21 @@ def on_query(statement: str) -> None: loaded = reader._pool_store.load_all()[0] assert len(updates) == 1 assert (loaded.model_timeout_seconds, loaded.model_timeout_revision) == (3600, 1) + + +def test_model_timeout_policy_records_verified_principal(tmp_path: Path) -> None: + """Policy history preserves the opaque principal supplied by the auth boundary.""" + from contextual_orchestrator.server import SecurityConfig + + security = SecurityConfig(admin_token="example_admin", inference_token="example_inference") + principal_id = security.principal_id({"Authorization": "Bearer example_admin"}) + model_agent = ModelAgent("timeout_agent", "example-model") + database_path = str(tmp_path / "agent-pool.db") + orchestrator = TaskOrchestrator([model_agent], agents_db=database_path) + orchestrator.patch_agent( + "default", model_agent.id, {"model_timeout_seconds": 7200}, actor_id=principal_id + ) + with sqlite3.connect(database_path) as connection: + recorded = connection.execute("SELECT actor_id FROM model_timeout_history").fetchone()[0] + assert recorded == principal_id + assert "example_admin" not in recorded From e0dc420968a550602cebb06566f021e3a477ad3f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:34:05 +0900 Subject: [PATCH 20/78] test(gateway): authorize the timeout history actor fixture Signed-off-by: Seongho Bae --- tests/test_model_timeout_policy.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_model_timeout_policy.py b/tests/test_model_timeout_policy.py index a765a5c51..ddf833550 100644 --- a/tests/test_model_timeout_policy.py +++ b/tests/test_model_timeout_policy.py @@ -214,7 +214,9 @@ def test_model_timeout_policy_records_verified_principal(tmp_path: Path) -> None from contextual_orchestrator.server import SecurityConfig security = SecurityConfig(admin_token="example_admin", inference_token="example_inference") - principal_id = security.principal_id({"Authorization": "Bearer example_admin"}) + headers = {"authorization": "Bearer example_admin"} + security.authorize(headers, "admin") + principal_id = security.principal_id(headers) model_agent = ModelAgent("timeout_agent", "example-model") database_path = str(tmp_path / "agent-pool.db") orchestrator = TaskOrchestrator([model_agent], agents_db=database_path) From c1b372df194a61f7a975fa9664e6c70d00fa2d65 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:34:19 +0900 Subject: [PATCH 21/78] test(gateway): supply caller address to actor authorization Signed-off-by: Seongho Bae --- tests/test_model_timeout_policy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_model_timeout_policy.py b/tests/test_model_timeout_policy.py index ddf833550..3bd0383ec 100644 --- a/tests/test_model_timeout_policy.py +++ b/tests/test_model_timeout_policy.py @@ -215,7 +215,7 @@ def test_model_timeout_policy_records_verified_principal(tmp_path: Path) -> None security = SecurityConfig(admin_token="example_admin", inference_token="example_inference") headers = {"authorization": "Bearer example_admin"} - security.authorize(headers, "admin") + security.authorize(headers, "admin", "127.0.0.1") principal_id = security.principal_id(headers) model_agent = ModelAgent("timeout_agent", "example-model") database_path = str(tmp_path / "agent-pool.db") From c08a5fd55a301f1467929b3a8676c42ff0b8857d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:34:56 +0900 Subject: [PATCH 22/78] feat(gateway): retain opaque timeout policy actor evidence Signed-off-by: Seongho Bae --- contextual_orchestrator/orchestrator.py | 31 ++++++++++++++++++------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index e47f77f6e..ef8c4db41 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -3418,6 +3418,9 @@ def __init__(self, path: str) -> None: "previous_seconds REAL, timeout_seconds REAL, " "created_at REAL NOT NULL)" ) + history_columns = {row[1] for row in conn.execute("PRAGMA table_info(model_timeout_history)")} + if "actor_id" not in history_columns: + conn.execute("ALTER TABLE model_timeout_history ADD COLUMN actor_id TEXT") conn.commit() except Exception: conn.rollback() @@ -3459,7 +3462,10 @@ def _migrate_legacy_groups(conn: sqlite3.Connection) -> None: ) conn.execute("DROP TABLE agent_pool_legacy_payloads") - def save(self, agent: "ModelAgent", *, timeout_previous: "ModelAgent | None" = None) -> int | None: + def save( + self, agent: "ModelAgent", *, timeout_previous: "ModelAgent | None" = None, + actor_id: str | None = None, + ) -> int | None: """Persist one normalized model-agent definition.""" with self._lock: conn = self._connect(self._path) @@ -3483,7 +3489,7 @@ def save(self, agent: "ModelAgent", *, timeout_previous: "ModelAgent | None" = N "UPDATE agent_pool SET model_timeout_seconds = ? WHERE agent_id = ?", (agent.model_timeout_seconds, agent.id), ) - revision = self._append_timeout_history(conn, timeout_previous, agent) + revision = self._append_timeout_history(conn, timeout_previous, agent, actor_id) conn.commit() return revision config = agent.to_config() @@ -3596,7 +3602,7 @@ def save(self, agent: "ModelAgent", *, timeout_previous: "ModelAgent | None" = N (agent.id, contract.contract_id), ) if timeout_previous is not None: - revision = self._append_timeout_history(conn, timeout_previous, agent) + revision = self._append_timeout_history(conn, timeout_previous, agent, actor_id) conn.commit() return revision if timeout_previous is not None else None finally: @@ -3604,13 +3610,14 @@ def save(self, agent: "ModelAgent", *, timeout_previous: "ModelAgent | None" = N @staticmethod def _append_timeout_history( - conn: sqlite3.Connection, previous: "ModelAgent", updated: "ModelAgent" + conn: sqlite3.Connection, previous: "ModelAgent", updated: "ModelAgent", + actor_id: str | None, ) -> int: """Write the policy change using the same uncommitted configuration transaction.""" cursor = conn.execute( "INSERT INTO model_timeout_history " - "(agent_id, previous_seconds, timeout_seconds, created_at) VALUES (?, ?, ?, ?)", - (updated.id, previous.model_timeout_seconds, updated.model_timeout_seconds, time.time()), + "(agent_id, previous_seconds, timeout_seconds, created_at, actor_id) VALUES (?, ?, ?, ?, ?)", + (updated.id, previous.model_timeout_seconds, updated.model_timeout_seconds, time.time(), actor_id), ) return int(cursor.lastrowid) @@ -6074,7 +6081,10 @@ def get_access_report( "verifier": run.get("verification"), } - def patch_agent(self, agent_pool_id: str, worker_agent_id: str, patch: dict[str, Any]) -> dict[str, Any]: + def patch_agent( + self, agent_pool_id: str, worker_agent_id: str, patch: dict[str, Any], *, + actor_id: str | None = None, + ) -> dict[str, Any]: """Apply governance updates without invalidating the active effort catalog.""" if not patch: # pragma: no cover raise ValueError("patch request body must contain updates") @@ -6121,9 +6131,14 @@ def patch_agent(self, agent_pool_id: str, worker_agent_id: str, patch: dict[str, if "model_timeout_seconds" in patch: if set(patch) != {"model_timeout_seconds"}: raise ValueError("model timeout policy must be updated separately") + if actor_id is not None and ( + type(actor_id) is not str or len(actor_id) != 64 + or any(character not in "0123456789abcdef" for character in actor_id) + ): + raise ValueError("timeout actor must be an opaque principal digest") if self._pool_store is None: raise ValueError("model timeout policy requires a durable agent store") - revision = self._pool_store.save(patched, timeout_previous=current) + revision = self._pool_store.save(patched, timeout_previous=current, actor_id=actor_id) patched = replace(patched, model_timeout_revision=revision) updated_candidates = [patched if agent.id == worker_agent_id else agent for agent in self.candidates] updated_agents = [agent for agent in updated_candidates if not agent.disabled] From fc2340204c42775773b8e7e8ec66235420f8b38e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:35:33 +0900 Subject: [PATCH 23/78] test(gateway): validate timeout actor migration and secret boundaries Signed-off-by: Seongho Bae --- tests/test_model_timeout_policy.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/test_model_timeout_policy.py b/tests/test_model_timeout_policy.py index 3bd0383ec..a4116b96b 100644 --- a/tests/test_model_timeout_policy.py +++ b/tests/test_model_timeout_policy.py @@ -227,3 +227,32 @@ def test_model_timeout_policy_records_verified_principal(tmp_path: Path) -> None recorded = connection.execute("SELECT actor_id FROM model_timeout_history").fetchone()[0] assert recorded == principal_id assert "example_admin" not in recorded + + +@pytest.mark.parametrize("invalid_actor", [True, "example_admin", "A" * 64, "a" * 63]) +def test_model_timeout_policy_rejects_raw_actor_values(tmp_path: Path, invalid_actor: object) -> None: + """Reject raw or malformed actor values before writing policy or history.""" + model_agent = ModelAgent("timeout_agent", "example-model") + database_path = str(tmp_path / "agent-pool.db") + orchestrator = TaskOrchestrator([model_agent], agents_db=database_path) + with pytest.raises(ValueError, match="opaque principal"): + orchestrator.patch_agent( + "default", model_agent.id, {"model_timeout_seconds": 7200}, actor_id=invalid_actor + ) + assert orchestrator._agent(model_agent.id).model_timeout_seconds is None + with sqlite3.connect(database_path) as connection: + assert connection.execute("SELECT COUNT(*) FROM model_timeout_history").fetchone() == (0,) + + +def test_model_timeout_policy_migrates_unknown_actor_history(tmp_path: Path) -> None: + """Historical changes without actor evidence remain explicitly unattributed.""" + model_agent = ModelAgent("timeout_agent", "example-model") + database_path = str(tmp_path / "agent-pool.db") + orchestrator = TaskOrchestrator([model_agent], agents_db=database_path) + orchestrator.patch_agent("default", model_agent.id, {"model_timeout_seconds": 7200}) + with sqlite3.connect(database_path) as connection: + connection.execute("ALTER TABLE model_timeout_history DROP COLUMN actor_id") + restored = TaskOrchestrator([model_agent], agents_db=database_path) + assert restored._agent(model_agent.id).model_timeout_seconds == 7200 + with sqlite3.connect(database_path) as connection: + assert connection.execute("SELECT actor_id FROM model_timeout_history").fetchall() == [(None,)] From 9c01b57cf90edf4315bdaa1cd38d423d88f9639c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:36:21 +0900 Subject: [PATCH 24/78] docs(gateway): distinguish timeout actor evidence from HTTP authorization Signed-off-by: Seongho Bae --- .../doctoring/model-timeout-policy-evidence.md | 18 ++++++++++++++++++ docs/product-technical-gap-baseline.md | 6 +++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/docs/doctoring/model-timeout-policy-evidence.md b/docs/doctoring/model-timeout-policy-evidence.md index 9233e8330..a93771462 100644 --- a/docs/doctoring/model-timeout-policy-evidence.md +++ b/docs/doctoring/model-timeout-policy-evidence.md @@ -88,6 +88,24 @@ This proves that interleaving, not all distributed serving coherence. Policy restore, authenticated actor attribution, complete concurrent-write failure coverage and actual runtime enforcement are still unfinished and unshipped. +`c08a5fd5` adds nullable opaque actor evidence to the same policy-history +transaction. The corrected RED at `c1b372df` reached the missing actor argument; +earlier `cbab94c8` and `e0dc4209` failed because of test-header casing and a +missing authorization argument, not product behavior. A test first authorizes +the administrator fixture with SecurityConfig, then passes its principal digest +to the configuration boundary and checks the stored value contains no bearer. +This is component composition, not an authenticated HTTP policy-write E2E. + +The caller-supplied actor must have the existing 64-character lowercase digest +shape when present. Legacy/internal records can remain null and are explicitly +unattributed. Static-token mode identifies a deployment principal, not an +individual human; individual attribution needs the configured identity resolver. +The actor migration does not invent identities for historical rows. HTTP write +admission must require authenticated actor evidence when enabled; it remains +closed until runtime enforcement and restore acceptance are complete. +At `fc234020`, 45 related tests passed in 7.18 seconds, exit 0, including +raw/malformed actor rejection and migration of an existing unattributed row. + ## Remaining delivery gates - Complete revision-based change/history and restore with authenticated actor evidence. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 636652c4f..ca292e727 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -18,7 +18,11 @@ not yet cover authenticated actor evidence or cross-process serving refresh. Subsequent `d911a38e` adds history-revision comparisons for ABA conflicts; `e5e9c96f` reads values and revisions in one database snapshot. Their committed RED cases reproduced stale-value acceptance and mixed revisions respectively; -the latter head has 39 related passes. Restore, +the latter head has 39 related passes. +Actor evidence is now stored as an optional opaque principal digest at local +`c08a5fd5`; this is not yet authenticated HTTP policy-write E2E and null +historical actors remain unknown. Static-token identity is deployment-scoped. +Restore, precedence, cancellation semantics, released Rust runtime integration and actual administrator visual/E2E evidence also remain open. From 2919652e8af45ab06baa4b9563991ba5994c54b6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:37:45 +0900 Subject: [PATCH 25/78] test(gateway): require revision-bound timeout restoration Signed-off-by: Seongho Bae --- tests/test_model_timeout_policy.py | 35 ++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/test_model_timeout_policy.py b/tests/test_model_timeout_policy.py index a4116b96b..b8ffda506 100644 --- a/tests/test_model_timeout_policy.py +++ b/tests/test_model_timeout_policy.py @@ -256,3 +256,38 @@ def test_model_timeout_policy_migrates_unknown_actor_history(tmp_path: Path) -> assert restored._agent(model_agent.id).model_timeout_seconds == 7200 with sqlite3.connect(database_path) as connection: assert connection.execute("SELECT actor_id FROM model_timeout_history").fetchall() == [(None,)] + + +def test_model_timeout_policy_restore_creates_a_new_revision(tmp_path: Path) -> None: + """Restore reuses a model's historical value without rewriting its history.""" + model_agent = ModelAgent("timeout_agent", "example-model") + database_path = str(tmp_path / "agent-pool.db") + orchestrator = TaskOrchestrator([model_agent], agents_db=database_path) + first = orchestrator.patch_agent("default", model_agent.id, {"model_timeout_seconds": 7200}) + cleared = orchestrator.patch_agent("default", model_agent.id, {"model_timeout_seconds": None}) + restored = orchestrator.restore_model_timeout( + "default", model_agent.id, first["model_timeout_revision"], + expected_revision=cleared["model_timeout_revision"], actor_id="a" * 64, + ) + assert restored["model_timeout_seconds"] == 7200 + assert restored["model_timeout_revision"] > cleared["model_timeout_revision"] + with sqlite3.connect(database_path) as connection: + assert connection.execute( + "SELECT previous_seconds, timeout_seconds, restored_from_revision, actor_id " + "FROM model_timeout_history ORDER BY policy_revision DESC LIMIT 1" + ).fetchone() == (None, 7200.0, first["model_timeout_revision"], "a" * 64) + assert connection.execute("SELECT COUNT(*) FROM model_timeout_history").fetchone() == (3,) + + +def test_model_timeout_policy_restore_rejects_stale_revision(tmp_path: Path) -> None: + """A restore based on an obsolete view cannot overwrite a newer policy.""" + model_agent = ModelAgent("timeout_agent", "example-model") + orchestrator = TaskOrchestrator([model_agent], agents_db=str(tmp_path / "agent-pool.db")) + first = orchestrator.patch_agent("default", model_agent.id, {"model_timeout_seconds": 7200}) + orchestrator.patch_agent("default", model_agent.id, {"model_timeout_seconds": 3600}) + with pytest.raises(ValueError, match="reload"): + orchestrator.restore_model_timeout( + "default", model_agent.id, first["model_timeout_revision"], + expected_revision=first["model_timeout_revision"], actor_id="a" * 64, + ) + assert orchestrator._agent(model_agent.id).model_timeout_seconds == 3600 From 37bca9cadb7a5fc898ffe0487f4cd5b40c5368ee Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:38:53 +0900 Subject: [PATCH 26/78] feat(gateway): restore model timeout as a revision-checked change Signed-off-by: Seongho Bae --- contextual_orchestrator/orchestrator.py | 71 +++++++++++++++++++++++-- 1 file changed, 66 insertions(+), 5 deletions(-) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index ef8c4db41..6e192dec1 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -3421,6 +3421,11 @@ def __init__(self, path: str) -> None: history_columns = {row[1] for row in conn.execute("PRAGMA table_info(model_timeout_history)")} if "actor_id" not in history_columns: conn.execute("ALTER TABLE model_timeout_history ADD COLUMN actor_id TEXT") + if "restored_from_revision" not in history_columns: + conn.execute( + "ALTER TABLE model_timeout_history ADD COLUMN restored_from_revision INTEGER " + "REFERENCES model_timeout_history(policy_revision)" + ) conn.commit() except Exception: conn.rollback() @@ -3465,6 +3470,7 @@ def _migrate_legacy_groups(conn: sqlite3.Connection) -> None: def save( self, agent: "ModelAgent", *, timeout_previous: "ModelAgent | None" = None, actor_id: str | None = None, + restored_from_revision: int | None = None, ) -> int | None: """Persist one normalized model-agent definition.""" with self._lock: @@ -3489,7 +3495,7 @@ def save( "UPDATE agent_pool SET model_timeout_seconds = ? WHERE agent_id = ?", (agent.model_timeout_seconds, agent.id), ) - revision = self._append_timeout_history(conn, timeout_previous, agent, actor_id) + revision = self._append_timeout_history(conn, timeout_previous, agent, actor_id, restored_from_revision) conn.commit() return revision config = agent.to_config() @@ -3602,7 +3608,7 @@ def save( (agent.id, contract.contract_id), ) if timeout_previous is not None: - revision = self._append_timeout_history(conn, timeout_previous, agent, actor_id) + revision = self._append_timeout_history(conn, timeout_previous, agent, actor_id, restored_from_revision) conn.commit() return revision if timeout_previous is not None else None finally: @@ -3612,15 +3618,40 @@ def save( def _append_timeout_history( conn: sqlite3.Connection, previous: "ModelAgent", updated: "ModelAgent", actor_id: str | None, + restored_from_revision: int | None, ) -> int: """Write the policy change using the same uncommitted configuration transaction.""" + if restored_from_revision is not None: + historical = conn.execute( + "SELECT timeout_seconds FROM model_timeout_history WHERE agent_id = ? AND policy_revision = ?", + (updated.id, restored_from_revision), + ).fetchone() + if historical is None or historical[0] != updated.model_timeout_seconds: + raise ValueError("restored timeout must match this model's historical revision") cursor = conn.execute( "INSERT INTO model_timeout_history " - "(agent_id, previous_seconds, timeout_seconds, created_at, actor_id) VALUES (?, ?, ?, ?, ?)", - (updated.id, previous.model_timeout_seconds, updated.model_timeout_seconds, time.time(), actor_id), + "(agent_id, previous_seconds, timeout_seconds, created_at, actor_id, restored_from_revision) " + "VALUES (?, ?, ?, ?, ?, ?)", + (updated.id, previous.model_timeout_seconds, updated.model_timeout_seconds, + time.time(), actor_id, restored_from_revision), ) return int(cursor.lastrowid) + def timeout_at_revision(self, agent_id: str, policy_revision: int) -> float | None: + """Read a historical value only when its revision belongs to this model.""" + with self._lock: + conn = self._connect(self._path) + try: + row = conn.execute( + "SELECT timeout_seconds FROM model_timeout_history WHERE agent_id = ? AND policy_revision = ?", + (agent_id, policy_revision), + ).fetchone() + finally: + conn.close() + if row is None: + raise KeyError("model timeout revision not found") + return row[0] + def load_all(self) -> list["ModelAgent"]: """Load every persisted model-agent definition.""" with self._lock: @@ -6084,6 +6115,8 @@ def get_access_report( def patch_agent( self, agent_pool_id: str, worker_agent_id: str, patch: dict[str, Any], *, actor_id: str | None = None, + expected_timeout_revision: int | None = None, + restored_from_revision: int | None = None, ) -> dict[str, Any]: """Apply governance updates without invalidating the active effort catalog.""" if not patch: # pragma: no cover @@ -6131,6 +6164,11 @@ def patch_agent( if "model_timeout_seconds" in patch: if set(patch) != {"model_timeout_seconds"}: raise ValueError("model timeout policy must be updated separately") + if expected_timeout_revision is not None and ( + type(expected_timeout_revision) is not int + or expected_timeout_revision != current.model_timeout_revision + ): + raise ValueError("model timeout policy changed; reload before updating") if actor_id is not None and ( type(actor_id) is not str or len(actor_id) != 64 or any(character not in "0123456789abcdef" for character in actor_id) @@ -6138,7 +6176,10 @@ def patch_agent( raise ValueError("timeout actor must be an opaque principal digest") if self._pool_store is None: raise ValueError("model timeout policy requires a durable agent store") - revision = self._pool_store.save(patched, timeout_previous=current, actor_id=actor_id) + revision = self._pool_store.save( + patched, timeout_previous=current, actor_id=actor_id, + restored_from_revision=restored_from_revision, + ) patched = replace(patched, model_timeout_revision=revision) updated_candidates = [patched if agent.id == worker_agent_id else agent for agent in self.candidates] updated_agents = [agent for agent in updated_candidates if not agent.disabled] @@ -6184,6 +6225,26 @@ def patch_agent( ) return self._agent_to_admin_payload(patched) + def restore_model_timeout( + self, agent_pool_id: str, worker_agent_id: str, source_revision: int, *, + expected_revision: int, actor_id: str, + ) -> dict[str, Any]: + """Restore a model-owned historical value as a new revision, never rewrite history.""" + self._agent_in_pool(agent_pool_id, worker_agent_id) + if type(source_revision) is not int or not 0 < source_revision <= _AGENT_POOL_INTEGER_MAX: + raise ValueError("source_revision must be a positive integer") + if type(expected_revision) is not int or expected_revision < 0: + raise ValueError("expected_revision must be a non-negative integer") + if actor_id is None: + raise ValueError("restore requires an opaque principal digest") + if self._pool_store is None: + raise ValueError("model timeout policy requires a durable agent store") + value = self._pool_store.timeout_at_revision(worker_agent_id, source_revision) + return self.patch_agent( + agent_pool_id, worker_agent_id, {"model_timeout_seconds": value}, actor_id=actor_id, + expected_timeout_revision=expected_revision, restored_from_revision=source_revision, + ) + def list_model_groups(self) -> list[dict[str, Any]]: """Return operator-defined logical models and measured member evidence.""" names = sorted({canonical_group_name(agent.group_name) for agent in self.candidates if agent.group_name}) From 62ba3c3bdd6bd37ebaa97d8452a05ab81b80f72e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:39:50 +0900 Subject: [PATCH 27/78] test(gateway): cover timeout restore ownership and rollback Signed-off-by: Seongho Bae --- tests/test_model_timeout_policy.py | 52 ++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/tests/test_model_timeout_policy.py b/tests/test_model_timeout_policy.py index b8ffda506..e3a2308fe 100644 --- a/tests/test_model_timeout_policy.py +++ b/tests/test_model_timeout_policy.py @@ -291,3 +291,55 @@ def test_model_timeout_policy_restore_rejects_stale_revision(tmp_path: Path) -> expected_revision=first["model_timeout_revision"], actor_id="a" * 64, ) assert orchestrator._agent(model_agent.id).model_timeout_seconds == 3600 + + +def test_model_timeout_policy_restore_rejects_foreign_history(tmp_path: Path) -> None: + """A history identifier from another model does not authorize restoration.""" + first = ModelAgent("first_agent", "first-model") + second = ModelAgent("second_agent", "second-model") + orchestrator = TaskOrchestrator([first, second], agents_db=str(tmp_path / "agent-pool.db")) + changed = orchestrator.patch_agent("default", first.id, {"model_timeout_seconds": 7200}) + with pytest.raises(KeyError, match="revision not found"): + orchestrator.restore_model_timeout( + "default", second.id, changed["model_timeout_revision"], + expected_revision=0, actor_id="a" * 64, + ) + assert orchestrator._agent(second.id).model_timeout_seconds is None + + +def test_model_timeout_policy_restore_audit_failure_rolls_back(tmp_path: Path) -> None: + """A rejected restore must preserve its prior value and revision.""" + model_agent = ModelAgent("timeout_agent", "example-model") + database_path = str(tmp_path / "agent-pool.db") + orchestrator = TaskOrchestrator([model_agent], agents_db=database_path) + first = orchestrator.patch_agent("default", model_agent.id, {"model_timeout_seconds": 7200}) + latest = orchestrator.patch_agent("default", model_agent.id, {"model_timeout_seconds": None}) + with sqlite3.connect(database_path) as connection: + connection.execute( + "CREATE TRIGGER reject_restore BEFORE INSERT ON model_timeout_history " + "BEGIN SELECT RAISE(ABORT, 'restore audit unavailable'); END" + ) + with pytest.raises(sqlite3.IntegrityError, match="restore audit unavailable"): + orchestrator.restore_model_timeout( + "default", model_agent.id, first["model_timeout_revision"], + expected_revision=latest["model_timeout_revision"], actor_id="a" * 64, + ) + restored = TaskOrchestrator([model_agent], agents_db=database_path) + assert restored._agent(model_agent.id).model_timeout_seconds is None + assert restored._agent(model_agent.id).model_timeout_revision == latest["model_timeout_revision"] + assert orchestrator._agent(model_agent.id).model_timeout_revision == latest["model_timeout_revision"] + + +@pytest.mark.parametrize("source_revision, expected_revision", [(True, 0), (0, 0), (2**63, 0), (1, True), (1, -1)]) +def test_model_timeout_policy_restore_validates_revision_types( + tmp_path: Path, source_revision: object, expected_revision: object +) -> None: + """Reject malformed revision identifiers before history access or mutation.""" + model_agent = ModelAgent("timeout_agent", "example-model") + orchestrator = TaskOrchestrator([model_agent], agents_db=str(tmp_path / "agent-pool.db")) + with pytest.raises(ValueError): + orchestrator.restore_model_timeout( + "default", model_agent.id, source_revision, + expected_revision=expected_revision, actor_id="a" * 64, + ) + assert orchestrator._agent(model_agent.id).model_timeout_revision == 0 From 6aece30da1cef26406a723a8bce82f617ab2140c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:40:28 +0900 Subject: [PATCH 28/78] docs(gateway): record revision-bound timeout restore evidence Signed-off-by: Seongho Bae --- docs/doctoring/model-timeout-policy-evidence.md | 15 +++++++++++++++ docs/product-technical-gap-baseline.md | 4 +++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/docs/doctoring/model-timeout-policy-evidence.md b/docs/doctoring/model-timeout-policy-evidence.md index a93771462..962e48e70 100644 --- a/docs/doctoring/model-timeout-policy-evidence.md +++ b/docs/doctoring/model-timeout-policy-evidence.md @@ -106,6 +106,21 @@ closed until runtime enforcement and restore acceptance are complete. At `fc234020`, 45 related tests passed in 7.18 seconds, exit 0, including raw/malformed actor rejection and migration of an existing unattributed row. +Restoration was first missing at `2919652e` (2 failed, 25 deselected, 2.26 +seconds). Local `37bca9ca` adds a model-scoped historical lookup and restores +its value through the same revision-checked policy transaction. The new history +row references its source revision and supplied actor; existing rows remain +unchanged. Both the expected current revision and the model owning the source +history are checked. No HTTP restore endpoint is admitted yet. + +At `62ba3c3b`, 54 related tests passed in 6.65 seconds, exit 0. They include +restore success as a new revision, stale-view rejection, foreign-model history +rejection, malformed revision rejection and history-insertion failure rollback +for both durable value and in-memory revision. This is local storage/domain +evidence. Authenticated HTTP restore, user-facing history/restore controls, +multi-process serving refresh, complete concurrency and actual model execution +enforcement remain required before release. + ## Remaining delivery gates - Complete revision-based change/history and restore with authenticated actor evidence. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index ca292e727..407eed2f8 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -22,7 +22,9 @@ the latter head has 39 related passes. Actor evidence is now stored as an optional opaque principal digest at local `c08a5fd5`; this is not yet authenticated HTTP policy-write E2E and null historical actors remain unknown. Static-token identity is deployment-scoped. -Restore, +Local `37bca9ca` adds revision-checked restoration as a new, source-linked +history entry; `62ba3c3b` has 54 related passes including foreign-model and +audit-failure rejection. Authenticated HTTP restore, precedence, cancellation semantics, released Rust runtime integration and actual administrator visual/E2E evidence also remain open. From e0eab787bd84fa462e921d8d34a26440547e1053 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:44:54 +0900 Subject: [PATCH 29/78] test(gateway): reject stale timeout in ordinary pool edits Signed-off-by: Seongho Bae --- tests/test_model_timeout_policy.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_model_timeout_policy.py b/tests/test_model_timeout_policy.py index e3a2308fe..11e4397a6 100644 --- a/tests/test_model_timeout_policy.py +++ b/tests/test_model_timeout_policy.py @@ -8,6 +8,21 @@ from contextual_orchestrator import ModelAgent, TaskOrchestrator +def test_ordinary_patch_rejects_stale_timeout_snapshot(tmp_path: Path) -> None: + """A priority edit cannot silently clear another writer's audited timeout.""" + model_agent = ModelAgent("timeout_agent", "example-model") + database_path = str(tmp_path / "agent-pool.db") + writer = TaskOrchestrator([model_agent], agents_db=database_path) + stale = TaskOrchestrator([model_agent], agents_db=database_path) + writer.patch_agent("default", model_agent.id, {"model_timeout_seconds": 7200}) + with pytest.raises(ValueError, match="reload"): + stale.patch_agent("default", model_agent.id, {"priority": 7}) + assert stale._agent(model_agent.id).priority == model_agent.priority + restored = TaskOrchestrator([model_agent], agents_db=database_path) + assert restored._agent(model_agent.id).model_timeout_seconds == 7200 + assert restored._agent(model_agent.id).model_timeout_revision == 1 + + def test_model_timeout_policy_defaults_to_null() -> None: """An ordinary model has no administrator-imposed execution limit.""" model_agent = ModelAgent("timeout_agent", "example-model") From 9701dec2155f61b480ac64fd5889e1fdb5dbdbdb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:45:27 +0900 Subject: [PATCH 30/78] fix(gateway): guard timeout policy in every durable pool save Signed-off-by: Seongho Bae --- contextual_orchestrator/orchestrator.py | 27 +++++++++++++------------ 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 6e192dec1..d6e589135 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -3476,21 +3476,22 @@ def save( with self._lock: conn = self._connect(self._path) try: + conn.execute("BEGIN IMMEDIATE") + previous = timeout_previous if timeout_previous is not None else agent + revision = conn.execute( + "SELECT COALESCE(MAX(policy_revision), 0) FROM model_timeout_history WHERE agent_id = ?", + (agent.id,), + ).fetchone()[0] + if revision != previous.model_timeout_revision: + raise ValueError("model timeout policy changed; reload before updating") + row = conn.execute( + "SELECT model_timeout_seconds FROM agent_pool WHERE agent_id = ?", + (agent.id,), + ).fetchone() + if row is not None and row[0] != previous.model_timeout_seconds: + raise ValueError("model timeout policy changed; reload before updating") if timeout_previous is not None: - conn.execute("BEGIN IMMEDIATE") - revision = conn.execute( - "SELECT COALESCE(MAX(policy_revision), 0) FROM model_timeout_history WHERE agent_id = ?", - (agent.id,), - ).fetchone()[0] - if revision != timeout_previous.model_timeout_revision: - raise ValueError("model timeout policy changed; reload before updating") - row = conn.execute( - "SELECT model_timeout_seconds FROM agent_pool WHERE agent_id = ?", - (agent.id,), - ).fetchone() if row is not None: - if row[0] != timeout_previous.model_timeout_seconds: - raise ValueError("model timeout policy changed; reload before updating") conn.execute( "UPDATE agent_pool SET model_timeout_seconds = ? WHERE agent_id = ?", (agent.model_timeout_seconds, agent.id), From 28313ef3e91aea23618fec2d4f844144d81d0ef5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:46:07 +0900 Subject: [PATCH 31/78] docs(gateway): record ordinary save policy protection Signed-off-by: Seongho Bae --- docs/doctoring/model-timeout-policy-evidence.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/doctoring/model-timeout-policy-evidence.md b/docs/doctoring/model-timeout-policy-evidence.md index 962e48e70..fdf13cbe1 100644 --- a/docs/doctoring/model-timeout-policy-evidence.md +++ b/docs/doctoring/model-timeout-policy-evidence.md @@ -148,6 +148,21 @@ the original failures and JUnit reports. These temporary paths are not public release artifacts. The local policy delta remains unpushed pending the gates above; the remote PR's completed full-suite result applies only to `661ce8db`. +## Ordinary-save policy protection + +At `e0eab787`, a stale orchestrator's priority edit was accepted after another +writer set a 7200-second policy (one failed test, 1.12 seconds). The ordinary +pool UPDATE included the stale timeout without appending policy history. +At `9701dec2`, every pool save checks the policy value and revision inside the +existing immediate transaction before writing. A stale ordinary edit now fails +before publishing its candidate; the durable policy and history remain intact. +The policy, pool database, and governance suites passed 60 tests in 4.09 seconds. +This is focused storage-boundary evidence, not distributed refresh, runtime +deadline enforcement, HTTP authorization, or a new full-suite result. + +A fresh visual-inspection attempt on 2026-09-06 could not proceed because the +Mac was locked. Earlier remote PR screenshots do not verify these local changes. + ## Source reference ContextualWisdomLab. (n.d.). *Finite outbound request-timeout boundaries* From 1edf0fba458b91d51b042adf7dd02480f6787b1d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:47:37 +0900 Subject: [PATCH 32/78] test(gateway): preserve serving state when pool edits fail Signed-off-by: Seongho Bae --- tests/test_model_timeout_policy.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_model_timeout_policy.py b/tests/test_model_timeout_policy.py index 11e4397a6..493f26a4b 100644 --- a/tests/test_model_timeout_policy.py +++ b/tests/test_model_timeout_policy.py @@ -23,6 +23,27 @@ def test_ordinary_patch_rejects_stale_timeout_snapshot(tmp_path: Path) -> None: assert restored._agent(model_agent.id).model_timeout_revision == 1 +@pytest.mark.parametrize("operation", ["remove", "set_group", "delete_group"]) +def test_rejected_pool_change_preserves_serving_snapshot(tmp_path: Path, operation: str) -> None: + """A rejected durable edit must not publish a removal or membership change.""" + model_agent = ModelAgent("timeout_agent", "example-model", group_name="test_group") + other_agent = ModelAgent("other_agent", "other-model") + database_path = str(tmp_path / "agent-pool.db") + writer = TaskOrchestrator([model_agent, other_agent], agents_db=database_path) + stale = TaskOrchestrator([model_agent, other_agent], agents_db=database_path) + before_candidates, before_agents = list(stale.candidates), list(stale.agents) + writer.patch_agent("default", model_agent.id, {"model_timeout_seconds": 7200}) + with pytest.raises(ValueError, match="reload"): + if operation == "remove": + stale.remove_agent("default", model_agent.id) + elif operation == "set_group": + stale.set_model_group("new_group", [model_agent.id]) + else: + stale.delete_model_group("test_group") + assert stale.candidates == before_candidates + assert stale.agents == before_agents + + def test_model_timeout_policy_defaults_to_null() -> None: """An ordinary model has no administrator-imposed execution limit.""" model_agent = ModelAgent("timeout_agent", "example-model") From befe04ce2e949794dfa064570ae83f4bfc148aec Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:48:05 +0900 Subject: [PATCH 33/78] fix(gateway): publish pool edits only after durable saves Signed-off-by: Seongho Bae --- contextual_orchestrator/orchestrator.py | 30 ++++++++++++------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index d6e589135..0a08a51be 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -6296,22 +6296,22 @@ def set_model_group(self, group_name: str, member_agent_ids: list[str]) -> dict[ else agent for agent in self.candidates ] - self.candidates = updated - self.agents = [agent for agent in updated if not agent.disabled] changed = { before.id for before, after in zip(previous_candidates, updated) if before.group_name != after.group_name } - self._routers_reset_members(changed) - for agent_id in changed: - self._routers_register_member(agent_id) for agent in updated: if agent.id in requested: if self._pool_store is not None: self._pool_store.save(agent) elif agent.id in previous and self._pool_store is not None: self._pool_store.save(agent) + self.candidates = updated + self.agents = [agent for agent in updated if not agent.disabled] + self._routers_reset_members(changed) + for agent_id in changed: + self._routers_register_member(agent_id) self._routers_forget_members({agent.id for agent in updated}) self._append_audit_event("model_group_set", {"group_name": name, "member_agent_ids": sorted(requested)}) return self.get_model_group(name) @@ -6321,15 +6321,16 @@ def delete_model_group(self, group_name: str) -> dict[str, Any]: current = self.get_model_group(group_name) name = current["group_name"] member_ids = set(current["member_agent_ids"]) - self._routers_reset_members(member_ids) - for agent_id in member_ids: - self._routers_register_member(agent_id) - self.candidates = [replace(agent, group_name="") if agent.id in member_ids else agent for agent in self.candidates] - self.agents = [agent for agent in self.candidates if not agent.disabled] + updated = [replace(agent, group_name="") if agent.id in member_ids else agent for agent in self.candidates] if self._pool_store is not None: - for agent in self.candidates: + for agent in updated: if agent.id in member_ids: self._pool_store.save(agent) + self.candidates = updated + self.agents = [agent for agent in updated if not agent.disabled] + self._routers_reset_members(member_ids) + for agent_id in member_ids: + self._routers_register_member(agent_id) self._routers_forget_members({agent.id for agent in self.candidates}) self._append_audit_event("model_group_deleted", {"group_name": name}) return {"group_name": name, "deleted": True} @@ -6425,14 +6426,13 @@ def remove_agent(self, agent_pool_id: str, worker_agent_id: str) -> dict[str, An agent for agent in self.candidates if agent.id != worker_agent_id ] self._require_role_effort_pool(remaining_candidates) + if self._pool_store is not None: + # Persist the tombstone before removing the serving candidate. + self._pool_store.save(replace(target, disabled=True, group_name="")) self.candidates = remaining_candidates self.agents = [agent for agent in self.candidates if not agent.disabled] self._rebuild_budget_meter() self._routers_forget_members({agent.id for agent in self.candidates}) - if self._pool_store is not None: - # Disabled tombstone (not a row delete): it overlays the seed file on restart - # and startup drops disabled agents, so removal survives even for seed agents. - self._pool_store.save(replace(target, disabled=True, group_name="")) self._append_audit_event( "agent_removed", {"agent_pool_id": agent_pool_id, "worker_agent_id": worker_agent_id, "model": target.model}, From 8a412b89e39c99342cc7da799691df671b8a09a0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:48:57 +0900 Subject: [PATCH 34/78] docs(gateway): record rejected edit publication evidence Signed-off-by: Seongho Bae --- docs/doctoring/model-timeout-policy-evidence.md | 14 ++++++++++++++ docs/product-technical-gap-baseline.md | 7 +++++++ 2 files changed, 21 insertions(+) diff --git a/docs/doctoring/model-timeout-policy-evidence.md b/docs/doctoring/model-timeout-policy-evidence.md index fdf13cbe1..4ad8833cf 100644 --- a/docs/doctoring/model-timeout-policy-evidence.md +++ b/docs/doctoring/model-timeout-policy-evidence.md @@ -163,6 +163,20 @@ deadline enforcement, HTTP authorization, or a new full-suite result. A fresh visual-inspection attempt on 2026-09-06 could not proceed because the Mac was locked. Earlier remote PR screenshots do not verify these local changes. +## Serving publication after rejected pool edits + +At `1edf0fba`, three regressions reproduced removal, group assignment and group +deletion changing the in-memory serving candidates despite a rejected stale +policy save (3 failed, 35 deselected, 0.70 seconds). At `befe04ce`, these callers +publish candidate lists and reset routing state only after durable saves. +Policy, pool, governance, model-group and mixed-role-effort suites passed +103 tests in 15.34 seconds, exit 0. No full-suite or transport claim follows. + +Multi-model group/discovery writes still commit one row at a time. A later +failure may leave earlier durable rows changed even though serving publication +is withheld. Batch rollback and concurrent serving refresh remain open gates; +the single-target regressions above do not prove either requirement. + ## Source reference ContextualWisdomLab. (n.d.). *Finite outbound request-timeout boundaries* diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 407eed2f8..547edf0ca 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -28,6 +28,13 @@ audit-failure rejection. Authenticated HTTP restore, precedence, cancellation semantics, released Rust runtime integration and actual administrator visual/E2E evidence also remain open. +Local `9701dec2` guards ordinary pool saves against stale policy value/revision +overwrites (60 focused passes). `befe04ce` additionally delays removal and +group-change serving publication until durable saves succeed, following three +reproduced failures; 103 related tests passed in 15.34 seconds. Multi-model +batch rollback and cross-process serving refresh remain unproven. Neither +change supplies model deadline enforcement or a new full-suite result. + See [the evidence record](doctoring/model-timeout-policy-evidence.md) for exact revisions, retained failures, corrected test-evidence limitations, owner boundaries and the full remaining acceptance gates. Local configuration work From ad337e1857f3c5b819224c8cf322b97b077cc3d8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:50:29 +0900 Subject: [PATCH 35/78] test(gateway): require rollback on late pool batch conflicts Signed-off-by: Seongho Bae --- tests/test_model_timeout_policy.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/test_model_timeout_policy.py b/tests/test_model_timeout_policy.py index 493f26a4b..793d588df 100644 --- a/tests/test_model_timeout_policy.py +++ b/tests/test_model_timeout_policy.py @@ -1,6 +1,7 @@ """Administrator-owned model timeout policy must survive configuration changes.""" from pathlib import Path +from dataclasses import replace import sqlite3 import pytest @@ -44,6 +45,31 @@ def test_rejected_pool_change_preserves_serving_snapshot(tmp_path: Path, operati assert stale.agents == before_agents +@pytest.mark.parametrize("operation", ["set_group", "delete_group", "discovery"]) +def test_late_batch_conflict_rolls_back_all_models(tmp_path: Path, operation: str) -> None: + """A stale second model cannot leave the first model partially committed.""" + seeds = [ModelAgent("first_agent", "first-model", group_name="test_group"), + ModelAgent("second_agent", "second-model", group_name="test_group")] + database_path = str(tmp_path / "agent-pool.db") + writer = TaskOrchestrator(seeds, agents_db=database_path) + writer.sync_discovered_agents(seeds) + stale = TaskOrchestrator(seeds, agents_db=database_path) + before = list(stale.candidates) + writer.patch_agent("default", "second_agent", {"model_timeout_seconds": 7200}) + with pytest.raises(ValueError, match="reload"): + if operation == "set_group": + stale.set_model_group("new_group", [agent.id for agent in seeds]) + elif operation == "delete_group": + stale.delete_model_group("test_group") + else: + stale.sync_discovered_agents([replace(agent, priority=7) for agent in seeds]) + assert stale.candidates == before + restored = TaskOrchestrator(seeds, agents_db=database_path) + assert restored._agent("first_agent") == seeds[0] + assert restored._agent("second_agent").model_timeout_seconds == 7200 + assert restored._agent("second_agent").group_name == "test_group" + + def test_model_timeout_policy_defaults_to_null() -> None: """An ordinary model has no administrator-imposed execution limit.""" model_agent = ModelAgent("timeout_agent", "example-model") From 36fc35df98f92e81eccedb024e067e9e3f0b9d4d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:51:04 +0900 Subject: [PATCH 36/78] fix(gateway): commit pool batch changes in one transaction Signed-off-by: Seongho Bae --- contextual_orchestrator/orchestrator.py | 316 +++++++++++++----------- 1 file changed, 166 insertions(+), 150 deletions(-) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 0a08a51be..931151b4c 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -3467,153 +3467,176 @@ def _migrate_legacy_groups(conn: sqlite3.Connection) -> None: ) conn.execute("DROP TABLE agent_pool_legacy_payloads") + @contextmanager + def _write_transaction(self) -> Iterable[sqlite3.Connection]: + """Commit the complete pool operation or roll back every affected row.""" + with self._lock: + conn = self._connect(self._path) + try: + conn.execute("BEGIN IMMEDIATE") + yield conn + conn.commit() + finally: + conn.close() + def save( self, agent: "ModelAgent", *, timeout_previous: "ModelAgent | None" = None, actor_id: str | None = None, restored_from_revision: int | None = None, ) -> int | None: """Persist one normalized model-agent definition.""" - with self._lock: - conn = self._connect(self._path) - try: - conn.execute("BEGIN IMMEDIATE") - previous = timeout_previous if timeout_previous is not None else agent - revision = conn.execute( - "SELECT COALESCE(MAX(policy_revision), 0) FROM model_timeout_history WHERE agent_id = ?", - (agent.id,), - ).fetchone()[0] - if revision != previous.model_timeout_revision: - raise ValueError("model timeout policy changed; reload before updating") - row = conn.execute( - "SELECT model_timeout_seconds FROM agent_pool WHERE agent_id = ?", - (agent.id,), - ).fetchone() - if row is not None and row[0] != previous.model_timeout_seconds: - raise ValueError("model timeout policy changed; reload before updating") - if timeout_previous is not None: - if row is not None: - conn.execute( - "UPDATE agent_pool SET model_timeout_seconds = ? WHERE agent_id = ?", - (agent.model_timeout_seconds, agent.id), - ) - revision = self._append_timeout_history(conn, timeout_previous, agent, actor_id, restored_from_revision) - conn.commit() - return revision - config = agent.to_config() - conn.execute( - """ - UPDATE agent_pool SET - model_name = ?, base_url = ?, api_key_env = ?, credential_key = ?, - priority = ?, disabled = ?, provider_name = ?, - local_credential_key = ?, auth_scheme = ?, - max_output_tokens = ?, context_window = ?, - reasoning_effort_supported = ?, stream_usage_supported = ?, - model_timeout_seconds = ? - WHERE agent_id = ? - """, - ( - config["model"], - config["base_url"], - config["api_key_env"], - config["credential_key"], - config["priority"], - int(config["disabled"]), - config["provider_name"], - config["local_credential_key"], - config["auth_scheme"], - config["max_output_tokens"], - config["context_window"], - config["reasoning_effort_supported"], - int(config["stream_usage_supported"]), - config["model_timeout_seconds"], - agent.id, - ), - ) - if conn.execute("SELECT changes()").fetchone()[0] == 0: - self._insert_agent(conn, agent) - else: - conn.execute("DELETE FROM agent_pool_tags WHERE agent_id = ?", (agent.id,)) - conn.execute( - "DELETE FROM agent_pool_provider_exclusions WHERE agent_id = ?", - (agent.id,), - ) - conn.executemany( - "INSERT INTO agent_pool_tags (agent_id, tag_position, tag_name) VALUES (?, ?, ?)", - [(agent.id, position, tag) for position, tag in enumerate(agent.tags)], - ) - conn.executemany( - """ - INSERT INTO agent_pool_provider_exclusions - (agent_id, exclusion_position, provider_name) - VALUES (?, ?, ?) - """, - [ - (agent.id, position, provider) - for position, provider in enumerate(agent.provider_exclusions) - ], - ) - # Model-group membership is a normalized relation beside the pool. - conn.execute("DELETE FROM model_group_member WHERE agent_id = ?", (agent.id,)) - if agent.group_name: - conn.execute( - "INSERT OR IGNORE INTO model_group (group_name) VALUES (?)", - (agent.group_name,), - ) - conn.execute( - "INSERT INTO model_group_member (agent_id, group_name) VALUES (?, ?)", - (agent.id, agent.group_name), - ) - conn.execute( - "DELETE FROM model_group WHERE NOT EXISTS (" - "SELECT 1 FROM model_group_member " - "WHERE model_group_member.group_name = model_group.group_name)" - ) - conn.execute("DELETE FROM endpoint_equivalence_member WHERE agent_id = ?", (agent.id,)) + with self._write_transaction() as conn: + return self._save_in_transaction( + conn, agent, timeout_previous=timeout_previous, actor_id=actor_id, + restored_from_revision=restored_from_revision, + ) + + def save_many(self, agents: Iterable["ModelAgent"]) -> None: + """Persist a group or discovery operation without partial model updates.""" + with self._write_transaction() as conn: + for agent in agents: + self._save_in_transaction(conn, agent) + + def _save_in_transaction( + self, conn: sqlite3.Connection, agent: "ModelAgent", *, + timeout_previous: "ModelAgent | None" = None, + actor_id: str | None = None, + restored_from_revision: int | None = None, + ) -> int | None: + """Apply existing normalized writes inside the caller's transaction.""" + previous = timeout_previous if timeout_previous is not None else agent + revision = conn.execute( + "SELECT COALESCE(MAX(policy_revision), 0) FROM model_timeout_history WHERE agent_id = ?", + (agent.id,), + ).fetchone()[0] + if revision != previous.model_timeout_revision: + raise ValueError("model timeout policy changed; reload before updating") + row = conn.execute( + "SELECT model_timeout_seconds FROM agent_pool WHERE agent_id = ?", + (agent.id,), + ).fetchone() + if row is not None and row[0] != previous.model_timeout_seconds: + raise ValueError("model timeout policy changed; reload before updating") + if timeout_previous is not None: + if row is not None: conn.execute( - "DELETE FROM endpoint_equivalence_contract WHERE NOT EXISTS (" - "SELECT 1 FROM endpoint_equivalence_member " - "WHERE endpoint_equivalence_member.contract_id = " - "endpoint_equivalence_contract.contract_id)" + "UPDATE agent_pool SET model_timeout_seconds = ? WHERE agent_id = ?", + (agent.model_timeout_seconds, agent.id), ) - if agent.endpoint_equivalence is not None: - contract = EndpointEquivalenceContract(**agent.endpoint_equivalence) - conn.execute( - "INSERT INTO endpoint_equivalence_contract VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) " - "ON CONFLICT(contract_id) DO UPDATE SET model_revision=excluded.model_revision, " - "reasoning_effort_profile=excluded.reasoning_effort_profile, " - "structured_output_contract=excluded.structured_output_contract, " - "accuracy_class=excluded.accuracy_class, data_residency_policy=excluded.data_residency_policy, " - "retention_policy=excluded.retention_policy, context_limit=excluded.context_limit, " - "pricing_evidence_id=excluded.pricing_evidence_id, hedge_eligible=excluded.hedge_eligible, " - "cancellation_supported=excluded.cancellation_supported, " - "execution_policy=excluded.execution_policy", - ( - contract.contract_id, contract.model_revision, - contract.reasoning_effort_profile, contract.structured_output_contract, - contract.accuracy_class, contract.data_residency_policy, - contract.retention_policy, contract.context_limit, - contract.pricing_evidence_id, int(contract.hedge_eligible), - int(contract.cancellation_supported), contract.execution_policy, - ), - ) - conn.execute( - "DELETE FROM endpoint_equivalence_capability WHERE contract_id = ?", - (contract.contract_id,), - ) - conn.executemany( - "INSERT INTO endpoint_equivalence_capability (contract_id, capability_name) VALUES (?, ?)", - [(contract.contract_id, name) for name in contract.capability_set], - ) - conn.execute( - "INSERT INTO endpoint_equivalence_member (agent_id, contract_id) VALUES (?, ?)", - (agent.id, contract.contract_id), - ) - if timeout_previous is not None: - revision = self._append_timeout_history(conn, timeout_previous, agent, actor_id, restored_from_revision) - conn.commit() - return revision if timeout_previous is not None else None - finally: - conn.close() + revision = self._append_timeout_history(conn, timeout_previous, agent, actor_id, restored_from_revision) + return revision + config = agent.to_config() + conn.execute( + """ + UPDATE agent_pool SET + model_name = ?, base_url = ?, api_key_env = ?, credential_key = ?, + priority = ?, disabled = ?, provider_name = ?, + local_credential_key = ?, auth_scheme = ?, + max_output_tokens = ?, context_window = ?, + reasoning_effort_supported = ?, stream_usage_supported = ?, + model_timeout_seconds = ? + WHERE agent_id = ? + """, + ( + config["model"], + config["base_url"], + config["api_key_env"], + config["credential_key"], + config["priority"], + int(config["disabled"]), + config["provider_name"], + config["local_credential_key"], + config["auth_scheme"], + config["max_output_tokens"], + config["context_window"], + config["reasoning_effort_supported"], + int(config["stream_usage_supported"]), + config["model_timeout_seconds"], + agent.id, + ), + ) + if conn.execute("SELECT changes()").fetchone()[0] == 0: + self._insert_agent(conn, agent) + else: + conn.execute("DELETE FROM agent_pool_tags WHERE agent_id = ?", (agent.id,)) + conn.execute( + "DELETE FROM agent_pool_provider_exclusions WHERE agent_id = ?", + (agent.id,), + ) + conn.executemany( + "INSERT INTO agent_pool_tags (agent_id, tag_position, tag_name) VALUES (?, ?, ?)", + [(agent.id, position, tag) for position, tag in enumerate(agent.tags)], + ) + conn.executemany( + """ + INSERT INTO agent_pool_provider_exclusions + (agent_id, exclusion_position, provider_name) + VALUES (?, ?, ?) + """, + [ + (agent.id, position, provider) + for position, provider in enumerate(agent.provider_exclusions) + ], + ) + # Model-group membership is a normalized relation beside the pool. + conn.execute("DELETE FROM model_group_member WHERE agent_id = ?", (agent.id,)) + if agent.group_name: + conn.execute( + "INSERT OR IGNORE INTO model_group (group_name) VALUES (?)", + (agent.group_name,), + ) + conn.execute( + "INSERT INTO model_group_member (agent_id, group_name) VALUES (?, ?)", + (agent.id, agent.group_name), + ) + conn.execute( + "DELETE FROM model_group WHERE NOT EXISTS (" + "SELECT 1 FROM model_group_member " + "WHERE model_group_member.group_name = model_group.group_name)" + ) + conn.execute("DELETE FROM endpoint_equivalence_member WHERE agent_id = ?", (agent.id,)) + conn.execute( + "DELETE FROM endpoint_equivalence_contract WHERE NOT EXISTS (" + "SELECT 1 FROM endpoint_equivalence_member " + "WHERE endpoint_equivalence_member.contract_id = " + "endpoint_equivalence_contract.contract_id)" + ) + if agent.endpoint_equivalence is not None: + contract = EndpointEquivalenceContract(**agent.endpoint_equivalence) + conn.execute( + "INSERT INTO endpoint_equivalence_contract VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) " + "ON CONFLICT(contract_id) DO UPDATE SET model_revision=excluded.model_revision, " + "reasoning_effort_profile=excluded.reasoning_effort_profile, " + "structured_output_contract=excluded.structured_output_contract, " + "accuracy_class=excluded.accuracy_class, data_residency_policy=excluded.data_residency_policy, " + "retention_policy=excluded.retention_policy, context_limit=excluded.context_limit, " + "pricing_evidence_id=excluded.pricing_evidence_id, hedge_eligible=excluded.hedge_eligible, " + "cancellation_supported=excluded.cancellation_supported, " + "execution_policy=excluded.execution_policy", + ( + contract.contract_id, contract.model_revision, + contract.reasoning_effort_profile, contract.structured_output_contract, + contract.accuracy_class, contract.data_residency_policy, + contract.retention_policy, contract.context_limit, + contract.pricing_evidence_id, int(contract.hedge_eligible), + int(contract.cancellation_supported), contract.execution_policy, + ), + ) + conn.execute( + "DELETE FROM endpoint_equivalence_capability WHERE contract_id = ?", + (contract.contract_id,), + ) + conn.executemany( + "INSERT INTO endpoint_equivalence_capability (contract_id, capability_name) VALUES (?, ?)", + [(contract.contract_id, name) for name in contract.capability_set], + ) + conn.execute( + "INSERT INTO endpoint_equivalence_member (agent_id, contract_id) VALUES (?, ?)", + (agent.id, contract.contract_id), + ) + if timeout_previous is not None: + revision = self._append_timeout_history(conn, timeout_previous, agent, actor_id, restored_from_revision) + return revision if timeout_previous is not None else None @staticmethod def _append_timeout_history( @@ -6301,12 +6324,8 @@ def set_model_group(self, group_name: str, member_agent_ids: list[str]) -> dict[ for before, after in zip(previous_candidates, updated) if before.group_name != after.group_name } - for agent in updated: - if agent.id in requested: - if self._pool_store is not None: - self._pool_store.save(agent) - elif agent.id in previous and self._pool_store is not None: - self._pool_store.save(agent) + if self._pool_store is not None: + self._pool_store.save_many(agent for agent in updated if agent.id in requested | previous) self.candidates = updated self.agents = [agent for agent in updated if not agent.disabled] self._routers_reset_members(changed) @@ -6323,9 +6342,7 @@ def delete_model_group(self, group_name: str) -> dict[str, Any]: member_ids = set(current["member_agent_ids"]) updated = [replace(agent, group_name="") if agent.id in member_ids else agent for agent in self.candidates] if self._pool_store is not None: - for agent in updated: - if agent.id in member_ids: - self._pool_store.save(agent) + self._pool_store.save_many(agent for agent in updated if agent.id in member_ids) self.candidates = updated self.agents = [agent for agent in updated if not agent.disabled] self._routers_reset_members(member_ids) @@ -6398,8 +6415,7 @@ def sync_discovered_agents(self, discovered_agents: list[ModelAgent]) -> dict[st effective_discovered_agents.append(agent) self._require_role_effort_pool(updated_candidates) if self._pool_store is not None: - for agent in effective_discovered_agents: - self._pool_store.save(agent) + self._pool_store.save_many(effective_discovered_agents) self.candidates = updated_candidates self.agents = [candidate for candidate in self.candidates if not candidate.disabled] self._rebuild_budget_meter() From 81145d41a6012916bac20546a27db8a623ddeb47 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:51:56 +0900 Subject: [PATCH 37/78] docs(gateway): record atomic pool batch evidence Signed-off-by: Seongho Bae --- docs/doctoring/model-timeout-policy-evidence.md | 16 ++++++++++++++++ docs/product-technical-gap-baseline.md | 6 ++++++ 2 files changed, 22 insertions(+) diff --git a/docs/doctoring/model-timeout-policy-evidence.md b/docs/doctoring/model-timeout-policy-evidence.md index 4ad8833cf..8270c7f37 100644 --- a/docs/doctoring/model-timeout-policy-evidence.md +++ b/docs/doctoring/model-timeout-policy-evidence.md @@ -177,6 +177,22 @@ failure may leave earlier durable rows changed even though serving publication is withheld. Batch rollback and concurrent serving refresh remain open gates; the single-target regressions above do not prove either requirement. +## Multi-model transaction rollback + +At `ad337e18`, group assignment, group deletion and discovery each reproduced +a partial durable write when the second model had a stale timeout revision: +3 failed, 38 deselected, 2.78 seconds. The first model remained changed after +restart even though the operation failed and memory remained unchanged. + +At `36fc35df`, single and batch saves reuse the same per-connection normalized +write body. Each group/discovery operation uses one immediate transaction; +any exception closes the uncommitted connection and rolls back earlier rows. +The three reproductions and policy/pool/governance/group/mixed-role/bootstrap +boundary suites passed 122 tests in 5.04 seconds, exit 0. This supersedes the +per-row group/discovery rollback gap above, but not cross-process serving +refresh, separate bootstrap operations, audit streams outside the pool, or +actual model deadline enforcement. No new full-suite result is claimed. + ## Source reference ContextualWisdomLab. (n.d.). *Finite outbound request-timeout boundaries* diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 547edf0ca..216ff2e96 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -35,6 +35,12 @@ reproduced failures; 103 related tests passed in 15.34 seconds. Multi-model batch rollback and cross-process serving refresh remain unproven. Neither change supplies model deadline enforcement or a new full-suite result. +Subsequent `ad337e18` reproduced partial persistence in all three group/discovery +batch paths when the second model conflicted. `36fc35df` now commits each batch +in one transaction using the existing normalized writes; 122 related tests +passed in 5.04 seconds. Cross-process refresh and end-to-end timeout delivery +remain open; separate bootstrap operations are not one atomic batch. + See [the evidence record](doctoring/model-timeout-policy-evidence.md) for exact revisions, retained failures, corrected test-evidence limitations, owner boundaries and the full remaining acceptance gates. Local configuration work From a2951f676a0c0d49760f42d9ec2f059f7ec22283 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:53:59 +0900 Subject: [PATCH 38/78] test(gateway): verify timeout conflict on authenticated HTTP edits Signed-off-by: Seongho Bae --- tests/test_agent_pool_db.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/test_agent_pool_db.py b/tests/test_agent_pool_db.py index 78b556392..d1d2d27f7 100644 --- a/tests/test_agent_pool_db.py +++ b/tests/test_agent_pool_db.py @@ -453,6 +453,36 @@ def _call(url: str, method: str, token: str, payload: dict | None = None) -> tup return exc.code, json.loads(exc.read().decode("utf-8")) +def test_http_stale_policy_rejects_admin_edit_without_overwrite(tmp_path) -> None: + """Actual authenticated HTTP edits cannot overwrite another writer's policy.""" + seeds = _seed() + database_path = str(tmp_path / "pool.db") + writer = TaskOrchestrator(seeds, agents_db=database_path) + serving = TaskOrchestrator(seeds, agents_db=database_path) + server = build_server(serving, port=0, security=SecurityConfig(auth_token="pool_token")) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + url = f"http://127.0.0.1:{server.server_address[1]}/api/v1/agent_pools/default/worker_agents/general_agent" + before = list(serving.candidates) + try: + writer.patch_agent("default", "general_agent", {"model_timeout_seconds": 7200}) + status, _ = _call(url, "PATCH", "wrong_token", {"priority": 7}) + assert status == 401 + status, payload = _call(url, "PATCH", "pool_token", {"priority": 7}) + assert status == 400 + assert "reload" in json.dumps(payload) + status, _ = _call(url, "PATCH", "pool_token", {"model_timeout_seconds": 3600}) + assert status == 400 # New policy writes stay closed until runtime delivery exists. + assert serving.candidates == before + restored = TaskOrchestrator(seeds, agents_db=database_path) + assert restored._agent("general_agent").model_timeout_seconds == 7200 + assert restored._agent("general_agent").model_timeout_revision == 1 + finally: + server.shutdown() + server.server_close() + thread.join() + + def test_http_create_and_delete_worker_agents() -> None: token = "pool_token" orchestrator = TaskOrchestrator(_seed()) From 3612ab92c6b28ff3af8c000e8b5cf9dc709485d2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:54:30 +0900 Subject: [PATCH 39/78] docs(gateway): distinguish HTTP conflict checks from timeout activation Signed-off-by: Seongho Bae --- docs/doctoring/model-timeout-policy-evidence.md | 16 ++++++++++++++++ docs/product-technical-gap-baseline.md | 6 ++++++ 2 files changed, 22 insertions(+) diff --git a/docs/doctoring/model-timeout-policy-evidence.md b/docs/doctoring/model-timeout-policy-evidence.md index 8270c7f37..b3e462ed7 100644 --- a/docs/doctoring/model-timeout-policy-evidence.md +++ b/docs/doctoring/model-timeout-policy-evidence.md @@ -193,6 +193,22 @@ per-row group/discovery rollback gap above, but not cross-process serving refresh, separate bootstrap operations, audit streams outside the pool, or actual model deadline enforcement. No new full-suite result is claimed. +## Authenticated HTTP conflict boundary + +At `a2951f67`, an actual loopback HTTP server and a separate orchestrator sharing +the pool database verify: an invalid bearer gets 401; an authenticated stale +priority PATCH gets 400 with reload guidance; a direct timeout PATCH remains +400 because the new write field is not admitted. Serving candidates remain +unchanged and restart retains the other writer's 7200-second policy/revision 1. +The focused test passed in 4.01 seconds (1 passed, 20 deselected, exit 0). +This proves existing HTTP rejection, not authenticated policy-write success, +automatic serving refresh, actual inference selection, or deadline enforcement. + +The EgressWeave protected-main SHA was rechecked as `bd0339bf` and its GitHub +release listing returned no entries. DeepWiki returned repository-not-found; +neither result proves absence from every registry or absence of another writer. +Runtime-owner coordination remains open; no consumer transport clone was added. + ## Source reference ContextualWisdomLab. (n.d.). *Finite outbound request-timeout boundaries* diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 216ff2e96..dd8b10e24 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -41,6 +41,12 @@ in one transaction using the existing normalized writes; 122 related tests passed in 5.04 seconds. Cross-process refresh and end-to-end timeout delivery remain open; separate bootstrap operations are not one atomic batch. +`a2951f67` adds actual authenticated HTTP evidence: stale ordinary edits fail +without overwriting the other writer's policy, invalid credentials get 401, +and the unfinished timeout write field remains rejected (1 focused pass, +4.01 seconds). This is rejection-boundary evidence, not successful policy +activation or inference enforcement. + See [the evidence record](doctoring/model-timeout-policy-evidence.md) for exact revisions, retained failures, corrected test-evidence limitations, owner boundaries and the full remaining acceptance gates. Local configuration work From e794ed359b9e79cdc49f4653b6c5aa6e0d5ac2b5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 22:56:57 +0900 Subject: [PATCH 40/78] docs(gateway): link canonical runtime acceptance request Signed-off-by: Seongho Bae --- docs/doctoring/model-timeout-policy-evidence.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/doctoring/model-timeout-policy-evidence.md b/docs/doctoring/model-timeout-policy-evidence.md index b3e462ed7..4bcbbfa92 100644 --- a/docs/doctoring/model-timeout-policy-evidence.md +++ b/docs/doctoring/model-timeout-policy-evidence.md @@ -209,6 +209,21 @@ release listing returned no entries. DeepWiki returned repository-not-found; neither result proves absence from every registry or absence of another writer. Runtime-owner coordination remains open; no consumer transport clone was added. +## Canonical runtime dependency request + +The owner PRD/TRD at `bd0339bf` explicitly require finite request-phase waits +and describe a Python runtime. Model response lifetime therefore requires an +explicit Proposed contract change, not merely passing null into current APIs. +The required Rust owner behavior, separate total/read deadlines, cancellation +causes, resource/security invariants and five behavioral RED families are +recorded in [the existing timeout-policy review lane](https://github.com/ContextualWisdomLab/EgressWeave/pull/220#issuecomment-5559695131). +This is a dependency request, not accepted architecture or implementation. + +The visible owner worktrees were inspected read-only: Actions concurrency, +draft admission and #235 gateway migration are separate deltas. Their untracked +lock/index/desktop files were preserved. No owner branch was taken over; no +consumer source copy, release adoption or deadline activation occurred. + ## Source reference ContextualWisdomLab. (n.d.). *Finite outbound request-timeout boundaries* From b80e64be8ff1e72a32f05e2a962b1223240ab801 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 23:48:41 +0900 Subject: [PATCH 41/78] test(admin): require durable timeout policy read without runtime activation Signed-off-by: Seongho Bae --- tests/test_agent_pool_db.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/test_agent_pool_db.py b/tests/test_agent_pool_db.py index d1d2d27f7..1b309fc80 100644 --- a/tests/test_agent_pool_db.py +++ b/tests/test_agent_pool_db.py @@ -483,6 +483,39 @@ def test_http_stale_policy_rejects_admin_edit_without_overwrite(tmp_path) -> Non thread.join() +def test_http_timeout_policy_reads_durable_state_without_activation(tmp_path) -> None: + """Operators can distinguish stored policy from a stale serving snapshot.""" + seeds = _seed() + database_path = str(tmp_path / "pool.db") + writer = TaskOrchestrator(seeds, agents_db=database_path) + serving = TaskOrchestrator(seeds, agents_db=database_path) + server = build_server(serving, port=0, security=SecurityConfig(auth_token="pool_token")) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + base = f"http://127.0.0.1:{server.server_address[1]}/api/v1/agent_pools/default/worker_agents/general_agent" + try: + assert _call(base + "/timeout_policy", "GET", "wrong_token")[0] == 401 + status, initial = _call(base + "/timeout_policy", "GET", "pool_token") + assert status == 200 + assert initial["configured_seconds"] is None + assert initial["revision"] == 0 + writer.patch_agent("default", "general_agent", {"model_timeout_seconds": 7200}) + status, policy = _call(base + "/timeout_policy", "GET", "pool_token") + assert status == 200 + assert policy == { + "configured_seconds": 7200.0, "revision": 1, "unit": "seconds", + "serving_snapshot_seconds": None, "serving_snapshot_revision": 0, + "enforcement_available": False, + } + assert serving._agent("general_agent").model_timeout_revision == 0 + missing = base.replace("general_agent", "missing_agent") + assert _call(missing + "/timeout_policy", "GET", "pool_token")[0] == 404 + finally: + server.shutdown() + server.server_close() + thread.join() + + def test_http_create_and_delete_worker_agents() -> None: token = "pool_token" orchestrator = TaskOrchestrator(_seed()) From 2e12ed46eee08c57510a8e9d1aa8509566f80e83 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 23:49:08 +0900 Subject: [PATCH 42/78] feat(admin): read durable timeout policy without activating execution limits Signed-off-by: Seongho Bae --- contextual_orchestrator/orchestrator.py | 19 +++++++++++++++++++ contextual_orchestrator/server.py | 7 +++++++ 2 files changed, 26 insertions(+) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 931151b4c..1859eb7d7 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -6249,6 +6249,25 @@ def patch_agent( ) return self._agent_to_admin_payload(patched) + def get_model_timeout_policy(self, agent_pool_id: str, worker_agent_id: str) -> dict[str, Any]: + """Read configured policy without publishing or claiming runtime enforcement.""" + serving = self._agent_in_pool(agent_pool_id, worker_agent_id) + configured = serving + if self._pool_store is not None: + # Reuse the transactional configuration snapshot, never refresh routing here. + configured = next( + (agent for agent in self._pool_store.load_all() if agent.id == worker_agent_id), + serving, + ) + return { + "configured_seconds": configured.model_timeout_seconds, + "revision": configured.model_timeout_revision, + "unit": "seconds", + "serving_snapshot_seconds": serving.model_timeout_seconds, + "serving_snapshot_revision": serving.model_timeout_revision, + "enforcement_available": False, + } + def restore_model_timeout( self, agent_pool_id: str, worker_agent_id: str, source_revision: int, *, expected_revision: int, actor_id: str, diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index 7c6060dfd..bd98641b8 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -6238,6 +6238,13 @@ def do_GET(self) -> None: # noqa: N802 return if path.startswith("/api/v1/agent_pools/"): segments = [part for part in path.split("/") if part] + if (len(segments) == 7 and segments[:3] == ["api", "v1", "agent_pools"] + and segments[4] == "worker_agents" and segments[6] == "timeout_policy"): + try: + self._send(orchestrator.get_model_timeout_policy(segments[3], segments[5])) + except KeyError: + self._send_error(404, "agent_not_found", "Model configuration was not found.") + return if len(segments) == 6 and segments[:3] == ["api", "v1", "agent_pools"] and segments[4] == "worker_agents": agent_pool_id = segments[3] worker_agent_id = segments[-1] From 6774dab460f3d4a789f3c46936427f4c05176c7b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 23:50:33 +0900 Subject: [PATCH 43/78] test(admin): bind timeout policy reads to admin scope and OpenAPI Signed-off-by: Seongho Bae --- contextual_orchestrator/api_contract.py | 33 +++++++++++++++++++++++++ contextual_orchestrator/orchestrator.py | 2 +- tests/test_agent_pool_db.py | 13 +++++++++- tests/test_model_timeout_policy.py | 4 +++ 4 files changed, 50 insertions(+), 2 deletions(-) diff --git a/contextual_orchestrator/api_contract.py b/contextual_orchestrator/api_contract.py index 03f89ba3a..358bc9fc6 100644 --- a/contextual_orchestrator/api_contract.py +++ b/contextual_orchestrator/api_contract.py @@ -471,6 +471,39 @@ "responses": {"200": {"description": "Agent pool collection"}}, } }, + "/api/v1/agent_pools/{agent_pool_id}/worker_agents/{worker_agent_id}/timeout_policy": { + "get": { + "operationId": "get_model_timeout_policy", + "summary": "Read configured timeout and serving snapshot without activating a limit", + "security": [{"admin_bearer_auth": []}], + "parameters": [ + {"name": name, "in": "path", "required": True, "schema": {"type": "string"}} + for name in ("agent_pool_id", "worker_agent_id") + ], + "responses": { + "200": { + "description": "Stored policy and local snapshot; runtime enforcement is not integrated", + "content": {"application/json": {"schema": { + "type": "object", "additionalProperties": False, + "required": ["configured_seconds", "revision", "unit", + "serving_snapshot_seconds", "serving_snapshot_revision", + "enforcement_available"], + "properties": { + "configured_seconds": {"type": ["number", "null"], "exclusiveMinimum": 0}, + "revision": {"type": "integer", "minimum": 0}, + "unit": {"const": "seconds"}, + "serving_snapshot_seconds": {"type": ["number", "null"], "exclusiveMinimum": 0}, + "serving_snapshot_revision": {"type": "integer", "minimum": 0}, + "enforcement_available": {"const": False}, + }, + }}}, + }, + "401": {"description": "Authentication required"}, + "403": {"description": "Administrator scope required"}, + "404": {"description": "Model configuration not found"}, + }, + }, + }, "/api/v1/agent_pools/{agent_pool_id}/worker_agents/{worker_agent_id}": { "patch": { "operationId": "patch_worker_agent", diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 1859eb7d7..9879b5266 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -6254,7 +6254,7 @@ def get_model_timeout_policy(self, agent_pool_id: str, worker_agent_id: str) -> serving = self._agent_in_pool(agent_pool_id, worker_agent_id) configured = serving if self._pool_store is not None: - # Reuse the transactional configuration snapshot, never refresh routing here. + # ponytail: one full snapshot per admin read; indexed lookup if pool size warrants it. configured = next( (agent for agent in self._pool_store.load_all() if agent.id == worker_agent_id), serving, diff --git a/tests/test_agent_pool_db.py b/tests/test_agent_pool_db.py index 1b309fc80..bf801154b 100644 --- a/tests/test_agent_pool_db.py +++ b/tests/test_agent_pool_db.py @@ -489,12 +489,15 @@ def test_http_timeout_policy_reads_durable_state_without_activation(tmp_path) -> database_path = str(tmp_path / "pool.db") writer = TaskOrchestrator(seeds, agents_db=database_path) serving = TaskOrchestrator(seeds, agents_db=database_path) - server = build_server(serving, port=0, security=SecurityConfig(auth_token="pool_token")) + server = build_server(serving, port=0, security=SecurityConfig( + admin_token="pool_token", inference_token="inference_token", + )) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() base = f"http://127.0.0.1:{server.server_address[1]}/api/v1/agent_pools/default/worker_agents/general_agent" try: assert _call(base + "/timeout_policy", "GET", "wrong_token")[0] == 401 + assert _call(base + "/timeout_policy", "GET", "inference_token")[0] == 401 status, initial = _call(base + "/timeout_policy", "GET", "pool_token") assert status == 200 assert initial["configured_seconds"] is None @@ -510,6 +513,14 @@ def test_http_timeout_policy_reads_durable_state_without_activation(tmp_path) -> assert serving._agent("general_agent").model_timeout_revision == 0 missing = base.replace("general_agent", "missing_agent") assert _call(missing + "/timeout_policy", "GET", "pool_token")[0] == 404 + from contextual_orchestrator.api_contract import OPENAPI_SPEC + operation = OPENAPI_SPEC["paths"][ + "/api/v1/agent_pools/{agent_pool_id}/worker_agents/{worker_agent_id}/timeout_policy" + ]["get"] + assert operation["security"] == [{"admin_bearer_auth": []}] + schema = operation["responses"]["200"]["content"]["application/json"]["schema"] + assert set(schema["required"]) == set(policy) + assert schema["properties"]["enforcement_available"] == {"const": False} finally: server.shutdown() server.server_close() diff --git a/tests/test_model_timeout_policy.py b/tests/test_model_timeout_policy.py index 793d588df..bdf9c977d 100644 --- a/tests/test_model_timeout_policy.py +++ b/tests/test_model_timeout_policy.py @@ -74,6 +74,10 @@ def test_model_timeout_policy_defaults_to_null() -> None: """An ordinary model has no administrator-imposed execution limit.""" model_agent = ModelAgent("timeout_agent", "example-model") assert model_agent.to_config()["model_timeout_seconds"] is None + policy = TaskOrchestrator([model_agent]).get_model_timeout_policy("default", model_agent.id) + assert policy["configured_seconds"] is None + assert policy["revision"] == 0 + assert policy["enforcement_available"] is False def test_model_timeout_policy_accepts_large_finite_seconds(tmp_path: Path) -> None: From b0d1fda59ccf055e8b6617cb0aa61d43f38dcec4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 23:51:31 +0900 Subject: [PATCH 44/78] docs(admin): distinguish policy visibility from runtime timeout enforcement Signed-off-by: Seongho Bae --- .../model-timeout-policy-evidence.md | 18 +++++++++++++++ docs/product-technical-gap-baseline.md | 22 ++++++++++++++----- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/docs/doctoring/model-timeout-policy-evidence.md b/docs/doctoring/model-timeout-policy-evidence.md index 4bcbbfa92..46d979b70 100644 --- a/docs/doctoring/model-timeout-policy-evidence.md +++ b/docs/doctoring/model-timeout-policy-evidence.md @@ -224,6 +224,24 @@ draft admission and #235 gateway migration are separate deltas. Their untracked lock/index/desktop files were preserved. No owner branch was taken over; no consumer source copy, release adoption or deadline activation occurred. +## Read-only operator policy view + +`b80e64be` records the missing-route RED: authenticated GET returned 400 rather +than exposing a fresh policy view. `2e12ed46` reuses the store's transactional +snapshot for `GET /api/v1/agent_pools/{agent_pool_id}/worker_agents/{worker_agent_id}/timeout_policy`. +It reports configured seconds/revision separately from the local serving +snapshot, with seconds as the explicit unit and `enforcement_available=false`. +Reading does not refresh routing, change policy or activate an execution limit. +The existing full-snapshot read is linear in pool size; this is an operator +read, not a routing hot path or an independently measured latency improvement. + +At `6774dab4`, 88 pool/policy/security tests pass in 9.52 seconds. Actual HTTP +checks cover null defaults, a second writer's 7200-second revision, preservation +of the stale serving snapshot, wrong and inference-only credentials, and a +missing model. OpenAPI declares the admin-only contract and explicit inactive +enforcement state. This does not implement history pagination, write/clear/ +restore HTTP operations, runtime integration or the administrator UI. + ## Source reference ContextualWisdomLab. (n.d.). *Finite outbound request-timeout boundaries* diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index dd8b10e24..3a2e3c0e0 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -2713,7 +2713,7 @@ failed after the verdict was already correctly published and enforced — a downstream status-publish step, not the review pipeline itself; not yet investigated. -**Separate, newly found, NOT YET FIXED bug** (traced via `.github#1276`'s +**Historical incident, not current-runtime proof** (traced via `.github#1276`'s `noema-review` run, flagged by the user as "still not working" after the above fixes landed): `TaskOrchestrator._invoke()` (`contextual_orchestrator/orchestrator.py:6353-6537`, the per-agent @@ -2728,7 +2728,19 @@ nothing is truly hung, just still working through an unbounded internal retry chain. Reproduced once (`.github` run `33312258611`, job `99259327051`); org-wide evidence since the pingora/Strix fixes landed shows this is now occasional, not the dominant failure mode (most -`.github`-hosted PRs' `noema-review`/dispatch runs succeed). Correct fix -is an overall deadline on `_invoke`'s candidate/retry loop, not another -timeout increase on the sidecar's client side — deferred rather than -rushed into this heavily-tested core file without dedicated validation. +`.github`-hosted PRs' `noema-review`/dispatch runs succeed in that historical +observation). The earlier proposed universal deadline is superseded by the +explicit model-timeout requirement: model execution defaults to null, and only +an administrator-configured model limit may bound its complete execution. +Readiness probes have a separate finite operational contract. The old run does +not prove current deployment behavior or justify imposing a global inference cap. + +### Read-only timeout policy visibility — local, not runtime activation + +Source `6774dab4` exposes an admin-only timeout-policy GET that separates fresh +configured seconds/revision from the local serving snapshot. It reports seconds +and `enforcement_available=false`; reads do not activate limits or refresh +routing. The actual HTTP and related pool/policy/security checks pass 88 tests +in 9.52 seconds. History navigation, HTTP set/clear/restore, released Rust +runtime integration and actual administrator UI acceptance remain open. See +[policy evidence](doctoring/model-timeout-policy-evidence.md#read-only-operator-policy-view). From 80b3aa361ae1519b499f1f98329cedc6b0414629 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 23:53:30 +0900 Subject: [PATCH 45/78] test(admin): require model-scoped stable timeout history pagination Signed-off-by: Seongho Bae --- tests/test_model_timeout_policy.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/test_model_timeout_policy.py b/tests/test_model_timeout_policy.py index bdf9c977d..befb3e0cf 100644 --- a/tests/test_model_timeout_policy.py +++ b/tests/test_model_timeout_policy.py @@ -80,6 +80,32 @@ def test_model_timeout_policy_defaults_to_null() -> None: assert policy["enforcement_available"] is False +def test_timeout_history_cursor_preserves_model_scope_and_new_insertions(tmp_path: Path) -> None: + """Paging older audit entries cannot repeat a new write or leak another model.""" + seeds = [ModelAgent("first_agent", "first-model"), ModelAgent("second_agent", "second-model")] + orchestrator = TaskOrchestrator(seeds, agents_db=str(tmp_path / "pool.db")) + first = orchestrator.patch_agent("default", "first_agent", {"model_timeout_seconds": 7200}) + orchestrator.patch_agent("default", "second_agent", {"model_timeout_seconds": 900}) + cleared = orchestrator.patch_agent("default", "first_agent", {"model_timeout_seconds": None}) + page = orchestrator.list_model_timeout_history("default", "first_agent", page_size=1) + assert page["history_available"] is True + assert page["items"][0]["revision"] == cleared["model_timeout_revision"] + assert page["items"][0]["configured_seconds"] is None + restored = orchestrator.restore_model_timeout( + "default", "first_agent", first["model_timeout_revision"], + expected_revision=cleared["model_timeout_revision"], actor_id="a" * 64, + ) + older = orchestrator.list_model_timeout_history( + "default", "first_agent", page_size=1, before_revision=page["next_before_revision"], + ) + assert [row["revision"] for row in older["items"]] == [first["model_timeout_revision"]] + assert older["next_before_revision"] is None + newest = orchestrator.list_model_timeout_history("default", "first_agent")["items"][0] + assert newest["revision"] == restored["model_timeout_revision"] + assert newest["actor_id"] == "a" * 64 + assert newest["restored_from_revision"] == first["model_timeout_revision"] + + def test_model_timeout_policy_accepts_large_finite_seconds(tmp_path: Path) -> None: """Valid seconds must not accidentally use SQLite's signed integer binding.""" model_agent = ModelAgent("timeout_agent", "example-model") From 5dc69bc1d7a0e4ca36800cf285961a7f5966ad45 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 23:54:21 +0900 Subject: [PATCH 46/78] feat(admin): expose bounded model timeout audit history Signed-off-by: Seongho Bae --- contextual_orchestrator/orchestrator.py | 42 +++++++++++++++++++++++++ contextual_orchestrator/server.py | 14 +++++++++ 2 files changed, 56 insertions(+) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 9879b5266..f0cea8135 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -3426,6 +3426,10 @@ def __init__(self, path: str) -> None: "ALTER TABLE model_timeout_history ADD COLUMN restored_from_revision INTEGER " "REFERENCES model_timeout_history(policy_revision)" ) + conn.execute( + "CREATE INDEX IF NOT EXISTS model_timeout_history_agent_revision " + "ON model_timeout_history(agent_id, policy_revision)" + ) conn.commit() except Exception: conn.rollback() @@ -3661,6 +3665,24 @@ def _append_timeout_history( ) return int(cursor.lastrowid) + def timeout_history(self, agent_id: str, page_size: int, before_revision: int | None) -> list[dict[str, Any]]: + """Read one bounded, model-scoped audit page plus a continuation sentinel.""" + with self._lock: + conn = self._connect(self._path) + try: + rows = conn.execute( + "SELECT policy_revision, previous_seconds, timeout_seconds, created_at, " + "actor_id, restored_from_revision FROM model_timeout_history " + "WHERE agent_id = ? AND (? IS NULL OR policy_revision < ?) " + "ORDER BY policy_revision DESC LIMIT ?", + (agent_id, before_revision, before_revision, page_size + 1), + ).fetchall() + finally: + conn.close() + fields = ("revision", "previous_seconds", "configured_seconds", "changed_at", + "actor_id", "restored_from_revision") + return [dict(zip(fields, row)) for row in rows] + def timeout_at_revision(self, agent_id: str, policy_revision: int) -> float | None: """Read a historical value only when its revision belongs to this model.""" with self._lock: @@ -6268,6 +6290,26 @@ def get_model_timeout_policy(self, agent_pool_id: str, worker_agent_id: str) -> "enforcement_available": False, } + def list_model_timeout_history( + self, agent_pool_id: str, worker_agent_id: str, *, page_size: int = 20, + before_revision: int | None = None, + ) -> dict[str, Any]: + """Page older model policy changes without offset drift during new writes.""" + self._agent_in_pool(agent_pool_id, worker_agent_id) + if type(page_size) is not int or not 1 <= page_size <= 100: + raise ValueError("page_size must be an integer between 1 and 100") + if before_revision is not None and ( + type(before_revision) is not int or not 1 <= before_revision <= _AGENT_POOL_INTEGER_MAX + ): + raise ValueError("before_revision must be a positive stored revision") + rows = self._pool_store.timeout_history(worker_agent_id, page_size, before_revision) if self._pool_store else [] + items = rows[:page_size] + return { + "items": items, + "next_before_revision": items[-1]["revision"] if len(rows) > page_size else None, + "history_available": self._pool_store is not None, + } + def restore_model_timeout( self, agent_pool_id: str, worker_agent_id: str, source_revision: int, *, expected_revision: int, actor_id: str, diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index bd98641b8..64bf92096 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -6238,6 +6238,20 @@ def do_GET(self) -> None: # noqa: N802 return if path.startswith("/api/v1/agent_pools/"): segments = [part for part in path.split("/") if part] + if (len(segments) == 8 and segments[:3] == ["api", "v1", "agent_pools"] + and segments[4] == "worker_agents" and segments[6:] == ["timeout_policy", "history"]): + page_size = self._parse_positive_int( + (query.get("page_size") or [None])[0], "page_size", 20, 100, + ) + before_revision = self._parse_optional_int(query, "before_revision") + try: + self._send(orchestrator.list_model_timeout_history( + segments[3], segments[5], page_size=page_size, + before_revision=before_revision, + )) + except KeyError: + self._send_error(404, "agent_not_found", "Model configuration was not found.") + return if (len(segments) == 7 and segments[:3] == ["api", "v1", "agent_pools"] and segments[4] == "worker_agents" and segments[6] == "timeout_policy"): try: From f9505a5c54ed957d41489015b47eaf21f61416bb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 23:55:15 +0900 Subject: [PATCH 47/78] test(admin): verify timeout history HTTP bounds and document schema Signed-off-by: Seongho Bae --- contextual_orchestrator/api_contract.py | 39 +++++++++++++++++++++++++ tests/test_agent_pool_db.py | 10 +++++++ tests/test_model_timeout_policy.py | 14 +++++++++ 3 files changed, 63 insertions(+) diff --git a/contextual_orchestrator/api_contract.py b/contextual_orchestrator/api_contract.py index 358bc9fc6..b97f1be08 100644 --- a/contextual_orchestrator/api_contract.py +++ b/contextual_orchestrator/api_contract.py @@ -471,6 +471,45 @@ "responses": {"200": {"description": "Agent pool collection"}}, } }, + "/api/v1/agent_pools/{agent_pool_id}/worker_agents/{worker_agent_id}/timeout_policy/history": { + "get": { + "operationId": "list_model_timeout_history", + "summary": "Read older model timeout changes with a stable revision cursor", + "security": [{"admin_bearer_auth": []}], + "parameters": [ + {"name": "agent_pool_id", "in": "path", "required": True, "schema": {"type": "string"}}, + {"name": "worker_agent_id", "in": "path", "required": True, "schema": {"type": "string"}}, + {"name": "page_size", "in": "query", "schema": {"type": "integer", "minimum": 1, "maximum": 100, "default": 20}}, + {"name": "before_revision", "in": "query", "schema": {"type": "integer", "minimum": 1, "maximum": _AGENT_POOL_INTEGER_MAX}}, + ], + "responses": { + "200": {"description": "Descending revisions; changed_at is Unix seconds, actor_id is an opaque digest or null", + "content": {"application/json": {"schema": { + "type": "object", "additionalProperties": False, + "required": ["items", "next_before_revision", "history_available"], + "properties": { + "history_available": {"type": "boolean"}, + "next_before_revision": {"type": ["integer", "null"], "minimum": 1}, + "items": {"type": "array", "maxItems": 100, "items": { + "type": "object", "additionalProperties": False, + "required": ["revision", "previous_seconds", "configured_seconds", "changed_at", "actor_id", "restored_from_revision"], + "properties": { + "revision": {"type": "integer", "minimum": 1}, + "previous_seconds": {"type": ["number", "null"]}, + "configured_seconds": {"type": ["number", "null"]}, + "changed_at": {"type": "number"}, + "actor_id": {"type": ["string", "null"]}, + "restored_from_revision": {"type": ["integer", "null"], "minimum": 1}, + }, + }}, + }, + }}}}, + "400": {"description": "Invalid page size or revision cursor"}, + "401": {"description": "Administrator authentication required"}, + "404": {"description": "Model configuration not found"}, + }, + }, + }, "/api/v1/agent_pools/{agent_pool_id}/worker_agents/{worker_agent_id}/timeout_policy": { "get": { "operationId": "get_model_timeout_policy", diff --git a/tests/test_agent_pool_db.py b/tests/test_agent_pool_db.py index bf801154b..5ea3e619f 100644 --- a/tests/test_agent_pool_db.py +++ b/tests/test_agent_pool_db.py @@ -521,6 +521,16 @@ def test_http_timeout_policy_reads_durable_state_without_activation(tmp_path) -> schema = operation["responses"]["200"]["content"]["application/json"]["schema"] assert set(schema["required"]) == set(policy) assert schema["properties"]["enforcement_available"] == {"const": False} + history_url = base + "/timeout_policy/history" + assert _call(history_url, "GET", "inference_token")[0] == 401 + status, history = _call(history_url + "?page_size=1", "GET", "pool_token") + assert status == 200 + assert history["history_available"] is True + assert history["items"][0]["configured_seconds"] == 7200 + assert history["items"][0]["revision"] == 1 + assert history["next_before_revision"] is None + for query in ("page_size=0", "page_size=101", "before_revision=-1", "before_revision=9223372036854775808"): + assert _call(history_url + "?" + query, "GET", "pool_token")[0] == 400 finally: server.shutdown() server.server_close() diff --git a/tests/test_model_timeout_policy.py b/tests/test_model_timeout_policy.py index befb3e0cf..87fc92648 100644 --- a/tests/test_model_timeout_policy.py +++ b/tests/test_model_timeout_policy.py @@ -78,6 +78,20 @@ def test_model_timeout_policy_defaults_to_null() -> None: assert policy["configured_seconds"] is None assert policy["revision"] == 0 assert policy["enforcement_available"] is False + assert TaskOrchestrator([model_agent]).list_model_timeout_history("default", model_agent.id) == { + "items": [], "next_before_revision": None, "history_available": False, + } + + +@pytest.mark.parametrize("field,value", [ + ("page_size", True), ("page_size", 0), ("page_size", 101), ("page_size", 1.5), + ("before_revision", True), ("before_revision", 0), ("before_revision", 2**63), +]) +def test_timeout_history_rejects_invalid_bounds(field: str, value: object) -> None: + """Untrusted bounds cannot disable the read limit or overflow SQLite bindings.""" + agent = ModelAgent("timeout_agent", "example-model") + with pytest.raises(ValueError): + TaskOrchestrator([agent]).list_model_timeout_history("default", agent.id, **{field: value}) def test_timeout_history_cursor_preserves_model_scope_and_new_insertions(tmp_path: Path) -> None: From b12883440da510a8903b3ca9dddb09a4a11a7fe7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 23:56:07 +0900 Subject: [PATCH 48/78] docs(admin): record bounded read-only timeout audit evidence Signed-off-by: Seongho Bae --- docs/doctoring/model-timeout-policy-evidence.md | 15 +++++++++++++-- docs/product-technical-gap-baseline.md | 7 +++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/docs/doctoring/model-timeout-policy-evidence.md b/docs/doctoring/model-timeout-policy-evidence.md index 46d979b70..cf13feb07 100644 --- a/docs/doctoring/model-timeout-policy-evidence.md +++ b/docs/doctoring/model-timeout-policy-evidence.md @@ -239,8 +239,19 @@ At `6774dab4`, 88 pool/policy/security tests pass in 9.52 seconds. Actual HTTP checks cover null defaults, a second writer's 7200-second revision, preservation of the stale serving snapshot, wrong and inference-only credentials, and a missing model. OpenAPI declares the admin-only contract and explicit inactive -enforcement state. This does not implement history pagination, write/clear/ -restore HTTP operations, runtime integration or the administrator UI. +enforcement state. That revision does not implement history pagination, write/ +clear/restore HTTP operations, runtime integration or the administrator UI. + +`80b3aa36` retains a missing-history-method RED. `5dc69bc1` adds a model-scoped +descending revision cursor and a model/revision index. At `f9505a5c`, 96 related +pool/policy/security tests pass in 7.70 seconds, including actual admin-only +HTTP history reads and invalid-bound rejection. A page contains at most 100 +records. Newer insertions do not repeat or displace records on an older-page +cursor; another model's revisions never enter the result. Restore provenance, +nullable legacy actor digests and original timestamps are retained. An +in-memory-only pool reports history unavailable, not durable empty-history proof. +These are read-only operations; set/clear/restore HTTP actions, execution +enforcement and UI acceptance remain incomplete. ## Source reference diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 3a2e3c0e0..87a3663c3 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -2741,6 +2741,9 @@ Source `6774dab4` exposes an admin-only timeout-policy GET that separates fresh configured seconds/revision from the local serving snapshot. It reports seconds and `enforcement_available=false`; reads do not activate limits or refresh routing. The actual HTTP and related pool/policy/security checks pass 88 tests -in 9.52 seconds. History navigation, HTTP set/clear/restore, released Rust -runtime integration and actual administrator UI acceptance remain open. See +in 9.52 seconds. Source `f9505a5c` adds model-scoped audit history with stable +older-revision cursors and at most 100 records per read; 96 related tests pass +in 7.70 seconds, including HTTP authorization and invalid-bound checks. HTTP +set/clear/restore, released Rust runtime integration and actual administrator +UI acceptance remain open. See [policy evidence](doctoring/model-timeout-policy-evidence.md#read-only-operator-policy-view). From 10225f43d5c74c1bd6ca367e8ff5cfdd485b44f9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 00:19:54 +0900 Subject: [PATCH 49/78] test(server): require concurrent error log response correlation Signed-off-by: Seongho Bae --- tests/test_security_hardening.py | 41 ++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/test_security_hardening.py b/tests/test_security_hardening.py index 55987348b..b5bad2973 100644 --- a/tests/test_security_hardening.py +++ b/tests/test_security_hardening.py @@ -21,6 +21,47 @@ def build() -> TaskOrchestrator: return TaskOrchestrator([ModelAgent("general_agent", "mock-generalist", tags=("reasoning", "writing"))]) +def test_concurrent_error_responses_share_only_their_own_log_id(caplog) -> None: + """Concurrent HTTP failures correlate without logging credentials or bodies.""" + from concurrent.futures import ThreadPoolExecutor + + def reject_verification(token: str, scope: str) -> bool: + raise RuntimeError("verification unavailable") + + server = build_server(build(), port=0, security=SecurityConfig(bearer_verifier=reject_verification)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + base = f"http://127.0.0.1:{server.server_address[1]}" + try: + with ThreadPoolExecutor(max_workers=2) as executor: + responses = list(executor.map( + lambda _: request_json( + f"{base}/v1/models", "GET", + headers={"authorization": "Bearer private_test_credential"}, + ), + range(2), + )) + request_ids = [] + for status, payload, _ in responses: + assert status == 500 + assert payload["error"]["code"] == "internal_error" + request_id = payload["error"]["detail"]["request_id"] + assert payload["error_detail"]["request_id"] == request_id + assert len(request_id) == 32 + request_ids.append(request_id) + assert len(set(request_ids)) == 2 + records = [record.getMessage() for record in caplog.records if "request_failed" in record.getMessage()] + assert sorted(records) == sorted( + f"request_failed status=500 code=internal_error request_id={request_id}" + for request_id in request_ids + ) + assert "private_test_credential" not in caplog.text + finally: + server.shutdown() + thread.join(timeout=5) + server.server_close() + + def test_external_bearer_verifier_is_fail_closed_and_scoped() -> None: seen: list[tuple[str, str]] = [] From 4b73933975a03b6e0b07ce0beae4e05b9bf3a63b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 00:20:18 +0900 Subject: [PATCH 50/78] test(server): inject generic handler failure after auth boundary Signed-off-by: Seongho Bae --- tests/test_security_hardening.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/test_security_hardening.py b/tests/test_security_hardening.py index b5bad2973..39c160ac0 100644 --- a/tests/test_security_hardening.py +++ b/tests/test_security_hardening.py @@ -25,15 +25,12 @@ def test_concurrent_error_responses_share_only_their_own_log_id(caplog) -> None: """Concurrent HTTP failures correlate without logging credentials or bodies.""" from concurrent.futures import ThreadPoolExecutor - def reject_verification(token: str, scope: str) -> bool: - raise RuntimeError("verification unavailable") - - server = build_server(build(), port=0, security=SecurityConfig(bearer_verifier=reject_verification)) + server = build_server(build(), port=0, security=SecurityConfig(auth_token="private_test_credential")) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() base = f"http://127.0.0.1:{server.server_address[1]}" try: - with ThreadPoolExecutor(max_workers=2) as executor: + with patch.object(server.RequestHandlerClass, "_authorize", side_effect=RuntimeError("test failure")), ThreadPoolExecutor(max_workers=2) as executor: responses = list(executor.map( lambda _: request_json( f"{base}/v1/models", "GET", From 0e7c03bdc933d67b54f095c2a0cc4b22c4e31534 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 00:20:36 +0900 Subject: [PATCH 51/78] fix(server): correlate sanitized error logs with response IDs Signed-off-by: Seongho Bae --- contextual_orchestrator/server.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index 64bf92096..fc5962318 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -8221,8 +8221,18 @@ def _send_error( message: str, detail: dict[str, Any] | None = None, ) -> None: - _LOGGER.warning("request_failed status=%s code=%s", status, code) - self._send(_error_payload(code, message, {"request_id": uuid.uuid4().hex, **(detail or {})}), status) + error_detail = {"request_id": uuid.uuid4().hex, **(detail or {})} + request_id = error_detail["request_id"] + # Preserve response details, but never log arbitrary caller-supplied text. + safe_request_id = ( + request_id if isinstance(request_id, str) and len(request_id) == 32 + and all(character in "0123456789abcdef" for character in request_id) + else "" + ) + _LOGGER.warning( + "request_failed status=%s code=%s request_id=%s", status, code, safe_request_id + ) + self._send(_error_payload(code, message, error_detail), status) def _write_response(self, writer: Callable[[], None]) -> bool: """Run a response-writing callback, swallowing a dead-peer disconnect. From 8c20f1e1c206d73ba186cc3b5dd9d7e1dfaac7bc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 00:21:06 +0900 Subject: [PATCH 52/78] test(server): keep untrusted error details out of correlation logs Signed-off-by: Seongho Bae --- tests/test_security_hardening.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/tests/test_security_hardening.py b/tests/test_security_hardening.py index 39c160ac0..797c2399b 100644 --- a/tests/test_security_hardening.py +++ b/tests/test_security_hardening.py @@ -30,7 +30,10 @@ def test_concurrent_error_responses_share_only_their_own_log_id(caplog) -> None: thread.start() base = f"http://127.0.0.1:{server.server_address[1]}" try: - with patch.object(server.RequestHandlerClass, "_authorize", side_effect=RuntimeError("test failure")), ThreadPoolExecutor(max_workers=2) as executor: + with ( + patch.object(server.RequestHandlerClass, "_authorize", side_effect=RuntimeError("test failure")), + ThreadPoolExecutor(max_workers=2) as executor, + ): responses = list(executor.map( lambda _: request_json( f"{base}/v1/models", "GET", @@ -59,6 +62,28 @@ def test_concurrent_error_responses_share_only_their_own_log_id(caplog) -> None: server.server_close() +def test_error_log_bounds_existing_response_request_ids(caplog) -> None: + """Existing response details survive, but arbitrary IDs never enter logs.""" + server = build_server(build(), port=0) + handler = object.__new__(server.RequestHandlerClass) + try: + for request_id in ("a" * 32, "secret\nforged_log", "g" * 32, 42, None): + caplog.clear() + detail = {"request_id": request_id, "diagnostic": "private_diagnostic"} + with patch.object(handler, "_send") as send: + handler._send_error(400, "invalid_request", "invalid request", detail) + assert send.call_args.args[0]["error_detail"] == detail + assert send.call_args.args[1] == 400 + expected_id = request_id if request_id == "a" * 32 else "" + assert caplog.records[-1].getMessage() == ( + f"request_failed status=400 code=invalid_request request_id={expected_id}" + ) + assert "private_diagnostic" not in caplog.text + assert "forged_log" not in caplog.text + finally: + server.server_close() + + def test_external_bearer_verifier_is_fail_closed_and_scoped() -> None: seen: list[tuple[str, str]] = [] From 443aa5fb4255419fbec4b0aa8541b196b431b25f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 00:22:01 +0900 Subject: [PATCH 53/78] docs(server): record Strix timeout and correlation regression evidence Signed-off-by: Seongho Bae --- .../model-timeout-policy-evidence.md | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/docs/doctoring/model-timeout-policy-evidence.md b/docs/doctoring/model-timeout-policy-evidence.md index cf13feb07..4a90c0d48 100644 --- a/docs/doctoring/model-timeout-policy-evidence.md +++ b/docs/doctoring/model-timeout-policy-evidence.md @@ -253,6 +253,38 @@ in-memory-only pool reports history unavailable, not durable empty-history proof These are read-only operations; set/clear/restore HTTP actions, execution enforcement and UI acceptance remain incomplete. +## Strix HTTP 500 and error correlation + +The [Strix run 34031339200](https://github.com/ContextualWisdomLab/.github/actions/runs/34031339200) +installed CO `414f22973658c4ddc3d4320fcf7acd9b4e8ba991` (job log +1280–1281). Artifact `9991542931` contains sidecar failures at +14:53:36.102 and 14:53:54.577 UTC on 2026-09-06: `TimeoutError` at +`_open_provider`, then generic HTTP 500. At the installed revision, the +reported line 2281 waits for response headers. Strix reports failures within +12 milliseconds of those events. This is temporal correlation, not an exact +request-ID join: the sidecar does not contain either terminal response ID. +The 5377-second wrapper duration is not one model request's timeout. + +`10225f43` first tested the authentication verifier failure incorrectly as a +500; the existing fail-closed boundary correctly returned 401. The corrected +`4b739339` test injects a generic handler failure and retains the actual RED: +concurrent responses contain unique IDs absent from their corresponding logs. +`0e7c03bd` reuses each error response's ID in the existing sanitized status/code +log. No new telemetry system, transport timer, provider retry or fallback is +introduced. Existing response details remain unchanged; noncanonical override +IDs are omitted from logs rather than copied as arbitrary text. + +At `8c20f1e1`, 142 security, provider-error and passthrough tests pass in 14.36 +seconds. They cover concurrent HTTP 500 correlation, existing ID preservation, +and rejection of arbitrary diagnostic text from logs. This is focused evidence, +not a full-suite or protected-release claim. SSE-specific error events remain a +separate correlation gap; this change addresses the ordinary HTTP error path. +The underlying raw transport exception/fallback boundary still needs repair +analysis separately from the default-null/model-lifetime contract. + +An actual screen-access attempt still returned a locked Mac. Administrator UI +visual acceptance remains unverified; paper figure inspection is not UI proof. + ## Source reference ContextualWisdomLab. (n.d.). *Finite outbound request-timeout boundaries* From 4b1c108fbc38878f3083600764daf0d5bbbec4bd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 00:36:19 +0900 Subject: [PATCH 54/78] test(server): reproduce SDK replay of terminal tool errors Signed-off-by: Seongho Bae --- tests/test_tool_execution_fallback.py | 41 +++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/test_tool_execution_fallback.py b/tests/test_tool_execution_fallback.py index db453dee3..754081127 100644 --- a/tests/test_tool_execution_fallback.py +++ b/tests/test_tool_execution_fallback.py @@ -708,6 +708,47 @@ def _open_provider(self, request, destination=None, *, timeout=None): # type: i raise _provider_tool_stop_http_error() +@pytest.mark.parametrize( + ("error_code", "expected_calls"), + [("tool_execution_stopped", 1), ("conflict", 3)], +) +def test_sdk_http_retry_respects_explicit_tool_stop(monkeypatch, error_code, expected_calls) -> None: + """The optional exact SDK probe uses real loopback HTTP, never a provider.""" + import asyncio + + sdk = pytest.importorskip("openai", reason="run this integration probe with openai==2.54.0") + assert sdk.__version__ == "2.54.0" + server = build_server(TaskOrchestrator([ModelAgent("local_worker", "mock-local")]), port=0) + received_calls = [] + + def respond(handler): + handler._read_json() + received_calls.append(error_code) + handler._send_error(409, error_code, "request stopped", {"retryable": False}) + + monkeypatch.setattr(server.RequestHandlerClass, "do_POST", respond) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + + async def request_completion(): + async with sdk.AsyncOpenAI( + api_key="local_test_only", base_url=f"http://127.0.0.1:{server.server_address[1]}/v1", + ) as client: + with pytest.raises(sdk.ConflictError) as raised: + await client.chat.completions.create( + model="local_test_model", messages=[{"role": "user", "content": "fixture"}], + ) + assert raised.value.body["code"] == error_code + + try: + asyncio.run(request_completion()) + assert len(received_calls) == expected_calls + finally: + server.shutdown() + thread.join(timeout=5) + server.server_close() + + def test_http_fail_closed_tool_error_has_dedicated_contract() -> None: error = ToolExecutionError( "request may have completed token=must-not-leak", From fc94faab0d1cfb5079a382c529b424947198f472 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 00:36:53 +0900 Subject: [PATCH 55/78] fix(server): prevent SDK replay of explicit tool stops Signed-off-by: Seongho Bae --- contextual_orchestrator/server.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index fc5962318..e834b64cf 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -8232,7 +8232,12 @@ def _send_error( _LOGGER.warning( "request_failed status=%s code=%s request_id=%s", status, code, safe_request_id ) - self._send(_error_payload(code, message, error_detail), status) + payload = _error_payload(code, message, error_detail) + if code == TOOL_FALLBACK_STOPPED_CODE: + # The SDK retries ordinary 409s; this explicit stop must not replay. + self._send(payload, status, extra_headers={"x-should-retry": "false"}) + else: + self._send(payload, status) def _write_response(self, writer: Callable[[], None]) -> bool: """Run a response-writing callback, swallowing a dead-peer disconnect. From 5c82b9c011c56c978b7b601c54ee38ebd9e72da8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 00:37:32 +0900 Subject: [PATCH 56/78] test(server): verify tool stop header without optional SDK Signed-off-by: Seongho Bae --- tests/test_tool_execution_fallback.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_tool_execution_fallback.py b/tests/test_tool_execution_fallback.py index 754081127..c89f3af50 100644 --- a/tests/test_tool_execution_fallback.py +++ b/tests/test_tool_execution_fallback.py @@ -650,6 +650,7 @@ def test_tool_retry_backoff_requires_finite_nonnegative_number(value: object) -> def _post_fallback_json( port: int, payload: dict[str, object], + response_headers: dict[str, str] | None = None, ) -> tuple[int, dict[str, object]]: request = urllib.request.Request( f"http://127.0.0.1:{port}/v1/chat/completions", @@ -665,6 +666,8 @@ def _post_fallback_json( with urllib.request.urlopen(request, timeout=5) as response: return response.status, json.loads(response.read().decode("utf-8")) except urllib.error.HTTPError as error: + if response_headers is not None: + response_headers.update(error.headers) return error.code, json.loads(error.read().decode("utf-8")) @@ -792,6 +795,7 @@ def test_http_fail_closed_tool_error_has_dedicated_contract() -> None: def test_provider_http_tool_stop_preserves_409_and_does_not_fail_over() -> None: + response_headers: dict[str, str] = {} agents = [ ModelAgent( "primary_worker", @@ -822,6 +826,7 @@ def test_provider_http_tool_stop_preserves_409_and_does_not_fail_over() -> None: "mode": "route", "messages": [{"role": "user", "content": "send this message"}], }, + response_headers, ) finally: server.shutdown() @@ -830,6 +835,7 @@ def test_provider_http_tool_stop_preserves_409_and_does_not_fail_over() -> None: assert status == 409 assert body["error"]["code"] == "tool_execution_stopped" assert body["error"]["detail"]["failure_kind"] == "ambiguous_outcome" + assert response_headers["x-should-retry"] == "false" assert client.calls == ["primary_worker"] assert "provider.example" not in json.dumps(body) From ea5555dfe9c4cfe7c0fabf1c35659bf805d954e6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 00:38:25 +0900 Subject: [PATCH 57/78] docs(server): record explicit tool stop SDK retry contract Signed-off-by: Seongho Bae --- .../model-timeout-policy-evidence.md | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/docs/doctoring/model-timeout-policy-evidence.md b/docs/doctoring/model-timeout-policy-evidence.md index 4a90c0d48..40678734a 100644 --- a/docs/doctoring/model-timeout-policy-evidence.md +++ b/docs/doctoring/model-timeout-policy-evidence.md @@ -282,6 +282,36 @@ separate correlation gap; this change addresses the ordinary HTTP error path. The underlying raw transport exception/fallback boundary still needs repair analysis separately from the default-null/model-lifetime contract. +### Explicit tool-stop SDK replay regression + +The installed Strix SDK is OpenAI Python 2.54.0 (job log 1778 and 1905). +Its [versioned retry implementation](https://github.com/openai/openai-python/blob/v2.54.0/src/openai/_base_client.py) +retries HTTP 409 and 5xx without consulting JSON `retryable` details. It honors +the nonstandard `x-should-retry: false` response header. An offline SDK probe +confirmed three attempts without that header versus one with it for 409, +500, 502 and 503. This evidence applies to that SDK, not every consumer. + +`4b1c108f` then reproduced the problem with the actual CO error-response +writer over loopback HTTP and the exact asynchronous SDK: the explicit +`tool_execution_stopped` 409 was requested three times instead of once; +the ordinary `conflict` 409 correctly retained three attempts. The run +finished with one failure and one pass in 9.88 seconds. No provider was called. + +`fc94faab` adds the no-retry header only for the existing explicit tool-stop +error code, reusing the response writer's existing extra-header support. +Status, error payload and other error codes remain unchanged. At `5c82b9c0`, +161 related tests, including the SDK integration cases, pass in 11.82 seconds: +`uv run --offline --with openai==2.54.0 pytest -q tests/test_tool_execution_fallback.py tests/test_provider_reliability.py tests/test_security_hardening.py`. +The existing provider-to-orchestrator-to-HTTP terminal-stop test now also +checks the header without needing the optional SDK. Normal suites without +the SDK explicitly skip the two SDK cases; they are not counted as passes. + +This repairs the existing tool-stop response's SDK compatibility. It does +not show that the Strix incident took the tool-stop branch, classify raw +ambiguous passthrough timeouts, prevent higher-level Strix retries, or alter +the model-lifetime contract. The previous 443aa5fb full-suite receipt does +not validate these subsequent commits. + An actual screen-access attempt still returned a locked Mac. Administrator UI visual acceptance remains unverified; paper figure inspection is not UI proof. From 08e519345daee4abefa034d0b480026f641b7661 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 00:53:57 +0900 Subject: [PATCH 58/78] test(gateway): require typed nonreplayable transport outcomes Signed-off-by: Seongho Bae --- tests/test_passthrough_provider_failover.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/test_passthrough_provider_failover.py b/tests/test_passthrough_provider_failover.py index 87751f2ac..69e3d9835 100644 --- a/tests/test_passthrough_provider_failover.py +++ b/tests/test_passthrough_provider_failover.py @@ -1289,9 +1289,10 @@ def test_only_temporary_dns_failures_advance( ) -def test_ambiguous_timeout_is_not_replayed() -> None: - """A timeout may follow provider acceptance, so passthrough fails closed.""" - failure = TimeoutError("provider outcome unknown") +@pytest.mark.parametrize("error_type", [TimeoutError, ConnectionError]) +def test_ambiguous_timeout_is_not_replayed(error_type) -> None: + """Unknown transport outcomes remain terminal and expose no raw diagnostics.""" + failure = error_type("provider outcome unknown token=private_test_value") client = SequencedProxyClient( { "primary_agent": failure, @@ -1299,9 +1300,15 @@ def test_ambiguous_timeout_is_not_replayed() -> None: } ) - with pytest.raises(TimeoutError, match="outcome unknown"): + with pytest.raises(ProviderUpstreamError) as raised: _build(client).proxy_completion({"messages": [{"role": "user", "content": "x"}]}) + assert raised.value.error_code == "provider_outcome_unknown" + assert raised.value.client_status == 502 + assert raised.value.provider_status is None + assert raised.value.retryable is False + assert raised.value.transport == "passthrough" + assert "private_test_value" not in str(raised.value) assert [agent_id for agent_id, _ in client.calls] == ["primary_agent"] From 260767da037fa358ee9226216703b936211c95ce Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 00:54:17 +0900 Subject: [PATCH 59/78] test(server): prevent SDK replay of unknown provider outcomes Signed-off-by: Seongho Bae --- tests/test_tool_execution_fallback.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/test_tool_execution_fallback.py b/tests/test_tool_execution_fallback.py index c89f3af50..8865802d1 100644 --- a/tests/test_tool_execution_fallback.py +++ b/tests/test_tool_execution_fallback.py @@ -712,10 +712,11 @@ def _open_provider(self, request, destination=None, *, timeout=None): # type: i @pytest.mark.parametrize( - ("error_code", "expected_calls"), - [("tool_execution_stopped", 1), ("conflict", 3)], + ("error_code", "status_code", "expected_calls"), + [("tool_execution_stopped", 409, 1), ("conflict", 409, 3), + ("provider_outcome_unknown", 502, 1)], ) -def test_sdk_http_retry_respects_explicit_tool_stop(monkeypatch, error_code, expected_calls) -> None: +def test_sdk_http_retry_respects_explicit_tool_stop(monkeypatch, error_code, status_code, expected_calls) -> None: """The optional exact SDK probe uses real loopback HTTP, never a provider.""" import asyncio @@ -727,7 +728,7 @@ def test_sdk_http_retry_respects_explicit_tool_stop(monkeypatch, error_code, exp def respond(handler): handler._read_json() received_calls.append(error_code) - handler._send_error(409, error_code, "request stopped", {"retryable": False}) + handler._send_error(status_code, error_code, "request stopped", {"retryable": False}) monkeypatch.setattr(server.RequestHandlerClass, "do_POST", respond) thread = threading.Thread(target=server.serve_forever, daemon=True) @@ -737,11 +738,12 @@ async def request_completion(): async with sdk.AsyncOpenAI( api_key="local_test_only", base_url=f"http://127.0.0.1:{server.server_address[1]}/v1", ) as client: - with pytest.raises(sdk.ConflictError) as raised: + with pytest.raises(sdk.APIStatusError) as raised: await client.chat.completions.create( model="local_test_model", messages=[{"role": "user", "content": "fixture"}], ) assert raised.value.body["code"] == error_code + assert raised.value.status_code == status_code try: asyncio.run(request_completion()) From fe1e85a0eeff0011bf0dfd82b6eca2f86f0e60b2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 00:54:46 +0900 Subject: [PATCH 60/78] fix(gateway): surface unknown outcomes without enabling replay Signed-off-by: Seongho Bae --- contextual_orchestrator/orchestrator.py | 12 ++++++++++++ contextual_orchestrator/provider_errors.py | 3 +++ contextual_orchestrator/server.py | 6 +++--- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index f0cea8135..a54dee1ac 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -50,6 +50,7 @@ from .endpoint_race import EndpointAttempt, EndpointEquivalenceContract, race_first_valid from .reasoning_effort_profile import EffortProfileError from .provider_errors import ( + PROVIDER_OUTCOME_UNKNOWN_CODE, ProviderUpstreamError, classify_provider_failure, provider_error_body, @@ -4577,6 +4578,17 @@ def proxy_completion( result = send_once(candidate, endpoint, candidate_payload) except Exception as exc: # noqa: BLE001 - provider trust boundary if not _is_passthrough_failover_error(exc): + if isinstance(exc, (TimeoutError, ConnectionError)): + # No acceptance evidence: classify without enabling replay. + raise ProviderUpstreamError( + agent_id=candidate.id, + model=candidate.model, + error_code=PROVIDER_OUTCOME_UNKNOWN_CODE, + message="the provider request outcome is unknown; automatic replay is unsafe", + client_status=502, + retryable=False, + transport="passthrough", + ) from None if isinstance(exc, (urllib.error.HTTPError, ProviderUpstreamError)): raise classify_provider_failure( exc, diff --git a/contextual_orchestrator/provider_errors.py b/contextual_orchestrator/provider_errors.py index 3b862792d..8b4a4929c 100644 --- a/contextual_orchestrator/provider_errors.py +++ b/contextual_orchestrator/provider_errors.py @@ -27,6 +27,7 @@ from typing import Any __all__ = [ + "PROVIDER_OUTCOME_UNKNOWN_CODE", "MAX_PROVIDER_ERROR_BODY_BYTES", "MAX_SAFE_MESSAGE_CHARS", "PROVIDER_STATUS_SURFACES", @@ -36,6 +37,8 @@ "safe_provider_message", ] +PROVIDER_OUTCOME_UNKNOWN_CODE = "provider_outcome_unknown" + #: Upper bound for any provider-supplied message that reaches a caller. MAX_SAFE_MESSAGE_CHARS = 300 diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index e834b64cf..5d3fac1a2 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -57,7 +57,7 @@ sse_stream_body, ) from .pii_protection import DEFAULT_PURPOSE_BY_SCOPE, PURPOSES_BY_SCOPE -from .provider_errors import ProviderUpstreamError +from .provider_errors import PROVIDER_OUTCOME_UNKNOWN_CODE, ProviderUpstreamError from .tool_fallback import ToolFallbackStoppedError from .model_group import canonical_group_name from .release_authorization import verify_release_authority_snapshot @@ -8233,8 +8233,8 @@ def _send_error( "request_failed status=%s code=%s request_id=%s", status, code, safe_request_id ) payload = _error_payload(code, message, error_detail) - if code == TOOL_FALLBACK_STOPPED_CODE: - # The SDK retries ordinary 409s; this explicit stop must not replay. + if code in {TOOL_FALLBACK_STOPPED_CODE, PROVIDER_OUTCOME_UNKNOWN_CODE}: + # The SDK retries ordinary 409/5xx; explicit unsafe outcomes must not replay. self._send(payload, status, extra_headers={"x-should-retry": "false"}) else: self._send(payload, status) From 76d1caab5c51de272d1487fef28347f8c8d949ed Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 00:55:56 +0900 Subject: [PATCH 61/78] test(gateway): verify SDK to HTTP unknown outcome boundary Signed-off-by: Seongho Bae --- tests/test_passthrough_provider_failover.py | 41 +++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/test_passthrough_provider_failover.py b/tests/test_passthrough_provider_failover.py index 69e3d9835..49b0b7372 100644 --- a/tests/test_passthrough_provider_failover.py +++ b/tests/test_passthrough_provider_failover.py @@ -1312,6 +1312,47 @@ def test_ambiguous_timeout_is_not_replayed(error_type) -> None: assert [agent_id for agent_id, _ in client.calls] == ["primary_agent"] +def test_sdk_passthrough_unknown_outcome_never_replays() -> None: + """Exact SDK to real HTTP to passthrough preserves one unknown-outcome attempt.""" + import asyncio + import threading + from contextual_orchestrator.server import SecurityConfig, build_server + + sdk = pytest.importorskip("openai", reason="run this integration probe with openai==2.54.0") + assert sdk.__version__ == "2.54.0" + transport = SequencedProxyClient({ + "primary_agent": TimeoutError("token=private_test_value"), + "fallback_agent": {"model": "fallback-model"}, + }) + server = build_server(_build(transport), port=0, security=SecurityConfig(auth_token="local_test_only")) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + + async def request_completion(): + async with sdk.AsyncOpenAI( + api_key="local_test_only", base_url=f"http://127.0.0.1:{server.server_address[1]}/v1", + ) as client: + with pytest.raises(sdk.APIStatusError) as raised: + await client.chat.completions.create( + model="contextual-orchestrator", + messages=[{"role": "user", "content": "inspect locally"}], + tools=[{"type": "function", "function": {"name": "inspect", "parameters": {"type": "object"}}}], + ) + assert raised.value.status_code == 502 + assert raised.value.body["code"] == "provider_outcome_unknown" + assert raised.value.body["detail"]["retryable"] is False + assert raised.value.response.headers["x-should-retry"] == "false" + assert "private_test_value" not in str(raised.value) + + try: + asyncio.run(request_completion()) + assert [agent_id for agent_id, _ in transport.calls] == ["primary_agent"] + finally: + server.shutdown() + thread.join(timeout=5) + server.server_close() + + def test_virtual_effort_profile_selects_a_supported_provider() -> None: """Mixed pools skip unsupported candidates instead of aborting valid routing.""" client = SequencedProxyClient( From 4ceafa0d05fa19ba51f286d64f92900e9ac87ed6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 00:57:12 +0900 Subject: [PATCH 62/78] docs(gateway): record unknown outcome no-replay evidence Signed-off-by: Seongho Bae --- .../model-timeout-policy-evidence.md | 25 +++++++++++++++++++ docs/product-technical-gap-baseline.md | 15 +++++++++++ 2 files changed, 40 insertions(+) diff --git a/docs/doctoring/model-timeout-policy-evidence.md b/docs/doctoring/model-timeout-policy-evidence.md index 40678734a..2347d13e9 100644 --- a/docs/doctoring/model-timeout-policy-evidence.md +++ b/docs/doctoring/model-timeout-policy-evidence.md @@ -312,6 +312,31 @@ ambiguous passthrough timeouts, prevent higher-level Strix retries, or alter the model-lifetime contract. The previous 443aa5fb full-suite receipt does not validate these subsequent commits. +### Unknown passthrough outcome without replay + +`08e51934` retains two RED cases: TimeoutError and ConnectionError escape +unclassified from the non-failover branch. `260767da` retains an independent +HTTP/SDK RED: a proposed unknown-outcome 502 produces three requests despite +JSON retryable false. Existing explicit-stop and ordinary-conflict controls +pass in that run (one failure, two passes, 4.61 seconds). + +`fe1e85a0` uses the existing provider-error type with a distinct +`provider_outcome_unknown` code, no provider status, retryable false, and +package-owned text. Only the existing non-failover passthrough branch's +TimeoutError/ConnectionError handling changes; generic programming errors, +accepted failover statuses and model timeout policy do not change. The same +error code activates the existing no-retry-header mechanism. No tool execution +is inferred, and no new retry or fallback is introduced. + +At `76d1caab`, a complete SDK 2.54.0 → actual loopback HTTP → orchestrator → +mock provider test passes with one primary attempt, zero fallback attempts, +typed 502, retryable false, the no-retry header, and no private exception +text. All 212 related SDK-enabled tests pass in 14.63 seconds. These commits +still require a fresh full suite and protected review. Optional SDK skips in +the normal suite must remain distinct from this explicit SDK-enabled run. +This does not prove higher-level Strix replay prevention or live upstream +cleanup, and it does not remove the legacy runtime's finite timeout. + An actual screen-access attempt still returned a locked Mac. Administrator UI visual acceptance remains unverified; paper figure inspection is not UI proof. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 87a3663c3..5423823dd 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -2747,3 +2747,18 @@ in 7.70 seconds, including HTTP authorization and invalid-bound checks. HTTP set/clear/restore, released Rust runtime integration and actual administrator UI acceptance remain open. See [policy evidence](doctoring/model-timeout-policy-evidence.md#read-only-operator-policy-view). + +### Unknown request outcome — local error-contract repair + +At `76d1caab`, a passthrough timeout or connection failure with unknown +acceptance no longer escapes as a generic internal error. The existing +single-attempt/no-fallback decision is retained, with a caller-safe +`provider_outcome_unknown` response, `retryable=false` and an explicit +SDK no-retry header. No tool execution is inferred from a model timeout. +OpenAI SDK 2.54.0 → actual loopback HTTP → orchestrator → mock provider +verifies one primary call, no fallback, and no raw diagnostic disclosure. +The SDK-enabled related run passes 212 tests in 14.63 seconds. This is +local evidence, not a protected release or live-provider result; full-suite +verification of these new commits remains pending. Higher-level Strix +retries, SSE errors, default-null full-response lifetime and UI acceptance +remain open. See [incident and SDK evidence](doctoring/model-timeout-policy-evidence.md). From a35a3c6c902f727026183257e0d5d31a95c9da75 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 01:15:43 +0900 Subject: [PATCH 63/78] test(logging): require bounded upstream status evidence Signed-off-by: Seongho Bae --- tests/test_provider_reliability.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_provider_reliability.py b/tests/test_provider_reliability.py index caa59f369..70beda5c5 100644 --- a/tests/test_provider_reliability.py +++ b/tests/test_provider_reliability.py @@ -27,6 +27,7 @@ ModelClient, ProviderRequestTooLargeError, ProviderResponseError, + _log_provider_attempt_failed, is_transient_error, ) from contextual_orchestrator.provider_errors import ( # noqa: E402 @@ -40,6 +41,26 @@ def _http_error(code: int) -> urllib.error.HTTPError: return urllib.error.HTTPError("https://provider.example/chat/completions", code, "err", None, None) +@pytest.mark.parametrize("status", [425, 429, 503, None, 0, True, "429", 600]) +def test_attempt_log_preserves_only_valid_numeric_upstream_status(caplog, status) -> None: + """Logs distinguish missing status without reading bodies or exposing diagnostics.""" + body = io.BytesIO(b"private_response_body") + failure = urllib.error.HTTPError( + "https://provider.example/private", status, + "https://provider.example/private private_failure_text", None, body, + ) + agent = ModelAgent("local_worker", "mock-local") + with caplog.at_level("DEBUG", logger="contextual_orchestrator.orchestrator"): + _log_provider_attempt_failed(agent, 0, failure, True) + expected = status if type(status) is int and 100 <= status <= 599 else None + assert f"provider_status={expected}" in caplog.text + assert "error_message=" in caplog.text + assert "provider.example" not in caplog.text + assert "private_failure_text" not in caplog.text + assert "private_response_body" not in caplog.text + assert body.tell() == 0 + + def _stopped_http_error() -> urllib.error.HTTPError: return urllib.error.HTTPError( "https://provider.example/chat/completions", From b20f994585425386f9270a047b43f60f63e094f9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 01:16:01 +0900 Subject: [PATCH 64/78] fix(logging): retain numeric provider status without raw diagnostics Signed-off-by: Seongho Bae --- contextual_orchestrator/orchestrator.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index a54dee1ac..36c2b2353 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -1360,16 +1360,23 @@ def _log_provider_attempt(agent: ModelAgent, attempt: int, retry_limit: int) -> def _log_provider_attempt_failed( agent: ModelAgent, attempt: int, exc: Exception, transient: bool ) -> None: - """DEBUG-log one failed provider attempt with a redacted, bounded error message.""" + """DEBUG-log typed status evidence without reading or stringifying provider content.""" if _LOGGER.isEnabledFor(logging.DEBUG): + provider_status = ( + exc.code if isinstance(exc, urllib.error.HTTPError) + else exc.provider_status if isinstance(exc, ProviderUpstreamError) + else None + ) + if type(provider_status) is not int or not 100 <= provider_status <= 599: + provider_status = None _LOGGER.debug( - "provider_attempt_failed agent_id=%s model=%s attempt=%d error_type=%s transient=%s error_message=%s", + "provider_attempt_failed agent_id=%s model=%s attempt=%d error_type=%s transient=%s provider_status=%s error_message=", agent.id, agent.model, attempt + 1, type(exc).__name__, transient, - redact_text(str(exc))[:500], + provider_status, ) From 0b949aa2b69872ae5232f3b5664baef323946cec Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 01:16:32 +0900 Subject: [PATCH 65/78] test(logging): preserve typed status and avoid exception stringification Signed-off-by: Seongho Bae --- tests/test_provider_reliability.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_provider_reliability.py b/tests/test_provider_reliability.py index 70beda5c5..7b4e3bdaf 100644 --- a/tests/test_provider_reliability.py +++ b/tests/test_provider_reliability.py @@ -61,6 +61,27 @@ def test_attempt_log_preserves_only_valid_numeric_upstream_status(caplog, status assert body.tell() == 0 +def test_attempt_log_handles_typed_and_non_http_failures_without_stringifying(caplog) -> None: + """A typed status survives; a non-HTTP failure remains explicitly unknown.""" + class UnprintableFailure(RuntimeError): + def __str__(self): + raise AssertionError("failure text must not be evaluated") + + failures = [ + (ProviderUpstreamError( + agent_id="local_worker", model="mock-local", error_code="service_unavailable", + message="private_failure_text", client_status=503, provider_status=503, + ), 503), + (UnprintableFailure(), None), + ] + for failure, expected in failures: + caplog.clear() + with caplog.at_level("DEBUG", logger="contextual_orchestrator.orchestrator"): + _log_provider_attempt_failed(ModelAgent("local_worker", "mock-local"), 0, failure, True) + assert f"provider_status={expected}" in caplog.text + assert "private_failure_text" not in caplog.text + + def _stopped_http_error() -> urllib.error.HTTPError: return urllib.error.HTTPError( "https://provider.example/chat/completions", From 73f89777287e2c0bcd21a630019a76512de99f02 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 01:17:35 +0900 Subject: [PATCH 66/78] docs(logging): record bounded upstream status evidence Signed-off-by: Seongho Bae --- .../model-timeout-policy-evidence.md | 21 +++++++++++++++++++ docs/product-technical-gap-baseline.md | 7 +++++++ 2 files changed, 28 insertions(+) diff --git a/docs/doctoring/model-timeout-policy-evidence.md b/docs/doctoring/model-timeout-policy-evidence.md index 2347d13e9..205f23bea 100644 --- a/docs/doctoring/model-timeout-policy-evidence.md +++ b/docs/doctoring/model-timeout-policy-evidence.md @@ -337,6 +337,27 @@ the normal suite must remain distinct from this explicit SDK-enabled run. This does not prove higher-level Strix replay prevention or live upstream cleanup, and it does not remove the legacy runtime's finite timeout. +### Noema final 429 and missing attempt-status evidence + +The separate Naruon Noema run `34039160343`, job `101508094555`, installed +CO `414f22973658c4ddc3d4320fcf7acd9b4e8ba991`. Its caller made one attempt, +reported 222.9 seconds, and ended with gateway HTTP 429. Artifact `9992599042` +records two approximately 90-second timeout attempts and subsequent HTTPError +attempts across several providers before the final rate-limit classification. +It omits individual HTTP status codes and request IDs; therefore neither +every candidate returning 429 nor a particular account's quota exhaustion +is established. This is distinct from the earlier generic HTTP 500 incident. + +`a35a3c6c` retains eight RED cases for missing upstream status in the shared +attempt logger. `b20f9945` records only actual HTTPError or typed-provider +status integers from 100 through 599. Absent or invalid status becomes None, +never a fabricated zero. The helper no longer stringifies the exception and +does not read its body. At `0b949aa2`, 89 related tests pass in 15.28 seconds, +including 425 versus 429, invalid values, typed status, no body reads and an +exception whose string conversion deliberately raises. This local diagnostic +repair does not make a provider available, change retries, establish request +correlation, or retrospectively fill missing evidence in the old artifact. + An actual screen-access attempt still returned a locked Mac. Administrator UI visual acceptance remains unverified; paper figure inspection is not UI proof. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 5423823dd..1cdcd606e 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -2762,3 +2762,10 @@ local evidence, not a protected release or live-provider result; full-suite verification of these new commits remains pending. Higher-level Strix retries, SSE errors, default-null full-response lifetime and UI acceptance remain open. See [incident and SDK evidence](doctoring/model-timeout-policy-evidence.md). + +The separate Naruon Noema 429 incident lacks per-attempt upstream status; +final gateway status alone cannot establish every candidate's failure cause. +Local `0b949aa2` adds bounded numeric status to the existing common failed- +attempt log without reading provider text or bodies (89 related tests pass, +15.28 seconds). Full verification and release of this diagnostic addition +remain pending; provider availability itself is not repaired by better logs. From b81739fcfa7fecef95116397bbe3237e97a957a9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 01:30:07 +0900 Subject: [PATCH 67/78] test(logging): require omission rather than partial diagnostic masking Signed-off-by: Seongho Bae --- tests/test_orchestrator_debug_logging.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_orchestrator_debug_logging.py b/tests/test_orchestrator_debug_logging.py index 93f5824f7..84192c2bc 100644 --- a/tests/test_orchestrator_debug_logging.py +++ b/tests/test_orchestrator_debug_logging.py @@ -67,8 +67,8 @@ def test_judged_ranking_log_does_not_label_quality_as_throughput() -> None: assert "success_rps=" not in output -def test_send_with_retry_debug_logs_redact_secret_shaped_error_message() -> None: - """THE key secret-leak test: a fake credential shape must never reach captured logs.""" +def test_send_with_retry_debug_logs_omit_secret_shaped_error_message() -> None: + """Only typed metadata survives; even partially redacted provider text is omitted.""" class LeakyClient(ModelClient): def __init__(self) -> None: @@ -87,7 +87,8 @@ def _send(self, agent: ModelAgent, payload: dict, destination=None) -> str: # t else: # pragma: no cover raise AssertionError("a failed provider request must raise") output = buffer.getvalue() - assert "[REDACTED]" in output + assert "provider_status=None error_message=" in output + assert "upstream rejected request" not in output assert _FAKE_SECRET not in output From 43165156d3799cee6bb23c0afdad22da07744831 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 01:31:01 +0900 Subject: [PATCH 68/78] docs(logging): record full regression and canonical sanitizer prerequisite Signed-off-by: Seongho Bae --- .../model-timeout-policy-evidence.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/doctoring/model-timeout-policy-evidence.md b/docs/doctoring/model-timeout-policy-evidence.md index 205f23bea..1611de664 100644 --- a/docs/doctoring/model-timeout-policy-evidence.md +++ b/docs/doctoring/model-timeout-policy-evidence.md @@ -358,6 +358,25 @@ exception whose string conversion deliberately raises. This local diagnostic repair does not make a provider available, change retries, establish request correlation, or retrospectively fill missing evidence in the old artifact. +The full suite at `73f89777` retained one failure, 3462 passes and six skips +in 690.53 seconds: an older debug-log regression required `[REDACTED]` to +appear. The stronger contract omits the complete provider message instead. +`b81739fc` updates that test to require the omission marker and absence of +both the original sentence and fake credential; 108 related tests pass in +15.88 seconds. A fresh full suite is required, not a flake rerun claim. + +Integration gate: the central sanitizer at protected `9aad23c0` drops the +new failed-attempt format and removes request IDs. Central Draft +[PR #1978](https://github.com/ContextualWisdomLab/.github/pull/1978), head +`5dad3fe874f93b19b232f4452e7190b72655f18e`, carries the compatible parser +(blob `9df99e1b7064ca4779a071cbd4c0d5b75afd8b82`). A coordinator's local +paired run passed real producer `73f89777` timestamped logs through that +exact parser blob: statuses 425/429/503 survived; fixture URL/text/body did +not, and the body stream remained unread. This is independent local evidence, +not protected publication or a hosted artifact result. The canonical consumer +must be protected and verified before adopting the new producer pin. No +sanitizer implementation is copied into CO. + An actual screen-access attempt still returned a locked Mac. Administrator UI visual acceptance remains unverified; paper figure inspection is not UI proof. From e1ff51b08ed420be3cc8ae7a30d310532ff50aae Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 11:05:54 +0900 Subject: [PATCH 69/78] test(timeout): preserve policy across unrelated writes --- tests/test_model_timeout_policy.py | 80 ++++++++++++++++++------------ 1 file changed, 47 insertions(+), 33 deletions(-) diff --git a/tests/test_model_timeout_policy.py b/tests/test_model_timeout_policy.py index 87fc92648..04eee978e 100644 --- a/tests/test_model_timeout_policy.py +++ b/tests/test_model_timeout_policy.py @@ -9,65 +9,79 @@ from contextual_orchestrator import ModelAgent, TaskOrchestrator -def test_ordinary_patch_rejects_stale_timeout_snapshot(tmp_path: Path) -> None: - """A priority edit cannot silently clear another writer's audited timeout.""" +def test_ordinary_patch_preserves_newer_timeout_policy(tmp_path: Path) -> None: + """An unrelated edit must not reject or overwrite another writer's policy.""" model_agent = ModelAgent("timeout_agent", "example-model") database_path = str(tmp_path / "agent-pool.db") writer = TaskOrchestrator([model_agent], agents_db=database_path) stale = TaskOrchestrator([model_agent], agents_db=database_path) writer.patch_agent("default", model_agent.id, {"model_timeout_seconds": 7200}) - with pytest.raises(ValueError, match="reload"): - stale.patch_agent("default", model_agent.id, {"priority": 7}) - assert stale._agent(model_agent.id).priority == model_agent.priority + patched = stale.patch_agent("default", model_agent.id, {"priority": 7}) + assert patched["priority"] == 7 restored = TaskOrchestrator([model_agent], agents_db=database_path) + assert restored._agent(model_agent.id).priority == 7 assert restored._agent(model_agent.id).model_timeout_seconds == 7200 assert restored._agent(model_agent.id).model_timeout_revision == 1 @pytest.mark.parametrize("operation", ["remove", "set_group", "delete_group"]) -def test_rejected_pool_change_preserves_serving_snapshot(tmp_path: Path, operation: str) -> None: - """A rejected durable edit must not publish a removal or membership change.""" +def test_pool_change_preserves_concurrent_timeout_policy( + tmp_path: Path, operation: str +) -> None: + """Unrelated durable edits neither reject nor overwrite a newer policy.""" model_agent = ModelAgent("timeout_agent", "example-model", group_name="test_group") other_agent = ModelAgent("other_agent", "other-model") database_path = str(tmp_path / "agent-pool.db") writer = TaskOrchestrator([model_agent, other_agent], agents_db=database_path) stale = TaskOrchestrator([model_agent, other_agent], agents_db=database_path) - before_candidates, before_agents = list(stale.candidates), list(stale.agents) writer.patch_agent("default", model_agent.id, {"model_timeout_seconds": 7200}) - with pytest.raises(ValueError, match="reload"): - if operation == "remove": - stale.remove_agent("default", model_agent.id) - elif operation == "set_group": - stale.set_model_group("new_group", [model_agent.id]) - else: - stale.delete_model_group("test_group") - assert stale.candidates == before_candidates - assert stale.agents == before_agents + if operation == "remove": + stale.remove_agent("default", model_agent.id) + elif operation == "set_group": + stale.set_model_group("new_group", [model_agent.id]) + else: + stale.delete_model_group("test_group") + restored = TaskOrchestrator([model_agent, other_agent], agents_db=database_path) + if operation == "remove": + with pytest.raises(KeyError): + restored._agent(model_agent.id) + else: + persisted = restored._agent(model_agent.id) + assert persisted.model_timeout_seconds == 7200 + assert persisted.model_timeout_revision == 1 + assert persisted.group_name == ("new_group" if operation == "set_group" else "") @pytest.mark.parametrize("operation", ["set_group", "delete_group", "discovery"]) -def test_late_batch_conflict_rolls_back_all_models(tmp_path: Path, operation: str) -> None: - """A stale second model cannot leave the first model partially committed.""" - seeds = [ModelAgent("first_agent", "first-model", group_name="test_group"), - ModelAgent("second_agent", "second-model", group_name="test_group")] +def test_batch_change_preserves_concurrent_timeout_policy( + tmp_path: Path, operation: str +) -> None: + """A batch edit preserves another writer's per-model timeout revision.""" + seeds = [ + ModelAgent("first_agent", "first-model", group_name="test_group"), + ModelAgent("second_agent", "second-model", group_name="test_group"), + ] database_path = str(tmp_path / "agent-pool.db") writer = TaskOrchestrator(seeds, agents_db=database_path) writer.sync_discovered_agents(seeds) stale = TaskOrchestrator(seeds, agents_db=database_path) - before = list(stale.candidates) writer.patch_agent("default", "second_agent", {"model_timeout_seconds": 7200}) - with pytest.raises(ValueError, match="reload"): - if operation == "set_group": - stale.set_model_group("new_group", [agent.id for agent in seeds]) - elif operation == "delete_group": - stale.delete_model_group("test_group") - else: - stale.sync_discovered_agents([replace(agent, priority=7) for agent in seeds]) - assert stale.candidates == before + if operation == "set_group": + stale.set_model_group("new_group", [agent.id for agent in seeds]) + elif operation == "delete_group": + stale.delete_model_group("test_group") + else: + stale.sync_discovered_agents([replace(agent, priority=7) for agent in seeds]) restored = TaskOrchestrator(seeds, agents_db=database_path) - assert restored._agent("first_agent") == seeds[0] - assert restored._agent("second_agent").model_timeout_seconds == 7200 - assert restored._agent("second_agent").group_name == "test_group" + second = restored._agent("second_agent") + assert second.model_timeout_seconds == 7200 + assert second.model_timeout_revision == 1 + if operation == "set_group": + assert {restored._agent(agent.id).group_name for agent in seeds} == {"new_group"} + elif operation == "delete_group": + assert {restored._agent(agent.id).group_name for agent in seeds} == {""} + else: + assert {restored._agent(agent.id).priority for agent in seeds} == {7} def test_model_timeout_policy_defaults_to_null() -> None: From b2a38b4e926f2b37406f02b2132d87248b1bea0e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 11:06:20 +0900 Subject: [PATCH 70/78] fix(timeout): isolate policy writes from pool updates --- contextual_orchestrator/orchestrator.py | 41 +++++++++++++++---------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 36c2b2353..5f54572cf 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -3516,26 +3516,35 @@ def _save_in_transaction( restored_from_revision: int | None = None, ) -> int | None: """Apply existing normalized writes inside the caller's transaction.""" - previous = timeout_previous if timeout_previous is not None else agent - revision = conn.execute( - "SELECT COALESCE(MAX(policy_revision), 0) FROM model_timeout_history WHERE agent_id = ?", - (agent.id,), - ).fetchone()[0] - if revision != previous.model_timeout_revision: - raise ValueError("model timeout policy changed; reload before updating") - row = conn.execute( - "SELECT model_timeout_seconds FROM agent_pool WHERE agent_id = ?", - (agent.id,), - ).fetchone() - if row is not None and row[0] != previous.model_timeout_seconds: - raise ValueError("model timeout policy changed; reload before updating") if timeout_previous is not None: + revision = conn.execute( + "SELECT COALESCE(MAX(policy_revision), 0) FROM model_timeout_history WHERE agent_id = ?", + (agent.id,), + ).fetchone()[0] + row = conn.execute( + "SELECT model_timeout_seconds FROM agent_pool WHERE agent_id = ?", + (agent.id,), + ).fetchone() + if ( + revision != timeout_previous.model_timeout_revision + or ( + row is not None + and row[0] != timeout_previous.model_timeout_seconds + ) + ): + raise ValueError("model timeout policy changed; reload before updating") if row is not None: conn.execute( "UPDATE agent_pool SET model_timeout_seconds = ? WHERE agent_id = ?", (agent.model_timeout_seconds, agent.id), ) - revision = self._append_timeout_history(conn, timeout_previous, agent, actor_id, restored_from_revision) + revision = self._append_timeout_history( + conn, + timeout_previous, + agent, + actor_id, + restored_from_revision, + ) return revision config = agent.to_config() conn.execute( @@ -3545,8 +3554,7 @@ def _save_in_transaction( priority = ?, disabled = ?, provider_name = ?, local_credential_key = ?, auth_scheme = ?, max_output_tokens = ?, context_window = ?, - reasoning_effort_supported = ?, stream_usage_supported = ?, - model_timeout_seconds = ? + reasoning_effort_supported = ?, stream_usage_supported = ? WHERE agent_id = ? """, ( @@ -3563,7 +3571,6 @@ def _save_in_transaction( config["context_window"], config["reasoning_effort_supported"], int(config["stream_usage_supported"]), - config["model_timeout_seconds"], agent.id, ), ) From 9179873737e8e608751bfa687b0d0b10c20d9983 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 11:42:16 +0900 Subject: [PATCH 71/78] test(timeout): verify isolated writes and disabled tombstones Signed-off-by: Seongho Bae --- tests/test_agent_pool_db.py | 15 +++++++++------ tests/test_model_timeout_policy.py | 11 +++++++---- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/tests/test_agent_pool_db.py b/tests/test_agent_pool_db.py index 5ea3e619f..fbeb1eba2 100644 --- a/tests/test_agent_pool_db.py +++ b/tests/test_agent_pool_db.py @@ -453,8 +453,8 @@ def _call(url: str, method: str, token: str, payload: dict | None = None) -> tup return exc.code, json.loads(exc.read().decode("utf-8")) -def test_http_stale_policy_rejects_admin_edit_without_overwrite(tmp_path) -> None: - """Actual authenticated HTTP edits cannot overwrite another writer's policy.""" +def test_http_unrelated_admin_edit_preserves_newer_timeout_policy(tmp_path) -> None: + """Authenticated priority edits preserve policy without relaxing access gates.""" seeds = _seed() database_path = str(tmp_path / "pool.db") writer = TaskOrchestrator(seeds, agents_db=database_path) @@ -468,13 +468,16 @@ def test_http_stale_policy_rejects_admin_edit_without_overwrite(tmp_path) -> Non writer.patch_agent("default", "general_agent", {"model_timeout_seconds": 7200}) status, _ = _call(url, "PATCH", "wrong_token", {"priority": 7}) assert status == 401 - status, payload = _call(url, "PATCH", "pool_token", {"priority": 7}) - assert status == 400 - assert "reload" in json.dumps(payload) + assert serving.candidates == before + status, _ = _call(url, "PATCH", "pool_token", {"priority": 7}) + assert status == 200 + assert serving._agent("general_agent").priority == 7 + after_priority_edit = list(serving.candidates) status, _ = _call(url, "PATCH", "pool_token", {"model_timeout_seconds": 3600}) assert status == 400 # New policy writes stay closed until runtime delivery exists. - assert serving.candidates == before + assert serving.candidates == after_priority_edit restored = TaskOrchestrator(seeds, agents_db=database_path) + assert restored._agent("general_agent").priority == 7 assert restored._agent("general_agent").model_timeout_seconds == 7200 assert restored._agent("general_agent").model_timeout_revision == 1 finally: diff --git a/tests/test_model_timeout_policy.py b/tests/test_model_timeout_policy.py index 04eee978e..56a005a89 100644 --- a/tests/test_model_timeout_policy.py +++ b/tests/test_model_timeout_policy.py @@ -42,13 +42,16 @@ def test_pool_change_preserves_concurrent_timeout_policy( else: stale.delete_model_group("test_group") restored = TaskOrchestrator([model_agent, other_agent], agents_db=database_path) + persisted = restored._agent(model_agent.id) + assert persisted.model_timeout_seconds == 7200 + assert persisted.model_timeout_revision == 1 if operation == "remove": + assert persisted.disabled is True + assert persisted.group_name == "" + assert model_agent.id not in {agent.id for agent in restored.agents} with pytest.raises(KeyError): - restored._agent(model_agent.id) + stale._agent(model_agent.id) else: - persisted = restored._agent(model_agent.id) - assert persisted.model_timeout_seconds == 7200 - assert persisted.model_timeout_revision == 1 assert persisted.group_name == ("new_group" if operation == "set_group" else "") From f50755b4b5e64c6c8afba81a43f1450f4f527bc4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 11:44:17 +0900 Subject: [PATCH 72/78] style(tests): clean timeout integration checks Signed-off-by: Seongho Bae --- tests/test_agent_pool_db.py | 8 ++++---- tests/test_model_timeout_policy.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_agent_pool_db.py b/tests/test_agent_pool_db.py index fbeb1eba2..b61a5767a 100644 --- a/tests/test_agent_pool_db.py +++ b/tests/test_agent_pool_db.py @@ -10,19 +10,19 @@ import json import os import sqlite3 -from pathlib import Path import sys import tempfile import threading import urllib.error import urllib.request +from pathlib import Path import pytest sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from contextual_orchestrator import ModelAgent, TaskOrchestrator # noqa: E402 -from contextual_orchestrator.server import SecurityConfig, build_server # noqa: E402 +from contextual_orchestrator import ModelAgent, TaskOrchestrator +from contextual_orchestrator.server import SecurityConfig, build_server def _seed() -> list[ModelAgent]: @@ -569,7 +569,7 @@ def test_http_create_and_delete_worker_agents() -> None: status, read = _call(f"{base}/general_agent", "GET", token) assert status == 200 and read["stream_usage_supported"] is True - status, dup = _call(base, "POST", token, NEW_AGENT) + status, _ = _call(base, "POST", token, NEW_AGENT) assert status == 400 # duplicate rejected status, wrong_pool = _call( diff --git a/tests/test_model_timeout_policy.py b/tests/test_model_timeout_policy.py index 56a005a89..6659ce2b9 100644 --- a/tests/test_model_timeout_policy.py +++ b/tests/test_model_timeout_policy.py @@ -1,8 +1,8 @@ """Administrator-owned model timeout policy must survive configuration changes.""" -from pathlib import Path -from dataclasses import replace import sqlite3 +from dataclasses import replace +from pathlib import Path import pytest From 78a71c0ee949710a82cd1a1def24887612431016 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 13:44:28 +0900 Subject: [PATCH 73/78] fix(router): distinguish pre-send admission timeouts Signed-off-by: Seongho Bae --- AGENTS.md | 7 +++ contextual_orchestrator/orchestrator.py | 8 +++- tests/test_passthrough_provider_failover.py | 50 +++++++++++++++++++++ 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 52b268819..84bb78f0b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -67,6 +67,13 @@ push or open a PR. ### This repo: the org LLM gateway +- Classify retry safety by the failure boundary, not a generic timeout name. + Only a direct local-slot admission failure proves no upstream send began; + wrapped exceptions and post-send timeouts do not authorize replay. Test the + real slot-to-transport path with per-candidate transport call counts and keep + unknown-outcome no-replay controls alongside it. Transport spies are not wire + delivery evidence. Preserve the default-null model timeout. + - Endpoint races require a complete operator-reviewed equivalence contract. Never infer equivalence from provider/model names, and never treat missing loser usage as free or zero-cost execution. diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 5f54572cf..05d0aa1d3 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -1041,6 +1041,10 @@ def _local_provider_state(base_url: str) -> _LocalProviderState: return _LOCAL_PROVIDER_STATES.setdefault(key, _LocalProviderState()) +class _LocalProviderAdmissionTimeout(TimeoutError): + """A local slot expired before any upstream request could be sent.""" + + @contextmanager def _local_provider_slot( agent: ModelAgent, @@ -1068,7 +1072,7 @@ def _local_provider_slot( remaining = None if deadline is None else deadline - time.monotonic() if remaining is not None and remaining <= 0: - raise TimeoutError("local provider endpoint is busy past its request deadline") + raise _LocalProviderAdmissionTimeout("local provider endpoint is busy past its request deadline") state.condition.wait(remaining) try: @@ -1639,6 +1643,8 @@ def _is_request_too_large_error(exc: BaseException) -> bool: def _is_passthrough_failover_error(exc: BaseException) -> bool: """Recognize failures proving that a passthrough request was not accepted.""" + if isinstance(exc, _LocalProviderAdmissionTimeout): + return True if _is_request_too_large_error(exc): return True current: BaseException | None = exc diff --git a/tests/test_passthrough_provider_failover.py b/tests/test_passthrough_provider_failover.py index 49b0b7372..05760f94b 100644 --- a/tests/test_passthrough_provider_failover.py +++ b/tests/test_passthrough_provider_failover.py @@ -1312,6 +1312,56 @@ def test_ambiguous_timeout_is_not_replayed(error_type) -> None: assert [agent_id for agent_id, _ in client.calls] == ["primary_agent"] +@pytest.mark.parametrize("wrapper_type", [RuntimeError, TimeoutError]) +def test_wrapped_admission_timeout_does_not_authorize_replay(wrapper_type) -> None: + """Only the direct pre-send exception carries the local admission proof.""" + from contextual_orchestrator.orchestrator import ( + _LocalProviderAdmissionTimeout, + _is_passthrough_failover_error, + ) + + wrapped = wrapper_type("unknown outer operation") + wrapped.__cause__ = _LocalProviderAdmissionTimeout("earlier slot failure") + assert not _is_passthrough_failover_error(wrapped) + + +@pytest.mark.parametrize("after_send", [False, True]) +def test_local_admission_timeout_preserves_send_boundary(monkeypatch, after_send: bool) -> None: + """Only a failed slot acquisition may advance without replaying a sent request.""" + from contextlib import nullcontext + from contextual_orchestrator.orchestrator import _local_provider_slot + + client = ModelClient(timeout=0.001) + router = _build(client) + router.agents[0] = replace( + router.agents[0], base_url="local://127.0.0.1:19441/v1" + ) + router.agents[1] = replace( + router.agents[1], base_url="local://127.0.0.1:19442/v1" + ) + sent = [] + + def raw_send(agent, *args, **kwargs): + sent.append(agent.id) + if after_send: + raise TimeoutError("response not received after transport invocation") + return {"model": agent.model, "choices": []} + + monkeypatch.setattr(client, "_send_raw_with_retry", raw_send) + slot = nullcontext() if after_send else _local_provider_slot(router.agents[0], 1, None) + with slot: + if after_send: + with pytest.raises(ProviderUpstreamError) as raised: + router.proxy_completion({"messages": [{"role": "user", "content": "x"}]}) + assert raised.value.error_code == "provider_outcome_unknown" + assert raised.value.retryable is False + assert sent == ["primary_agent"] + else: + result = router.proxy_completion({"messages": [{"role": "user", "content": "x"}]}) + assert result["model"] == "fallback-model" + assert sent == ["fallback_agent"] + + def test_sdk_passthrough_unknown_outcome_never_replays() -> None: """Exact SDK to real HTTP to passthrough preserves one unknown-outcome attempt.""" import asyncio From 1ccc9599415096433214cd9ad0df611eddfc8fbb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 14:07:19 +0900 Subject: [PATCH 74/78] test(sdk): require pinned integration probes in locked CI Signed-off-by: Seongho Bae --- pyproject.toml | 1 + tests/test_passthrough_provider_failover.py | 2 +- tests/test_tool_execution_fallback.py | 4 +- uv.lock | 178 ++++++++++++++++++++ 4 files changed, 182 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index de9f90de1..3fdfeec92 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ queue = [ [dependency-groups] dev = [ "hypothesis>=6.100", + "openai==2.54.0", "pytest>=8.0", ] diff --git a/tests/test_passthrough_provider_failover.py b/tests/test_passthrough_provider_failover.py index 05760f94b..96ffc9129 100644 --- a/tests/test_passthrough_provider_failover.py +++ b/tests/test_passthrough_provider_failover.py @@ -1368,7 +1368,7 @@ def test_sdk_passthrough_unknown_outcome_never_replays() -> None: import threading from contextual_orchestrator.server import SecurityConfig, build_server - sdk = pytest.importorskip("openai", reason="run this integration probe with openai==2.54.0") + import openai as sdk assert sdk.__version__ == "2.54.0" transport = SequencedProxyClient({ "primary_agent": TimeoutError("token=private_test_value"), diff --git a/tests/test_tool_execution_fallback.py b/tests/test_tool_execution_fallback.py index 8865802d1..0391a9f19 100644 --- a/tests/test_tool_execution_fallback.py +++ b/tests/test_tool_execution_fallback.py @@ -717,10 +717,10 @@ def _open_provider(self, request, destination=None, *, timeout=None): # type: i ("provider_outcome_unknown", 502, 1)], ) def test_sdk_http_retry_respects_explicit_tool_stop(monkeypatch, error_code, status_code, expected_calls) -> None: - """The optional exact SDK probe uses real loopback HTTP, never a provider.""" + """The pinned SDK probe uses real loopback HTTP, never a provider.""" import asyncio - sdk = pytest.importorskip("openai", reason="run this integration probe with openai==2.54.0") + import openai as sdk assert sdk.__version__ == "2.54.0" server = build_server(TaskOrchestrator([ModelAgent("local_worker", "mock-local")]), port=0) received_calls = [] diff --git a/uv.lock b/uv.lock index 7a11636f2..1f220ad32 100644 --- a/uv.lock +++ b/uv.lock @@ -422,6 +422,7 @@ test = [ [package.dev-dependencies] dev = [ { name = "hypothesis" }, + { name = "openai" }, { name = "pytest" }, ] @@ -448,6 +449,7 @@ provides-extras = ["test", "api", "db", "fuzz", "queue"] [package.metadata.requires-dev] dev = [ { name = "hypothesis", specifier = ">=6.100" }, + { name = "openai", specifier = "==2.54.0" }, { name = "pytest", specifier = ">=8.0" }, ] @@ -508,6 +510,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/b5/c2c5fce26f0ee40d21bafe7f191d29a34b35a65ac4fe8a1191d1983612e9/cryptography-50.0.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:c99c003e088647b8a5b7c145d6f78c335f6348332b62e142d411c4b63d1460b9", size = 3813796, upload-time = "2026-07-31T14:25:02.298Z" }, ] +[[package]] +name = "distro" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722, upload-time = "2023-12-24T09:54:32.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, +] + [[package]] name = "exceptiongroup" version = "1.3.1" @@ -662,6 +673,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + [[package]] name = "hypothesis" version = "6.165.10" @@ -772,6 +811,105 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "jiter" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/1f/10936e16d8860c70698a1aa939a46aa0224813b782bce4e000e637da0b2d/jiter-0.16.0.tar.gz", hash = "sha256:7b24c3492c5f4f84a37946ad9cf504910cf6a782d6a4e0689b6673c5894b4a1c", size = 176431, upload-time = "2026-06-29T13:05:13.657Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/d8/b959609e44012a42b1f3e5ba98ea3b33c7e41e6d4b77cd8f00fd19b1d3ad/jiter-0.16.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:c5fc4f8def331036a7b8e981b4347ebe409981edbc8308a5ea842b8c3614fa6c", size = 310082, upload-time = "2026-06-29T13:02:31.356Z" }, + { url = "https://files.pythonhosted.org/packages/c6/3d/4d7f5667ea0e0548534ba880b84bb3d12924fd133aa83ad6c6c80fca3d76/jiter-0.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5a71d0d2014c3275043e1170bf3d4e771493cb0dcf07be54c567155f4d8ee64b", size = 315643, upload-time = "2026-06-29T13:02:33.204Z" }, + { url = "https://files.pythonhosted.org/packages/9b/83/bed2dcb5c9f3e1ccfcbc67dda48265fe7d5ad0c9cadda5fe95f6e3b87f94/jiter-0.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:741eed508c233a76313a1c7b001f8f21b82f14327e9196ae8bd29a2cc164ae84", size = 341363, upload-time = "2026-06-29T13:02:34.853Z" }, + { url = "https://files.pythonhosted.org/packages/f4/2f/6bb3c3dda668ebc0445689c81a2b0f26a82b10843d67ed9c9b2c3edc177f/jiter-0.16.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3fb7bc819187b56dc48aa5c833aaf92257da8e07efdb9306156667bd2eeb491c", size = 365483, upload-time = "2026-06-29T13:02:36.295Z" }, + { url = "https://files.pythonhosted.org/packages/92/35/8a045ccb39164e70dcdae696413b661771f148b68b12b175c3a04d901937/jiter-0.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7c9610fd25ebccb43fca584136f5c2fbb26802447eccd430dfdbab95a0fd5126", size = 461219, upload-time = "2026-06-29T13:02:38.116Z" }, + { url = "https://files.pythonhosted.org/packages/e7/99/22292dbbf0ed0c610cfe5ddc7f3bd67237a412f121318f865196e62a07bd/jiter-0.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4a1d68ff7ca1d3b5dee20a97a3decda7d5f15003823bf6d140c81f8561d3bc5c", size = 374905, upload-time = "2026-06-29T13:02:40.357Z" }, + { url = "https://files.pythonhosted.org/packages/29/ac/2f55ccb1f0eeafa6d89d24caf52f6f0944a59290ee199e9ade62177dca42/jiter-0.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb08c276dd02dac3a284acdd02cacc630d2e3cd6572a4b85519f35cbd133c3de", size = 348320, upload-time = "2026-06-29T13:02:41.923Z" }, + { url = "https://files.pythonhosted.org/packages/50/e3/7d88b9174c40064fabc07c84a9b62e6b10f5644562ec0e0a29392edbe978/jiter-0.16.0-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:8fc4d94713c4697347e38faf7d6ef91547c142219bdcfc7220c4870879974244", size = 356519, upload-time = "2026-06-29T13:02:43.436Z" }, + { url = "https://files.pythonhosted.org/packages/27/57/c4a33aeef513a9d5e26e31534e0bcc752d6ea0e54c94ddb7b68bade669c2/jiter-0.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a0f05e229edb29e68cdd0ccb83cea13b64263416120cf943767a6fd72e6787f", size = 394204, upload-time = "2026-06-29T13:02:44.987Z" }, + { url = "https://files.pythonhosted.org/packages/9d/70/c6c23e76ebb3766b111bc399437bbc9f870a76e2a92e10b2a5f561d57372/jiter-0.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:2c842cbf374a8daf50b2c04212995bee34ca2ac2cdc29a901b4cdb072c9c4131", size = 521477, upload-time = "2026-06-29T13:02:46.724Z" }, + { url = "https://files.pythonhosted.org/packages/2a/d3/0001c8c0c5976af2625bb1cfb1895e8ec693b6589fe4574b8e6fc2c85501/jiter-0.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5ed466aee31294d7cdcd4d37dfe5c42c97bc29d9a5f00eacf24504358309cb9b", size = 552187, upload-time = "2026-06-29T13:02:48.144Z" }, + { url = "https://files.pythonhosted.org/packages/f6/76/311b718e07e85740e48619c0632b36f7e0b8d113984499e436452ed13a9a/jiter-0.16.0-cp310-cp310-win32.whl", hash = "sha256:b42e9ff5376819c053da25809a8d4b6fa6e473b4856ebe42e298ac958be3d7f9", size = 206513, upload-time = "2026-06-29T13:02:49.515Z" }, + { url = "https://files.pythonhosted.org/packages/db/7f/ac680eeb0777dc0eb7dc824800ba27880d7f6bc712e362d34ad8ee559f36/jiter-0.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:10438939205546132189c8e74a2d536a707841f3a25cd7c74ee91fe503407a26", size = 199505, upload-time = "2026-06-29T13:02:50.829Z" }, + { url = "https://files.pythonhosted.org/packages/4e/3f/fae6cc967d120ec89e31c5418a51176d8278b3087fbb384a9176754f353c/jiter-0.16.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:67fddeda1688f0cce2d2ae83ccf8a80f79936f2d2997d6cc2261f82fdb54a4d3", size = 309289, upload-time = "2026-06-29T13:02:52.301Z" }, + { url = "https://files.pythonhosted.org/packages/c8/e3/97c6c3562c077f6247d6e6ce5c82562500b6316c0d928e97e106b7a1321a/jiter-0.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c90c0f63df322be920eda6ce622e3083d8906ba267f8220fe7873213b8b4430e", size = 315181, upload-time = "2026-06-29T13:02:53.964Z" }, + { url = "https://files.pythonhosted.org/packages/7b/89/d8d073f8aa2667e46c6c0873f86fe4a512bba4293cc730f626a076211a62/jiter-0.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64c0203212098470032aabcde9356fc168f377aade3e43def61dfe17e92f2037", size = 340939, upload-time = "2026-06-29T13:02:55.412Z" }, + { url = "https://files.pythonhosted.org/packages/87/c9/db4fda3ed73fb864139305e935e5b8b38a5a24692a5a9dd356c22f1b9c8d/jiter-0.16.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:12288303c9844e61e1651d02a9a6f6633e47d39f897d6991d1427161ce6b746e", size = 364932, upload-time = "2026-06-29T13:02:57.28Z" }, + { url = "https://files.pythonhosted.org/packages/a2/74/52b5e86241057f52ddd7c9a580f90effb51f9d06239f6fc612279b91a838/jiter-0.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cf109d010b4b05a105afb3d43be36a21322d345ad3111e13d15f680afef0e5b", size = 461132, upload-time = "2026-06-29T13:02:58.994Z" }, + { url = "https://files.pythonhosted.org/packages/a9/87/544a700f7447c1f31c5d7833821a4daa5683165c2d5a094fbf5b5800c3dc/jiter-0.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:62c1b7fe1f77925acf5af68b6140b8810fa87dfd4dc0a9c8568ec2fa2a10429c", size = 374857, upload-time = "2026-06-29T13:03:00.455Z" }, + { url = "https://files.pythonhosted.org/packages/40/cd/0fcc3f7d39183674d5bfa9ec640faaeb506c60be7c8f94625dfba366e37c/jiter-0.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8597d23c87f59294f83bcb6229b9ed1fccee13dbba967b46930d2f1759466fee", size = 347053, upload-time = "2026-06-29T13:03:02.045Z" }, + { url = "https://files.pythonhosted.org/packages/5c/ae/c7e64e7932ad597fa395b61440b249ada6366716e25c6e08dd2afbd021e6/jiter-0.16.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:3126a5dbad56401989ac769aca0cb56005bfb3e2366eea0ca99d1a91c3c1ee03", size = 356153, upload-time = "2026-06-29T13:03:03.706Z" }, + { url = "https://files.pythonhosted.org/packages/d4/1c/1c719044f14da814e1a060191ab19b96f3e99207bc5b4bfc6d6be34b3f80/jiter-0.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c4b4717bdb35ae456f831a6b08d01880fff399887a6bbc526a583a406e484eea", size = 393956, upload-time = "2026-06-29T13:03:05.165Z" }, + { url = "https://files.pythonhosted.org/packages/3b/dc/7b2f303a2847207e265503853a2d964a55354cffd62a5f2936c155486798/jiter-0.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:adff21bc78edfe086c15eb495b900306076de378dc2337c132401fc39bd79c91", size = 521081, upload-time = "2026-06-29T13:03:06.886Z" }, + { url = "https://files.pythonhosted.org/packages/c2/5f/501cf6e1e09caeb420195179ffc6f62aca603f1220ec53fd80d0d70b3e56/jiter-0.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dab907db06fc593645e73109acf4581ba5b548897d28b9348dc41ddc8343b2d3", size = 552085, upload-time = "2026-06-29T13:03:08.339Z" }, + { url = "https://files.pythonhosted.org/packages/79/54/aa5be86520113b79455c3877f3d1f07a348098df4083ba3688e9537e52dd/jiter-0.16.0-cp311-cp311-win32.whl", hash = "sha256:560b2cf3fb03240cd34f27409a238547488708f05b7c3924f571a60422251ec7", size = 206755, upload-time = "2026-06-29T13:03:09.653Z" }, + { url = "https://files.pythonhosted.org/packages/64/ec/2feb893eb330bd69b413866f4d5daada33c3962f1c6f270c91ca2d87fdf9/jiter-0.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:e431cfc9caf44c1d5459ff77d4e64cbf85fddb6a35dad836a15c6a9ec23087c1", size = 199155, upload-time = "2026-06-29T13:03:10.979Z" }, + { url = "https://files.pythonhosted.org/packages/b9/9c/ca040d94415048a3666fc237774df8151c96f8d2b661cbe3b184acc95876/jiter-0.16.0-cp311-cp311-win_arm64.whl", hash = "sha256:2a8e9e39cf083016137aa5cadafe3188adc2ba6ba1fbf1e5d18889ad3e9ad056", size = 194403, upload-time = "2026-06-29T13:03:12.341Z" }, + { url = "https://files.pythonhosted.org/packages/83/2b/52ace16ed031354f0539749a49e4bf33797d82bea5137910835fa4b09793/jiter-0.16.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:67c3bc1760f8c99d805dcab4e644027142a53b1d5d861f18780ebdbd5d40b72a", size = 306943, upload-time = "2026-06-29T13:03:14.035Z" }, + { url = "https://files.pythonhosted.org/packages/94/2e/34957c2c1b661c252ba9bcc60ae0bddc27e0f7202c6073326a13c5390eec/jiter-0.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5af7780e4a26bd7d0d989592bf9ef12ebf806b74ab709223ecca37c749872ea9", size = 307779, upload-time = "2026-06-29T13:03:15.418Z" }, + { url = "https://files.pythonhosted.org/packages/88/6c/59bd309cab4460c54cf1079f3eb7fe7af6a4c895c5c957a53378693bad2b/jiter-0.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5bf78d0e05e45cfdd66558893938d59afe3d1b1a824a202039b20e607d25a72", size = 335826, upload-time = "2026-06-29T13:03:17.11Z" }, + { url = "https://files.pythonhosted.org/packages/3b/8c/f5ef7b65f0df47afa16596969defb281ebb86e96df346d62be6fd853d620/jiter-0.16.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f4444a83f946605990c98f625cdd3d2725bfb818158760c5748c653170a20e0e", size = 362573, upload-time = "2026-06-29T13:03:18.781Z" }, + { url = "https://files.pythonhosted.org/packages/2b/0b/ace4354da061ee38844a0c27dc2c21eecd27aea119e8da324bea987522d0/jiter-0.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3a23f0e4f957e1be65752d2dfac9a5a06b1917af8dc85deb639c3b9d02e31290", size = 457979, upload-time = "2026-06-29T13:03:20.293Z" }, + { url = "https://files.pythonhosted.org/packages/55/40/c0253d3772eb9dcd8e6606ee9b2d53ec8e5b814589c47f140aa585f21eaa/jiter-0.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c22a488f7b9218e245a0025a9ba6b100e2e54700831cf4cf16833a27fba3ad01", size = 372302, upload-time = "2026-06-29T13:03:21.739Z" }, + { url = "https://files.pythonhosted.org/packages/a8/d2/4839422241aa12860ce597b20068727094ba0bc480723c74924ca5bad483/jiter-0.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46add52f4ad47a08bfb1219f3e673da972191489a33016edefdb5ea55bfa8c48", size = 343805, upload-time = "2026-06-29T13:03:23.384Z" }, + { url = "https://files.pythonhosted.org/packages/e2/59/e196888a05befdda7dbe299b722d56f2f6eec65402bc34c0a3306d595feb/jiter-0.16.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:9c8a956fd72c2cf1e730d01ea080341f13aa0a97a4a33b51abebe725b7ae9ca9", size = 351107, upload-time = "2026-06-29T13:03:24.815Z" }, + { url = "https://files.pythonhosted.org/packages/ec/74/4cd9e0fca65232136400354b630fbfcd2de634e22ccbb96567725981b548/jiter-0.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:561926e0573ffe4a32498420a76d64b16c513e1ab413b9d28158a8764ac701e5", size = 388441, upload-time = "2026-06-29T13:03:26.266Z" }, + { url = "https://files.pythonhosted.org/packages/d9/8c/554691e48bc711299c0a293dd8a6179e24b2d66a54dc295421fcf64569c0/jiter-0.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:44d019fa8cdaf89bf29c71b39e3712143fdd0ac76725c6ef954f9957a5ea8730", size = 516354, upload-time = "2026-06-29T13:03:28.02Z" }, + { url = "https://files.pythonhosted.org/packages/a4/cb/01e9d69dc2cc6759d4f91e230b34489c4fdb2518992650633f9e20bece89/jiter-0.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:0df91907609837f33341b8e6fe73b95991fdaa57caf1a0fbd343dffe826f386f", size = 547880, upload-time = "2026-06-29T13:03:29.534Z" }, + { url = "https://files.pythonhosted.org/packages/79/70/2953195f1c6ad00f49fa67e13df7e60acb3dd4f387101bc15abccddd905e/jiter-0.16.0-cp312-cp312-win32.whl", hash = "sha256:51d7b836acb0108d7c77df1742332cac2a1fa04a74d6dacec46e7091f0e91274", size = 203473, upload-time = "2026-06-29T13:03:31.025Z" }, + { url = "https://files.pythonhosted.org/packages/2d/05/2909a8b10699a4d560f8c502b6b2c5f3991b682b1922c1eedda242b225bd/jiter-0.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:1878349266f8ee36ecb1375cc5ba2f115f35fd9f0a1a4119e725e379126647f7", size = 196905, upload-time = "2026-06-29T13:03:32.472Z" }, + { url = "https://files.pythonhosted.org/packages/e9/a9/6b82bb1c8d7790d602489b967b982a909e5d092875a6c2ade96444c8dfc5/jiter-0.16.0-cp312-cp312-win_arm64.whl", hash = "sha256:2ed5738ae4af18271a51a528b8811b0cbfa4a1858de9d83359e4169855d6a331", size = 190618, upload-time = "2026-06-29T13:03:34.672Z" }, + { url = "https://files.pythonhosted.org/packages/91/c0/555fc60473d30d66894ba825e63615e3be7524fac23858356afa7a38906c/jiter-0.16.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:41977aa5654023948c2dae2a81cbf9c43343954bef1cd59a154dd15a4d84c195", size = 306203, upload-time = "2026-06-29T13:03:36.243Z" }, + { url = "https://files.pythonhosted.org/packages/d0/2b/c3eaf16f5d7c9bad66ea32f40a95bd169b29a91217fcc7f081375157e99c/jiter-0.16.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d28bb3c26762358dadf3e5bf0bccd29ae987d65e6988d2e6f49829c76b003c09", size = 306489, upload-time = "2026-06-29T13:03:37.846Z" }, + { url = "https://files.pythonhosted.org/packages/96/3f/02fdfc6705cad96127d883af5c34e4867f554f29ec7705ec1a46156400a9/jiter-0.16.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0542a7189c26920778658fc8fcf2af8bae05bae9924577f71804acef37996536", size = 335453, upload-time = "2026-06-29T13:03:39.221Z" }, + { url = "https://files.pythonhosted.org/packages/b2/a6/e4bda5920d4b0d7c5dfb7174ce4a6b2e4d3e11c9162c452ef0eab4cdbdbd/jiter-0.16.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8fb8de1e23a0cb2a7f53c335049c7b72b6db41aa6227cdcc0972a1de5cb39450", size = 361625, upload-time = "2026-06-29T13:03:40.597Z" }, + { url = "https://files.pythonhosted.org/packages/b7/97/4e6b59b2c6e55cbb3e183595f81ad65dcfb21c915fee5e19e335df21bc55/jiter-0.16.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b72d0b2990ca754a9102779ac98d8597b7cb31678958562214a007f909eab78e", size = 456958, upload-time = "2026-06-29T13:03:42.074Z" }, + { url = "https://files.pythonhosted.org/packages/15/e0/97e9557686d2f94f4b93786eccb7eed28e9228ad132ea8237f44727314a7/jiter-0.16.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5f91b1c27fc22a57993d5a5cb8a627cb8ed4b10502716fac1ffbfe1d19d84e8", size = 372017, upload-time = "2026-06-29T13:03:43.658Z" }, + { url = "https://files.pythonhosted.org/packages/0f/94/db768b6938e0df35c86beeba3dfbbb025c9ee5c19e1aa271f2396e50864d/jiter-0.16.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c682bea068a90b764577bdb78a60a4c1d1606daf9cd4c893832a37c7cc9d9026", size = 343320, upload-time = "2026-06-29T13:03:45.226Z" }, + { url = "https://files.pythonhosted.org/packages/c1/d6/5a59d938244a30735fe62d9433fd325f9021ea29d89780ea4596ea93bc89/jiter-0.16.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:8d031aabecc4f1b6276adfb42e3aabb77c89d468bf616600e8d3a11328929053", size = 350520, upload-time = "2026-06-29T13:03:46.671Z" }, + { url = "https://files.pythonhosted.org/packages/67/f8/c4a857f49c9af125f6bbcac7e3eee7f7978ed89682833062e2dbf62576b1/jiter-0.16.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:eab2cd170150e70153de16896a1774e3a1dca80154c56b54d7a812c479a7165e", size = 387550, upload-time = "2026-06-29T13:03:48.361Z" }, + { url = "https://files.pythonhosted.org/packages/8b/d6/5fbc2f7d6b67b754caa61a993a2e626e815dec47ffc2f9e35f01adfebec7/jiter-0.16.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:6edb63a46e65a82c26800a868e49b2cac30dd5a4218b88d74bc2c848c8ad60bb", size = 515424, upload-time = "2026-06-29T13:03:49.881Z" }, + { url = "https://files.pythonhosted.org/packages/ed/54/284f0164b64a5fed915fea6ba7e9ba9b3d8d37c67d59cf2e3bb99d45cdfe/jiter-0.16.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:659039cc50b5addcc35fcc87ae2c1833b7c0a8e5326ef631a75e4478447bcf84", size = 546981, upload-time = "2026-06-29T13:03:51.363Z" }, + { url = "https://files.pythonhosted.org/packages/13/c5/2a467585a576594384e1d2c43e1224deaafc085f24e243529cf98beef8e1/jiter-0.16.0-cp313-cp313-win32.whl", hash = "sha256:c9c53be232c2e206ef9cdbad81a48bfa74c3d3f08bcf8124630a8a748aad993e", size = 202853, upload-time = "2026-06-29T13:03:53.015Z" }, + { url = "https://files.pythonhosted.org/packages/88/6a/de61d04b9eec69c71719968d2f716532a3bc121170c44a39e14979c6be81/jiter-0.16.0-cp313-cp313-win_amd64.whl", hash = "sha256:baad945ed47f163ad833314f8e3288c396118934f94e7bbb9e243ce4b341a4fd", size = 196160, upload-time = "2026-06-29T13:03:54.447Z" }, + { url = "https://files.pythonhosted.org/packages/19/4b/b390ed59bafb3f31d008d1218578f10327714484b334439947f7e5b11e7f/jiter-0.16.0-cp313-cp313-win_arm64.whl", hash = "sha256:3c1fd2dbe1b0af19e987f03fe66c5f5bd105a2229c1aff4ab14890b24f41d21a", size = 189862, upload-time = "2026-06-29T13:03:55.754Z" }, + { url = "https://files.pythonhosted.org/packages/a7/89/bc4f1b57d5da938fd344a466396541e586d161320d70bffd929aaafcd8f4/jiter-0.16.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:b2c61484666ad42726029af0c00ef4541f0f3b5cdc550221f56c2343208018ee", size = 308239, upload-time = "2026-06-29T13:03:57.205Z" }, + { url = "https://files.pythonhosted.org/packages/65/7a/c415453e5213001bf3b411ff65dec3d303b0e76a4a2cfea9768cd4960994/jiter-0.16.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:63efadc657488f45db1c676d81e704cac2abf3fdb892def1faea61db053127e2", size = 308928, upload-time = "2026-06-29T13:03:58.643Z" }, + { url = "https://files.pythonhosted.org/packages/11/fc/1f4fb7ebf9a724c7741994f4aae18fba1e2f3133df14521a79194952c34a/jiter-0.16.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf0d73f50e7b6935677854f6e8e31d499ca7064dd24734f703e060f5b237d883", size = 336998, upload-time = "2026-06-29T13:04:00.071Z" }, + { url = "https://files.pythonhosted.org/packages/a0/8d/72cadaac05ccfa7cc3a0a2232862e6c72443ca40cf300ba8b57f9f18b69b/jiter-0.16.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bf3ea07d9bc8e7d03a9fbc051295462e6dbc295b894fd72457c3136e3e43d898", size = 362112, upload-time = "2026-06-29T13:04:01.52Z" }, + { url = "https://files.pythonhosted.org/packages/58/4a/c4b0d5f651fda90a24ffce9f8d56cde462a2e09d31ae3de3c68cef34c04e/jiter-0.16.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:26798522707abb47d767db536e4148ceac1b14446bf028ee85e579a2e043cfe5", size = 459807, upload-time = "2026-06-29T13:04:03.214Z" }, + { url = "https://files.pythonhosted.org/packages/80/58/ef77879ea9aa56b50824edc5a445e226422c7a8d211f3fd2a56bcb9493cf/jiter-0.16.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bc837c1b9631be10abfe0191537fe8009838204cec7e44827401ace390ddb567", size = 373181, upload-time = "2026-06-29T13:04:04.629Z" }, + { url = "https://files.pythonhosted.org/packages/49/2e/ffbc3f254e4d8a66da3062c624a7df4b7c2b2cf9e1fe43cf394b3e104041/jiter-0.16.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49060fd70737fad59d33ba9dcc0d83247dc9e77187de26053a19c16c9f32bd69", size = 344927, upload-time = "2026-06-29T13:04:06.067Z" }, + { url = "https://files.pythonhosted.org/packages/9a/f6/0be5dc6d64a89f80aa8fec984f94dedb2973e251edcae55841d60786d578/jiter-0.16.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:adbb8edeadd431bc4477879d5d371ece7cb1334486584e0f252656dd7ffada29", size = 352754, upload-time = "2026-06-29T13:04:07.477Z" }, + { url = "https://files.pythonhosted.org/packages/da/6e/7d31243b3b91cd261dd19e9d3557fc3251a80883d3d8049c86174e7ab7af/jiter-0.16.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:31aaee5b80f672c1dc21272bcfb9cbdcfc1ea04ff50f00ed5af500b80c44fa93", size = 390553, upload-time = "2026-06-29T13:04:08.92Z" }, + { url = "https://files.pythonhosted.org/packages/25/33/51ae371fde3c88897520f62b4d5f8b27ad7103e2bb10812ff52195609853/jiter-0.16.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:6722bcef4ffc86c835574b1b2fac6b33b9fb4a889c781e67950e891591f3c55a", size = 516900, upload-time = "2026-06-29T13:04:10.407Z" }, + { url = "https://files.pythonhosted.org/packages/a0/45/6449b3d123ea439ba79507c657288f461d55049e7bcbdc2cf8eb8210f491/jiter-0.16.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:5ab4f50ff971b611d656554ea10b75f80097392c827bc32923c6eeb6386c8b00", size = 548754, upload-time = "2026-06-29T13:04:12.046Z" }, + { url = "https://files.pythonhosted.org/packages/9b/e7/fd2fb11ae3e2649333da3aa170d04d7b3000bbdc3b270f6513382fdf4e04/jiter-0.16.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:710cc51d4ebdcd3c1f70b232c1db1ea1344a075770422bbd4bede5708335acbe", size = 122381, upload-time = "2026-06-29T13:04:13.413Z" }, + { url = "https://files.pythonhosted.org/packages/26/80/f0b147a62c315a164ed2168908286ca302310824c218d3aae52b06c0c9a9/jiter-0.16.0-cp314-cp314-win32.whl", hash = "sha256:57b37fc887a32d44798e4d8ebfa7c9683ff3da1d5bf38f08d1bb3573ccb39106", size = 204578, upload-time = "2026-06-29T13:04:14.813Z" }, + { url = "https://files.pythonhosted.org/packages/5e/e6/4758a14304b4523a6f5adb2419340086aa3593bd4327c2b25b5948a90548/jiter-0.16.0-cp314-cp314-win_amd64.whl", hash = "sha256:cbd18dd5e2df96b580487b5745adf57ef64ad89ba2d9662fc3c19386acce7db8", size = 198154, upload-time = "2026-06-29T13:04:16.272Z" }, + { url = "https://files.pythonhosted.org/packages/26/be/41fa54a2e7ea41d6c99f1dc5b1f0fd4cb474680304b5d268dd518e81da3a/jiter-0.16.0-cp314-cp314-win_arm64.whl", hash = "sha256:a32d2027a9fa67f109ff245a3252ece3ccc32cc56703e1deab6cc846a59e0585", size = 191458, upload-time = "2026-06-29T13:04:17.707Z" }, + { url = "https://files.pythonhosted.org/packages/81/6b/59127338b86d9fe4d99418f5a15118bea778103ee0fe9d9dd7e0af174e95/jiter-0.16.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2577196f4474ef3fc4779a088a23b0897bbf86f9ea3679c372d45b8383b43207", size = 316739, upload-time = "2026-06-29T13:04:19.663Z" }, + { url = "https://files.pythonhosted.org/packages/2d/95/49461034d5388196d3dabf98748935f017b7785d8f3f5349f834bcc4ed0d/jiter-0.16.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:616e89e008a93c01104161c75b4988e58716b01d62307ebfe161e52a56d2a818", size = 340911, upload-time = "2026-06-29T13:04:21.257Z" }, + { url = "https://files.pythonhosted.org/packages/cd/97/a4369f2fb82cb3dda13b98622f31249b2e014b223fe64ee534413ad72294/jiter-0.16.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e2e9efbe042210df657bade597f66d6d75723e3d8f45a12ea6d8167ff8bbce3", size = 361747, upload-time = "2026-06-29T13:04:22.677Z" }, + { url = "https://files.pythonhosted.org/packages/28/51/49b6ed456261646e1906016a6760367a28aacd3c24805e4e5fe64116c1db/jiter-0.16.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f4d9e473a5ce7d27fef8b848df4dc16e283893d3f53b4a585e72c9595f3c284", size = 460225, upload-time = "2026-06-29T13:04:24.441Z" }, + { url = "https://files.pythonhosted.org/packages/33/b5/5689aff4f66c5b60be63106e591dbfcba2190df97d2c9c7cf052361ddb98/jiter-0.16.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d30a4a1c87713060c8d1cc59a7b6c8fb6b8ef0a6900368014c76c87922a2929", size = 373169, upload-time = "2026-06-29T13:04:25.884Z" }, + { url = "https://files.pythonhosted.org/packages/a2/96/3ae1b85ee0d6d6cab254fb7f8da018272b932bbf2d69b07e98aa2a96c746/jiter-0.16.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bae96332410f866e5900d809298b1ed82735932986c672495f9701daacd80620", size = 350332, upload-time = "2026-06-29T13:04:27.302Z" }, + { url = "https://files.pythonhosted.org/packages/15/32/c99d7bafd78986556c95bf60ce84c6cc98786eac56066c12d7f828bb6747/jiter-0.16.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:da3d7ec75dc83bb18bca888b5edfae0656a26849056c59e05a7728badd17e7af", size = 353377, upload-time = "2026-06-29T13:04:28.731Z" }, + { url = "https://files.pythonhosted.org/packages/0e/4b/f99a8e571287c3dec766bcc18528bbe8e8fb5365522ab5e6d64c93e87066/jiter-0.16.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ee6162b77d49a9939229df666dfa8af3e656b6701b54c4c84966d740e189264e", size = 387746, upload-time = "2026-06-29T13:04:30.319Z" }, + { url = "https://files.pythonhosted.org/packages/75/69/c78a5b3f71040e34eb5917df26fb7ae9a2174cad1ccbf277512507c53a6e/jiter-0.16.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:63ffdbdae7d4499f4cda14eadc12ddcabef0fc0c081191bdc2247489cb698077", size = 517292, upload-time = "2026-06-29T13:04:31.709Z" }, + { url = "https://files.pythonhosted.org/packages/c2/f7/095b38eda4c70d03651c403f29a5590f16d12ddc5d544aac9f9cddf72277/jiter-0.16.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:a111256a7193bea0759267b10385e5870949c239ed7b6ddbaaf57573edb38734", size = 549259, upload-time = "2026-06-29T13:04:33.721Z" }, + { url = "https://files.pythonhosted.org/packages/2e/c5/6a0207d90e5f656d95af98ebd0934f382d37674416f215aeda2ff8063e51/jiter-0.16.0-cp314-cp314t-win32.whl", hash = "sha256:de5ba8763e56b793561f43bed197c9ea55776daa5e9a6b91eed68a909bc9cdbf", size = 206523, upload-time = "2026-06-29T13:04:35.068Z" }, + { url = "https://files.pythonhosted.org/packages/a5/31/c757d5f30a8980fd945ce7b98be10be9e4ff59c7c42f5fd86804c2e87db8/jiter-0.16.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b8a3f9a6008048fe9def7bf465180564a6e458047d2ce499149cfbe73c3ae9db", size = 200366, upload-time = "2026-06-29T13:04:36.61Z" }, + { url = "https://files.pythonhosted.org/packages/7c/a2/d88de6d313d734a544a7901353ad5db67cb38dcfcd91713b7979dafc345d/jiter-0.16.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0fa25b09b13075c46f5bc174f2690525a925a4fc2f7c82969a2bbabff22386ce", size = 190516, upload-time = "2026-06-29T13:04:38.004Z" }, + { url = "https://files.pythonhosted.org/packages/06/d3/8e278946d43eeca2585b4dd0834a887cd71136329b837f3a16ed86a8b4b0/jiter-0.16.0-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:850ccb1d7eedb4200f4014b1c0e8a577de114fc3cd88faad646dcc9bc4bb12ad", size = 304518, upload-time = "2026-06-29T13:05:00.172Z" }, + { url = "https://files.pythonhosted.org/packages/72/43/28d4ef495028bf0506a413d4db3f4eb3e7288a382e0f065f306a17bbeb5e/jiter-0.16.0-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:e34e97bda77eb63242a410243c071e28ac7e0d8c0948c5ee658498690a4b2f2f", size = 310207, upload-time = "2026-06-29T13:05:02.123Z" }, + { url = "https://files.pythonhosted.org/packages/e0/ca/c366b1012da1d640de975d9683acd44e4d150d9068845d0ca2610435253f/jiter-0.16.0-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b7dc85ea77d4abbae8bad0d3538678aedee75bceec4e2f6c8dfb1c74772e5aa5", size = 342771, upload-time = "2026-06-29T13:05:03.55Z" }, + { url = "https://files.pythonhosted.org/packages/16/52/50cc4056fc1ae02e7154704e7ecc89df0afb8300222cfe8a52d3f67e4730/jiter-0.16.0-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17ca7fae79f6d99cd9a042b75f917eaada7b895cfc7dd2ee3a16089dcaec7a85", size = 346468, upload-time = "2026-06-29T13:05:05.452Z" }, + { url = "https://files.pythonhosted.org/packages/98/ab/664fd8c4be028b2bedd3d2ff08769c4ede23d0dbc87a77c62384a0515b5d/jiter-0.16.0-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:f17d61a28b4b3e0e3e2ba98490c70501403b4d196f78732439160e7fd3678127", size = 303106, upload-time = "2026-06-29T13:05:07.118Z" }, + { url = "https://files.pythonhosted.org/packages/1a/07/421f1d5b65493a76e16027b848aba6a7d28073ae75944fa4289cc914d39f/jiter-0.16.0-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:96e38eea538c8ddf853a35727c7be0741c76c13f04148ac5c116222f50ece3b3", size = 304658, upload-time = "2026-06-29T13:05:08.708Z" }, + { url = "https://files.pythonhosted.org/packages/0a/db/bba1155f01a01c3c37a89425d571da751bbedf5c54247b831a04cb971798/jiter-0.16.0-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d284fb8d94d5855d60c44fefcab4bf966f1da6fada73992b01f6f0c9bc0c6702", size = 339719, upload-time = "2026-06-29T13:05:10.41Z" }, + { url = "https://files.pythonhosted.org/packages/78/f7/18a1afcd64f35314b68c1f23afcd9994d0bc13e65cc77517afff4e83986d/jiter-0.16.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64d613743df53199b1aa256a7d328340da6d7078aac7705a7db9d7a791e9cfd2", size = 343885, upload-time = "2026-06-29T13:05:12.087Z" }, +] + [[package]] name = "jsonschema" version = "4.26.0" @@ -970,6 +1108,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b4/07/458c344f0f0c178f4481dad5cca790626ffe4c34eabf9467069d06ee4999/numpy-2.5.2-cp315-cp315t-win_arm64.whl", hash = "sha256:5f8e00be2ec6f45f4e8a41a527f68d44a7d96fee92a650e4d8b1326f77f61e6e", size = 10748103, upload-time = "2026-08-09T13:48:24.21Z" }, ] +[[package]] +name = "openai" +version = "2.54.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "httpx" }, + { name = "jiter" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/50/9a/8c75e8c8a5b407a0586faeb2afac91674ff955c191ecc1d6d3b6669f6788/openai-2.54.0.tar.gz", hash = "sha256:e3e6f8bc1ba30ddf381ace1a14340eed381cb984a1a59bd0f34b5be3b5d49cfa", size = 1100285, upload-time = "2026-08-11T18:46:59.035Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/a8/bb76c7356de8ad57f59d5ff993d434df0607f07f08bcc9c9a5c275e399c0/openai-2.54.0-py3-none-any.whl", hash = "sha256:89089789197ccdb87f173a03145ed1598d00795220c93e96cf712b1cbf5e5f2b", size = 1660351, upload-time = "2026-08-11T18:46:56.684Z" }, +] + [[package]] name = "opentelemetry-api" version = "1.44.0" @@ -1625,6 +1782,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/10/85/48f0abdcef5cce4e034c7a5b0ceeceba0b01bf0d942824f4bb720afe2dec/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:8e65860d238379ed982fd9ba690579b5e95af2f4840f99c772816dbe573cb826", size = 586486, upload-time = "2026-06-30T07:17:51.141Z" }, ] +[[package]] +name = "sniffio" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload-time = "2024-02-25T23:20:04.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, +] + [[package]] name = "sortedcontainers" version = "2.4.0" @@ -1750,6 +1916,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, ] +[[package]] +name = "tqdm" +version = "4.70.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/21/3b/6c24bec5be5e743ffd99576daa5cc077722fc7d5bbc00bd133fa0c698dc6/tqdm-4.70.0.tar.gz", hash = "sha256:55b0b0dbd97462d06ebee91e4dac24ed4d4702be82b24f07e6c1d27e08cea220", size = 795438, upload-time = "2026-07-27T11:33:15.271Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/1c/01bfd571a64e7f270e6bab5e33777debe0edc56759233ce84f27dec92d14/tqdm-4.70.0-py3-none-any.whl", hash = "sha256:7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953", size = 80184, upload-time = "2026-07-27T11:33:13.167Z" }, +] + [[package]] name = "typing-extensions" version = "4.16.0" From a95d3c8c1160c821d4e1ed7a2fbe50eaed176b7f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 14:14:34 +0900 Subject: [PATCH 75/78] fix(audit): record timeout policy changes and history access Signed-off-by: Seongho Bae --- contextual_orchestrator/orchestrator.py | 9 +++++ contextual_orchestrator/server.py | 5 +++ docs/product-technical-gap-baseline.md | 17 +++++++++ tests/test_model_timeout_policy.py | 50 +++++++++++++++++++++++++ 4 files changed, 81 insertions(+) diff --git a/contextual_orchestrator/orchestrator.py b/contextual_orchestrator/orchestrator.py index 05d0aa1d3..dbe85ddcb 100644 --- a/contextual_orchestrator/orchestrator.py +++ b/contextual_orchestrator/orchestrator.py @@ -6263,6 +6263,15 @@ def patch_agent( updated_agents = [agent for agent in updated_candidates if not agent.disabled] self.candidates = updated_candidates self.agents = updated_agents + self._append_audit_event( + "model_timeout_policy_changed", + { + "agent_pool_id": agent_pool_id, + "worker_agent_id": worker_agent_id, + "revision": revision, + "restored_from_revision": restored_from_revision, + }, + ) return self._agent_to_admin_payload(patched) if self._pool_store is not None: self._pool_store.save(patched) diff --git a/contextual_orchestrator/server.py b/contextual_orchestrator/server.py index 5d3fac1a2..4bd91ede3 100644 --- a/contextual_orchestrator/server.py +++ b/contextual_orchestrator/server.py @@ -8038,6 +8038,11 @@ def register_video_job(agent: ModelAgent, provider_result: dict[str, Any]) -> di @staticmethod def _admin_purpose(path: str) -> str: """Select the least-privileged purpose for an admin GET route.""" + segments = [part for part in path.split("/") if part] + if (len(segments) == 8 and segments[:3] == ["api", "v1", "agent_pools"] + and segments[4] == "worker_agents" + and segments[6:] == ["timeout_policy", "history"]): + return "audit_replay" if ( path == "/admin/state" or path == "/api/v1/workflow_runs" diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 1cdcd606e..e71f2ea29 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1,5 +1,22 @@ # Contextual Orchestrator: Product & Technical Gap Baseline +## 2026-09-07 timeout audit visibility repair + +On PR #1053 base `1ccc9599415096433214cd9ad0df611eddfc8fbb`, successful +timeout set, clear, and restore operations had durable policy history but no +operator audit event. A local regression reproduced the missing events; the +repair adds committed revision references to the existing audit stream. +History GET access now uses the existing durable replay-authorization path. +An authenticated loopback HTTP regression reproduced missing access auditing +and now checks both successful access and HTTP 503 when audit recording fails. + +The timeout-policy and agent-pool suites passed 73 tests in 12.30s before this +documentation update. No live provider, protected merge, or deployment is +claimed. The HTTP test spies on the durable audit call; it does not prove +storage survival after a crash. Policy history remains atomic with the policy +update; the separate operator event is not a new cross-store transaction. +Default-null model timeout and explicit administrator control are unchanged. + ## 2026-09-06 model-specific timeout policy: local, not delivered PR #1053's remote `661ce8db` has a completed 3400-pass/2-skip regression suite diff --git a/tests/test_model_timeout_policy.py b/tests/test_model_timeout_policy.py index 6659ce2b9..265ae7ad7 100644 --- a/tests/test_model_timeout_policy.py +++ b/tests/test_model_timeout_policy.py @@ -239,6 +239,56 @@ def test_model_timeout_policy_requires_durable_store() -> None: assert orchestrator._agent(model_agent.id).model_timeout_seconds is None +def test_timeout_history_read_requires_durable_authorization_audit(tmp_path: Path, monkeypatch) -> None: + """A real history GET records replay access, and fails closed without audit.""" + import threading + from urllib.error import HTTPError + from urllib.request import Request, urlopen + from contextual_orchestrator.server import SecurityConfig, build_server + + router = TaskOrchestrator([ModelAgent("timeout_agent", "example-model")], + agents_db=str(tmp_path / "pool.db")) + recorded = [] + monkeypatch.setattr(router, "record_authorization_decision", lambda **kwargs: recorded.append(kwargs)) + server = build_server(router, port=0, security=SecurityConfig(auth_token="local_test_only")) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + url = f"http://127.0.0.1:{server.server_port}/api/v1/agent_pools/default/worker_agents/timeout_agent/timeout_policy/history" + request = Request(url, headers={"Authorization": "Bearer local_test_only"}) + try: + with urlopen(request, timeout=5) as response: + assert response.status == 200 + assert any(row.get("purpose") == "audit_replay" and row.get("durable") is True + and row["allowed"] is True for row in recorded) + def reject_audit(**kwargs): + raise RuntimeError("audit unavailable") + monkeypatch.setattr(router, "record_authorization_decision", reject_audit) + with pytest.raises(HTTPError) as error: + urlopen(request, timeout=5) + assert error.value.code == 503 + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +def test_timeout_changes_are_visible_in_operator_audit(tmp_path: Path) -> None: + """Set, clear, and restore expose only committed revision references.""" + agent = ModelAgent("timeout_agent", "example-model") + router = TaskOrchestrator([agent], agents_db=str(tmp_path / "pool.db")) + router.patch_agent("default", agent.id, {"model_timeout_seconds": 7200}) + router.patch_agent("default", agent.id, {"model_timeout_seconds": None}) + router.restore_model_timeout("default", agent.id, 1, expected_revision=2, actor_id="a" * 64) + events = [event for event in reversed(router.list_recent_audit_events()) + if event["event_type"] == "model_timeout_policy_changed"] + assert [event["event_detail"]["revision"] for event in events] == [1, 2, 3] + assert events[-1]["event_detail"]["restored_from_revision"] == 1 + with pytest.raises(ValueError, match="reload"): + router.patch_agent("default", agent.id, {"model_timeout_seconds": None}, + expected_timeout_revision=1) + assert len(router.list_recent_audit_events()) == 3 + + def test_model_timeout_policy_records_atomic_history(tmp_path: Path) -> None: """Committed revisions retain their old and new limits in order.""" model_agent = ModelAgent("timeout_agent", "example-model") From 7685d3f66f26655c89915c060559ad0b14ecdbf7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 14:42:21 +0900 Subject: [PATCH 76/78] fix(admin): make model timeout audit entries readable Signed-off-by: Seongho Bae --- contextual_orchestrator/admin.py | 44 +++++++++++++++++++++++++++++--- tests/test_admin_contract.py | 33 ++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 4 deletions(-) diff --git a/contextual_orchestrator/admin.py b/contextual_orchestrator/admin.py index b1227afc1..9a84a978e 100644 --- a/contextual_orchestrator/admin.py +++ b/contextual_orchestrator/admin.py @@ -34,6 +34,15 @@ "models_table_scroll_hint": "Model details table. Scroll horizontally to review latency and success.", "no_agents_configured": "Add a model connection to start routing requests.", "no_audit_events": "Run a workflow to create your first audit event.", + "audit_timeout_changed": "Model time limit updated", + "audit_timeout_restored": "Model time limit restored", + "audit_model_reference": "Model", + "audit_revision_reference": "Revision", + "audit_restored_reference": "Restored from revision", + "audit_date_unknown": "Date unavailable", + "audit_event_heading": "Event", + "audit_detail_heading": "Detail", + "audit_created_heading": "Created", "no_policy_evidence": "No policy evidence is loaded. Open Audit to review recorded events.", "no_recent_errors": "No current alerts. Open Audit to review recent changes.", "prompt_placeholder": "Describe the task you want to route, then run the trace.", @@ -297,6 +306,15 @@ "models_table_scroll_hint": "모델 상세 표입니다. 지연 시간과 성공률을 보려면 가로로 스크롤하세요.", "no_agents_configured": "요청 라우팅을 시작하려면 모델 연결을 추가하세요.", "no_audit_events": "첫 감사 이벤트를 만들려면 워크플로를 실행하세요.", + "audit_timeout_changed": "모델 시간 제한 변경", + "audit_timeout_restored": "모델 시간 제한 복원", + "audit_model_reference": "모델", + "audit_revision_reference": "변경 버전", + "audit_restored_reference": "복원한 버전", + "audit_date_unknown": "날짜 확인 불가", + "audit_event_heading": "변경 내용", + "audit_detail_heading": "상세", + "audit_created_heading": "기록 시각", "no_policy_evidence": "불러온 정책 근거가 없습니다. 기록된 이벤트를 검토하려면 감사를 여세요.", "no_recent_errors": "현재 알림이 없습니다. 최근 변경 사항을 검토하려면 감사를 여세요.", "prompt_placeholder": "라우팅할 작업을 설명한 다음 트레이스를 실행하세요.", @@ -879,6 +897,9 @@ @media (prefers-reduced-motion: reduce) { *, *::before, *::after { scroll-behavior: auto !important; } } + .audit-table { min-width: 0; table-layout: fixed; } + .audit-table th, .audit-table td { white-space: normal; overflow-wrap: anywhere; vertical-align: top; } + .audit-table th:nth-child(2) { width: 45%; } @@ -1067,7 +1088,7 @@