diff --git a/.github/workflows/reasoning-workload-verify.yml b/.github/workflows/reasoning-workload-verify.yml new file mode 100644 index 000000000..fc7b15c0f --- /dev/null +++ b/.github/workflows/reasoning-workload-verify.yml @@ -0,0 +1,68 @@ +name: Reasoning control quality + +on: + pull_request: + paths: + - contextual_orchestrator/_reasoning_*.py + - contextual_orchestrator/reasoning_*.py + - tests/reasoning_fakes.py + - tests/test_reasoning_*.py + - .github/workflows/reasoning-workload-verify.yml + push: + branches: [main] + paths: + - contextual_orchestrator/_reasoning_*.py + - contextual_orchestrator/reasoning_*.py + - tests/reasoning_fakes.py + - tests/test_reasoning_*.py + - .github/workflows/reasoning-workload-verify.yml + +permissions: + contents: read + +concurrency: + group: reasoning-control-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + verify: + name: Reasoning control statement, branch, and docstring coverage + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - name: Checkout exact source revision without credentials + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # actions/checkout@v7 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install hash-locked verification dependencies + run: | + python -m pip install --require-hashes -r fuzz/requirements-property.txt + python -m pip install --require-hashes -r requirements-opencode-review-ci.txt + + - name: Run full tests and changed-module quality gates + run: | + python -m compileall -q contextual_orchestrator tests + python -m pytest -q + python -m coverage erase + python -m coverage run --branch --source=contextual_orchestrator -m pytest -q + python -m coverage report \ + --include='contextual_orchestrator/_reasoning_*.py,contextual_orchestrator/reasoning_*.py' \ + --show-missing \ + --fail-under=100 + python -m interrogate -f 100 \ + contextual_orchestrator/_reasoning_workload.py \ + contextual_orchestrator/_reasoning_policy.py \ + contextual_orchestrator/_reasoning_state.py \ + contextual_orchestrator/_reasoning_orchestrator_hooks.py \ + contextual_orchestrator/_reasoning_workflow.py \ + contextual_orchestrator/_reasoning_client_hooks.py \ + contextual_orchestrator/reasoning_control.py \ + contextual_orchestrator/reasoning_runtime.py + git diff --check diff --git a/CHANGELOG.md b/CHANGELOG.md index 245b2debf..7a35b3148 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and - Add a transport-neutral, versioned model fallback policy that validates explicit cost tiers and deterministically exhausts eligible free candidates before any paid fallback. - Filter fallback candidates by repository visibility, required capability, and configured credential name without retaining or serializing secret values. - Add a standard-library CLI for immutable cross-repository workflow integration, with complete statement, branch, and public-docstring coverage for the fallback policy. +- Add explicit per-agent reasoning capability profiles and provider-neutral canonical levels from `none` through `max` without inferring capability from model names. +- Add role-aware adaptive and fixed-effort policies across routing, conducted workflows, generated planning, model verification, streaming, Responses passthrough, and Batch requests. +- Add bounded verifier-triggered worker escalation and fixed-effort ablation evidence with provider-reported reasoning-token accounting. +- Expose reasoning profiles through agent configuration and admin projections while preserving them across runtime agent replacement and durable re-save. ### Security @@ -28,15 +32,23 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and - Remove fallback-policy environment-value inspection; trusted callers now declare only validated available credential names, and the policy CLI rejects the former secret-bearing environment selector. - Reject boolean or floating-point schema versions, unsafe programmatic agent selectors, non-string candidate identifiers, and mutable credential-control collections as controlled validation errors instead of leaking Python type exceptions or permitting post-validation policy changes. - Document narrowly scoped Semgrep suppressions for parameter-bound database queries, the explicit development-only TLS verification opt-out, and provider URLs that pass the egress guard. +- Validate custom reasoning payload paths and scalar templates, preserve caller-owned reasoning fields, and record only bounded decision evidence rather than private intermediate reasoning text. +- Reject mutable direct-constructor reasoning controls, omitted payload-rule values, and NaN or infinite provider mappings before any reasoning payload is projected. ### Changed - Pin Atheris by Python interpreter so the Python 3.11 fuzz job and the newer central coverage-evidence image both install a published, hash-locked wheel. +- Treat reasoning effort as a third test-time-compute axis alongside model routing and workflow topology, with failover models projecting one canonical decision onto their declared capabilities. +- Keep synthesizer effort below analysis roles by default and require multiple high-impact signals before adaptive policy reaches the model-specific maximum. +- Make adaptive-reasoning activation explicit and idempotent: package import no longer installs reasoning hooks, while the product CLI activates them before loading agent configuration. - Run repository Tests, Fuzz, and Security workflows for stacked pull requests targeting any branch, bind every checkout to the literal contributor-head SHA, and keep checkout credentials non-persistent so local evidence cannot silently become absent or synthetic-merge-only evidence. ### Documentation - Add APA 7 doctoring for Python environment-marker semantics, Atheris artifact availability and hashes, and the supported-platform uncertainty boundary. +- Add architecture, operations, test strategy, provider mapping, governance, and APA 7 research evidence for adaptive reasoning control. +- Record the standalone/MSA activation boundary and import-purity regression contract for optional reasoning control. +- Define the immutable direct-constructor and strict interoperable-JSON boundary for reasoning profile control data. - Add provider-response resource-bound doctoring covering the 8 MiB fail-closed limit, HTTP framing preflight, `text/event-stream` media-type enforcement, bounded SSE reads, OpenAI-compatible `[DONE]` completion evidence, malformed-event and premature-EOF handling, batch-output partitioning, incident handling, and operational rollback. - Add provider-stream UTF-8 doctoring grounding strict SSE/JSON decoding and redacted malformed-input handling in the WHATWG HTML Standard and RFC 8259, with verification, failure, rollback, and authority boundaries. - Add provider-JSON trust-boundary doctoring grounding strict UTF-8 object decoding, duplicate-name and non-finite-number rejection, finite-runtime numeric enforcement for extreme exponents, Batch JSONL validation, redacted parser failures, request-path authority, operator recovery, and rollback in RFC 8259, current Python documentation, and the OpenAI Batch API contract. diff --git a/contextual_orchestrator/__init__.py b/contextual_orchestrator/__init__.py index 276216d2e..1aea777cc 100644 --- a/contextual_orchestrator/__init__.py +++ b/contextual_orchestrator/__init__.py @@ -54,8 +54,31 @@ load_fallback_manifest, ) from .orchestrator import ModelAgent, TaskOrchestrator, WorkflowStep, load_agents +from .reasoning_control import ( + CANONICAL_REASONING_LEVELS, + PayloadRule, + ReasoningAblationCell, + ReasoningDecision, + ReasoningPolicy, + ReasoningProfile, + ReasoningWorkload, + adapt_reasoning_decision, + apply_reasoning_payload, + escalate_reasoning_decision, + extract_reasoning_tokens, + select_reasoning_decision, + sum_usage_tokens, +) +from .reasoning_runtime import ( + agent_reasoning_profile, + configure_agent_reasoning, + configure_orchestrator_reasoning, + current_reasoning_decision, + enable_reasoning_control, + orchestrator_reasoning_policy, + reasoning_override, +) from .token_counting import HeuristicTokenCounter, build_token_counter - __all__ = [ "ModelAgent", "TaskOrchestrator", @@ -114,4 +137,25 @@ "SkippedCandidate", "build_fallback_plan", "load_fallback_manifest", + # adaptive provider reasoning control + "CANONICAL_REASONING_LEVELS", + "PayloadRule", + "ReasoningAblationCell", + "ReasoningDecision", + "ReasoningPolicy", + "ReasoningProfile", + "ReasoningWorkload", + "adapt_reasoning_decision", + "apply_reasoning_payload", + "escalate_reasoning_decision", + "extract_reasoning_tokens", + "select_reasoning_decision", + "sum_usage_tokens", + "agent_reasoning_profile", + "configure_agent_reasoning", + "configure_orchestrator_reasoning", + "current_reasoning_decision", + "enable_reasoning_control", + "orchestrator_reasoning_policy", + "reasoning_override", ] diff --git a/contextual_orchestrator/__main__.py b/contextual_orchestrator/__main__.py index 5f68c3b74..43c11a5ee 100644 --- a/contextual_orchestrator/__main__.py +++ b/contextual_orchestrator/__main__.py @@ -9,6 +9,7 @@ from .credentials import register_credential from .orchestrator import ModelClient, TaskOrchestrator, load_agents +from .reasoning_runtime import enable_reasoning_control from .server import SecurityConfig, serve @@ -55,12 +56,21 @@ def _register_credential_command(argv: list[str]) -> None: print(json.dumps({"registered": args.name, "backend": "kv"}, ensure_ascii=False)) +def _enable_reasoning_runtime() -> None: + """Activate optional reasoning hooks for the executable product runtime.""" + enable_reasoning_control() + + def main() -> None: """Parse CLI options and run bootstrap, prompt completion, or the HTTP server.""" if len(sys.argv) > 1 and sys.argv[1] == "register-credential": _register_credential_command(sys.argv[2:]) return + # Product execution opts in explicitly. Merely importing the package remains + # side-effect free, while configuration loading below understands profiles. + _enable_reasoning_runtime() + parser = argparse.ArgumentParser(description="Route or conduct chat requests across model agents.") parser.add_argument("prompt", nargs="?", help="User prompt for CLI mode.") parser.add_argument("--agents", default="examples/agents.mock.json", help="Agent config JSON.") diff --git a/contextual_orchestrator/_reasoning_client_hooks.py b/contextual_orchestrator/_reasoning_client_hooks.py new file mode 100644 index 000000000..130e08eef --- /dev/null +++ b/contextual_orchestrator/_reasoning_client_hooks.py @@ -0,0 +1,179 @@ +"""Provider-client hooks for reasoning payload and batch projection.""" + +from __future__ import annotations + +from typing import Any, Iterator + +from .reasoning_control import ( + ReasoningPolicy, + ReasoningWorkload, + adapt_reasoning_decision, + apply_reasoning_payload, + select_reasoning_decision, +) +from ._reasoning_state import ( + _ACTIVE_DECISION, + _ACTIVE_POLICY, + _BATCH_DECISIONS, + _append_event, + _decision_scope, + _infer_role, + _input_text, + _message_text, + _resolve_decision, + agent_reasoning_profile, +) +from ._reasoning_workflow import _rewrite_batch_payload + + +def install_client_hooks(model_client_type: type[Any]) -> None: + """Install chat, stream, passthrough, and Batch reasoning hooks.""" + original_client_chat = model_client_type.chat + original_client_stream_chat = model_client_type.stream_chat + original_client_proxy_send = model_client_type.proxy_send + original_client_batch_chat = model_client_type.batch_chat + original_client_send = model_client_type._send + original_client_stream_send = model_client_type._stream_send + original_client_send_raw = model_client_type._send_raw + original_client_batch_upload = model_client_type._batch_upload + + def client_chat( + self: Any, + agent: Any, + messages: list[dict[str, str]], + temperature: float = 0.2, + ) -> str: + """Keep one role-aware decision active through chat payload construction.""" + role = _infer_role(messages) + decision = _resolve_decision(agent, _message_text(messages), role) + with _decision_scope(decision): + output = original_client_chat(self, agent, messages, temperature) + _append_event(agent, role, decision) + return output + + def client_stream_chat( + self: Any, + agent: Any, + messages: list[dict[str, str]], + temperature: float = 0.2, + ) -> Iterator[str]: + """Keep one decision active until a streaming provider response completes.""" + role = _infer_role(messages) + decision = _resolve_decision(agent, _message_text(messages), role) + with _decision_scope(decision): + yield from original_client_stream_chat(self, agent, messages, temperature) + _append_event(agent, role, decision) + + def client_proxy_send( + self: Any, + agent: Any, + endpoint: str, + payload: dict[str, Any], + ) -> dict[str, Any]: + """Apply a worker decision to full-shape chat or Responses passthrough.""" + decision = _resolve_decision(agent, _input_text(payload), "worker") + with _decision_scope(decision): + output = original_client_proxy_send(self, agent, endpoint, payload) + _append_event(agent, "worker", decision) + return output + + def client_send(self: Any, agent: Any, payload: dict[str, Any]) -> str: + """Project the active decision into a chat-completions payload.""" + profile = agent_reasoning_profile(agent) + decision = adapt_reasoning_decision(profile, _ACTIVE_DECISION.get()) + return original_client_send( + self, + agent, + apply_reasoning_payload(payload, profile, decision, "chat/completions"), + ) + + def client_stream_send( + self: Any, + agent: Any, + payload: dict[str, Any], + ) -> Iterator[str]: + """Project the active decision into a streaming chat payload.""" + profile = agent_reasoning_profile(agent) + decision = adapt_reasoning_decision(profile, _ACTIVE_DECISION.get()) + yield from original_client_stream_send( + self, + agent, + apply_reasoning_payload(payload, profile, decision, "chat/completions"), + ) + + def client_send_raw( + self: Any, + agent: Any, + endpoint: str, + payload: dict[str, Any], + ) -> dict[str, Any]: + """Project the active decision into chat or Responses passthrough payloads.""" + profile = agent_reasoning_profile(agent) + decision = adapt_reasoning_decision(profile, _ACTIVE_DECISION.get()) + return original_client_send_raw( + self, + agent, + endpoint, + apply_reasoning_payload(payload, profile, decision, endpoint), + ) + + def client_batch_chat( + self: Any, + agent: Any, + requests: dict[str, list[dict[str, str]]], + temperature: float = 0.2, + poll_interval: float = 5.0, + poll_timeout: float = 3600.0, + ) -> dict[str, dict[str, Any]]: + """Select and retain one direct-route decision for each batch item.""" + policy = _ACTIVE_POLICY.get() or ReasoningPolicy() + profile = agent_reasoning_profile(agent) + decisions = { + custom_id: decision + for custom_id, messages in requests.items() + if ( + decision := select_reasoning_decision( + profile, + policy, + _message_text(messages), + "worker", + workload=ReasoningWorkload(), + ) + ) + is not None + } + token = _BATCH_DECISIONS.set(decisions) + try: + results = original_client_batch_chat( + self, + agent, + requests, + temperature, + poll_interval, + poll_timeout, + ) + finally: + _BATCH_DECISIONS.reset(token) + for custom_id in results: + _append_event(agent, "worker", decisions.get(custom_id)) + return results + + def client_batch_upload(self: Any, agent: Any, payload: bytes) -> str: + """Rewrite provider Batch JSONL immediately before the secured upload.""" + profile = agent_reasoning_profile(agent) + decisions = _BATCH_DECISIONS.get() + if profile is not None and decisions: + payload = _rewrite_batch_payload(payload, decisions, profile) + return original_client_batch_upload(self, agent, payload) + + model_client_type.chat = client_chat + model_client_type.stream_chat = client_stream_chat + model_client_type.proxy_send = client_proxy_send + model_client_type.batch_chat = client_batch_chat + model_client_type._send = client_send + model_client_type._stream_send = client_stream_send + model_client_type._send_raw = client_send_raw + model_client_type._batch_upload = client_batch_upload + + +__all__ = ["install_client_hooks"] diff --git a/contextual_orchestrator/_reasoning_config_hooks.py b/contextual_orchestrator/_reasoning_config_hooks.py new file mode 100644 index 000000000..d7adfe400 --- /dev/null +++ b/contextual_orchestrator/_reasoning_config_hooks.py @@ -0,0 +1,75 @@ +"""Agent, policy-snapshot, and orchestrator-construction hooks.""" + +from __future__ import annotations + +from typing import Any, Mapping + +from .reasoning_control import ReasoningPolicy, ReasoningProfile +from ._reasoning_state import ( + _POLICY_OBJECTS, + agent_reasoning_profile, + configure_agent_reasoning, + configure_orchestrator_reasoning, +) + + +def install_config_hooks( + model_agent_type: type[Any], + orchestrator_type: type[Any], + policy_type: type[Any], +) -> None: + """Install configuration round-trip and policy snapshot hooks.""" + original_agent_from_dict = model_agent_type.from_dict + original_agent_to_config = model_agent_type.to_config + original_policy_as_dict = policy_type.as_dict + original_orchestrator_init = orchestrator_type.__init__ + + def agent_from_dict(cls: type[Any], value: Mapping[str, Any]) -> Any: + """Load normal agent fields plus an optional explicit reasoning profile.""" + if not isinstance(value, Mapping): + return original_agent_from_dict(value) + cleaned = dict(value) + profile_data = cleaned.pop("reasoning_profile", None) + agent = original_agent_from_dict(cleaned) + if profile_data is not None: + configure_agent_reasoning(agent, ReasoningProfile.from_dict(profile_data)) + return agent + + def agent_to_config(self: Any) -> dict[str, Any]: + """Serialize the optional reasoning profile with the normal agent contract.""" + value = original_agent_to_config(self) + profile = agent_reasoning_profile(self) + if profile is not None: + value["reasoning_profile"] = profile.to_dict() + return value + + def policy_as_dict(self: Any) -> dict[str, Any]: + """Include reasoning policy evidence in an orchestration policy snapshot.""" + value = original_policy_as_dict(self) + policy = _POLICY_OBJECTS.get(self) + if policy is not None: + value["reasoning_control"] = policy.to_dict() + return value + + def orchestrator_init(self: Any, *args: Any, reasoning_policy: Any = None, **kwargs: Any) -> None: + """Initialize the core and attach an optional JSON or typed reasoning policy.""" + original_orchestrator_init(self, *args, **kwargs) + if reasoning_policy is None: + policy = ReasoningPolicy() + elif isinstance(reasoning_policy, ReasoningPolicy): + policy = reasoning_policy + elif isinstance(reasoning_policy, Mapping): + policy = ReasoningPolicy.from_dict(reasoning_policy) + else: + raise TypeError("reasoning_policy must be a mapping, ReasoningPolicy, or None") + configure_orchestrator_reasoning(self, policy) + + + model_agent_type.from_dict = classmethod(agent_from_dict) + model_agent_type.to_config = agent_to_config + model_agent_type.reasoning_profile = property(agent_reasoning_profile) + policy_type.as_dict = policy_as_dict + orchestrator_type.__init__ = orchestrator_init + + +__all__ = ["install_config_hooks"] diff --git a/contextual_orchestrator/_reasoning_orchestrator_hooks.py b/contextual_orchestrator/_reasoning_orchestrator_hooks.py new file mode 100644 index 000000000..af9f32752 --- /dev/null +++ b/contextual_orchestrator/_reasoning_orchestrator_hooks.py @@ -0,0 +1,362 @@ +"""Orchestrator hooks for role control, traces, retries, and ablation.""" + +from __future__ import annotations + +from contextvars import ContextVar +from typing import Any, Callable, Iterator, Mapping, Sequence + +from .reasoning_control import ( + ReasoningAblationCell, + ReasoningPolicy, + ReasoningProfile, + ReasoningWorkload, + WorkflowReasoningCursor, + select_reasoning_decision, + sum_usage_tokens, +) +from ._reasoning_state import ( + _ACTIVE_POLICY, + _EVENT_CAPTURE, + _OVERRIDE_DECISION, + _annotate_trace, + _decision_scope, + _input_text, + _message_text, + agent_reasoning_profile, + configure_agent_reasoning, + configure_orchestrator_reasoning, + current_reasoning_workload, + orchestrator_reasoning_policy, +) +from ._reasoning_workflow import _capture_batch, _retry_rejected_worker_once + +_WORKFLOW_CURSOR: ContextVar[WorkflowReasoningCursor | None] = ContextVar( + "contextual_orchestrator_reasoning_workflow_cursor", + default=None, +) + + +def _update_generated_plan_cursor(steps: Any) -> None: + """Apply a validated generated list size to the active workflow cursor.""" + cursor = _WORKFLOW_CURSOR.get() + if cursor is not None and isinstance(steps, list): + cursor.set_plan_size(len(steps)) + + +def install_orchestrator_hooks(orchestrator_type: type[Any]) -> None: + """Install role-aware invocation, workflow evidence, retry, and ablation hooks.""" + original_invoke = orchestrator_type._invoke + original_route_once = orchestrator_type.route_once + original_conduct = orchestrator_type.conduct + original_stream_route = orchestrator_type.stream_route + original_batch_route = orchestrator_type.batch_route + original_proxy_completion = orchestrator_type.proxy_completion + original_plan_generated = orchestrator_type._plan_generated + original_model_judge = orchestrator_type._model_judge_verification + original_patch_agent = getattr(orchestrator_type, "patch_agent", None) + original_agent_to_admin_payload = getattr(orchestrator_type, "_agent_to_admin_payload", None) + + def agent_to_admin_payload(self: Any, agent: Any) -> dict[str, Any]: + """Expose explicit reasoning capability in the admin-safe agent view.""" + value = original_agent_to_admin_payload(self, agent) + profile = agent_reasoning_profile(agent) + if profile is not None: + value["reasoning_profile"] = profile.to_dict() + return value + + def patch_agent( + self: Any, + agent_pool_id: str, + worker_agent_id: str, + patch: dict[str, Any], + ) -> dict[str, Any]: + """Preserve or explicitly update capability when the core replaces an agent.""" + current = self._agent(worker_agent_id) + previous = agent_reasoning_profile(current) + explicit = "reasoning_profile" in patch + requested: ReasoningProfile | None = previous + if explicit: + raw_profile = patch["reasoning_profile"] + if raw_profile is None: + requested = None + elif isinstance(raw_profile, ReasoningProfile): + requested = raw_profile + elif isinstance(raw_profile, Mapping): + requested = ReasoningProfile.from_dict(raw_profile) + else: + raise TypeError( + "reasoning_profile patch must be a mapping, ReasoningProfile, or None" + ) + core_patch = dict(patch) + core_patch.pop("reasoning_profile", None) + original_patch_agent(self, agent_pool_id, worker_agent_id, core_patch) + replacement = self._agent(worker_agent_id) + configure_agent_reasoning(replacement, requested) + pool_store = getattr(self, "_pool_store", None) + if pool_store is not None: + pool_store.save(replacement) + return self._agent_to_admin_payload(replacement) + + def orchestrator_invoke( + self: Any, + primary: Any, + messages: list[dict[str, str]], + *, + text: str, + role: str, + ) -> tuple[str, str, dict[str, Any] | None]: + """Select once from role, task, and graph evidence, then project on failover.""" + policy = orchestrator_reasoning_policy(self) + profile = agent_reasoning_profile(primary) + workload = current_reasoning_workload() + cursor = _WORKFLOW_CURSOR.get() + if workload is None and cursor is not None: + workload = cursor.observe(messages) + decision = _OVERRIDE_DECISION.get() or select_reasoning_decision( + profile, + policy, + text, + role, + workload=workload, + ) + policy_token = _ACTIVE_POLICY.set(policy) + try: + with _decision_scope(decision): + output, served_id, usage = original_invoke( + self, + primary, + messages, + text=text, + role=role, + ) + finally: + _ACTIVE_POLICY.reset(policy_token) + events = _EVENT_CAPTURE.get() + if events: + for event in reversed(events): + if ( + event["role"] == role + and event["agent_id"] == served_id + and event.get("usage") is None + ): + event["usage"] = usage + break + return output, served_id, usage + + def _capture_workflow( + self: Any, + operation: Callable[[Any, list[dict[str, str]]], dict[str, Any]], + messages: list[dict[str, str]], + *, + allow_escalation: bool, + workflow_step_count: int, + ) -> dict[str, Any]: + """Capture, structurally annotate, and optionally repair one visible workflow.""" + events: list[dict[str, Any]] = [] + event_token = _EVENT_CAPTURE.set(events) + policy = orchestrator_reasoning_policy(self) + policy_token = _ACTIVE_POLICY.set(policy) + cursor_token = _WORKFLOW_CURSOR.set(WorkflowReasoningCursor(workflow_step_count)) + try: + result = operation(self, messages) + trace = result.get("trace") + if isinstance(trace, list): + _annotate_trace(trace, events) + result["reasoning_control"] = policy.to_dict() + if allow_escalation: + _retry_rejected_worker_once(self, result, _message_text(messages)) + finally: + _WORKFLOW_CURSOR.reset(cursor_token) + _ACTIVE_POLICY.reset(policy_token) + _EVENT_CAPTURE.reset(event_token) + return result + + def route_once(self: Any, messages: list[dict[str, str]]) -> dict[str, Any]: + """Capture reasoning evidence for the single-step route path.""" + return _capture_workflow( + self, + original_route_once, + messages, + allow_escalation=False, + workflow_step_count=1, + ) + + def conduct(self: Any, messages: list[dict[str, str]]) -> dict[str, Any]: + """Capture graph-aware deep-workflow evidence and one verifier-driven retry.""" + return _capture_workflow( + self, + original_conduct, + messages, + allow_escalation=True, + workflow_step_count=4, + ) + + def stream_route( + self: Any, + messages: list[dict[str, str]], + workflow_run_id: str | None = None, + ) -> Iterator[str]: + """Keep direct-route worker effort active across one streamed response.""" + task = _message_text(messages) + agent = self._select_agent(task, "worker") + policy = orchestrator_reasoning_policy(self) + decision = select_reasoning_decision( + agent_reasoning_profile(agent), + policy, + task, + "worker", + workload=ReasoningWorkload(), + ) + policy_token = _ACTIVE_POLICY.set(policy) + try: + with _decision_scope(decision): + yield from original_stream_route(self, messages, workflow_run_id=workflow_run_id) + finally: + _ACTIVE_POLICY.reset(policy_token) + + def batch_route(self: Any, prompts: list[str]) -> list[dict[str, Any]]: + """Capture per-item reasoning evidence for the provider Batch route.""" + return _capture_batch(self, original_batch_route, prompts) + + def proxy_completion( + self: Any, + body: dict[str, Any], + *, + endpoint: str = "chat/completions", + ) -> dict[str, Any]: + """Apply direct-route defaults to full-shape chat and Responses requests.""" + task = _input_text(body) + agent = self._select_agent(task, "worker") + policy = orchestrator_reasoning_policy(self) + decision = select_reasoning_decision( + agent_reasoning_profile(agent), + policy, + task, + "worker", + workload=ReasoningWorkload(), + ) + policy_token = _ACTIVE_POLICY.set(policy) + try: + with _decision_scope(decision): + return original_proxy_completion(self, body, endpoint=endpoint) + finally: + _ACTIVE_POLICY.reset(policy_token) + + def plan_generated(self: Any, task: str) -> Any: + """Allocate thinker effort for explicit workflow decomposition.""" + planner = self._select_agent(task, "thinker") + policy = orchestrator_reasoning_policy(self) + maximum_steps = int(getattr(self.policy, "max_workflow_steps", 4)) + planning_workload = ReasoningWorkload( + workflow_step_index=0, + workflow_step_count=maximum_steps, + recursion_depth=0, + decomposition_count=maximum_steps, + accessible_step_count=0, + ) + decision = select_reasoning_decision( + agent_reasoning_profile(planner), + policy, + task, + "thinker", + workload=planning_workload, + ) + policy_token = _ACTIVE_POLICY.set(policy) + try: + with _decision_scope(decision): + steps = original_plan_generated(self, task) + finally: + _ACTIVE_POLICY.reset(policy_token) + _update_generated_plan_cursor(steps) + return steps + + def model_judge(self: Any, task: str, fallback: dict[str, Any]) -> dict[str, Any]: + """Apply verifier-role effort to the optional model verdict.""" + judge = self._select_agent(task, "verifier") + policy = orchestrator_reasoning_policy(self) + decision = select_reasoning_decision( + agent_reasoning_profile(judge), + policy, + task, + "verifier", + workload=ReasoningWorkload(), + ) + policy_token = _ACTIVE_POLICY.set(policy) + try: + with _decision_scope(decision): + return original_model_judge(self, task, fallback) + finally: + _ACTIVE_POLICY.reset(policy_token) + + def run_reasoning_ablation( + self: Any, + prompts: Sequence[str], + *, + mode: str = "auto", + levels: Sequence[str] | None = None, + ) -> dict[str, Any]: + """Measure fixed effort cells under one prompt set without persisting outputs.""" + if not prompts: + raise ValueError("reasoning ablation requires at least one prompt") + candidate_levels = tuple(levels or ("minimal", "low", "medium", "high")) + previous = orchestrator_reasoning_policy(self) + cells: list[ReasoningAblationCell] = [] + try: + for level in candidate_levels: + fixed = ReasoningPolicy( + strategy="fixed", + fixed_level=level, + max_escalations=0, + ) + configure_orchestrator_reasoning(self, fixed) + accepted = 0 + reasoning_tokens = 0 + total_tokens = 0 + for prompt in prompts: + result = self._dispatch([{"role": "user", "content": prompt}], mode) + accepted += int(bool(result.get("verification", {}).get("accepted"))) + trace = result.get("trace") + if isinstance(trace, list): + cell_reasoning, cell_total = sum_usage_tokens(trace) + reasoning_tokens += cell_reasoning + total_tokens += cell_total + cells.append( + ReasoningAblationCell( + level=level, + prompt_count=len(prompts), + accepted_count=accepted, + reasoning_tokens=reasoning_tokens, + total_tokens=total_tokens, + ) + ) + finally: + configure_orchestrator_reasoning(self, previous) + return { + "mode": mode, + "prompt_count": len(prompts), + "cells": [cell.to_dict() for cell in cells], + "quality_measure": ( + "workflow verifier acceptance; task-specific benchmark scorers remain authoritative" + ), + } + + orchestrator_type._invoke = orchestrator_invoke + orchestrator_type.route_once = route_once + orchestrator_type.conduct = conduct + orchestrator_type.stream_route = stream_route + orchestrator_type.batch_route = batch_route + orchestrator_type.proxy_completion = proxy_completion + orchestrator_type._plan_generated = plan_generated + orchestrator_type._model_judge_verification = model_judge + orchestrator_type.run_reasoning_ablation = run_reasoning_ablation + if original_agent_to_admin_payload is not None: + orchestrator_type._agent_to_admin_payload = agent_to_admin_payload + if original_patch_agent is not None and original_agent_to_admin_payload is not None: + orchestrator_type.patch_agent = patch_agent + + +__all__ = [ + "_WORKFLOW_CURSOR", + "_update_generated_plan_cursor", + "install_orchestrator_hooks", +] diff --git a/contextual_orchestrator/_reasoning_payload.py b/contextual_orchestrator/_reasoning_payload.py new file mode 100644 index 000000000..0b0edf2ca --- /dev/null +++ b/contextual_orchestrator/_reasoning_payload.py @@ -0,0 +1,163 @@ +"""Provider payload projection and reasoning-token accounting.""" + +from __future__ import annotations + +import copy +from typing import Any, Mapping, MutableMapping, Sequence + +from ._reasoning_profile import JsonScalar, PayloadRule, ReasoningProfile +from ._reasoning_policy import ReasoningDecision, _nearest_supported + +def apply_reasoning_payload( + payload: Mapping[str, Any], + profile: ReasoningProfile | None, + decision: ReasoningDecision | None, + endpoint: str, +) -> dict[str, Any]: + """Return a copied payload with provider reasoning fields set if unowned.""" + if not isinstance(payload, Mapping): + raise ValueError("provider payload must be an object") + output = copy.deepcopy(dict(payload)) + if profile is None or decision is None: + return output + level = _nearest_supported(profile.bounded_levels, decision.level) + normalized = _normalize_endpoint(endpoint) + rules = _rules_for(profile, normalized) + if _any_complete_path(output, tuple(rule.path for rule in rules)): + return output + mapping = dict(profile.level_values) or {item: item for item in profile.supported_levels} + for rule in rules: + _set_nested_if_absent(output, rule.path, _render_value(rule.value, level, mapping)) + return output + + +def extract_reasoning_tokens(usage: Mapping[str, Any] | None) -> int | None: + """Extract provider-reported reasoning tokens from known usage shapes.""" + if not isinstance(usage, Mapping): + return None + direct = usage.get("reasoning_tokens") + if isinstance(direct, int) and not isinstance(direct, bool) and direct >= 0: + return direct + for key in ("output_tokens_details", "completion_tokens_details"): + details = usage.get(key) + if isinstance(details, Mapping): + value = details.get("reasoning_tokens") + if isinstance(value, int) and not isinstance(value, bool) and value >= 0: + return value + return None + + +def sum_usage_tokens(trace: Sequence[Mapping[str, Any]]) -> tuple[int, int]: + """Return reasoning and total token sums from a workflow trace.""" + reasoning_total = 0 + total = 0 + for step in trace: + usage = step.get("usage") + if not isinstance(usage, Mapping): + continue + reasoning_total += extract_reasoning_tokens(usage) or 0 + value = usage.get("total_tokens") + if isinstance(value, int) and not isinstance(value, bool) and value >= 0: + total += value + return reasoning_total, total + + + +def _normalize_endpoint(endpoint: str) -> str: + """Normalize supported OpenAI-compatible endpoint names.""" + if not isinstance(endpoint, str): + raise ValueError("endpoint must be a string") + value = endpoint.strip().lower().strip("/") + if value.startswith("v1/"): + value = value[3:] + if value not in {"chat/completions", "responses"}: + raise ValueError(f"unsupported reasoning endpoint: {endpoint}") + return value + + +def _rules_for(profile: ReasoningProfile, endpoint: str) -> tuple[PayloadRule, ...]: + """Return explicit or preset rules for an endpoint.""" + if endpoint == "chat/completions" and profile.chat_rules: + return profile.chat_rules + if endpoint == "responses" and profile.responses_rules: + return profile.responses_rules + if profile.preset in {"openai_effort", "nvidia_reasoning_effort"}: + return ( + (PayloadRule(("reasoning_effort",), "$mapped"),) + if endpoint == "chat/completions" + else (PayloadRule(("reasoning", "effort"), "$mapped"),) + ) + if profile.preset == "nvidia_nemotron_thinking" and endpoint == "chat/completions": + return ( + PayloadRule(("chat_template_kwargs", "enable_thinking"), "$enabled"), + PayloadRule(("chat_template_kwargs", "low_effort"), "$low_effort"), + ) + if profile.preset == "gemini_thinking_level" and endpoint == "chat/completions": + return ( + PayloadRule(("extra_body", "google", "thinking_config", "thinking_level"), "$mapped"), + ) + return () + + +def _render_value(template: JsonScalar, level: str, mapping: Mapping[str, JsonScalar]) -> JsonScalar: + """Render one fixed template without expression evaluation.""" + if not isinstance(template, str) or not template.startswith("$"): + return template + if template == "$level": + return level + if template == "$mapped": + if level not in mapping: + raise ValueError(f"reasoning level has no provider mapping: {level}") + return mapping[level] + if template == "$enabled": + return level != "none" + if template == "$low_effort": + return level in {"minimal", "low"} + if template == "$int": + value = mapping.get(level) + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError("$int requires an integer level mapping") + return value + raise ValueError(f"unsupported reasoning template: {template}") + + +def _any_complete_path(payload: Mapping[str, Any], paths: Sequence[tuple[str, ...]]) -> bool: + """Return whether the caller owns any complete target path, including ``None``.""" + for path in paths: + cursor: Any = payload + for segment in path: + if not isinstance(cursor, Mapping) or segment not in cursor: + break + cursor = cursor[segment] + else: + return True + return False + + +def _set_nested_if_absent(target: MutableMapping[str, Any], path: tuple[str, ...], value: JsonScalar) -> None: + """Set a nested value without overwriting caller-owned fields.""" + cursor: MutableMapping[str, Any] = target + for segment in path[:-1]: + current = cursor.get(segment) + if current is None: + child: dict[str, Any] = {} + cursor[segment] = child + cursor = child + elif isinstance(current, MutableMapping): + cursor = current + else: + raise ValueError(f"reasoning path conflicts with caller scalar: {segment}") + cursor.setdefault(path[-1], value) + + + +__all__ = [ + "apply_reasoning_payload", + "extract_reasoning_tokens", + "sum_usage_tokens", + "_any_complete_path", + "_normalize_endpoint", + "_render_value", + "_rules_for", + "_set_nested_if_absent", +] diff --git a/contextual_orchestrator/_reasoning_policy.py b/contextual_orchestrator/_reasoning_policy.py new file mode 100644 index 000000000..a569705c0 --- /dev/null +++ b/contextual_orchestrator/_reasoning_policy.py @@ -0,0 +1,338 @@ +"""Quality-aware, role-sensitive reasoning decision policy.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Mapping + +from ._reasoning_profile import CANONICAL_REASONING_LEVELS, ReasoningProfile +from ._reasoning_workload import ReasoningWorkload + +_COMPLEXITY_TERMS = ( + "analyze", + "architecture", + "compare", + "derive", + "evaluate", + "implement", + "migration", + "optimize", + "prove", + "reason", + "research", + "trade-off", + "tradeoff", + "verify", + "분석", + "아키텍처", + "비교", + "구현", + "연구", + "검증", +) +_HIGH_RISK_TERMS = ( + "authentication", + "authorization", + "financial", + "legal", + "medical", + "payment", + "privacy", + "safety", + "security", + "개인정보", + "법률", + "보안", + "의료", +) +_ROLE_BASELINE_OFFSET = { + "thinker": 1, + "worker": 0, + "verifier": 1, + "synthesizer": 0, +} + + +@dataclass(frozen=True) +class ReasoningPolicy: + """Quality policy for role-aware effort and bounded escalation.""" + + strategy: str = "adaptive" + fixed_level: str | None = None + max_escalations: int = 1 + + def __post_init__(self) -> None: + """Reject unknown strategies, ambiguous fixed mode, and retry loops.""" + if self.strategy not in {"disabled", "adaptive", "fixed"}: + raise ValueError("strategy must be disabled, adaptive, or fixed") + if self.strategy == "fixed" and self.fixed_level is None: + raise ValueError("fixed strategy requires fixed_level") + if self.fixed_level is not None and self.fixed_level not in CANONICAL_REASONING_LEVELS: + raise ValueError("fixed_level must be canonical") + if isinstance(self.max_escalations, bool) or self.max_escalations not in {0, 1}: + raise ValueError("max_escalations must be 0 or 1") + + @classmethod + def from_dict(cls, value: Mapping[str, Any]) -> "ReasoningPolicy": + """Parse a strict JSON-compatible reasoning policy.""" + if not isinstance(value, Mapping): + raise ValueError("reasoning policy must be an object") + unknown = set(value) - {"strategy", "fixed_level", "max_escalations"} + if unknown: + raise ValueError(f"unknown reasoning policy keys: {sorted(unknown)}") + return cls( + strategy=value.get("strategy", "adaptive"), + fixed_level=value.get("fixed_level"), + max_escalations=value.get("max_escalations", 1), + ) + + def to_dict(self) -> dict[str, Any]: + """Return the policy as stable audit data.""" + return { + "strategy": self.strategy, + "fixed_level": self.fixed_level, + "max_escalations": self.max_escalations, + } + + +@dataclass(frozen=True) +class ReasoningDecision: + """One explainable request-time reasoning-level decision.""" + + level: str + source: str + role: str + complexity_score: int + factors: tuple[str, ...] + escalation_index: int = 0 + workload: ReasoningWorkload | None = None + + def __post_init__(self) -> None: + """Validate canonical and non-negative decision evidence.""" + if self.level not in CANONICAL_REASONING_LEVELS: + raise ValueError("decision level must be canonical") + if not self.source or not self.role: + raise ValueError("decision source and role must not be empty") + if isinstance(self.complexity_score, bool) or not isinstance(self.complexity_score, int): + raise ValueError("complexity_score must be an integer") + if self.complexity_score < 0: + raise ValueError("complexity_score must be non-negative") + if not all(isinstance(item, str) and item for item in self.factors): + raise ValueError("decision factors must be non-empty strings") + if isinstance(self.escalation_index, bool) or not isinstance(self.escalation_index, int): + raise ValueError("escalation_index must be an integer") + if self.escalation_index < 0: + raise ValueError("escalation_index must be non-negative") + if self.workload is not None and not isinstance(self.workload, ReasoningWorkload): + raise ValueError("decision workload must be ReasoningWorkload or None") + + def to_dict(self) -> dict[str, Any]: + """Return API-safe evidence without private intermediate reasoning text.""" + value: dict[str, Any] = { + "level": self.level, + "source": self.source, + "role": self.role, + "complexity_score": self.complexity_score, + "factors": list(self.factors), + "escalation_index": self.escalation_index, + } + if self.workload is not None: + value["workload"] = self.workload.to_dict() + return value + + +@dataclass(frozen=True) +class ReasoningAblationCell: + """One measured fixed-effort cell in a reasoning ablation.""" + + level: str + prompt_count: int + accepted_count: int + reasoning_tokens: int + total_tokens: int + + def to_dict(self) -> dict[str, int | str]: + """Return a stable machine-readable ablation cell.""" + return { + "level": self.level, + "prompt_count": self.prompt_count, + "accepted_count": self.accepted_count, + "reasoning_tokens": self.reasoning_tokens, + "total_tokens": self.total_tokens, + } + + +def select_reasoning_decision( + profile: ReasoningProfile | None, + policy: ReasoningPolicy, + task: str, + role: str, + *, + workload: ReasoningWorkload | Mapping[str, Any] | None = None, +) -> ReasoningDecision | None: + """Select bounded effort from role, task evidence, and workflow topology. + + Latency is intentionally not an input. The policy allocates test-time compute + from role, semantic complexity, risk, decomposition, recursion, and access-list + fan-in, subject only to the model profile's explicit maximum. + """ + if profile is None or policy.strategy == "disabled": + return None + if not isinstance(task, str) or not isinstance(role, str) or not role: + raise ValueError("task and role must be strings and role must not be empty") + structure = _coerce_workload(workload) + if policy.strategy == "fixed": + requested = policy.fixed_level or profile.default_level + level = _nearest_supported(profile.bounded_levels, requested) + return ReasoningDecision( + level, + "fixed_policy", + role, + 0, + ("fixed_policy",), + workload=structure, + ) + + lowered = task.lower() + factors: list[str] = [] + score = _ROLE_BASELINE_OFFSET.get(role, 0) + if score: + factors.append(f"role:{role}") + term_hits = sum(1 for term in _COMPLEXITY_TERMS if term in lowered) + if term_hits >= 2: + score += 1 + factors.append("multiple_complexity_signals") + if len(task) > 800: + score += 1 + factors.append("long_context") + if task.count("\n") >= 8 or sum(task.count(marker) for marker in ("1.", "2.", "- ")) >= 4: + score += 1 + factors.append("multi_step_structure") + risk_hits = sum(1 for term in _HIGH_RISK_TERMS if term in lowered) + if risk_hits >= 2: + score += 1 + factors.append("multiple_high_impact_signals") + if structure is not None: + if ( + structure.workflow_step_index >= 2 + and ( + structure.workflow_step_count >= 4 + or structure.decomposition_count >= 3 + ) + ): + score += 1 + factors.append("decomposed_workflow") + if structure.recursion_depth >= 2: + score += 1 + factors.append("recursive_workflow_depth") + if structure.accessible_step_count >= 2: + score += 1 + factors.append("access_list_fan_in") + if ( + structure.workflow_step_index >= max(2, structure.workflow_step_count - 2) + and structure.accessible_step_count >= 1 + ): + score += 1 + factors.append("late_workflow_integration") + + base_index = CANONICAL_REASONING_LEVELS.index(profile.default_level) + requested_index = min( + base_index + score, + CANONICAL_REASONING_LEVELS.index(profile.maximum_level), + ) + requested = CANONICAL_REASONING_LEVELS[requested_index] + level = _nearest_supported(profile.bounded_levels, requested) + return ReasoningDecision( + level=level, + source="adaptive_policy", + role=role, + complexity_score=score, + factors=tuple(factors or ("profile_default",)), + workload=structure, + ) + + +def adapt_reasoning_decision( + profile: ReasoningProfile | None, + decision: ReasoningDecision | None, +) -> ReasoningDecision | None: + """Project one canonical decision onto a failover model's capabilities.""" + if profile is None or decision is None: + return None + level = _nearest_supported(profile.bounded_levels, decision.level) + if level == decision.level: + return decision + return ReasoningDecision( + level=level, + source=f"{decision.source}:capability_projection", + role=decision.role, + complexity_score=decision.complexity_score, + factors=decision.factors + ("projected_to_provider_capability",), + escalation_index=decision.escalation_index, + workload=decision.workload, + ) + + +def escalate_reasoning_decision( + profile: ReasoningProfile | None, + policy: ReasoningPolicy, + decision: ReasoningDecision | None, +) -> ReasoningDecision | None: + """Return the immediately higher supported level after a verifier rejection.""" + if profile is None or decision is None: + return None + if policy.max_escalations == 0 or decision.escalation_index >= policy.max_escalations: + return None + levels = profile.bounded_levels + current = _nearest_supported(levels, decision.level) + index = levels.index(current) + if index + 1 >= len(levels): + return None + return ReasoningDecision( + level=levels[index + 1], + source="verifier_escalation", + role=decision.role, + complexity_score=decision.complexity_score, + factors=decision.factors + ("verifier_rejected_prior_attempt",), + escalation_index=decision.escalation_index + 1, + workload=decision.workload, + ) + + +def _coerce_workload( + value: ReasoningWorkload | Mapping[str, Any] | None, +) -> ReasoningWorkload | None: + """Normalize optional typed or JSON-compatible structural evidence.""" + if value is None or isinstance(value, ReasoningWorkload): + return value + if isinstance(value, Mapping): + return ReasoningWorkload.from_mapping(value) + raise ValueError("workload must be a mapping, ReasoningWorkload, or None") + + +def _nearest_supported(levels: tuple[str, ...], requested: str) -> str: + """Project a canonical requested level to the closest supported lower level.""" + if not levels: + raise ValueError("no bounded reasoning levels are available") + if requested not in CANONICAL_REASONING_LEVELS: + raise ValueError(f"unknown canonical reasoning level: {requested}") + requested_index = CANONICAL_REASONING_LEVELS.index(requested) + lower = [ + level + for level in levels + if CANONICAL_REASONING_LEVELS.index(level) <= requested_index + ] + return lower[-1] if lower else levels[0] + + +__all__ = [ + "ReasoningAblationCell", + "ReasoningDecision", + "ReasoningPolicy", + "ReasoningWorkload", + "_coerce_workload", + "_nearest_supported", + "adapt_reasoning_decision", + "escalate_reasoning_decision", + "select_reasoning_decision", +] diff --git a/contextual_orchestrator/_reasoning_profile.py b/contextual_orchestrator/_reasoning_profile.py new file mode 100644 index 000000000..f14ac3445 --- /dev/null +++ b/contextual_orchestrator/_reasoning_profile.py @@ -0,0 +1,16 @@ +"""Compatibility facade for reasoning profile primitives and value objects.""" + +from ._reasoning_profile_types import ( + CANONICAL_REASONING_LEVELS, + JsonScalar, + PayloadRule, +) +from ._reasoning_profile_value import ReasoningProfile, _parse_rules + +__all__ = [ + "CANONICAL_REASONING_LEVELS", + "JsonScalar", + "PayloadRule", + "ReasoningProfile", + "_parse_rules", +] diff --git a/contextual_orchestrator/_reasoning_profile_types.py b/contextual_orchestrator/_reasoning_profile_types.py new file mode 100644 index 000000000..08753485f --- /dev/null +++ b/contextual_orchestrator/_reasoning_profile_types.py @@ -0,0 +1,87 @@ +"""Primitive types and validated nested payload rules for reasoning profiles.""" + +from __future__ import annotations + +from dataclasses import dataclass +import math +import re +from typing import Any, Mapping + +JsonScalar = str | int | float | bool | None + +CANONICAL_REASONING_LEVELS = ( + "none", + "minimal", + "low", + "medium", + "high", + "xhigh", + "max", +) + +PRESETS = { + "openai_effort", + "nvidia_reasoning_effort", + "nvidia_nemotron_thinking", + "gemini_thinking_level", + "custom", +} +TEMPLATES = {"$level", "$mapped", "$enabled", "$low_effort", "$int"} +_SAFE_SEGMENT = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + + +@dataclass(frozen=True) +class PayloadRule: + """One validated nested assignment used by a custom provider mapping.""" + + path: tuple[str, ...] + value: JsonScalar + + def __post_init__(self) -> None: + """Validate path depth, identifier syntax, and template vocabulary.""" + if not isinstance(self.path, tuple): + raise ValueError("reasoning payload path must be a tuple") + if not 1 <= len(self.path) <= 8: + raise ValueError("reasoning payload path must contain 1 to 8 segments") + if any(not isinstance(part, str) or not _SAFE_SEGMENT.fullmatch(part) for part in self.path): + raise ValueError("reasoning payload path contains an unsafe segment") + if isinstance(self.value, str) and self.value.startswith("$") and self.value not in TEMPLATES: + raise ValueError(f"unsupported reasoning payload template: {self.value}") + _validate_json_scalar(self.value, "reasoning payload value") + + @classmethod + def from_dict(cls, value: Mapping[str, Any]) -> "PayloadRule": + """Parse one strict JSON-compatible payload rule.""" + if not isinstance(value, Mapping): + raise ValueError("reasoning payload rule must be an object") + unknown = set(value) - {"path", "value"} + if unknown: + raise ValueError(f"unknown reasoning payload rule keys: {sorted(unknown)}") + path = value.get("path") + if not isinstance(path, (list, tuple)) or not path or not all(isinstance(item, str) for item in path): + raise ValueError("reasoning payload rule path must be a non-empty string array") + if "value" not in value: + raise ValueError("reasoning payload rule must include value") + return cls(tuple(path), value["value"]) + + def to_dict(self) -> dict[str, Any]: + """Return the rule as stable JSON-compatible data.""" + return {"path": list(self.path), "value": self.value} + + +def _validate_json_scalar(value: Any, field_name: str) -> JsonScalar: + """Return a strict JSON scalar, rejecting non-finite Python floats.""" + if not isinstance(value, (str, int, float, bool)) and value is not None: + raise ValueError(f"{field_name} must be JSON scalars") + if isinstance(value, float) and not math.isfinite(value): + raise ValueError(f"{field_name} must be finite JSON scalars") + return value + + +__all__ = [ + "CANONICAL_REASONING_LEVELS", + "JsonScalar", + "PayloadRule", + "PRESETS", + "TEMPLATES", +] diff --git a/contextual_orchestrator/_reasoning_profile_value.py b/contextual_orchestrator/_reasoning_profile_value.py new file mode 100644 index 000000000..282ddd517 --- /dev/null +++ b/contextual_orchestrator/_reasoning_profile_value.py @@ -0,0 +1,151 @@ +"""Validated model-level reasoning capability profiles.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Mapping + +from ._reasoning_profile_types import ( + CANONICAL_REASONING_LEVELS, + JsonScalar, + PayloadRule, + PRESETS, + _validate_json_scalar, +) + + +@dataclass(frozen=True) +class ReasoningProfile: + """Explicit reasoning capability and payload mapping for one model.""" + + preset: str = "openai_effort" + supported_levels: tuple[str, ...] = ("minimal", "low", "medium", "high") + default_level: str = "low" + maximum_level: str = "high" + level_values: tuple[tuple[str, JsonScalar], ...] = () + chat_rules: tuple[PayloadRule, ...] = () + responses_rules: tuple[PayloadRule, ...] = () + + def __post_init__(self) -> None: + """Validate the complete provider capability contract.""" + if not isinstance(self.supported_levels, tuple): + raise ValueError("supported_levels must be a tuple") + if not isinstance(self.level_values, tuple): + raise ValueError("level_values must be a tuple") + if not isinstance(self.chat_rules, tuple): + raise ValueError("chat_rules must be a tuple") + if not isinstance(self.responses_rules, tuple): + raise ValueError("responses_rules must be a tuple") + if self.preset not in PRESETS: + raise ValueError(f"unsupported reasoning preset: {self.preset}") + if not self.supported_levels: + raise ValueError("supported_levels must not be empty") + if len(set(self.supported_levels)) != len(self.supported_levels): + raise ValueError("supported_levels must not contain duplicates") + try: + indexes = [CANONICAL_REASONING_LEVELS.index(level) for level in self.supported_levels] + except ValueError as exc: + raise ValueError("supported_levels contains an unknown level") from exc + if indexes != sorted(indexes): + raise ValueError("supported_levels must follow canonical order") + if self.default_level not in self.supported_levels: + raise ValueError("default_level must be supported") + if self.maximum_level not in self.supported_levels: + raise ValueError("maximum_level must be supported") + if self.supported_levels.index(self.default_level) > self.supported_levels.index(self.maximum_level): + raise ValueError("default_level cannot exceed maximum_level") + if self.preset == "custom" and not (self.chat_rules or self.responses_rules): + raise ValueError("custom preset requires chat_rules or responses_rules") + mapping = dict(self.level_values) + if len(mapping) != len(self.level_values): + raise ValueError("level_values must not contain duplicate keys") + for mapped in mapping.values(): + _validate_json_scalar(mapped, "level_values values") + unknown = set(mapping) - set(self.supported_levels) + if unknown: + raise ValueError(f"level_values maps unsupported levels: {sorted(unknown)}") + rules = self.chat_rules + self.responses_rules + if any(rule.value in {"$mapped", "$int"} for rule in rules): + missing = set(self.supported_levels) - set(mapping) + if missing: + raise ValueError(f"mapped rules require every supported level: {sorted(missing)}") + + @property + def bounded_levels(self) -> tuple[str, ...]: + """Return supported levels no more expensive than ``maximum_level``.""" + return self.supported_levels[: self.supported_levels.index(self.maximum_level) + 1] + + @classmethod + def from_dict(cls, value: Mapping[str, Any]) -> "ReasoningProfile": + """Parse a strict JSON-compatible model capability profile.""" + if not isinstance(value, Mapping): + raise ValueError("reasoning_profile must be an object") + allowed = { + "preset", + "supported_levels", + "default_level", + "maximum_level", + "level_values", + "chat_rules", + "responses_rules", + } + unknown = set(value) - allowed + if unknown: + raise ValueError(f"unknown reasoning_profile keys: {sorted(unknown)}") + preset = value.get("preset", "openai_effort") + supported = value.get("supported_levels", ("minimal", "low", "medium", "high")) + default_level = value.get("default_level", "low") + maximum_level = value.get("maximum_level", "high") + if not isinstance(preset, str): + raise ValueError("reasoning preset must be a string") + if not isinstance(supported, (list, tuple)) or not all(isinstance(item, str) for item in supported): + raise ValueError("supported_levels must be a string array") + if not isinstance(default_level, str) or not isinstance(maximum_level, str): + raise ValueError("default_level and maximum_level must be strings") + raw_values = value.get("level_values", {}) + if not isinstance(raw_values, Mapping): + raise ValueError("level_values must be an object") + level_values: list[tuple[str, JsonScalar]] = [] + for level, mapped in raw_values.items(): + if not isinstance(level, str): + raise ValueError("level_values keys must be strings") + level_values.append( + (level, _validate_json_scalar(mapped, "level_values values")) + ) + return cls( + preset=preset, + supported_levels=tuple(supported), + default_level=default_level, + maximum_level=maximum_level, + level_values=tuple(level_values), + chat_rules=_parse_rules(value.get("chat_rules", ())), + responses_rules=_parse_rules(value.get("responses_rules", ())), + ) + + def to_dict(self) -> dict[str, Any]: + """Return the capability profile as stable JSON-compatible data.""" + value: dict[str, Any] = { + "preset": self.preset, + "supported_levels": list(self.supported_levels), + "default_level": self.default_level, + "maximum_level": self.maximum_level, + } + if self.level_values: + value["level_values"] = dict(self.level_values) + if self.chat_rules: + value["chat_rules"] = [rule.to_dict() for rule in self.chat_rules] + if self.responses_rules: + value["responses_rules"] = [rule.to_dict() for rule in self.responses_rules] + return value + + +def _parse_rules(value: Any) -> tuple[PayloadRule, ...]: + """Parse an optional payload-rule array.""" + if value is None: + return () + if not isinstance(value, (list, tuple)): + raise ValueError("payload rules must be an array") + return tuple(PayloadRule.from_dict(item) for item in value) + + +__all__ = ["ReasoningProfile", "_parse_rules"] diff --git a/contextual_orchestrator/_reasoning_state.py b/contextual_orchestrator/_reasoning_state.py new file mode 100644 index 000000000..6f23ed772 --- /dev/null +++ b/contextual_orchestrator/_reasoning_state.py @@ -0,0 +1,314 @@ +"""Identity registries and request-local state for reasoning control.""" + +from __future__ import annotations + +from contextlib import contextmanager +from contextvars import ContextVar +from typing import Any, Iterator, Mapping, Sequence +import weakref + +from .reasoning_control import ( + ReasoningDecision, + ReasoningPolicy, + ReasoningProfile, + ReasoningWorkload, + adapt_reasoning_decision, + extract_reasoning_tokens, + select_reasoning_decision, +) + + +class _WeakIdentityMap: + """Weak mapping keyed by object identity rather than value equality.""" + + def __init__(self) -> None: + """Create an empty identity registry.""" + self._entries: dict[int, tuple[weakref.ReferenceType[Any], Any]] = {} + + def set(self, key: Any, value: Any) -> None: + """Store ``value`` for one live object without equality collisions.""" + identity = id(key) + + def remove(reference: weakref.ReferenceType[Any]) -> None: + """Remove only the entry still owned by this exact weak reference.""" + current = self._entries.get(identity) + if current is not None and current[0] is reference: + self._entries.pop(identity, None) + + self._entries[identity] = (weakref.ref(key, remove), value) + + def get(self, key: Any, default: Any = None) -> Any: + """Return the value only when the stored weak reference is ``key``.""" + entry = self._entries.get(id(key)) + if entry is None: + return default + if entry[0]() is key: + return entry[1] + self._entries.pop(id(key), None) + return default + + def pop(self, key: Any, default: Any = None) -> Any: + """Remove and return the identity-owned entry, or ``default``.""" + entry = self._entries.get(id(key)) + if entry is None or entry[0]() is not key: + return default + self._entries.pop(id(key), None) + return entry[1] + + +_AGENT_PROFILES = _WeakIdentityMap() +_ORCHESTRATOR_POLICIES = _WeakIdentityMap() +_POLICY_OBJECTS = _WeakIdentityMap() +_ACTIVE_DECISION: ContextVar[ReasoningDecision | None] = ContextVar( + "contextual_orchestrator_reasoning_decision", default=None +) +_ACTIVE_POLICY: ContextVar[ReasoningPolicy | None] = ContextVar( + "contextual_orchestrator_reasoning_policy", default=None +) +_OVERRIDE_DECISION: ContextVar[ReasoningDecision | None] = ContextVar( + "contextual_orchestrator_reasoning_override", default=None +) +_WORKLOAD_OVERRIDE: ContextVar[ReasoningWorkload | None] = ContextVar( + "contextual_orchestrator_reasoning_workload_override", default=None +) +_EVENT_CAPTURE: ContextVar[list[dict[str, Any]] | None] = ContextVar( + "contextual_orchestrator_reasoning_events", default=None +) +_BATCH_DECISIONS: ContextVar[dict[str, ReasoningDecision] | None] = ContextVar( + "contextual_orchestrator_batch_reasoning", default=None +) + + +def configure_agent_reasoning(agent: Any, profile: ReasoningProfile | None) -> None: + """Attach or remove an explicit reasoning capability profile from an agent.""" + if profile is None: + _AGENT_PROFILES.pop(agent, None) + elif not isinstance(profile, ReasoningProfile): + raise TypeError("profile must be ReasoningProfile or None") + else: + _AGENT_PROFILES.set(agent, profile) + + +def agent_reasoning_profile(agent: Any) -> ReasoningProfile | None: + """Return an agent's explicit reasoning capability profile, if configured.""" + return _AGENT_PROFILES.get(agent) + + +def configure_orchestrator_reasoning(orchestrator: Any, policy: ReasoningPolicy | None) -> None: + """Attach a reasoning policy to an orchestrator and its policy snapshot object.""" + if policy is None: + _ORCHESTRATOR_POLICIES.pop(orchestrator, None) + current_policy = getattr(orchestrator, "policy", None) + if current_policy is not None: + _POLICY_OBJECTS.pop(current_policy, None) + return + if not isinstance(policy, ReasoningPolicy): + raise TypeError("policy must be ReasoningPolicy or None") + _ORCHESTRATOR_POLICIES.set(orchestrator, policy) + current_policy = getattr(orchestrator, "policy", None) + if current_policy is not None: + _POLICY_OBJECTS.set(current_policy, policy) + + +def orchestrator_reasoning_policy(orchestrator: Any) -> ReasoningPolicy: + """Return the configured policy or the default adaptive policy.""" + return _ORCHESTRATOR_POLICIES.get(orchestrator, ReasoningPolicy()) + + +def current_reasoning_decision() -> ReasoningDecision | None: + """Return the decision active for the current provider call context.""" + return _ACTIVE_DECISION.get() + + +def current_reasoning_workload() -> ReasoningWorkload | None: + """Return structural evidence explicitly bound to the current recomputation.""" + return _WORKLOAD_OVERRIDE.get() + + +@contextmanager +def reasoning_override(decision: ReasoningDecision | None) -> Iterator[None]: + """Temporarily force one canonical decision, primarily for bounded retries.""" + token = _OVERRIDE_DECISION.set(decision) + try: + yield + finally: + _OVERRIDE_DECISION.reset(token) + + +@contextmanager +def reasoning_workload_override(workload: ReasoningWorkload | None) -> Iterator[None]: + """Temporarily bind exact workflow topology to a recomputed trace step.""" + if workload is not None and not isinstance(workload, ReasoningWorkload): + raise TypeError("workload must be ReasoningWorkload or None") + token = _WORKLOAD_OVERRIDE.set(workload) + try: + yield + finally: + _WORKLOAD_OVERRIDE.reset(token) + + +@contextmanager +def _decision_scope(decision: ReasoningDecision | None) -> Iterator[None]: + """Make a decision visible to nested provider-payload hooks.""" + token = _ACTIVE_DECISION.set(decision) + try: + yield + finally: + _ACTIVE_DECISION.reset(token) + + +def _message_text(messages: Sequence[Mapping[str, Any]]) -> str: + """Return the latest user text from a chat message sequence.""" + for message in reversed(messages): + if message.get("role") == "user" and isinstance(message.get("content"), str): + return message["content"] + return "" + + +def _input_text(payload: Mapping[str, Any]) -> str: + """Extract bounded selection text from chat or Responses payloads.""" + messages = payload.get("messages") + if isinstance(messages, list): + return _message_text([item for item in messages if isinstance(item, Mapping)]) + value = payload.get("input") + if isinstance(value, str): + return value + parts: list[str] = [] + if isinstance(value, list): + for item in value: + if isinstance(item, str): + parts.append(item) + elif isinstance(item, Mapping): + content = item.get("content") + if isinstance(content, str): + parts.append(content) + elif isinstance(content, list): + for chunk in content: + if isinstance(chunk, Mapping) and isinstance(chunk.get("text"), str): + parts.append(chunk["text"]) + return " ".join(parts) + + +def _infer_role(messages: Sequence[Mapping[str, Any]], fallback: str = "worker") -> str: + """Infer an orchestration role from the repository's system-prompt contract.""" + for message in messages: + if message.get("role") != "system" or not isinstance(message.get("content"), str): + continue + content = message["content"] + for role in ("thinker", "worker", "verifier", "synthesizer"): + if f"Role: {role}" in content or f"role={role}" in content: + return role + return fallback + + +def _resolve_decision( + agent: Any, + task: str, + role: str, + workload: ReasoningWorkload | None = None, +) -> ReasoningDecision | None: + """Resolve override, active decision, or policy selection for one agent.""" + profile = agent_reasoning_profile(agent) + override = _OVERRIDE_DECISION.get() + if override is not None: + return adapt_reasoning_decision(profile, override) + active = _ACTIVE_DECISION.get() + if active is not None: + return adapt_reasoning_decision(profile, active) + policy = _ACTIVE_POLICY.get() or ReasoningPolicy() + effective_workload = _WORKLOAD_OVERRIDE.get() or workload + return select_reasoning_decision( + profile, + policy, + task, + role, + workload=effective_workload, + ) + + +def _reasoning_evidence( + profile: ReasoningProfile, + decision: ReasoningDecision, + usage: Mapping[str, Any] | None, +) -> dict[str, Any]: + """Build audit evidence without retaining private intermediate reasoning content.""" + return { + "decision": decision.to_dict(), + "profile": { + "preset": profile.preset, + "supported_levels": list(profile.supported_levels), + "maximum_level": profile.maximum_level, + }, + "reasoning_tokens": extract_reasoning_tokens(usage), + } + + +def _append_event(agent: Any, role: str, decision: ReasoningDecision | None) -> None: + """Append one successful invocation event when workflow capture is active.""" + events = _EVENT_CAPTURE.get() + profile = agent_reasoning_profile(agent) + if events is not None and profile is not None and decision is not None: + events.append( + { + "agent_id": getattr(agent, "id", ""), + "role": role, + "profile": profile, + "decision": decision, + "usage": None, + } + ) + + +def _annotate_trace(trace: list[dict[str, Any]], events: list[dict[str, Any]]) -> None: + """Attach each captured decision to the corresponding visible workflow step.""" + remaining = events + for step in trace: + role = step.get("role") + agent_id = step.get("served_agent_id", step.get("agent_id")) + match_index = next( + ( + index + for index, event in enumerate(remaining) + if event["role"] == role and event["agent_id"] == agent_id + ), + None, + ) + if match_index is None: + match_index = next( + (index for index, event in enumerate(remaining) if event["role"] == role), + None, + ) + if match_index is None: + continue + event = remaining.pop(match_index) + usage = step.get("usage") if isinstance(step.get("usage"), Mapping) else event.get("usage") + step["reasoning"] = _reasoning_evidence(event["profile"], event["decision"], usage) + + +__all__ = [ + "_ACTIVE_DECISION", + "_ACTIVE_POLICY", + "_AGENT_PROFILES", + "_BATCH_DECISIONS", + "_EVENT_CAPTURE", + "_OVERRIDE_DECISION", + "_POLICY_OBJECTS", + "_WORKLOAD_OVERRIDE", + "_WeakIdentityMap", + "_annotate_trace", + "_append_event", + "_decision_scope", + "_infer_role", + "_input_text", + "_message_text", + "_reasoning_evidence", + "_resolve_decision", + "agent_reasoning_profile", + "configure_agent_reasoning", + "configure_orchestrator_reasoning", + "current_reasoning_decision", + "current_reasoning_workload", + "orchestrator_reasoning_policy", + "reasoning_override", + "reasoning_workload_override", +] diff --git a/contextual_orchestrator/_reasoning_workflow.py b/contextual_orchestrator/_reasoning_workflow.py new file mode 100644 index 000000000..68169ae48 --- /dev/null +++ b/contextual_orchestrator/_reasoning_workflow.py @@ -0,0 +1,286 @@ +"""Workflow annotation, batch projection, and bounded verifier escalation.""" + +from __future__ import annotations + +import json +from typing import Any, Callable, Mapping, Sequence + +from .reasoning_control import ( + ReasoningDecision, + ReasoningProfile, + ReasoningWorkload, + adapt_reasoning_decision, + apply_reasoning_payload, + escalate_reasoning_decision, + workload_for_trace_row, +) +from ._reasoning_state import ( + _ACTIVE_POLICY, + _EVENT_CAPTURE, + _annotate_trace, + _reasoning_evidence, + agent_reasoning_profile, + orchestrator_reasoning_policy, + reasoning_override, + reasoning_workload_override, +) + + +def _step_messages( + task: str, + row: Mapping[str, Any], + trace: Sequence[Mapping[str, Any]], +) -> list[dict[str, str]]: + """Reconstruct the repository's access-list prompt for one retryable step.""" + accessed: list[str] = [] + for raw_index in row.get("access", []): + if isinstance(raw_index, int) and not isinstance(raw_index, bool) and 0 <= raw_index < len(trace): + accessed.append(f"Step {raw_index}: {trace[raw_index].get('output', '')}") + prior = "\n\n".join(accessed) if accessed else "(none)" + role = str(row.get("role", "worker")) + return [ + { + "role": "system", + "content": ( + f"Role: {role}. Complete only the assigned subtask. " + "Use only explicitly accessed prior outputs and do not invent evidence." + ), + }, + { + "role": "user", + "content": ( + f"Original task:\n{task}\n\nAccessed prior work:\n{prior}" + f"\n\nSubtask:\n{row.get('subtask', '')}" + ), + }, + ] + + +def _refresh_step_reasoning_from_event( + row: dict[str, Any], + role: str, + served_agent_id: str, + usage: Mapping[str, Any] | None, +) -> None: + """Refresh a recomputed step with the exact captured provider decision.""" + events = _EVENT_CAPTURE.get() or [] + event = next( + ( + item + for item in reversed(events) + if item["role"] == role and item["agent_id"] == served_agent_id + ), + None, + ) + if event is None: + return + effective_usage = usage if isinstance(usage, Mapping) else event.get("usage") + row["reasoning"] = _reasoning_evidence( + event["profile"], + event["decision"], + effective_usage, + ) + + +def _retry_rejected_worker_once(orchestrator: Any, result: dict[str, Any], task: str) -> None: + """Escalate one rejected worker and recompute affected downstream roles once.""" + verification = result.get("verification") + trace = result.get("trace") + if not isinstance(verification, Mapping) or verification.get("accepted") is not False: + return + if not isinstance(trace, list): + return + worker = next((row for row in reversed(trace) if row.get("role") == "worker"), None) + if not isinstance(worker, dict): + return + evidence = worker.get("reasoning") + decision_data = evidence.get("decision") if isinstance(evidence, Mapping) else None + if not isinstance(decision_data, Mapping): + return + try: + raw_workload = decision_data.get("workload") + workload = ( + ReasoningWorkload.from_mapping(raw_workload) + if isinstance(raw_workload, Mapping) + else workload_for_trace_row(worker, trace) + ) + prior = ReasoningDecision( + level=str(decision_data["level"]), + source=str(decision_data["source"]), + role="worker", + complexity_score=int(decision_data.get("complexity_score", 0)), + factors=tuple(decision_data.get("factors", ())), + escalation_index=int(decision_data.get("escalation_index", 0)), + workload=workload, + ) + except (KeyError, TypeError, ValueError): + return + agent_id = worker.get("served_agent_id", worker.get("agent_id")) + try: + agent = orchestrator._agent(agent_id) + except (KeyError, StopIteration, TypeError): + return + policy = orchestrator_reasoning_policy(orchestrator) + profile = agent_reasoning_profile(agent) + escalated = escalate_reasoning_decision(profile, policy, prior) + if escalated is None: + return + + with reasoning_override(escalated), reasoning_workload_override(workload): + output, served_id, usage = orchestrator._invoke( + agent, + _step_messages(task, worker, trace), + text=task, + role="worker", + ) + worker["output"] = output + worker["served_agent_id"] = served_id + if usage is None: + worker.pop("usage", None) + else: + worker["usage"] = usage + served_agent = orchestrator._agent(served_id) + served_profile = agent_reasoning_profile(served_agent) + served_decision = adapt_reasoning_decision(served_profile, escalated) + if served_profile is not None and served_decision is not None: + worker["reasoning"] = _reasoning_evidence(served_profile, served_decision, usage) + + verifier = next( + ( + row + for row in trace + if row.get("role") == "verifier" + and row.get("id", -1) > worker.get("id", -1) + ), + None, + ) + if isinstance(verifier, dict): + verifier_agent = orchestrator._agent(verifier.get("agent_id")) + verifier_workload = workload_for_trace_row(verifier, trace) + with reasoning_workload_override(verifier_workload): + verifier_output, verifier_served, verifier_usage = orchestrator._invoke( + verifier_agent, + _step_messages(task, verifier, trace), + text=task, + role="verifier", + ) + verifier["output"] = verifier_output + verifier["served_agent_id"] = verifier_served + if verifier_usage is None: + verifier.pop("usage", None) + else: + verifier["usage"] = verifier_usage + _refresh_step_reasoning_from_event( + verifier, + "verifier", + verifier_served, + verifier_usage, + ) + result["verification"] = orchestrator._judge_verifier_output( + verifier_output, + str( + next( + ( + row.get("output", "") + for row in trace + if row.get("role") == "thinker" + ), + "", + ) + ), + output, + ) + + accepted = bool(result.get("verification", {}).get("accepted")) + synthesizer = next( + (row for row in reversed(trace) if row.get("role") == "synthesizer"), + None, + ) + if accepted and isinstance(synthesizer, dict): + synth_agent = orchestrator._agent(synthesizer.get("agent_id")) + synth_workload = workload_for_trace_row(synthesizer, trace) + with reasoning_workload_override(synth_workload): + synth_output, synth_served, synth_usage = orchestrator._invoke( + synth_agent, + _step_messages(task, synthesizer, trace), + text=task, + role="synthesizer", + ) + synthesizer["output"] = synth_output + synthesizer["served_agent_id"] = synth_served + if synth_usage is None: + synthesizer.pop("usage", None) + else: + synthesizer["usage"] = synth_usage + _refresh_step_reasoning_from_event( + synthesizer, + "synthesizer", + synth_served, + synth_usage, + ) + result["answer"] = synth_output + else: + result["answer"] = output + result["reasoning_escalation"] = { + "attempted": True, + "from_level": prior.level, + "to_level": escalated.level, + "accepted_after_retry": accepted, + } + + +def _rewrite_batch_payload( + payload: bytes, + decisions: Mapping[str, ReasoningDecision], + profile: ReasoningProfile, +) -> bytes: + """Apply per-item decisions to an OpenAI Batch JSONL request body.""" + output: list[str] = [] + for raw_line in payload.decode("utf-8").splitlines(): + if not raw_line.strip(): + continue + row = json.loads(raw_line) + custom_id = row.get("custom_id") + body = row.get("body") + decision = decisions.get(custom_id) if isinstance(custom_id, str) else None + if isinstance(body, Mapping) and decision is not None: + row["body"] = apply_reasoning_payload( + body, + profile, + decision, + "chat/completions", + ) + output.append(json.dumps(row, ensure_ascii=False, separators=(",", ":"))) + return ("\n".join(output) + "\n").encode("utf-8") + + +def _capture_batch( + orchestrator: Any, + operation: Callable[[Any, list[str]], list[dict[str, Any]]], + prompts: list[str], +) -> list[dict[str, Any]]: + """Capture and annotate reasoning evidence for a batch-route operation.""" + events: list[dict[str, Any]] = [] + event_token = _EVENT_CAPTURE.set(events) + policy = orchestrator_reasoning_policy(orchestrator) + policy_token = _ACTIVE_POLICY.set(policy) + try: + records = operation(orchestrator, prompts) + for record in records: + trace = record.get("trace") + if isinstance(trace, list): + _annotate_trace(trace, events) + record["reasoning_control"] = policy.to_dict() + finally: + _ACTIVE_POLICY.reset(policy_token) + _EVENT_CAPTURE.reset(event_token) + return records + + +__all__ = [ + "_capture_batch", + "_refresh_step_reasoning_from_event", + "_retry_rejected_worker_once", + "_rewrite_batch_payload", + "_step_messages", +] diff --git a/contextual_orchestrator/_reasoning_workload.py b/contextual_orchestrator/_reasoning_workload.py new file mode 100644 index 000000000..9f508acc9 --- /dev/null +++ b/contextual_orchestrator/_reasoning_workload.py @@ -0,0 +1,220 @@ +"""Validated workflow-structure evidence for test-time reasoning allocation.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +import re +from typing import Any, Mapping, Sequence + +_ACCESS_SECTION = re.compile( + r"Accessed prior work:\n(?P
.*?)(?:\n\nSubtask:|\Z)", + re.DOTALL, +) +_STEP_LINE = re.compile(r"^Step\s+(?P\d+):", re.MULTILINE) + + +@dataclass(frozen=True) +class ReasoningWorkload: + """Structural evidence for one role invocation in a workflow graph. + + The object intentionally contains no latency field. Reasoning allocation is + based on workflow topology, role, task evidence, and operator caps rather + than response-speed targets. + """ + + workflow_step_index: int = 0 + workflow_step_count: int = 1 + recursion_depth: int = 0 + decomposition_count: int = 1 + accessible_step_count: int = 0 + + def __post_init__(self) -> None: + """Reject boolean pseudo-integers and impossible workflow topology.""" + values = { + "workflow_step_index": self.workflow_step_index, + "workflow_step_count": self.workflow_step_count, + "recursion_depth": self.recursion_depth, + "decomposition_count": self.decomposition_count, + "accessible_step_count": self.accessible_step_count, + } + if any(isinstance(value, bool) or not isinstance(value, int) for value in values.values()): + raise ValueError("reasoning workload values must be integers") + if self.workflow_step_count < 1 or self.decomposition_count < 1: + raise ValueError("workflow_step_count and decomposition_count must be positive") + if any( + value < 0 + for name, value in values.items() + if name not in {"workflow_step_count", "decomposition_count"} + ): + raise ValueError("reasoning workload values must be non-negative") + if self.workflow_step_index >= self.workflow_step_count: + raise ValueError("workflow_step_index must be within workflow_step_count") + if self.recursion_depth > self.workflow_step_index: + raise ValueError("recursion_depth cannot exceed workflow_step_index") + if self.accessible_step_count > self.workflow_step_index: + raise ValueError("accessible_step_count cannot exceed prior workflow steps") + if self.decomposition_count > self.workflow_step_count: + raise ValueError("decomposition_count cannot exceed workflow_step_count") + + @classmethod + def from_mapping(cls, value: Mapping[str, Any]) -> "ReasoningWorkload": + """Parse one strict JSON-compatible workflow workload object.""" + if not isinstance(value, Mapping): + raise ValueError("reasoning workload must be an object") + allowed = { + "workflow_step_index", + "workflow_step_count", + "recursion_depth", + "decomposition_count", + "accessible_step_count", + } + unknown = set(value) - allowed + if unknown: + raise ValueError(f"unknown reasoning workload keys: {sorted(unknown)}") + return cls(**{key: value[key] for key in allowed if key in value}) + + def to_dict(self) -> dict[str, int]: + """Return stable audit evidence for the structural allocation input.""" + return { + "workflow_step_index": self.workflow_step_index, + "workflow_step_count": self.workflow_step_count, + "recursion_depth": self.recursion_depth, + "decomposition_count": self.decomposition_count, + "accessible_step_count": self.accessible_step_count, + } + + +@dataclass +class WorkflowReasoningCursor: + """Track structural position while a route or conducted workflow executes.""" + + workflow_step_count: int + decomposition_count: int | None = None + next_step_index: int = 0 + depths: dict[int, int] = field(default_factory=dict) + + def __post_init__(self) -> None: + """Normalize the initial decomposition count and validate positivity.""" + if ( + isinstance(self.workflow_step_count, bool) + or not isinstance(self.workflow_step_count, int) + or self.workflow_step_count < 1 + ): + raise ValueError("workflow_step_count must be a positive integer") + if self.decomposition_count is None: + self.decomposition_count = self.workflow_step_count + if ( + isinstance(self.decomposition_count, bool) + or not isinstance(self.decomposition_count, int) + or self.decomposition_count < 1 + ): + raise ValueError("decomposition_count must be a positive integer") + if self.decomposition_count > self.workflow_step_count: + raise ValueError("decomposition_count cannot exceed workflow_step_count") + + def set_plan_size(self, step_count: int) -> None: + """Replace the provisional template size with a validated generated plan size.""" + if isinstance(step_count, bool) or not isinstance(step_count, int) or step_count < 1: + raise ValueError("generated workflow step_count must be a positive integer") + if self.next_step_index: + raise RuntimeError("workflow plan size cannot change after step execution begins") + self.workflow_step_count = step_count + self.decomposition_count = step_count + + def observe(self, messages: Sequence[Mapping[str, Any]]) -> ReasoningWorkload | None: + """Return and advance the structural evidence for the next workflow step.""" + if self.next_step_index >= self.workflow_step_count: + return None + step_index = self.next_step_index + access_ids, had_access_content = _access_ids(messages) + if not access_ids and had_access_content and self.workflow_step_count == 4: + access_ids = tuple(range(step_index)) + access_ids = tuple(sorted({value for value in access_ids if 0 <= value < step_index})) + recursion_depth = 0 + if access_ids: + recursion_depth = 1 + max(self.depths.get(value, 0) for value in access_ids) + workload = ReasoningWorkload( + workflow_step_index=step_index, + workflow_step_count=self.workflow_step_count, + recursion_depth=recursion_depth, + decomposition_count=int(self.decomposition_count), + accessible_step_count=len(access_ids), + ) + self.depths[step_index] = recursion_depth + self.next_step_index += 1 + return workload + + +def _access_ids(messages: Sequence[Mapping[str, Any]]) -> tuple[tuple[int, ...], bool]: + """Extract exact accessed step identifiers from the bounded prompt section.""" + user_text = "\n".join( + str(message.get("content", "")) + for message in messages + if message.get("role") == "user" and isinstance(message.get("content"), str) + ) + match = _ACCESS_SECTION.search(user_text) + if match is None: + return (), False + section = match.group("section").strip() + had_access_content = bool(section and section != "(none)") + return tuple(int(item.group("step_id")) for item in _STEP_LINE.finditer(section)), had_access_content + + +def workload_for_trace_row( + row: Mapping[str, Any], + trace: Sequence[Mapping[str, Any]], +) -> ReasoningWorkload: + """Reconstruct structural evidence for one existing trace row and its access list.""" + rows = [ + item + for item in trace + if isinstance(item.get("id"), int) + and not isinstance(item.get("id"), bool) + and item["id"] >= 0 + ] + if not rows: + raise ValueError("trace must contain at least one non-negative integer step id") + target_id = row.get("id") + if isinstance(target_id, bool) or not isinstance(target_id, int) or target_id < 0: + raise ValueError("trace row id must be a non-negative integer") + step_count = max(item["id"] for item in rows) + 1 + if target_id >= step_count: + raise ValueError("trace row id is outside the workflow") + depths: dict[int, int] = {} + target_access: tuple[int, ...] = () + for item in sorted(rows, key=lambda item: item["id"]): + step_id = item["id"] + raw_access = item.get("access", ()) + access = ( + tuple( + sorted( + { + value + for value in raw_access + if isinstance(value, int) + and not isinstance(value, bool) + and 0 <= value < step_id + } + ) + ) + if isinstance(raw_access, (list, tuple)) + else () + ) + depths[step_id] = 0 if not access else 1 + max(depths.get(value, 0) for value in access) + if step_id == target_id: + target_access = access + return ReasoningWorkload( + workflow_step_index=target_id, + workflow_step_count=step_count, + recursion_depth=depths.get(target_id, 0), + decomposition_count=len(rows), + accessible_step_count=len(target_access), + ) + + +__all__ = [ + "ReasoningWorkload", + "WorkflowReasoningCursor", + "_access_ids", + "workload_for_trace_row", +] diff --git a/contextual_orchestrator/reasoning_control.py b/contextual_orchestrator/reasoning_control.py new file mode 100644 index 000000000..544939044 --- /dev/null +++ b/contextual_orchestrator/reasoning_control.py @@ -0,0 +1,58 @@ +"""Provider-neutral control of model reasoning effort. + +The public facade combines explicit model capability profiles, role-sensitive +compute selection, workflow-structure evidence, provider payload projection, +bounded verifier escalation, and measured reasoning-token evidence without +retaining private model traces. +""" + +from ._reasoning_payload import ( + _any_complete_path, + _normalize_endpoint, + _render_value, + _rules_for, + _set_nested_if_absent, + apply_reasoning_payload, + extract_reasoning_tokens, + sum_usage_tokens, +) +from ._reasoning_policy import ( + _coerce_workload, + _nearest_supported, + ReasoningAblationCell, + ReasoningDecision, + ReasoningPolicy, + ReasoningWorkload, + adapt_reasoning_decision, + escalate_reasoning_decision, + select_reasoning_decision, +) +from ._reasoning_profile import ( + _parse_rules, + CANONICAL_REASONING_LEVELS, + PayloadRule, + ReasoningProfile, +) +from ._reasoning_workload import ( + WorkflowReasoningCursor, + _access_ids, + workload_for_trace_row, +) + +__all__ = [ + "CANONICAL_REASONING_LEVELS", + "PayloadRule", + "ReasoningAblationCell", + "ReasoningDecision", + "ReasoningPolicy", + "ReasoningProfile", + "ReasoningWorkload", + "WorkflowReasoningCursor", + "adapt_reasoning_decision", + "apply_reasoning_payload", + "escalate_reasoning_decision", + "extract_reasoning_tokens", + "select_reasoning_decision", + "sum_usage_tokens", + "workload_for_trace_row", +] diff --git a/contextual_orchestrator/reasoning_runtime.py b/contextual_orchestrator/reasoning_runtime.py new file mode 100644 index 000000000..d0a8274a3 --- /dev/null +++ b/contextual_orchestrator/reasoning_runtime.py @@ -0,0 +1,100 @@ +"""Runtime integration for provider-neutral adaptive reasoning control. + +Importing this module is side-effect free. Library consumers explicitly call +:func:`enable_reasoning_control` before constructing or loading runtime objects; +the product CLI performs that activation during command execution. The lower- +level installer remains available for isolated fakes and alternate runtimes. +""" + +from __future__ import annotations + +from typing import Any + +from ._reasoning_client_hooks import install_client_hooks +from ._reasoning_config_hooks import install_config_hooks +from ._reasoning_orchestrator_hooks import install_orchestrator_hooks +from ._reasoning_state import ( + _ACTIVE_DECISION, + _ACTIVE_POLICY, + _AGENT_PROFILES, + _BATCH_DECISIONS, + _EVENT_CAPTURE, + _OVERRIDE_DECISION, + _POLICY_OBJECTS, + _WORKLOAD_OVERRIDE, + _WeakIdentityMap, + _annotate_trace, + _append_event, + _decision_scope, + _infer_role, + _input_text, + _message_text, + _reasoning_evidence, + _resolve_decision, + agent_reasoning_profile, + configure_agent_reasoning, + configure_orchestrator_reasoning, + current_reasoning_decision, + current_reasoning_workload, + orchestrator_reasoning_policy, + reasoning_override, + reasoning_workload_override, +) +from ._reasoning_workflow import ( + _capture_batch, + _refresh_step_reasoning_from_event, + _retry_rejected_worker_once, + _rewrite_batch_payload, + _step_messages, +) + + +def install_reasoning_control( + model_agent_type: type[Any], + model_client_type: type[Any], + orchestrator_type: type[Any], + policy_type: type[Any], +) -> None: + """Install reasoning control on supplied runtime classes exactly once.""" + if getattr(model_client_type, "_reasoning_control_installed", False): + return + install_config_hooks(model_agent_type, orchestrator_type, policy_type) + install_client_hooks(model_client_type) + install_orchestrator_hooks(orchestrator_type) + model_client_type._reasoning_control_installed = True + + +def enable_reasoning_control() -> None: + """Explicitly activate reasoning control for the built-in runtime classes. + + Call this before loading agent configuration or constructing a + :class:`~contextual_orchestrator.orchestrator.TaskOrchestrator`. Repeated + calls are safe and do not wrap methods more than once. + """ + from .orchestrator import ( # Local import preserves package import purity. + ModelAgent, + ModelClient, + OrchestrationPolicy, + TaskOrchestrator, + ) + + install_reasoning_control( + ModelAgent, + ModelClient, + TaskOrchestrator, + OrchestrationPolicy, + ) + + +__all__ = [ + "agent_reasoning_profile", + "configure_agent_reasoning", + "configure_orchestrator_reasoning", + "current_reasoning_decision", + "current_reasoning_workload", + "enable_reasoning_control", + "install_reasoning_control", + "orchestrator_reasoning_policy", + "reasoning_override", + "reasoning_workload_override", +] diff --git a/docs/architecture.md b/docs/architecture.md index c0f63a81e..99e1ffc7b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -2,54 +2,94 @@ ## Sources Read -- Sakana AI launch article, "Sakana Fugu: One Model to Command Them All" (June 22, 2026): https://sakana.ai/fugu-release/ +- Sakana AI launch article, “Sakana Fugu: One Model to Command Them All” (June 22, 2026): https://sakana.ai/fugu-release/ - Sakana Fugu Technical Report: https://github.com/SakanaAI/fugu/blob/main/Fugu_technical_report.pdf - TRINITY: An Evolved LLM Coordinator: https://arxiv.org/abs/2512.04695 - Learning to Orchestrate Agents in Natural Language with the Conductor: https://arxiv.org/abs/2512.04388 +- Scaling LLM Test-Time Compute Optimally Can Be More Effective Than Scaling Model Parameters: https://arxiv.org/abs/2408.03314 +- RouteLLM: Learning to Route LLMs with Preference Data: https://arxiv.org/abs/2406.18665 +- FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance: https://arxiv.org/abs/2305.05176 +- ISO/IEC 23894:2023 and ISO/IEC 42001:2023. +- Current official OpenAI, NVIDIA NIM, and Gemini reasoning-control documentation. + +See `docs/doctoring/adaptive-reasoning-control.md` for the source-to-design trace and APA 7th references. ## What The Architecture Is -The public shape is a single model API. The internal shape is a model pool plus a learned coordinator that decides when to answer directly, when to delegate, how much context each worker receives, when to verify, and how to synthesize the final answer. +The public shape is a single model API. The internal shape is a model pool plus a learned-coordinator-compatible control plane that decides when to answer directly, when to delegate, how much context each worker receives, when to verify, how to synthesize, and how much model-native reasoning compute each call should receive. -The useful split is quality-latency, not separate products: +The useful split is compute allocation, not separate products: -- Low-latency routing: select one worker for the current query or turn. -- Deep orchestration: create a multi-step workflow when the task needs decomposition, independent attempts, verification, or synthesis. +- **Model allocation:** select one worker and exhaust eligible free fallbacks before paid candidates. +- **Topology allocation:** use low-overhead routing or a deeper workflow with bounded steps, subtasks, recursive planning, and access lists. +- **Reasoning allocation:** select the least costly supported reasoning level justified for each role and task, then escalate once only when verification rejects the worker result. TRINITY contributes the compact coordinator idea: a small model representation plus a lightweight head can choose agent and role over multiple turns. Its Thinker, Worker, and Verifier contracts are practical enough to implement directly. -Conductor contributes the workflow representation: each step is a natural-language subtask, an assigned worker, and an access list of prior step outputs. This is the key piece for preventing every worker from being dragged into the same transcript while still allowing deliberate collaboration. +Conductor contributes the workflow representation: each step is a natural-language subtask, an assigned worker, and an access list of prior step outputs. This prevents every worker from receiving the same transcript while permitting deliberate collaboration. + +Fugu combines these ideas into production constraints: -The Fugu report combines these ideas into production constraints: +- one compatible API hides model-pool orchestration; +- low-overhead routing and deeper quality-oriented orchestration remain selectable; +- the agent pool is swappable for provider, compliance, and availability constraints; +- recursive and multi-agent work needs bounded memory and explicit visibility. -- Fugu is optimized for latency by selecting a worker without expensive coordinator generation. -- Fugu-Ultra is optimized for quality by generating deeper workflows over a broader agent pool. -- The agent pool is swappable, allowing provider preference, model exclusion, and compliance controls. -- Multi-agent tool/function-call workflows need memory discipline: isolate agents inside the current workflow, but keep useful shared memory across turns. +Adaptive reasoning control adds the missing within-model compute dimension. Provider capability is explicit configuration. The runtime never assumes that two models accept the same effort vocabulary or payload path. ## Implementation Mapping This repository implements the interface and control plane, not the trained coordinator. -- `contextual_orchestrator.orchestrator.Agent`: one configured worker model. -- `Orchestrator.route_once`: the low-latency routing path. -- `Orchestrator.conduct`: the workflow path with planner, worker, verifier, and synthesizer steps. +- `contextual_orchestrator.orchestrator.ModelAgent`: one configured worker model. +- `TaskOrchestrator.route_once`: low-topology-compute routing. +- `TaskOrchestrator.conduct`: planner, worker, verifier, and synthesizer workflow. - `WorkflowStep.access`: Conductor-style visibility control. -- `ModelClient`: OpenAI-compatible HTTP client, with `mock://` for local checks. -- `contextual_orchestrator.server`: small `/v1/chat/completions` HTTP server. - -The deliberate simplification is the policy. The paper systems learn routing and topology from rewards; this lab uses deterministic keyword scoring so the repo runs without training data, GPUs, or vendor credentials. - -Add learned routing only when there is an evaluation set and logs proving the heuristic policy is the bottleneck. +- `model_fallback`: deterministic free-first candidate eligibility and ordering. +- `reasoning_control`: provider-neutral profiles, policies, decisions, payload rules, token evidence, and ablation cells. +- `reasoning_runtime`: idempotent integration across agent configuration, admin views, provider calls, failover, streaming, Responses passthrough, Batch, workflow traces, verifier escalation, and ablation. +- `provider_transport`: DNS-pinned HTTPS egress with original-host TLS verification, no environment proxy, and redirect rejection. +- `contextual_orchestrator.server`: compatible API and admin control plane. + +The deliberate simplification remains the coordinator policy. The paper systems learn routing and topology from rewards; this lab uses deterministic policy so the repository runs without training data, GPUs, or vendor credentials. Learned routing should be added only after replayable evaluations and logs show the deterministic policy is the bottleneck. + +## Adaptive Reasoning Data Flow + +```text +request + → route/conduct and role selection + → canonical ReasoningDecision + → candidate-specific capability projection + → endpoint-specific payload mapping + → provider call through pinned transport + → provider usage parsing + → bounded trace evidence + → verifier rejection? one next-level worker retry + → affected verifier/synthesizer recomputation +``` + +Caller-owned reasoning fields are never overwritten. Custom mappings are safe nested paths plus fixed scalar templates; they cannot execute expressions. Hidden private intermediate reasoning content is not persisted. + +## Failure and Compatibility Semantics + +- An agent without a reasoning profile preserves legacy request shape. +- An unsupported canonical level projects downward to the nearest declared level. +- A failover model remaps the canonical decision instead of inheriting another provider’s payload. +- Invalid custom paths or mappings fail at configuration time. +- A verifier retry is limited to one immediate higher supported level. +- No higher level, a disabled strategy, or a zero escalation cap means no retry. +- Admin patch operations preserve a profile across frozen-dataclass replacement and re-save it through the configured pool store. ## Product Planning Interpretation -The product is not a Fugu clone. It is a control-plane prototype for the same public shape: one compatible API with hidden orchestration. The enterprise value comes from exposing the hidden operating evidence: +The product is not a Fugu clone. It is a control plane for the same public shape: one compatible API with hidden but auditable orchestration. Enterprise value comes from exposing operating evidence: -- pool health and provider exclusion for Fugu-style configurability; -- latency-quality policy for the Fugu versus Fugu-Ultra tradeoff; -- thinker, worker, verifier, and synthesizer roles for TRINITY-style trace review; -- natural-language subtasks and access lists for Conductor-style auditability; -- replayable evaluation runs before any learned coordinator replaces the deterministic policy. +- pool health, visibility, cost tier, credential requirements, and provider exclusion; +- direct-versus-deep topology decisions; +- thinker, worker, verifier, and synthesizer traces; +- natural-language subtasks and access lists; +- role-specific reasoning decisions, caps, escalation, and token use; +- fixed-effort ablations before production policy changes; +- replayable task evaluations before any learned coordinator replaces deterministic policy. -See [product_planning.md](product_planning.md) for the product reboot. +See `docs/product_planning.md` for the broader product reboot and `docs/reasoning-control.md` for the subsystem contract. diff --git a/docs/doctoring/adaptive-reasoning-control.md b/docs/doctoring/adaptive-reasoning-control.md new file mode 100644 index 000000000..02515cfb4 --- /dev/null +++ b/docs/doctoring/adaptive-reasoning-control.md @@ -0,0 +1,51 @@ +# Adaptive Reasoning Control — Evidence Doctoring + +## Claim boundary + +The implementation claims provider-neutral control only for settings explicitly declared by a model profile. It does not claim that every model supports every canonical level, that higher effort always improves every task, or that verifier acceptance is equivalent to human-judged quality. + +## Source-to-design trace + +| Design decision | Source support | Implementation consequence | +|---|---|---| +| Separate direct routing from deeper orchestration | Fugu; Conductor; TRINITY | Existing route/conduct split remains independent from effort control. | +| Use roles and access-constrained workflows | Conductor; TRINITY | Thinker, worker, verifier, and synthesizer receive role-specific decisions; access lists remain authoritative. | +| Allocate test-time compute by task | Snell et al. | Adaptive policy starts cheaply and raises effort only for bounded complexity or risk evidence. | +| Route inexpensive models before costly models | FrugalGPT; RouteLLM | Reasoning control composes with the versioned free-first fallback policy rather than replacing it. | +| Support model-dependent effort values | Official OpenAI, NVIDIA NIM, and Gemini documentation | Capabilities and payload mappings are explicit profiles; model names are not parsed heuristically. | +| Observe reasoning-token consumption | Official provider usage contracts | Trace evidence records counts, never private intermediate reasoning text. | +| Manage AI risk and governance evidence | ISO/IEC 23894:2023; ISO/IEC 42001:2023 | Decisions, caps, overrides, escalation, and ablations are machine-readable and auditable. | + +## Activation and ownership boundary + +Adaptive reasoning is an optional runtime extension. Package import must remain free of reasoning-related class mutation so standalone consumers, central `.github` automation, naruon, and other CWL services can inspect or import the library without import-order-dependent behavior. + +The built-in executable explicitly activates the extension before it loads agent configuration. Programmatic consumers call `enable_reasoning_control()` before loading agents or constructing an orchestrator. The operation is idempotent and retains a lower-level typed installer for isolated alternative runtimes and deterministic test fakes. + +This boundary is an architectural control rather than an empirical research claim. It prevents optional capability activation from becoming an implicit global side effect, makes ownership observable at the application composition root, and permits a later replacement of hooks with composition or subclasses without changing the public activation contract. + +## Provider documentation reviewed + +- OpenAI API model and reasoning guidance: model-dependent effort sets and usage-level reasoning token details. +- NVIDIA NIM for LLMs: `reasoning_effort`, `enable_thinking`, `low_effort`, hard reasoning budgets, and model-dependent parallel reasoning. +- Google Gemini OpenAI compatibility: explicit mapping between OpenAI reasoning effort and Gemini thinking levels or budgets. + +These provider contracts evolve. Profiles are therefore operator-controlled data, not hard-coded model inventories. Unsupported or expired mappings must be removed or updated through reviewed configuration. + +## APA 7th references + +Chen, L., Zaharia, M., & Zou, J. (2023). FrugalGPT: How to use large language models while reducing cost and improving performance. *arXiv*. https://arxiv.org/abs/2305.05176 + +International Organization for Standardization. (2023a). *ISO/IEC 23894:2023 Information technology—Artificial intelligence—Guidance on risk management*. https://www.iso.org/standard/77304.html + +International Organization for Standardization. (2023b). *ISO/IEC 42001:2023 Information technology—Artificial intelligence—Management system*. https://www.iso.org/standard/42001.html + +Nielsen, S., Cetin, E., Schwendeman, P., Sun, Q., Xu, J., & Tang, Y. (2025). Learning to orchestrate agents in natural language with the Conductor. *arXiv*. https://arxiv.org/abs/2512.04388 + +Ong, I., Almahairi, A., Wu, V., Chiang, W.-L., Wu, T., Gonzalez, J. E., Kadous, M. W., & Stoica, I. (2024). RouteLLM: Learning to route LLMs with preference data. *arXiv*. https://arxiv.org/abs/2406.18665 + +Sakana AI. (2026, June 22). *Sakana Fugu: One model to command them all*. https://sakana.ai/fugu-release/ + +Snell, C., Lee, J., Xu, K., & Kumar, A. (2024). Scaling LLM test-time compute optimally can be more effective than scaling model parameters. *arXiv*. https://arxiv.org/abs/2408.03314 + +Xu, J., Sun, Q., Schwendeman, P., Nielsen, S., Cetin, E., & Tang, Y. (2025). TRINITY: An evolved LLM coordinator. *arXiv*. https://arxiv.org/abs/2512.04695 diff --git a/docs/library_research.md b/docs/library_research.md index 42c7fa95c..6445aee7e 100644 --- a/docs/library_research.md +++ b/docs/library_research.md @@ -1,58 +1,71 @@ # Library Research -The design researched existing libraries before adding code. The repository keeps the runtime dependency-free for the current lab, but the enterprise implementation target is explicit. +The design researched existing libraries and provider contracts before adding code. The repository keeps the runtime dependency-free for the current lab, but the enterprise implementation target is explicit. ## Selected Stack -| Area | Library | Decision | Evidence | +| Area | Library or contract | Decision | Evidence | |---|---|---|---| -| REST API | [FastAPI](https://github.com/fastapi/fastapi) | Use when the API moves beyond the current stdlib prototype. | FastAPI provides request validation with Pydantic models, standard status/response declarations, and OpenAPI/JSON Schema generation. Context7: `/fastapi/fastapi`. | -| Admin console | [React-admin](https://github.com/marmelab/react-admin) | Use for production CRUD/admin surfaces. | React-admin has `Admin`, `Resource`, `dataProvider`, `authProvider`, `i18nProvider`, dashboard, layout, and custom route hooks. Context7: `/marmelab/react-admin`. | -| i18n | [i18next](https://github.com/i18next/i18next) | Use for shared web translation runtime, especially outside React-admin defaults. | i18next supports resource bundles, `fallbackLng`, interpolation, language detection, and runtime `changeLanguage`. Context7: `/i18next/i18next`. | -| Persistence | [SQLAlchemy 2.x](https://docs.sqlalchemy.org/orm/) | Use for Python domain persistence. | Official docs cover ORM mapped classes and sessions. | -| Migrations | [Alembic](https://alembic.sqlalchemy.org/) | Use for schema migration lifecycle. | Alembic is the SQLAlchemy migration tool and supports autogenerated migrations from metadata. | -| Database | [PostgreSQL](https://www.postgresql.org/docs/current/sql-syntax-lexical.html) | Default relational store. | PostgreSQL identifiers allow letters, digits, and underscores; the project standardizes on unquoted lower snake_case. | -| API contract | [OpenAPI 3.1](https://spec.openapis.org/oas/v3.1.0.html) | Contract format for API review and client generation. | OAS defines a language-agnostic HTTP API description for humans and machines. | +| REST API | [FastAPI](https://github.com/fastapi/fastapi) | Use when the API moves beyond the current stdlib prototype. | FastAPI provides Pydantic validation, response declarations, and OpenAPI/JSON Schema generation. Context7: `/fastapi/fastapi`. | +| Admin console | [React-admin](https://github.com/marmelab/react-admin) | Use for production CRUD/admin surfaces. | React-admin provides resource, data-provider, authentication, i18n, dashboard, layout, and route extension points. Context7: `/marmelab/react-admin`. | +| i18n | [i18next](https://github.com/i18next/i18next) | Use for shared web translation runtime. | i18next supports resource bundles, fallback languages, interpolation, detection, and runtime language changes. Context7: `/i18next/i18next`. | +| Persistence | [SQLAlchemy 2.x](https://docs.sqlalchemy.org/orm/) | Use for Python domain persistence. | Official docs cover mapped classes, sessions, and transaction patterns. | +| Migrations | [Alembic](https://alembic.sqlalchemy.org/) | Use for schema migration lifecycle. | Alembic supports versioned SQLAlchemy migrations and metadata comparison. | +| Database | [PostgreSQL](https://www.postgresql.org/docs/current/sql-syntax-lexical.html) | Default relational store. | The project standardizes new objects on unquoted lower two-or-more-word `snake_case`. | +| API contract | [OpenAPI 3.1](https://spec.openapis.org/oas/v3.1.0.html) | Contract format for review and client generation. | OAS is a language-agnostic HTTP API description for humans and machines. | +| Reasoning control | Official OpenAI, NVIDIA NIM, and Gemini contracts | Use explicit profiles and stdlib JSON projection; do not add provider SDKs. | Providers expose different effort sets, thinking toggles, budgets, nesting, and endpoint support. | +| Orchestration research | Fugu, Conductor, TRINITY, RouteLLM, FrugalGPT, test-time-compute scaling | Preserve three independent compute axes and measure their trade-offs. | The papers support learned routing/topology, role assignment, recursive scaling, cost-aware routing, and task-dependent compute allocation. | +| AI governance | ISO/IEC 23894:2023; ISO/IEC 42001:2023 | Record policy, capability, decision, override, escalation, usage, and ablation evidence. | The standards support risk-management integration and an organization-level AI management system. | + +## Reasoning-Control Decision + +No new runtime dependency is required. A provider SDK would make the core less portable and would not solve the model-dependent capability problem. The selected design uses: + +- immutable Python dataclasses for validated profiles and decisions; +- `ContextVar` for request-local canonical decisions and bounded overrides; +- explicit nested JSON rules for provider payload mapping; +- weak identity registries so frozen value objects do not collide by equality; +- an idempotent runtime installer so standalone core use remains possible; +- existing pinned HTTPS and KV credential seams for provider calls. + +Deliberately skipped: + +- model-name parsing or hard-coded authoritative model inventories; +- arbitrary dictionaries copied into provider requests; +- expression languages for payload mutation; +- provider SDK dependencies; +- hidden reasoning-text persistence; +- an unbounded “retry until accepted” loop; +- a learned effort router before sufficient task-level evaluation evidence exists. ## Ponytail Decision No new dependency is added until it carries real product weight: -- Current prototype: stdlib server, handwritten OpenAPI, static admin UI. +- Current prototype: stdlib server, handwritten OpenAPI, static admin UI, stdlib reasoning-control extension. - First enterprise cut: FastAPI + React-admin + i18next + PostgreSQL + SQLAlchemy + Alembic. -- Do not add provider SDKs until raw OpenAI-compatible HTTP is insufficient. +- Do not add provider SDKs until raw OpenAI-compatible HTTP and validated custom mappings are demonstrably insufficient. -Skipped: custom admin framework, custom i18n engine, custom migration engine. +Skipped: custom admin framework, custom i18n engine, custom migration engine, and duplicated provider clients. ## Commercial Packaging Decision -For the KRW 2,000,000,000 commercial-readiness plan, keep Contextual -Orchestrator as one repository and one deployable product. Do not split the -orchestration core into a separate library, Git submodule, or package yet. +For the commercial-readiness plan, keep Contextual Orchestrator as one repository and one deployable product. Do not split the orchestration core into a separate library, Git submodule, or package yet. Reason: -- The buyer value is the integrated system: compatible API, admin evidence - surface, workflow trace, access-list reports, analytics snapshot, sales - readiness, and commercial readiness. -- A separate library would create release, versioning, and support overhead - before there is an external SDK consumer or independent orchestration-core - release cadence. -- A Git submodule would make due-diligence review harder because buyers need a - single evidence packet, not a multi-repo dependency chain. +- Buyer value is the integrated compatible API, routing, reasoning controls, admin evidence, workflow trace, access reports, analytics, and governance boundary. +- A separate library would add release, versioning, and support overhead before an external SDK consumer or independent core cadence exists. +- A Git submodule would make due-diligence review harder because buyers need one evidence packet rather than a multi-repository dependency chain. Extraction triggers: -- A second product or external customer needs the orchestration engine without - the admin control plane. -- The orchestration core needs a separately versioned API and compatibility - matrix. -- Security review requires a reusable, locked core package with independent - provenance. +- A second product or external customer needs the engine without the admin control plane. +- The core needs a separately versioned API and compatibility matrix. +- Security review requires a reusable locked package with independent provenance. -Until those triggers exist, Ponytail recommends strengthening the current -single-repo product instead of splitting it. +Until those triggers exist, strengthen the single-repo product while retaining focused, importable modules. ## Required For New Designs -Every new subsystem design must update this file before implementation starts. The entry must name the existing libraries researched, the selected library or stdlib alternative, and the custom code that was deliberately skipped. +Every new subsystem design must update this file before implementation. The entry must name the libraries, standards, papers, or official provider contracts researched; the selected library or stdlib alternative; and the custom code deliberately skipped. diff --git a/docs/reasoning-control.md b/docs/reasoning-control.md new file mode 100644 index 000000000..51551a3da --- /dev/null +++ b/docs/reasoning-control.md @@ -0,0 +1,177 @@ +# Adaptive Reasoning Control + +## Purpose + +Contextual Orchestrator controls three independent forms of test-time compute: + +1. **model routing** — choose one worker or a fallback candidate; +2. **workflow topology** — route directly or construct a Conductor/TRINITY-style multi-step workflow with explicit subtasks and access lists; +3. **reasoning effort** — request only the model-specific reasoning level justified for each role, task, and workflow position. + +The third axis is explicit in this subsystem. It is not inferred from a model name, provider brand, undocumented default, latency target, or response-speed objective. + +## Activation boundary + +Importing `contextual_orchestrator` does not activate or monkey-patch the optional reasoning extension. This preserves a predictable standalone library and lets another CWL service compose the package without import-order-dependent class mutation. + +The packaged CLI and HTTP-server entrypoint activate reasoning explicitly before agent configuration is loaded. Library consumers opt in before loading agents or constructing an orchestrator: + +```python +from contextual_orchestrator import enable_reasoning_control +from contextual_orchestrator.orchestrator import TaskOrchestrator, load_agents + +enable_reasoning_control() +orchestrator = TaskOrchestrator(load_agents("agents.json")) +``` + +Activation is idempotent. Calling it again does not wrap methods twice. Applications that do not call it retain the legacy runtime shape, while the explicit product entrypoint enables the full reasoning profile and policy contract. + +## Configuration contract + +Each model may declare a `reasoning_profile`: + +```json +{ + "id": "openai_reasoning_agent", + "model": "provider-model-id", + "base_url": "https://provider.example/v1", + "credential_key": "PROVIDER_API_KEY", + "reasoning_profile": { + "preset": "openai_effort", + "supported_levels": ["none", "low", "medium", "high", "xhigh"], + "default_level": "low", + "maximum_level": "high" + } +} +``` + +Canonical levels are ordered as: + +```text +none < minimal < low < medium < high < xhigh < max +``` + +A failover model receives the same canonical decision projected to its nearest supported level. Unsupported settings are never guessed. + +## Built-in mappings + +| Preset | Chat Completions | Responses | +|---|---|---| +| `openai_effort` | `reasoning_effort` | `reasoning.effort` | +| `nvidia_reasoning_effort` | `reasoning_effort` | `reasoning.effort` when the endpoint supports it | +| `nvidia_nemotron_thinking` | `chat_template_kwargs.enable_thinking` and `low_effort` | no implicit mapping | +| `gemini_thinking_level` | `extra_body.google.thinking_config.thinking_level` | no implicit mapping | +| `custom` | validated nested rules | validated nested rules | + +NVIDIA NIM also exposes model-dependent hard thinking budgets and parallel-reasoning modes. These are configured through strict `custom` rules and integer mappings rather than being sent to models that may not support them. + +```json +{ + "preset": "custom", + "supported_levels": ["low", "medium", "high"], + "default_level": "low", + "maximum_level": "high", + "level_values": {"low": 64, "medium": 256, "high": 1024}, + "chat_rules": [ + {"path": ["chat_template_kwargs", "enable_thinking"], "value": true}, + {"path": ["chat_template_kwargs", "reasoning_budget"], "value": "$int"} + ] +} +``` + +## Adaptive policy + +The default policy begins at the model profile's operator-declared default. It raises effort only for bounded evidence and never uses response speed as an allocation signal. + +Semantic and role evidence: + +- thinker and verifier roles receive one baseline increment; +- two or more complexity signals add one increment; +- long context or explicit multi-step task structure may add one increment; +- two or more high-impact signals may add one increment; +- the model's declared `maximum_level` is an absolute cap; +- synthesizers do not inherit the analysis role's effort automatically. + +Workflow-structure evidence: + +- `workflow_step_index` and `workflow_step_count` identify the call's position in the direct or deep workflow; +- `decomposition_count` records the number of validated workflow subtasks; +- `recursion_depth` is derived from the accessed-step dependency graph; +- `accessible_step_count` measures access-list fan-in without exposing hidden model reasoning; +- later integration steps with several dependencies may receive more effort than an early worker step; +- a direct route remains a one-step workload and does not inherit deep-workflow increments. + +A single keyword or one structural flag cannot force maximum effort. Operators may use a fixed policy for controlled experiments: + +```python +ReasoningPolicy(strategy="fixed", fixed_level="medium", max_escalations=0) +``` + +## Workflow workload evidence + +Each visible reasoning decision may carry this strict audit object: + +```json +{ + "workflow_step_index": 3, + "workflow_step_count": 4, + "recursion_depth": 3, + "decomposition_count": 4, + "accessible_step_count": 3 +} +``` + +The values are validated as non-boolean integers with internally consistent bounds. Generated workflows replace the provisional template size before execution. Conducted template workflows derive recursion and fan-in from the access-list prompts, while recomputed verifier and synthesizer steps reconstruct their workload from the trace itself. + +## Verification-driven escalation + +When a conducted workflow's verifier rejects the worker result, the runtime may retry exactly once at the next supported level. It then recomputes only the affected verifier and synthesizer steps. It does not restart the whole workflow, exceed the policy cap, or retry when no higher supported level exists. + +The retried worker retains the same structural workload that produced the rejected result. The recomputed verifier and synthesizer receive workload evidence reconstructed from their actual trace position and access lists. A stale trace identity that no longer resolves to a configured agent fails closed rather than being silently redirected. + +The workflow trace records: + +- canonical level; +- decision source and bounded factors; +- role and escalation index; +- validated workflow workload; +- provider profile preset, supported levels, and cap; +- provider-reported reasoning-token count when available. + +Hidden reasoning text is not retained. + +## Caller ownership and security + +Caller-supplied complete reasoning paths always win, including explicit `null`. Custom paths are limited to safe identifier segments and eight levels of nesting. Rules accept strict JSON scalars and fixed templates only; no expression evaluation occurs. A scalar that conflicts with a configured nested path fails closed. + +A rule must contain an explicit `value`; omission is not treated as JSON `null`. NaN, positive infinity, and negative infinity are rejected because they are Python float extensions rather than interoperable JSON numbers. JSON configuration arrays are copied into immutable tuples during parsing. Direct Python construction must likewise use tuples for rule paths, supported levels, level mappings, and endpoint-specific rule collections, so a caller cannot mutate validated control data after a profile is created. + +Provider egress continues to use the repository's DNS-pinned HTTPS transport, original-host TLS verification, redirect rejection, no environment proxy, and KV-backed credential boundary. + +## Batch and ablation + +Batch JSONL bodies are rewritten immediately before the secured upload. Decisions are selected per `custom_id`, not once for an entire batch. Each batch item is treated as an independent one-step route unless a future reviewed batch topology explicitly declares otherwise. + +`run_reasoning_ablation` evaluates fixed effort cells over one prompt set and reports verifier acceptance, total tokens, and reasoning tokens. Task-specific benchmark scorers remain authoritative; verifier acceptance is a workflow measure, not a universal quality claim. Ablation is the evidence path for deciding whether role or topology increments improve a specific evaluation set. + +## Verification evidence + +The exact branch head is verified by the permanent read-only `Reasoning control quality` workflow. It checks out the pull-request head SHA, runs the complete repository suite, measures every reasoning-control production module at 100% statement and branch coverage, enforces 100% public and nested-function docstrings, compiles all Python sources, and checks the Git diff. Every later head must rerun the same gate before its evidence is reusable. + +## References — APA 7th + +Chen, L., Zaharia, M., & Zou, J. (2023). FrugalGPT: How to use large language models while reducing cost and improving performance. *arXiv*. https://arxiv.org/abs/2305.05176 + +International Organization for Standardization. (2023a). *ISO/IEC 23894:2023 Information technology—Artificial intelligence—Guidance on risk management*. https://www.iso.org/standard/77304.html + +International Organization for Standardization. (2023b). *ISO/IEC 42001:2023 Information technology—Artificial intelligence—Management system*. https://www.iso.org/standard/42001.html + +Nielsen, S., Cetin, E., Schwendeman, P., Sun, Q., Xu, J., & Tang, Y. (2025). Learning to orchestrate agents in natural language with the Conductor. *arXiv*. https://arxiv.org/abs/2512.04388 + +Ong, I., Almahairi, A., Wu, V., Chiang, W.-L., Wu, T., Gonzalez, J. E., Kadous, M. W., & Stoica, I. (2024). RouteLLM: Learning to route LLMs with preference data. *arXiv*. https://arxiv.org/abs/2406.18665 + +Sakana AI. (2026, June 22). *Sakana Fugu: One model to command them all*. https://sakana.ai/fugu-release/ + +Snell, C., Lee, J., Xu, K., & Kumar, A. (2024). Scaling LLM test-time compute optimally can be more effective than scaling model parameters. *arXiv*. https://arxiv.org/abs/2408.03314 + +Xu, J., Sun, Q., Schwendeman, P., Nielsen, S., Cetin, E., & Tang, Y. (2025). TRINITY: An evolved LLM coordinator. *arXiv*. https://arxiv.org/abs/2512.04695 diff --git a/docs/superpowers/plans/2026-08-05-adaptive-reasoning-control.md b/docs/superpowers/plans/2026-08-05-adaptive-reasoning-control.md new file mode 100644 index 000000000..d6e810efc --- /dev/null +++ b/docs/superpowers/plans/2026-08-05-adaptive-reasoning-control.md @@ -0,0 +1,76 @@ +# Adaptive Reasoning Control Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development or superpowers:executing-plans task by task. Every behavior change follows red-green-refactor. + +**Goal:** Add explicit role-aware provider reasoning control, bounded verification escalation, and ablation without weakening routing, fallback, or provider-egress security. + +**Architecture:** Use focused stdlib modules behind `reasoning_control` and `reasoning_runtime` facades. Integrate through idempotent hooks on stable agent, client, orchestrator, and policy seams. Store capability by object identity and decisions in request-local contexts. + +**Tech Stack:** Python 3.10+, stdlib dataclasses, `ContextVar`, `weakref`, JSON, pytest, coverage.py. + +## Global Constraints + +- No new runtime dependency. +- No model-name capability inference. +- No provider secret from argv or runtime environment fallback. +- No caller-owned reasoning-field overwrite. +- No private intermediate reasoning persistence. +- Maximum one verifier-driven escalation. +- New database object names, if any, must be two-or-more-word `snake_case`. +- New production statement, branch, and docstring coverage must be 100%. +- Stack order: #96 → #94 → adaptive reasoning control. + +--- + +### Task 1: Capability profiles and payload rules + +**Files:** `_reasoning_profile.py`, `_reasoning_payload.py`, `reasoning_control.py`, control tests. + +- [x] Write failing tests for invalid presets, level ordering, unsafe paths, incomplete mappings, caller-owned fields, nested conflicts, and OpenAI/NVIDIA/Nemotron/Gemini/custom projections. +- [x] Verify failures are caused by missing behavior. +- [x] Implement immutable profiles, strict rules, endpoint normalization, fixed templates, and usage-token extraction. +- [x] Run focused tests and preserve 100% statement/branch coverage. + +### Task 2: Role-aware decision policy + +**Files:** `_reasoning_policy.py`, control tests. + +- [x] Write failing tests for disabled, fixed, adaptive, long-context, multi-step, multiple-complexity, multiple-high-impact, cap, failover projection, and next-level escalation behavior. +- [x] Implement canonical levels and least-cost bounded selection. +- [x] Verify fixed cells and decision serialization. + +### Task 3: Request-local runtime state + +**Files:** `_reasoning_state.py`, `reasoning_runtime.py`, runtime tests. + +- [x] Write failing tests proving equality-identical frozen dataclasses do not share profiles. +- [x] Implement weak identity registries and context-local decision/policy/override/event state. +- [x] Verify context cleanup and stale weak-reference branches. + +### Task 4: Provider-client integration + +**Files:** `_reasoning_client_hooks.py`, `_reasoning_workflow.py`, runtime tests. + +- [x] Write failing tests for chat, streaming, raw chat, Responses, per-item Batch decisions, and secured-upload rewriting. +- [x] Implement endpoint-specific projection and trace events. +- [x] Verify unprofiled models retain legacy payloads and caller settings win. + +### Task 5: Orchestrator integration and realistic recovery + +**Files:** `_reasoning_orchestrator_hooks.py`, runtime tests. + +- [x] Write a realistic failing test where low effort returns `41`, verification rejects it, medium returns `42`, and synthesis recovers the correct answer. +- [x] Implement route/conduct capture, generated planner and model-judge role decisions, one worker escalation, and downstream recomputation. +- [x] Add fixed-effort ablation and reasoning-token totals. +- [x] Verify lower effort uses fewer reasoning tokens while the bounded retry recovers correctness. + +### Task 6: Governance, persistence, documentation, and packaging + +**Files:** `__init__.py`, admin/persistence hooks, architecture, library research, doctoring, subsystem guide, CHANGELOG. + +- [x] Write a failing regression test proving frozen-agent replacement drops profile capability without a transfer hook. +- [x] Preserve or explicitly update profiles across agent patching, expose them in admin data, and re-save through the pool store. +- [x] Add APA 7 research and standards traceability. +- [x] Verify 100% statement/branch/docstring coverage and compile all new modules. +- [ ] Run full repository checks on the published exact head. +- [ ] Obtain independent exact-head approval and merge only after ancestor PRs integrate. diff --git a/docs/superpowers/specs/2026-08-05-adaptive-reasoning-control-design.md b/docs/superpowers/specs/2026-08-05-adaptive-reasoning-control-design.md new file mode 100644 index 000000000..f39aa562e --- /dev/null +++ b/docs/superpowers/specs/2026-08-05-adaptive-reasoning-control-design.md @@ -0,0 +1,96 @@ +# Adaptive Reasoning Control Design + +## Goal + +Add an explicit provider-neutral reasoning-compute layer that lowers expected cost while preserving difficult-task capability, standalone operation, and modular CWL MSA use. + +## Evidence and problem statement + +The repository already allocates test-time compute through model routing and workflow topology. Fugu, Conductor, and TRINITY support adaptive model/role/topology coordination, while test-time-compute and cost-aware routing research supports allocating compute by task evidence rather than using one maximum setting. Current provider APIs expose incompatible effort names, thinking toggles, budgets, and nesting. A single forwarded vendor field cannot control internal planner, worker, verifier, synthesizer, failover, streaming, Responses, and Batch calls. + +## Considered approaches + +### Infer support from model names + +Rejected. Aliases and provider behavior change; unsupported parameters fail requests; hidden inference is not auditable. + +### Forward caller fields only + +Rejected. This preserves expert control but does not optimize internal orchestration calls. + +### Explicit profiles plus bounded adaptive escalation + +Selected. Each agent declares supported levels and endpoint mappings. The runtime chooses a canonical decision, projects it onto a candidate's declared capability, preserves caller-owned fields, and permits one next-level worker retry after verifier rejection. + +## Architecture + +Focused modules keep responsibilities independently testable: + +- `_reasoning_profile.py`: validated capability profiles and payload rules; +- `_reasoning_policy.py`: adaptive/fixed selection, failover projection, escalation, and ablation value objects; +- `_reasoning_payload.py`: endpoint mapping and usage-token accounting; +- `reasoning_control.py`: public control facade; +- `_reasoning_state.py`: weak identity registries and request-local contexts; +- `_reasoning_workflow.py`: trace annotation, Batch JSONL projection, and downstream retry recomputation; +- `_reasoning_config_hooks.py`: agent round-trip and policy snapshot integration; +- `_reasoning_client_hooks.py`: chat, stream, passthrough, and Batch provider hooks; +- `_reasoning_orchestrator_hooks.py`: role invocation, admin visibility, replacement preservation, traces, retry, and ablation; +- `reasoning_runtime.py`: side-effect-free explicit activation and idempotent typed installation. + +No new runtime dependency is introduced. + +## Activation decision + +Importing the public package must not install optional reasoning hooks or mutate core classes. The package exports `enable_reasoning_control()` as an explicit composition-root action. The built-in CLI calls it before loading agent configuration; library and MSA consumers call it before creating runtime objects only when they need adaptive reasoning. + +Repeated activation is idempotent. Isolated alternate runtimes and test fakes may call the lower-level typed installer with their own class set. This preserves the extension's current implementation seams while preventing global import-order dependence and gives a stable public boundary for a future composition- or subclass-based implementation. + +## Data flow + +1. Explicitly activate the reasoning extension at the application composition root. +2. Resolve the selected agent's explicit profile. +3. Select a canonical level from policy, role, and bounded task signals. +4. Project the level to the candidate model during failover. +5. Enter a request-local decision scope. +6. Add endpoint-specific fields only when the caller does not own the complete path. +7. Call the provider through the existing pinned HTTPS and KV credential seams. +8. Capture usage and trace-safe decision evidence. +9. If verification rejects the worker, retry once at the next supported level and recompute affected downstream roles. + +## Compatibility and persistence + +- Importing `contextual_orchestrator` alone leaves reasoning hooks inactive. +- The product CLI activates reasoning before loading agent files. +- An unprofiled agent preserves legacy request shape after activation. +- Agent configuration round-trips `reasoning_profile`. +- Admin list/add/patch surfaces expose profile capability. +- Frozen-dataclass replacement preserves the prior profile unless an explicit profile patch changes or removes it. +- Durable agent-pool storage re-saves the replacement after the profile is attached. +- Caller payloads, provider retries, circuit breakers, route/conduct decisions, and security transport remain authoritative. + +## Security and privacy + +- No model-name inference or authoritative hard-coded inventory. +- No arbitrary payload dictionaries or expression evaluation. +- Safe path segments, bounded depth, JSON-scalar values, and strict templates only. +- Caller-owned complete paths are not overwritten, including explicit `null`. +- Hidden reasoning content is not stored; only level, factors, role, cap, escalation index, and token counts are recorded. +- Existing DNS pinning, SNI/certificate verification, proxy bypass, redirect rejection, and KV secret resolution remain unchanged. +- Optional reasoning activation is explicit rather than an import-time global mutation. + +## Testing and acceptance + +Tests must cover malformed profiles, provider presets, custom paths, caller ownership, adaptive/fixed policies, high-impact thresholds, failover projection, route/conduct, planner, verifier, streaming, Responses passthrough, Batch JSONL, admin visibility, durable profile re-save, bounded escalation, realistic arithmetic recovery, fixed-effort ablation, import purity, and idempotent explicit activation. + +Acceptance requires: + +- 100% statement coverage for every new production module; +- 100% branch coverage for every new production module; +- complete production and nested-function docstrings; +- package compile/import smoke tests; +- exact-head repository Tests, Fuzz, Security, Security Scan, SAST, package build/install, and independent review; +- no merge before stack ancestors are integrated. + +## Stack and release + +The development stack is security PR #96, then free-first fallback PR #94, then this reasoning-control PR. The feature PR remains Draft until ancestors merge and it is rebased or retargeted to the integrated exact base. The package remains `0.1.0`; no tag or release is authorized by this feature alone. diff --git a/pyproject.toml b/pyproject.toml index a54e74765..bdcd18cda 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,9 @@ branch = true show_missing = true fail_under = 100 +[tool.pytest.ini_options] +pythonpath = ["tests"] + [tool.interrogate] exclude = ["tests"] fail-under = 100 diff --git a/tests/reasoning_fakes.py b/tests/reasoning_fakes.py new file mode 100644 index 000000000..1a110c056 --- /dev/null +++ b/tests/reasoning_fakes.py @@ -0,0 +1,329 @@ +"""Deterministic runtime fakes shared by reasoning-control tests.""" + +from __future__ import annotations + +from dataclasses import dataclass, replace +import json +from typing import Any, Iterator + +from contextual_orchestrator.reasoning_control import ReasoningDecision, ReasoningPolicy, ReasoningProfile +from contextual_orchestrator.reasoning_runtime import ( + agent_reasoning_profile, + configure_agent_reasoning, + current_reasoning_decision, + install_reasoning_control, + orchestrator_reasoning_policy, + reasoning_override, +) + + +@dataclass(frozen=True) +class FakeAgent: + """Small hashable stand-in for the repository's model-agent value object.""" + + id: str + model: str + tags: tuple[str, ...] = () + disabled: bool = False + provider_exclusions: tuple[str, ...] = () + + @classmethod + def from_dict(cls, value: dict[str, Any]) -> "FakeAgent": + """Build an agent from normal configuration fields.""" + return cls( + id=value["id"], + model=value["model"], + tags=tuple(value.get("tags", ())), + ) + + def to_config(self) -> dict[str, Any]: + """Return normal configuration fields.""" + return {"id": self.id, "model": self.model, "tags": list(self.tags)} + + +@dataclass(frozen=True) +class FakePolicy: + """Stand-in for the repository's orchestration policy snapshot.""" + + verifier_required: bool = True + + def as_dict(self) -> dict[str, Any]: + """Return the base policy snapshot.""" + return {"verifier_required": self.verifier_required} + + +class FakeClient: + """Provider client seam that exposes payloads and deterministic usage.""" + + def __init__(self) -> None: + self.sent: list[dict[str, Any]] = [] + self._usage: dict[str, Any] | None = None + + def take_usage(self) -> dict[str, Any] | None: + """Return and clear the most recent deterministic usage record.""" + usage, self._usage = self._usage, None + return usage + + def chat(self, agent: FakeAgent, messages: list[dict[str, str]], temperature: float = 0.2) -> str: + """Construct a normal chat payload and delegate to the send seam.""" + payload = { + "model": agent.model, + "messages": messages, + "temperature": temperature, + "stream": False, + } + return self._send(agent, payload) + + def _send(self, agent: FakeAgent, payload: dict[str, Any]) -> str: + """Record one chat payload and produce effort-dependent deterministic output.""" + self.sent.append(payload) + effort = payload.get("reasoning_effort", "none") + reasoning_tokens = {"minimal": 2, "low": 4, "medium": 12, "high": 24}.get(effort, 0) + self._usage = { + "total_tokens": 20 + reasoning_tokens, + "completion_tokens_details": {"reasoning_tokens": reasoning_tokens}, + } + role = "worker" + system = messages_system(payload.get("messages", [])) + for candidate in ("thinker", "worker", "verifier", "synthesizer"): + if f"Role: {candidate}" in system: + role = candidate + if role == "worker": + return "42" if effort in {"medium", "high"} else "41" + if role == "verifier": + user = messages_user(payload.get("messages", [])) + return "verified accepted" if "42" in user else "reject incorrect result" + if role == "synthesizer": + return "final 42" + return "plan" + + def stream_chat( + self, + agent: FakeAgent, + messages: list[dict[str, str]], + temperature: float = 0.2, + ) -> Iterator[str]: + """Yield the same answer in two deterministic chunks.""" + value = self.chat(agent, messages, temperature) + yield value[:1] + yield value[1:] + + def _stream_send(self, agent: FakeAgent, payload: dict[str, Any]) -> Iterator[str]: + """Record a streaming payload and yield one marker.""" + self.sent.append(payload) + yield "stream" + + def proxy_send(self, agent: FakeAgent, endpoint: str, payload: dict[str, Any]) -> dict[str, Any]: + """Delegate full-shape passthrough to the raw send seam.""" + return self._send_raw(agent, endpoint, payload) + + def _send_raw(self, agent: FakeAgent, endpoint: str, payload: dict[str, Any]) -> dict[str, Any]: + """Record passthrough payload and return it for assertions.""" + self.sent.append(payload) + return {"endpoint": endpoint, "payload": payload} + + def batch_chat( + self, + agent: FakeAgent, + requests: dict[str, list[dict[str, str]]], + temperature: float = 0.2, + poll_interval: float = 5.0, + poll_timeout: float = 3600.0, + ) -> dict[str, dict[str, Any]]: + """Build Batch JSONL, pass through upload, and return deterministic results.""" + lines = [ + json.dumps( + { + "custom_id": custom_id, + "body": {"model": agent.model, "messages": messages, "temperature": temperature}, + } + ) + for custom_id, messages in requests.items() + ] + self._batch_upload(agent, "\n".join(lines).encode("utf-8")) + return { + custom_id: { + "content": "batch", + "usage": {"total_tokens": 10, "completion_tokens_details": {"reasoning_tokens": 2}}, + } + for custom_id in requests + } + + def _batch_upload(self, agent: FakeAgent, payload: bytes) -> str: + """Record decoded Batch JSONL bodies.""" + self.sent.extend(json.loads(line)["body"] for line in payload.decode().splitlines()) + return "file_1" + + +def messages_system(messages: list[dict[str, str]]) -> str: + """Return concatenated system content from a fake chat payload.""" + return "\n".join(item.get("content", "") for item in messages if item.get("role") == "system") + + +def messages_user(messages: list[dict[str, str]]) -> str: + """Return concatenated user content from a fake chat payload.""" + return "\n".join(item.get("content", "") for item in messages if item.get("role") == "user") + + +class FakeOrchestrator: + """Minimal orchestration core matching the extension's stable seams.""" + + def __init__(self, agents: list[FakeAgent], client: FakeClient | None = None) -> None: + self.agents = agents + self.client = client or FakeClient() + self.policy = FakePolicy() + + def _agent(self, agent_id: str) -> FakeAgent: + return next(agent for agent in self.agents if agent.id == agent_id) + + def _select_agent(self, _text: str, role: str) -> FakeAgent: + return next((agent for agent in self.agents if role in agent.tags), self.agents[0]) + + def _invoke( + self, + primary: FakeAgent, + messages: list[dict[str, str]], + *, + text: str, + role: str, + ) -> tuple[str, str, dict[str, Any] | None]: + output = self.client.chat(primary, messages) + return output, primary.id, self.client.take_usage() + + def route_once(self, messages: list[dict[str, str]]) -> dict[str, Any]: + task = messages_user(messages) + agent = self._select_agent(task, "worker") + output, served, usage = self._invoke(agent, messages, text=task, role="worker") + return { + "mode": "route", + "answer": output, + "trace": [ + { + "id": 0, + "role": "worker", + "agent_id": agent.id, + "served_agent_id": served, + "subtask": "Direct route", + "access": [], + "output": output, + "usage": usage, + } + ], + "verification": {"accepted": True}, + } + + def conduct(self, messages: list[dict[str, str]]) -> dict[str, Any]: + task = messages_user(messages) + trace: list[dict[str, Any]] = [] + outputs: dict[int, str] = {} + roles = ("thinker", "worker", "verifier", "synthesizer") + access = ((), (0,), (0, 1), (0, 1, 2)) + for index, role in enumerate(roles): + agent = self._select_agent(task, role) + prior = "\n".join(outputs[item] for item in access[index]) + step_messages = [ + {"role": "system", "content": f"Role: {role}. Complete the subtask."}, + { + "role": "user", + "content": f"Original task:\n{task}\n\nAccessed prior work:\n{prior}\n\nSubtask:\n{role}", + }, + ] + output, served, usage = self._invoke(agent, step_messages, text=task, role=role) + outputs[index] = output + trace.append( + { + "id": index, + "role": role, + "agent_id": agent.id, + "served_agent_id": served, + "subtask": role, + "access": list(access[index]), + "output": output, + "usage": usage, + } + ) + verification = self._judge_verifier_output(outputs[2], outputs[0], outputs[1]) + return { + "mode": "conduct", + "answer": outputs[3] if verification["accepted"] else outputs[1], + "trace": trace, + "verification": verification, + } + + def _dispatch(self, messages: list[dict[str, str]], mode: str) -> dict[str, Any]: + return self.route_once(messages) if mode == "route" else self.conduct(messages) + + def stream_route( + self, + messages: list[dict[str, str]], + workflow_run_id: str | None = None, + ) -> Iterator[str]: + agent = self._select_agent(messages_user(messages), "worker") + yield from self.client.stream_chat(agent, messages) + + def batch_route(self, prompts: list[str]) -> list[dict[str, Any]]: + agent = self._select_agent("", "worker") + requests = { + f"task_{index}": [{"role": "user", "content": prompt}] + for index, prompt in enumerate(prompts) + } + results = self.client.batch_chat(agent, requests) + return [ + { + "trace": [ + { + "id": 0, + "role": "worker", + "agent_id": agent.id, + "subtask": "batch", + "access": [], + "output": results[f"task_{index}"]["content"], + "usage": results[f"task_{index}"]["usage"], + } + ], + "verification": {"accepted": True}, + } + for index in range(len(prompts)) + ] + + def proxy_completion(self, body: dict[str, Any], *, endpoint: str = "chat/completions") -> dict[str, Any]: + agent = self._select_agent("", "worker") + return self.client.proxy_send(agent, endpoint, body) + + def _plan_generated(self, task: str) -> str: + agent = self._select_agent(task, "thinker") + return self.client.chat(agent, [{"role": "system", "content": "Role: thinker"}, {"role": "user", "content": task}]) + + def _model_judge_verification(self, task: str, fallback: dict[str, Any]) -> dict[str, Any]: + agent = self._select_agent(task, "verifier") + result = self.client.chat(agent, [{"role": "system", "content": "Role: verifier"}, {"role": "user", "content": task}]) + return {"accepted": "accepted" in result, "verifier_output": result, **fallback} + + def _judge_verifier_output(self, verifier: str, _thinker: str, worker: str) -> dict[str, Any]: + return {"accepted": "accepted" in verifier and worker == "42", "verifier_output": verifier} + + +def common_profile() -> ReasoningProfile: + """Return the profile shared by all fake role agents.""" + return ReasoningProfile( + supported_levels=("minimal", "low", "medium", "high"), + default_level="low", + maximum_level="high", + ) + + +def make_orchestrator() -> FakeOrchestrator: + """Build and configure four distinct role agents.""" + agents = [ + FakeAgent("thinker_agent", "thinker-model", ("thinker",)), + FakeAgent("worker_agent", "worker-model", ("worker",)), + FakeAgent("verifier_agent", "verifier-model", ("verifier",)), + FakeAgent("synth_agent", "synth-model", ("synthesizer",)), + ] + for agent in agents: + configure_agent_reasoning(agent, common_profile()) + return FakeOrchestrator(agents, reasoning_policy=ReasoningPolicy()) + + +install_reasoning_control(FakeAgent, FakeClient, FakeOrchestrator, FakePolicy) +install_reasoning_control(FakeAgent, FakeClient, FakeOrchestrator, FakePolicy) diff --git a/tests/test_reasoning_activation_coverage.py b/tests/test_reasoning_activation_coverage.py new file mode 100644 index 000000000..cb6588636 --- /dev/null +++ b/tests/test_reasoning_activation_coverage.py @@ -0,0 +1,51 @@ +"""Coverage regressions for explicit activation and generated-plan sizing.""" + +from __future__ import annotations + +from typing import Any + +import contextual_orchestrator._reasoning_orchestrator_hooks as orchestrator_hooks +import contextual_orchestrator.reasoning_runtime as reasoning_runtime +from contextual_orchestrator.reasoning_control import WorkflowReasoningCursor + + +def test_enable_reasoning_control_delegates_to_builtin_runtime_types( + monkeypatch: Any, +) -> None: + """Explicit activation must bind exactly the repository's four core classes.""" + captured: list[tuple[type[Any], ...]] = [] + + def capture(*runtime_types: type[Any]) -> None: + """Record the activation target without mutating process-global classes.""" + captured.append(runtime_types) + + monkeypatch.setattr(reasoning_runtime, "install_reasoning_control", capture) + reasoning_runtime.enable_reasoning_control() + + from contextual_orchestrator.orchestrator import ( + ModelAgent, + ModelClient, + OrchestrationPolicy, + TaskOrchestrator, + ) + + assert captured == [ + (ModelAgent, ModelClient, TaskOrchestrator, OrchestrationPolicy) + ] + + +def test_generated_plan_cursor_updates_only_for_validated_step_lists() -> None: + """Generated list plans replace the provisional size; other shapes do not.""" + cursor = WorkflowReasoningCursor(4) + token = orchestrator_hooks._WORKFLOW_CURSOR.set(cursor) + try: + orchestrator_hooks._update_generated_plan_cursor(("not", "a", "list")) + assert cursor.workflow_step_count == 4 + orchestrator_hooks._update_generated_plan_cursor([{"id": 0}, {"id": 1}]) + assert cursor.workflow_step_count == 2 + assert cursor.decomposition_count == 2 + finally: + orchestrator_hooks._WORKFLOW_CURSOR.reset(token) + + # With no active workflow, a generated list is intentionally a no-op. + orchestrator_hooks._update_generated_plan_cursor([{"id": 0}]) diff --git a/tests/test_reasoning_control.py b/tests/test_reasoning_control.py new file mode 100644 index 000000000..d90771bb1 --- /dev/null +++ b/tests/test_reasoning_control.py @@ -0,0 +1,236 @@ +"""Behavioral tests for adaptive reasoning policy and provider payload mapping.""" + +from __future__ import annotations + +import pytest + +from contextual_orchestrator.reasoning_control import ( + CANONICAL_REASONING_LEVELS, + PayloadRule, + ReasoningDecision, + ReasoningPolicy, + ReasoningProfile, + adapt_reasoning_decision, + apply_reasoning_payload, + escalate_reasoning_decision, + extract_reasoning_tokens, + select_reasoning_decision, + sum_usage_tokens, +) + + +def profile(preset: str = "openai_effort") -> ReasoningProfile: + """Return the common four-level profile used by tests.""" + return ReasoningProfile( + preset=preset, + supported_levels=("minimal", "low", "medium", "high"), + default_level="low", + maximum_level="high", + ) + + +def test_canonical_levels_are_cheapest_to_most_expensive() -> None: + assert CANONICAL_REASONING_LEVELS == ( + "none", + "minimal", + "low", + "medium", + "high", + "xhigh", + "max", + ) + + +@pytest.mark.parametrize( + ("kwargs", "message"), + [ + ({"preset": "unknown"}, "unsupported reasoning preset"), + ({"supported_levels": ()}, "must not be empty"), + ({"supported_levels": ("low", "minimal")}, "canonical order"), + ({"default_level": "xhigh"}, "default_level must be supported"), + ({"maximum_level": "minimal"}, "default_level cannot exceed"), + ({"preset": "custom"}, "custom preset requires"), + ], +) +def test_profile_rejects_ambiguous_or_unsupported_contracts(kwargs: dict[str, object], message: str) -> None: + with pytest.raises(ValueError, match=message): + ReasoningProfile(**kwargs) + + +def test_profile_round_trips_custom_mapping() -> None: + source = { + "preset": "custom", + "supported_levels": ["low", "medium", "high"], + "default_level": "low", + "maximum_level": "high", + "level_values": {"low": 128, "medium": 512, "high": 1024}, + "chat_rules": [{"path": ["thinking", "budget"], "value": "$int"}], + } + parsed = ReasoningProfile.from_dict(source) + assert ReasoningProfile.from_dict(parsed.to_dict()) == parsed + assert parsed.bounded_levels == ("low", "medium", "high") + + +def test_profile_rejects_unknown_keys_and_incomplete_mapped_rules() -> None: + with pytest.raises(ValueError, match="unknown reasoning_profile keys"): + ReasoningProfile.from_dict({"preset": "openai_effort", "typo": True}) + with pytest.raises(ValueError, match="require every supported level"): + ReasoningProfile( + preset="custom", + supported_levels=("low", "high"), + default_level="low", + maximum_level="high", + level_values=(("low", 128),), + chat_rules=(PayloadRule(("thinking", "budget"), "$int"),), + ) + + +def test_payload_rule_rejects_unsafe_path_and_unknown_template() -> None: + with pytest.raises(ValueError, match="unsafe segment"): + PayloadRule(("../secret",), "low") + with pytest.raises(ValueError, match="unsupported reasoning payload template"): + PayloadRule(("reasoning",), "$eval") + with pytest.raises(ValueError, match="unknown reasoning payload rule keys"): + PayloadRule.from_dict({"path": ["reasoning"], "value": "low", "extra": 1}) + + +@pytest.mark.parametrize( + ("preset", "endpoint", "expected"), + [ + ("openai_effort", "chat/completions", {"reasoning_effort": "medium"}), + ("openai_effort", "/v1/responses", {"reasoning": {"effort": "medium"}}), + ("nvidia_reasoning_effort", "chat/completions", {"reasoning_effort": "medium"}), + ( + "nvidia_nemotron_thinking", + "chat/completions", + {"chat_template_kwargs": {"enable_thinking": True, "low_effort": False}}, + ), + ( + "gemini_thinking_level", + "chat/completions", + {"extra_body": {"google": {"thinking_config": {"thinking_level": "medium"}}}}, + ), + ], +) +def test_provider_presets_map_one_canonical_decision( + preset: str, + endpoint: str, + expected: dict[str, object], +) -> None: + decision = ReasoningDecision("medium", "test", "worker", 1, ("test",)) + assert apply_reasoning_payload({}, profile(preset), decision, endpoint) == expected + + +def test_caller_owned_reasoning_field_is_never_overwritten() -> None: + decision = ReasoningDecision("high", "test", "worker", 2, ("test",)) + payload = {"reasoning": {"effort": "minimal"}, "input": "task"} + result = apply_reasoning_payload(payload, profile(), decision, "responses") + assert result == payload + assert result is not payload + + +def test_custom_integer_mapping_sets_nested_field() -> None: + custom = ReasoningProfile( + preset="custom", + supported_levels=("low", "medium"), + default_level="low", + maximum_level="medium", + level_values=(("low", 128), ("medium", 512)), + chat_rules=(PayloadRule(("thinking", "budget_tokens"), "$int"),), + ) + decision = ReasoningDecision("medium", "test", "worker", 1, ("test",)) + assert apply_reasoning_payload({}, custom, decision, "chat/completions") == { + "thinking": {"budget_tokens": 512} + } + + +def test_nested_mapping_conflict_fails_closed() -> None: + custom = ReasoningProfile( + preset="custom", + supported_levels=("low",), + default_level="low", + maximum_level="low", + chat_rules=(PayloadRule(("thinking", "effort"), "$level"),), + ) + decision = ReasoningDecision("low", "test", "worker", 0, ("test",)) + with pytest.raises(ValueError, match="conflicts with caller scalar"): + apply_reasoning_payload({"thinking": "caller scalar"}, custom, decision, "chat/completions") + + +def test_adaptive_policy_uses_default_for_simple_worker_and_more_for_verifier() -> None: + simple = select_reasoning_decision(profile(), ReasoningPolicy(), "Summarize this note.", "worker") + verifier = select_reasoning_decision(profile(), ReasoningPolicy(), "Verify this result.", "verifier") + assert simple is not None and simple.level == "low" + assert verifier is not None and verifier.level == "medium" + + +def test_adaptive_policy_requires_multiple_high_impact_signals_for_extra_compute() -> None: + one = select_reasoning_decision(profile(), ReasoningPolicy(), "Review authentication wording.", "worker") + two = select_reasoning_decision( + profile(), + ReasoningPolicy(), + "Analyze and verify authentication privacy security architecture failure modes.", + "worker", + ) + assert one is not None and one.level == "low" + assert two is not None and two.level == "high" + assert "multiple_high_impact_signals" in two.factors + + +def test_fixed_and_disabled_policies_are_deterministic() -> None: + fixed = select_reasoning_decision( + profile(), + ReasoningPolicy(strategy="fixed", fixed_level="medium", max_escalations=0), + "anything", + "worker", + ) + disabled = select_reasoning_decision(profile(), ReasoningPolicy(strategy="disabled"), "anything", "worker") + assert fixed is not None and fixed.level == "medium" and fixed.source == "fixed_policy" + assert disabled is None + + +def test_failover_projection_never_exceeds_provider_capability() -> None: + decision = ReasoningDecision("high", "adaptive_policy", "worker", 2, ("complex",)) + small = ReasoningProfile( + supported_levels=("minimal", "low"), + default_level="minimal", + maximum_level="low", + ) + projected = adapt_reasoning_decision(small, decision) + assert projected is not None and projected.level == "low" + assert projected.source.endswith("capability_projection") + + +def test_verifier_escalation_moves_exactly_one_supported_step() -> None: + prior = ReasoningDecision("low", "adaptive_policy", "worker", 0, ("default",)) + escalated = escalate_reasoning_decision(profile(), ReasoningPolicy(), prior) + assert escalated is not None and escalated.level == "medium" + assert escalated.escalation_index == 1 + assert escalate_reasoning_decision(profile(), ReasoningPolicy(), escalated) is None + assert escalate_reasoning_decision(profile(), ReasoningPolicy(max_escalations=0), prior) is None + + +def test_reasoning_token_usage_supports_responses_and_chat_shapes() -> None: + assert extract_reasoning_tokens({"reasoning_tokens": 3}) == 3 + assert extract_reasoning_tokens({"output_tokens_details": {"reasoning_tokens": 5}}) == 5 + assert extract_reasoning_tokens({"completion_tokens_details": {"reasoning_tokens": 7}}) == 7 + assert extract_reasoning_tokens({"reasoning_tokens": True}) is None + assert extract_reasoning_tokens(None) is None + + +def test_sum_usage_tokens_ignores_unknown_or_malformed_fields() -> None: + trace = [ + {"usage": {"total_tokens": 20, "output_tokens_details": {"reasoning_tokens": 5}}}, + {"usage": {"total_tokens": 10, "completion_tokens_details": {"reasoning_tokens": 3}}}, + {"usage": "unknown"}, + ] + assert sum_usage_tokens(trace) == (8, 30) + + +def test_policy_and_decision_validation_reject_ambiguous_values() -> None: + with pytest.raises(ValueError, match="fixed strategy requires"): + ReasoningPolicy(strategy="fixed") + with pytest.raises(ValueError, match="max_escalations"): + ReasoningPolicy(max_escalations=2) + with pytest.raises(ValueError, match="decision level"): + ReasoningDecision("ultra", "test", "worker", 0, ("test",)) diff --git a/tests/test_reasoning_control_coverage.py b/tests/test_reasoning_control_coverage.py new file mode 100644 index 000000000..e5a70182c --- /dev/null +++ b/tests/test_reasoning_control_coverage.py @@ -0,0 +1,254 @@ +"""Coverage-focused edge tests for the reasoning-control pure functions.""" + +from __future__ import annotations + +from collections import UserDict + +import pytest + +import contextual_orchestrator.reasoning_control as rc + + +def decision(level: str = "low") -> rc.ReasoningDecision: + """Return a minimal valid decision.""" + return rc.ReasoningDecision(level, "coverage", "worker", 0, ("coverage",)) + + +def one_level_profile(**kwargs: object) -> rc.ReasoningProfile: + """Return a one-level profile with optional overrides.""" + values: dict[str, object] = { + "supported_levels": ("low",), + "default_level": "low", + "maximum_level": "low", + } + values.update(kwargs) + return rc.ReasoningProfile(**values) + + +def test_payload_rule_defensive_validation() -> None: + with pytest.raises(ValueError, match="1 to 8"): + rc.PayloadRule((), "low") + with pytest.raises(ValueError, match="JSON scalar"): + rc.PayloadRule(("reasoning",), object()) + with pytest.raises(ValueError, match="must be an object"): + rc.PayloadRule.from_dict([]) # type: ignore[arg-type] + with pytest.raises(ValueError, match="non-empty string array"): + rc.PayloadRule.from_dict({"path": [], "value": "low"}) + with pytest.raises(ValueError, match="non-empty string array"): + rc.PayloadRule.from_dict({"path": [1], "value": "low"}) + + +def test_profile_defensive_validation() -> None: + with pytest.raises(ValueError, match="duplicates"): + rc.ReasoningProfile(supported_levels=("low", "low"), default_level="low", maximum_level="low") + with pytest.raises(ValueError, match="unknown level"): + rc.ReasoningProfile(supported_levels=("low", "ultra"), default_level="low", maximum_level="low") + with pytest.raises(ValueError, match="maximum_level must be supported"): + rc.ReasoningProfile(supported_levels=("low",), default_level="low", maximum_level="high") + with pytest.raises(ValueError, match="duplicate keys"): + rc.ReasoningProfile( + preset="custom", + supported_levels=("low",), + default_level="low", + maximum_level="low", + level_values=(("low", 1), ("low", 2)), + chat_rules=(rc.PayloadRule(("budget",), "$mapped"),), + ) + with pytest.raises(ValueError, match="unsupported levels"): + rc.ReasoningProfile( + supported_levels=("low",), + default_level="low", + maximum_level="low", + level_values=(("high", 3),), + ) + + +def test_profile_from_dict_defensive_validation() -> None: + with pytest.raises(ValueError, match="must be an object"): + rc.ReasoningProfile.from_dict([]) # type: ignore[arg-type] + with pytest.raises(ValueError, match="preset must be a string"): + rc.ReasoningProfile.from_dict({"preset": 1}) + with pytest.raises(ValueError, match="string array"): + rc.ReasoningProfile.from_dict({"supported_levels": "low"}) + with pytest.raises(ValueError, match="string array"): + rc.ReasoningProfile.from_dict({"supported_levels": [1]}) + with pytest.raises(ValueError, match="must be strings"): + rc.ReasoningProfile.from_dict({"default_level": 1}) + with pytest.raises(ValueError, match="level_values must be an object"): + rc.ReasoningProfile.from_dict({"level_values": []}) + + class OddMapping(UserDict): + """Mapping that can expose non-string keys for validation.""" + + with pytest.raises(ValueError, match="keys must be strings"): + rc.ReasoningProfile.from_dict({"level_values": OddMapping({1: "low"})}) + with pytest.raises(ValueError, match="JSON scalars"): + rc.ReasoningProfile.from_dict({"level_values": {"low": object()}}) + + +def test_profile_to_dict_includes_responses_rules() -> None: + profile = rc.ReasoningProfile( + preset="custom", + supported_levels=("low",), + default_level="low", + maximum_level="low", + responses_rules=(rc.PayloadRule(("reasoning", "effort"), "$level"),), + ) + assert profile.to_dict()["responses_rules"] == [ + {"path": ["reasoning", "effort"], "value": "$level"} + ] + + +def test_policy_from_dict_and_validation_edges() -> None: + with pytest.raises(ValueError, match="strategy must be"): + rc.ReasoningPolicy(strategy="unknown") + with pytest.raises(ValueError, match="canonical"): + rc.ReasoningPolicy(strategy="fixed", fixed_level="ultra") + with pytest.raises(ValueError, match="must be an object"): + rc.ReasoningPolicy.from_dict([]) # type: ignore[arg-type] + with pytest.raises(ValueError, match="unknown reasoning policy keys"): + rc.ReasoningPolicy.from_dict({"typo": True}) + parsed = rc.ReasoningPolicy.from_dict( + {"strategy": "fixed", "fixed_level": "high", "max_escalations": 0} + ) + assert parsed.to_dict() == { + "strategy": "fixed", + "fixed_level": "high", + "max_escalations": 0, + } + + +def test_decision_validation_edges() -> None: + with pytest.raises(ValueError, match="source and role"): + rc.ReasoningDecision("low", "", "worker", 0, ("x",)) + with pytest.raises(ValueError, match="must be an integer"): + rc.ReasoningDecision("low", "x", "worker", True, ("x",)) + with pytest.raises(ValueError, match="non-negative"): + rc.ReasoningDecision("low", "x", "worker", -1, ("x",)) + with pytest.raises(ValueError, match="non-empty strings"): + rc.ReasoningDecision("low", "x", "worker", 0, ("",)) + with pytest.raises(ValueError, match="escalation_index must be an integer"): + rc.ReasoningDecision("low", "x", "worker", 0, ("x",), True) + with pytest.raises(ValueError, match="escalation_index must be non-negative"): + rc.ReasoningDecision("low", "x", "worker", 0, ("x",), -1) + + +def test_selection_input_long_context_and_multi_step_branches() -> None: + profile = rc.ReasoningProfile( + supported_levels=("low", "medium", "high"), + default_level="low", + maximum_level="high", + ) + with pytest.raises(ValueError, match="task and role"): + rc.select_reasoning_decision(profile, rc.ReasoningPolicy(), 3, "worker") # type: ignore[arg-type] + long = rc.select_reasoning_decision(profile, rc.ReasoningPolicy(), "x" * 801, "worker") + structured = rc.select_reasoning_decision( + profile, + rc.ReasoningPolicy(), + "\n".join(str(index) for index in range(9)), + "worker", + ) + assert long is not None and "long_context" in long.factors + assert structured is not None and "multi_step_structure" in structured.factors + assert rc.select_reasoning_decision(None, rc.ReasoningPolicy(), "x", "worker") is None + + +def test_adaptation_and_escalation_none_and_ceiling_edges() -> None: + profile = one_level_profile() + assert rc.adapt_reasoning_decision(None, decision()) is None + assert rc.adapt_reasoning_decision(profile, None) is None + assert rc.adapt_reasoning_decision(profile, decision()) == decision() + assert rc.escalate_reasoning_decision(None, rc.ReasoningPolicy(), decision()) is None + assert rc.escalate_reasoning_decision(profile, rc.ReasoningPolicy(), None) is None + assert rc.escalate_reasoning_decision(profile, rc.ReasoningPolicy(), decision()) is None + + +def test_payload_input_none_profile_and_unknown_endpoint_edges() -> None: + with pytest.raises(ValueError, match="payload must be an object"): + rc.apply_reasoning_payload([], one_level_profile(), decision(), "chat/completions") # type: ignore[arg-type] + payload = {"model": "x"} + assert rc.apply_reasoning_payload(payload, None, decision(), "chat/completions") == payload + assert rc.apply_reasoning_payload(payload, one_level_profile(), None, "chat/completions") == payload + with pytest.raises(ValueError, match="endpoint must be a string"): + rc.apply_reasoning_payload({}, one_level_profile(), decision(), 1) # type: ignore[arg-type] + with pytest.raises(ValueError, match="unsupported reasoning endpoint"): + rc.apply_reasoning_payload({}, one_level_profile(), decision(), "embeddings") + + +def test_usage_negative_values_and_invalid_total_are_ignored() -> None: + assert rc.extract_reasoning_tokens({"reasoning_tokens": -1}) is None + assert rc.extract_reasoning_tokens({"output_tokens_details": {"reasoning_tokens": -1}}) is None + assert rc.sum_usage_tokens([{"usage": {"total_tokens": True}}]) == (0, 0) + + +def test_rule_parsing_projection_and_empty_rules_edges() -> None: + assert rc._parse_rules(None) == () + with pytest.raises(ValueError, match="must be an array"): + rc._parse_rules("bad") + with pytest.raises(ValueError, match="no bounded"): + rc._nearest_supported((), "low") + with pytest.raises(ValueError, match="unknown canonical"): + rc._nearest_supported(("low",), "ultra") + assert rc._nearest_supported(("medium", "high"), "minimal") == "medium" + custom = rc.ReasoningProfile( + preset="custom", + supported_levels=("low",), + default_level="low", + maximum_level="low", + chat_rules=(rc.PayloadRule(("literal",), 7),), + ) + assert rc.apply_reasoning_payload({}, custom, decision(), "chat/completions") == {"literal": 7} + empty_responses = rc.ReasoningProfile( + preset="nvidia_nemotron_thinking", + supported_levels=("low",), + default_level="low", + maximum_level="low", + ) + assert rc.apply_reasoning_payload({}, empty_responses, decision(), "responses") == {} + + +def test_explicit_response_rule_and_template_failure_edges() -> None: + explicit = rc.ReasoningProfile( + preset="custom", + supported_levels=("low",), + default_level="low", + maximum_level="low", + responses_rules=(rc.PayloadRule(("thinking",), "$level"),), + ) + assert rc.apply_reasoning_payload({}, explicit, decision(), "responses") == {"thinking": "low"} + with pytest.raises(ValueError, match="has no provider mapping"): + rc._render_value("$mapped", "low", {}) + with pytest.raises(ValueError, match="requires an integer"): + rc._render_value("$int", "low", {"low": True}) + with pytest.raises(ValueError, match="unsupported reasoning template"): + rc._render_value("$unknown", "low", {"low": "low"}) + assert rc._render_value("$enabled", "none", {}) is False + assert rc._render_value("$low_effort", "low", {}) is True + + +def test_any_complete_path_and_existing_mapping_branches() -> None: + assert rc._any_complete_path({"a": {"b": None}}, (("a", "b"),)) is True + assert rc._any_complete_path({"a": "scalar"}, (("a", "b"),)) is False + target = {"a": {}} + rc._set_nested_if_absent(target, ("a", "b"), 1) + rc._set_nested_if_absent(target, ("a", "b"), 2) + assert target == {"a": {"b": 1}} + + +def test_decision_and_ablation_cell_serialize_all_fields() -> None: + assert decision().to_dict() == { + "level": "low", + "source": "coverage", + "role": "worker", + "complexity_score": 0, + "factors": ["coverage"], + "escalation_index": 0, + } + cell = rc.ReasoningAblationCell("low", 2, 1, 3, 10) + assert cell.to_dict() == { + "level": "low", + "prompt_count": 2, + "accepted_count": 1, + "reasoning_tokens": 3, + "total_tokens": 10, + } diff --git a/tests/test_reasoning_import_boundary.py b/tests/test_reasoning_import_boundary.py new file mode 100644 index 000000000..42ce10ff1 --- /dev/null +++ b/tests/test_reasoning_import_boundary.py @@ -0,0 +1,73 @@ +"""Regression tests for explicit adaptive-reasoning runtime activation.""" + +from __future__ import annotations + +from pathlib import Path +import subprocess +import sys + + +_REPOSITORY_ROOT = Path(__file__).resolve().parents[1] + + +def _run_fresh_interpreter(source: str) -> subprocess.CompletedProcess[str]: + """Execute ``source`` in a fresh interpreter rooted at the repository.""" + return subprocess.run( + [sys.executable, "-c", source], + cwd=_REPOSITORY_ROOT, + check=False, + capture_output=True, + text=True, + ) + + +def test_package_import_does_not_activate_reasoning_hooks() -> None: + """Importing the public package must not mutate runtime classes.""" + result = _run_fresh_interpreter( + """ +import contextual_orchestrator +from contextual_orchestrator.orchestrator import ModelClient + +assert not hasattr(ModelClient, \"_reasoning_control_installed\"), ( + \"package import activated optional reasoning hooks\" +) +""" + ) + assert result.returncode == 0, result.stderr + + +def test_cli_activation_is_explicit_and_idempotent() -> None: + """The product CLI must opt in explicitly without double-wrapping methods.""" + result = _run_fresh_interpreter( + """ +import contextual_orchestrator +from contextual_orchestrator.__main__ import _enable_reasoning_runtime +from contextual_orchestrator.orchestrator import ModelClient + +assert not hasattr(ModelClient, \"_reasoning_control_installed\") +_enable_reasoning_runtime() +assert ModelClient._reasoning_control_installed is True +installed_chat = ModelClient.chat +_enable_reasoning_runtime() +assert ModelClient.chat is installed_chat +""" + ) + assert result.returncode == 0, result.stderr + + +def test_pytest_config_exposes_test_helpers_without_ci_path_injection() -> None: + """Plain local pytest must import shared fakes without workflow-only state.""" + project_config = (_REPOSITORY_ROOT / "pyproject.toml").read_text(encoding="utf-8") + tests_workflow = ( + _REPOSITORY_ROOT / ".github" / "workflows" / "tests.yml" + ).read_text(encoding="utf-8") + reasoning_workflow = ( + _REPOSITORY_ROOT + / ".github" + / "workflows" + / "reasoning-workload-verify.yml" + ).read_text(encoding="utf-8") + + assert '[tool.pytest.ini_options]\npythonpath = ["tests"]' in project_config + assert "PYTHONPATH:" not in tests_workflow + assert "PYTHONPATH:" not in reasoning_workflow diff --git a/tests/test_reasoning_profile_hardening.py b/tests/test_reasoning_profile_hardening.py new file mode 100644 index 000000000..67ef9f7ea --- /dev/null +++ b/tests/test_reasoning_profile_hardening.py @@ -0,0 +1,71 @@ +"""Fail-closed tests for immutable, JSON-safe reasoning profile controls.""" + +from __future__ import annotations + +import pytest + +from contextual_orchestrator.reasoning_control import PayloadRule, ReasoningProfile + + +def test_payload_rule_requires_immutable_path_control() -> None: + """A frozen rule must not retain a caller-mutable path list.""" + with pytest.raises(ValueError, match="path must be a tuple"): + PayloadRule(["reasoning", "effort"], "$level") # type: ignore[arg-type] + + +def test_payload_rule_distinguishes_missing_value_from_explicit_null() -> None: + """A missing assignment is invalid while an explicit JSON null is valid.""" + with pytest.raises(ValueError, match="must include value"): + PayloadRule.from_dict({"path": ["reasoning", "effort"]}) + assert PayloadRule.from_dict( + {"path": ["reasoning", "effort"], "value": None} + ).value is None + + +@pytest.mark.parametrize("invalid_value", [float("nan"), float("inf"), float("-inf")]) +def test_payload_rule_rejects_non_finite_json_numbers(invalid_value: float) -> None: + """Provider payload rules accept only numbers representable in strict JSON.""" + with pytest.raises(ValueError, match="finite JSON scalar"): + PayloadRule(("reasoning", "budget"), invalid_value) + + +@pytest.mark.parametrize( + ("field_name", "field_value"), + [ + ("supported_levels", ["low"]), + ("level_values", [("low", 1)]), + ("chat_rules", [PayloadRule(("reasoning",), "$level")]), + ("responses_rules", [PayloadRule(("reasoning",), "$level")]), + ], +) +def test_profile_rejects_mutable_direct_constructor_collections( + field_name: str, + field_value: object, +) -> None: + """Frozen profiles reject lists that could change after validation.""" + values: dict[str, object] = { + "preset": "openai_effort", + "supported_levels": ("low",), + "default_level": "low", + "maximum_level": "low", + "level_values": (), + "chat_rules": (), + "responses_rules": (), + } + values[field_name] = field_value + with pytest.raises(ValueError, match=f"{field_name} must be a tuple"): + ReasoningProfile(**values) # type: ignore[arg-type] + + +@pytest.mark.parametrize("invalid_value", [float("nan"), float("inf"), float("-inf")]) +def test_profile_rejects_non_finite_level_mapping(invalid_value: float) -> None: + """Mapped provider values cannot serialize as NaN or Infinity.""" + with pytest.raises(ValueError, match="finite JSON scalar"): + ReasoningProfile( + preset="custom", + supported_levels=("low",), + default_level="low", + maximum_level="low", + level_values=(("low", invalid_value),), + chat_rules=(PayloadRule(("reasoning", "budget"), "$mapped"),), + ) diff --git a/tests/test_reasoning_retry_usage_freshness.py b/tests/test_reasoning_retry_usage_freshness.py new file mode 100644 index 000000000..08b74dba7 --- /dev/null +++ b/tests/test_reasoning_retry_usage_freshness.py @@ -0,0 +1,76 @@ +"""Regression tests for current-invocation usage evidence after retries.""" + +from __future__ import annotations + +from typing import Any + +import contextual_orchestrator.reasoning_runtime as rr +from contextual_orchestrator.reasoning_control import ReasoningPolicy +from reasoning_fakes import ( + FakeAgent, + FakeClient, + FakeOrchestrator, + common_profile, +) + + +class _RetryDropsUsageClient(FakeClient): + """Expose usage for initial calls but not their retry replacements.""" + + def __init__(self) -> None: + super().__init__() + self.role_calls: dict[str, int] = {} + + def _send(self, agent: FakeAgent, payload: dict[str, Any]) -> str: + """Return normal outputs while dropping usage on recomputed roles.""" + output = super()._send(agent, payload) + system = " ".join( + item.get("content", "") + for item in payload.get("messages", []) + if item.get("role") == "system" + ) + role = next( + ( + name + for name in ("worker", "verifier", "synthesizer") + if f"Role: {name}" in system + ), + None, + ) + if role is not None: + call_count = self.role_calls.get(role, 0) + 1 + self.role_calls[role] = call_count + if call_count > 1: + self._usage = None + return output + + +def test_retry_removes_usage_from_replaced_invocations() -> None: + """Retry rows must not retain usage from provider calls they replaced.""" + agents = [ + FakeAgent("thinker_freshness", "m", ("thinker",)), + FakeAgent("worker_freshness", "m", ("worker",)), + FakeAgent("verifier_freshness", "m", ("verifier",)), + FakeAgent("synth_freshness", "m", ("synthesizer",)), + ] + for agent in agents: + rr.configure_agent_reasoning(agent, common_profile()) + client = _RetryDropsUsageClient() + orchestrator = FakeOrchestrator( + agents, + client=client, + reasoning_policy=ReasoningPolicy(), + ) + + result = orchestrator.conduct([{"role": "user", "content": "calculate"}]) + + assert result["answer"] == "final 42" + assert client.role_calls == { + "worker": 2, + "verifier": 2, + "synthesizer": 2, + } + for role in ("worker", "verifier", "synthesizer"): + row = next(item for item in result["trace"] if item["role"] == role) + assert "usage" not in row + assert row["reasoning"]["reasoning_tokens"] is None diff --git a/tests/test_reasoning_runtime.py b/tests/test_reasoning_runtime.py new file mode 100644 index 000000000..949ae4ce6 --- /dev/null +++ b/tests/test_reasoning_runtime.py @@ -0,0 +1,91 @@ +"""Integration tests for the idempotent reasoning-control runtime extension.""" + +from contextual_orchestrator.reasoning_control import ReasoningDecision, ReasoningPolicy +from contextual_orchestrator.reasoning_runtime import ( + agent_reasoning_profile, + current_reasoning_decision, + orchestrator_reasoning_policy, + reasoning_override, +) +from reasoning_fakes import FakeAgent, common_profile, make_orchestrator + +def test_agent_configuration_round_trip_preserves_reasoning_profile() -> None: + agent = FakeAgent.from_dict( + { + "id": "worker_agent", + "model": "worker-model", + "reasoning_profile": common_profile().to_dict(), + } + ) + assert agent_reasoning_profile(agent) == common_profile() + assert FakeAgent.from_dict(agent.to_config()).to_config() == agent.to_config() + + +def test_route_once_injects_payload_and_trace_evidence() -> None: + orchestrator = make_orchestrator() + result = orchestrator.route_once([{"role": "user", "content": "Summarize this note."}]) + assert orchestrator.client.sent[-1]["reasoning_effort"] == "low" + assert result["trace"][0]["reasoning"]["decision"]["level"] == "low" + assert result["trace"][0]["reasoning"]["reasoning_tokens"] == 4 + assert result["reasoning_control"]["strategy"] == "adaptive" + + +def test_rejected_low_effort_worker_escalates_once_and_recovers_true_answer() -> None: + orchestrator = make_orchestrator() + result = orchestrator.conduct([{"role": "user", "content": "Calculate the answer exactly."}]) + worker = next(row for row in result["trace"] if row["role"] == "worker") + assert result["reasoning_escalation"] == { + "attempted": True, + "from_level": "low", + "to_level": "medium", + "accepted_after_retry": True, + } + assert worker["output"] == "42" + assert worker["reasoning"]["decision"]["level"] == "medium" + assert result["answer"] == "final 42" + + +def test_stream_proxy_and_batch_paths_receive_reasoning_controls() -> None: + orchestrator = make_orchestrator() + assert "".join(orchestrator.stream_route([{"role": "user", "content": "stream"}])) == "41" + proxy = orchestrator.proxy_completion({"input": "Research architecture"}, endpoint="responses") + assert proxy["payload"]["reasoning"]["effort"] in {"medium", "high"} + records = orchestrator.batch_route(["simple", "Analyze and verify architecture"]) + assert orchestrator.client.sent[-2]["reasoning_effort"] == "low" + assert orchestrator.client.sent[-1]["reasoning_effort"] in {"medium", "high"} + assert all("reasoning" in record["trace"][0] for record in records) + + +def test_caller_owned_proxy_effort_survives_orchestrator_defaults() -> None: + orchestrator = make_orchestrator() + proxy = orchestrator.proxy_completion( + {"input": "complex", "reasoning": {"effort": "minimal"}}, + endpoint="responses", + ) + assert proxy["payload"]["reasoning"]["effort"] == "minimal" + + +def test_reasoning_override_is_scoped_and_projected() -> None: + orchestrator = make_orchestrator() + decision = ReasoningDecision("high", "test", "worker", 0, ("test",)) + with reasoning_override(decision): + assert current_reasoning_decision() is None + orchestrator.route_once([{"role": "user", "content": "task"}]) + assert orchestrator.client.sent[-1]["reasoning_effort"] == "high" + orchestrator.route_once([{"role": "user", "content": "task"}]) + assert orchestrator.client.sent[-1]["reasoning_effort"] == "low" + + +def test_policy_snapshot_and_ablation_are_machine_readable() -> None: + orchestrator = make_orchestrator() + assert orchestrator.policy.as_dict()["reasoning_control"]["strategy"] == "adaptive" + report = orchestrator.run_reasoning_ablation( + ["Calculate the answer exactly."], + mode="conduct", + levels=("low", "medium"), + ) + assert [cell["level"] for cell in report["cells"]] == ["low", "medium"] + assert report["cells"][0]["accepted_count"] == 0 + assert report["cells"][1]["accepted_count"] == 1 + assert report["cells"][0]["reasoning_tokens"] < report["cells"][1]["reasoning_tokens"] + assert orchestrator_reasoning_policy(orchestrator).strategy == "adaptive" diff --git a/tests/test_reasoning_runtime_coverage.py b/tests/test_reasoning_runtime_coverage.py new file mode 100644 index 000000000..3cd57c603 --- /dev/null +++ b/tests/test_reasoning_runtime_coverage.py @@ -0,0 +1,222 @@ +"""Coverage-focused edge tests for reasoning-runtime integration hooks.""" + +from __future__ import annotations + +from dataclasses import dataclass +import json +from typing import Any, Iterator, Mapping + +import pytest + +import contextual_orchestrator.reasoning_runtime as rr +from contextual_orchestrator.reasoning_control import ( + ReasoningDecision, + ReasoningPolicy, + ReasoningProfile, +) +from reasoning_fakes import ( + FakeAgent, + FakeClient, + FakeOrchestrator, + FakePolicy, + common_profile, + make_orchestrator, +) + +def test_registry_removal_type_guards_and_default_policy() -> None: + agent = FakeAgent("temporary_agent", "model") + rr.configure_agent_reasoning(agent, common_profile()) + rr.configure_agent_reasoning(agent, None) + assert rr.agent_reasoning_profile(agent) is None + with pytest.raises(TypeError, match="ReasoningProfile"): + rr.configure_agent_reasoning(agent, "bad") # type: ignore[arg-type] + + orchestrator = make_orchestrator() + rr.configure_orchestrator_reasoning(orchestrator, None) + assert rr.orchestrator_reasoning_policy(orchestrator) == ReasoningPolicy() + with pytest.raises(TypeError, match="ReasoningPolicy"): + rr.configure_orchestrator_reasoning(orchestrator, "bad") # type: ignore[arg-type] + + class NoPolicyObject: + """Object without the core policy attribute.""" + + target = NoPolicyObject() + rr.configure_orchestrator_reasoning(target, ReasoningPolicy(strategy="disabled")) + assert rr.orchestrator_reasoning_policy(target).strategy == "disabled" + rr.configure_orchestrator_reasoning(target, None) + + +def test_text_and_role_helpers_cover_nested_responses_input() -> None: + assert rr._message_text([{"role": "assistant", "content": "x"}]) == "" + assert rr._input_text({"messages": [{"role": "user", "content": "chat"}, 3]}) == "chat" + assert rr._input_text( + { + "input": [ + "a", + {"content": "b"}, + {"content": [{"text": "c"}, {"image": "ignored"}]}, + 4, + ] + } + ) == "a b c" + assert rr._input_text({"input": 5}) == "" + assert rr._infer_role( + [ + {"role": "assistant", "content": "Role: thinker"}, + {"role": "system", "content": 3}, + {"role": "system", "content": "role=verifier"}, + ] + ) == "verifier" + assert rr._infer_role([], "thinker") == "thinker" + + +def test_resolve_decision_uses_active_and_default_policy_paths() -> None: + agent = FakeAgent("resolve_agent", "model") + rr.configure_agent_reasoning(agent, common_profile()) + active = ReasoningDecision("high", "active", "worker", 0, ("active",)) + with rr._decision_scope(active): + assert rr._resolve_decision(agent, "x", "worker") == active + selected = rr._resolve_decision(agent, "x", "worker") + assert selected is not None and selected.level == "low" + unprofiled = FakeAgent("plain_agent", "model") + assert rr._resolve_decision(unprofiled, "x", "worker") is None + + +def test_event_capture_and_trace_annotation_fallbacks() -> None: + profiled = FakeAgent("captured_agent", "model") + rr.configure_agent_reasoning(profiled, common_profile()) + decision = ReasoningDecision("low", "test", "worker", 0, ("test",)) + token = rr._EVENT_CAPTURE.set([]) + try: + rr._append_event(profiled, "worker", decision) + events = rr._EVENT_CAPTURE.get() + assert events is not None and len(events) == 1 + finally: + rr._EVENT_CAPTURE.reset(token) + + rr._append_event(profiled, "worker", decision) + rr._append_event(FakeAgent("plain", "model"), "worker", decision) + trace = [ + {"role": "worker", "agent_id": "different", "usage": "bad"}, + {"role": "unknown", "agent_id": "none"}, + ] + rr._annotate_trace( + trace, + [ + { + "agent_id": "captured_agent", + "role": "worker", + "profile": common_profile(), + "decision": decision, + "usage": {"reasoning_tokens": 2}, + } + ], + ) + assert trace[0]["reasoning"]["reasoning_tokens"] == 2 + assert "reasoning" not in trace[1] + + +def test_step_message_access_filters_invalid_indexes() -> None: + trace = [{"output": "first"}, {"output": "second"}] + messages = rr._step_messages( + "task", + {"role": "worker", "subtask": "do", "access": [-1, "bad", 0, 9]}, + trace, + ) + assert "first" in messages[1]["content"] + assert "second" not in messages[1]["content"] + assert "(none)" in rr._step_messages("task", {}, trace)[1]["content"] + + +def test_retry_helper_returns_for_nonretryable_shapes() -> None: + orchestrator = make_orchestrator() + rr._retry_rejected_worker_once(orchestrator, {}, "task") + rr._retry_rejected_worker_once( + orchestrator, + {"verification": {"accepted": False}, "trace": "bad"}, + "task", + ) + rr._retry_rejected_worker_once( + orchestrator, + {"verification": {"accepted": False}, "trace": []}, + "task", + ) + rr._retry_rejected_worker_once( + orchestrator, + {"verification": {"accepted": False}, "trace": [{"role": "worker"}]}, + "task", + ) + rr._retry_rejected_worker_once( + orchestrator, + { + "verification": {"accepted": False}, + "trace": [{"role": "worker", "reasoning": {"decision": {"level": "bad"}}}], + }, + "task", + ) + original_agent_lookup = orchestrator._agent + orchestrator._agent = lambda agent_id: (_ for _ in ()).throw(KeyError(agent_id)) if agent_id == "missing" else original_agent_lookup(agent_id) + try: + rr._retry_rejected_worker_once( + orchestrator, + { + "verification": {"accepted": False}, + "trace": [ + { + "role": "worker", + "agent_id": "missing", + "reasoning": { + "decision": { + "level": "low", + "source": "x", + "complexity_score": 0, + "factors": ["x"], + } + }, + } + ], + }, + "task", + ) + finally: + orchestrator._agent = original_agent_lookup + ceiling = common_profile() + worker = orchestrator._select_agent("", "worker") + rr.configure_agent_reasoning( + worker, + ReasoningProfile( + supported_levels=("low",), + default_level="low", + maximum_level="low", + ), + ) + rr._retry_rejected_worker_once( + orchestrator, + { + "verification": {"accepted": False}, + "trace": [ + { + "role": "worker", + "agent_id": worker.id, + "reasoning": { + "decision": { + "level": "low", + "source": "x", + "complexity_score": 0, + "factors": ["x"], + } + }, + } + ], + }, + "task", + ) + rr.configure_agent_reasoning(worker, ceiling) + + + +def test_refresh_step_reasoning_without_event_is_a_noop() -> None: + """A direct retry helper call without capture leaves the row unchanged.""" + row: dict[str, Any] = {} + rr._refresh_step_reasoning_from_event(row, "verifier", "missing", None) + assert row == {} diff --git a/tests/test_reasoning_runtime_coverage_identity.py b/tests/test_reasoning_runtime_coverage_identity.py new file mode 100644 index 000000000..5ecf7d20e --- /dev/null +++ b/tests/test_reasoning_runtime_coverage_identity.py @@ -0,0 +1,354 @@ +"""Coverage-focused edge tests for reasoning-runtime integration hooks.""" + +from __future__ import annotations + +from dataclasses import dataclass +import json +from typing import Any, Iterator, Mapping + +import pytest + +import contextual_orchestrator.reasoning_runtime as rr +from contextual_orchestrator.reasoning_control import ( + ReasoningDecision, + ReasoningPolicy, + ReasoningProfile, +) +from reasoning_fakes import ( + FakeAgent, + FakeClient, + FakeOrchestrator, + FakePolicy, + common_profile, + make_orchestrator, +) + +@dataclass(frozen=True) +class EdgeAgent: + """Fresh agent type for testing non-list workflow traces after installation.""" + + id: str + model: str + + @classmethod + def from_dict(cls, value: Mapping[str, Any]) -> "EdgeAgent": + return cls(value["id"], value["model"]) + + def to_config(self) -> dict[str, Any]: + return {"id": self.id, "model": self.model} + + +@dataclass(frozen=True) +class EdgePolicy: + """Fresh policy type for isolated installer coverage.""" + + def as_dict(self) -> dict[str, Any]: + return {} + + +class EdgeClient: + """Fresh client type exposing all required installer seams.""" + + def chat(self, _agent: EdgeAgent, _messages: list[dict[str, str]], _temperature: float = 0.2) -> str: + return "x" + + def stream_chat(self, _agent: EdgeAgent, _messages: list[dict[str, str]], _temperature: float = 0.2) -> Iterator[str]: + yield "x" + + def proxy_send(self, _agent: EdgeAgent, endpoint: str, payload: dict[str, Any]) -> dict[str, Any]: + return {"endpoint": endpoint, **payload} + + def batch_chat(self, _agent: EdgeAgent, _requests: dict[str, list[dict[str, str]]], *_args: Any) -> dict[str, dict[str, Any]]: + return {} + + def _send(self, _agent: EdgeAgent, _payload: dict[str, Any]) -> str: + return "x" + + def _stream_send(self, _agent: EdgeAgent, _payload: dict[str, Any]) -> Iterator[str]: + yield "x" + + def _send_raw(self, _agent: EdgeAgent, _endpoint: str, payload: dict[str, Any]) -> dict[str, Any]: + return payload + + def _batch_upload(self, _agent: EdgeAgent, _payload: bytes) -> str: + return "file" + + +class EdgeOrchestrator: + """Fresh core whose workflows intentionally return non-list trace values.""" + + def __init__(self, agents: list[EdgeAgent], client: EdgeClient | None = None) -> None: + self.agents = agents + self.client = client or EdgeClient() + self.policy = EdgePolicy() + + def _select_agent(self, _text: str, _role: str) -> EdgeAgent: + return self.agents[0] + + def _agent(self, agent_id: str) -> EdgeAgent: + if self.agents[0].id != agent_id: + raise KeyError(agent_id) + return self.agents[0] + + def _invoke(self, primary: EdgeAgent, _messages: list[dict[str, str]], **_kwargs: Any) -> tuple[str, str, None]: + return "x", primary.id, None + + def route_once(self, _messages: list[dict[str, str]]) -> dict[str, Any]: + return {"trace": "bad", "verification": {"accepted": True}} + + def conduct(self, _messages: list[dict[str, str]]) -> dict[str, Any]: + return {"trace": "bad", "verification": {"accepted": True}} + + def stream_route(self, _messages: list[dict[str, str]], workflow_run_id: str | None = None) -> Iterator[str]: + yield "x" + + def batch_route(self, _prompts: list[str]) -> list[dict[str, Any]]: + return [{"trace": "bad"}] + + def proxy_completion(self, body: dict[str, Any], *, endpoint: str = "chat/completions") -> dict[str, Any]: + return {"endpoint": endpoint, **body} + + def _plan_generated(self, _task: str) -> str: + return "plan" + + def _model_judge_verification(self, _task: str, fallback: dict[str, Any]) -> dict[str, Any]: + return fallback + + def _judge_verifier_output(self, *_args: Any) -> dict[str, Any]: + return {"accepted": False} + + def _dispatch(self, messages: list[dict[str, str]], _mode: str) -> dict[str, Any]: + return self.route_once(messages) + + def _agent_to_admin_payload(self, agent: EdgeAgent) -> dict[str, Any]: + """Return the fake admin projection for one edge agent.""" + return {"id": agent.id, "model": agent.model} + + def patch_agent(self, _pool_id: str, agent_id: str, patch: dict[str, Any]) -> dict[str, Any]: + """Replace an edge agent while rejecting extension-only fields.""" + unknown = set(patch) - {"model"} + if unknown: + raise ValueError(f"unknown core patch fields: {sorted(unknown)}") + current = self._agent(agent_id) + replacement = EdgeAgent(current.id, str(patch.get("model", current.model))) + self.agents = [replacement if item.id == agent_id else item for item in self.agents] + return self._agent_to_admin_payload(replacement) + + +def test_fresh_installer_nonlist_workflow_and_batch_paths() -> None: + rr.install_reasoning_control(EdgeAgent, EdgeClient, EdgeOrchestrator, EdgePolicy) + agent = EdgeAgent("edge_agent", "edge-model") + rr.configure_agent_reasoning(agent, common_profile()) + orchestrator = EdgeOrchestrator([agent]) + assert orchestrator.route_once([{"role": "user", "content": "x"}])["trace"] == "bad" + assert orchestrator.conduct([{"role": "user", "content": "x"}])["trace"] == "bad" + assert orchestrator.batch_route(["x"])[0]["trace"] == "bad" + + +def test_weak_identity_map_stale_and_callback_edges() -> None: + import gc + import weakref + + registry = rr._WeakIdentityMap() + + class Key: + """Weak-referenceable identity-map key.""" + + first = Key() + second = Key() + registry._entries[id(first)] = (weakref.ref(second), "stale") + assert registry.get(first, "default") == "default" + assert registry.pop(first, "default") == "default" + + key = Key() + registry.set(key, "value") + registry.pop(key) + del key + gc.collect() + + +def test_input_and_role_inner_negative_branches() -> None: + assert rr._input_text({"input": [{"content": 3}]}) == "" + assert rr._infer_role([{"role": "system", "content": "no recognized role"}], "worker") == "worker" + + +def test_retry_can_fail_over_to_unprofiled_served_agent() -> None: + orchestrator = make_orchestrator() + worker = orchestrator._select_agent("", "worker") + plain = FakeAgent("unprofiled_served", "model") + orchestrator.agents.append(plain) + original_invoke = orchestrator._invoke + orchestrator._invoke = lambda _agent, _messages, **_kwargs: ("42", plain.id, None) + result = { + "verification": {"accepted": False}, + "trace": [ + { + "id": 0, + "role": "worker", + "agent_id": worker.id, + "subtask": "calculate", + "access": [], + "output": "41", + "reasoning": { + "decision": ReasoningDecision( + "low", "adaptive", "worker", 0, ("default",) + ).to_dict() + }, + } + ], + } + try: + rr._retry_rejected_worker_once(orchestrator, result, "task") + finally: + orchestrator._invoke = original_invoke + assert result["answer"] == "42" + assert result["trace"][0]["served_agent_id"] == plain.id + + +def test_agent_from_dict_without_profile_and_event_condition_edges() -> None: + plain = FakeAgent.from_dict({"id": "from_dict_plain", "model": "model"}) + assert rr.agent_reasoning_profile(plain) is None + + orchestrator = make_orchestrator() + worker = orchestrator._select_agent("", "worker") + token = rr._EVENT_CAPTURE.set( + [ + { + "agent_id": "wrong_agent", + "role": "worker", + "profile": common_profile(), + "decision": ReasoningDecision("low", "x", "worker", 0, ("x",)), + "usage": None, + }, + { + "agent_id": worker.id, + "role": "worker", + "profile": common_profile(), + "decision": ReasoningDecision("low", "x", "worker", 0, ("x",)), + "usage": {"total_tokens": 1}, + }, + ] + ) + try: + orchestrator._invoke( + worker, + [{"role": "user", "content": "x"}], + text="x", + role="worker", + ) + events = rr._EVENT_CAPTURE.get() + assert events is not None and events[0]["usage"] is None + finally: + rr._EVENT_CAPTURE.reset(token) + + +def test_identity_cleanup_callback_with_already_removed_entry() -> None: + registry = rr._WeakIdentityMap() + + class Key: + """Weak-referenceable key for callback invocation.""" + + key = Key() + registry.set(key, "value") + reference = registry._entries[id(key)][0] + callback = reference.__callback__ + registry.pop(key) + assert callback is not None + callback(reference) + + +def test_invoke_event_loop_exhausts_without_client_generated_event() -> None: + agent = EdgeAgent("edge_loop_agent", "model") + rr.configure_agent_reasoning(agent, common_profile()) + orchestrator = EdgeOrchestrator([agent]) + token = rr._EVENT_CAPTURE.set( + [ + { + "agent_id": "wrong", + "role": "worker", + "profile": common_profile(), + "decision": ReasoningDecision("low", "x", "worker", 0, ("x",)), + "usage": None, + }, + { + "agent_id": agent.id, + "role": "worker", + "profile": common_profile(), + "decision": ReasoningDecision("low", "x", "worker", 0, ("x",)), + "usage": {"total_tokens": 1}, + }, + ] + ) + try: + orchestrator._invoke( + agent, + [{"role": "user", "content": "x"}], + text="x", + role="worker", + ) + events = rr._EVENT_CAPTURE.get() + assert events is not None and events[0]["usage"] is None + finally: + rr._EVENT_CAPTURE.reset(token) + + +def test_agent_patch_preserves_profile_and_admin_visibility() -> None: + """A dataclass replacement must not silently drop its reasoning capability.""" + rr.install_reasoning_control(EdgeAgent, EdgeClient, EdgeOrchestrator, EdgePolicy) + agent = EdgeAgent("managed_agent", "before") + profile = common_profile() + rr.configure_agent_reasoning(agent, profile) + orchestrator = EdgeOrchestrator([agent]) + result = orchestrator.patch_agent("default", agent.id, {"model": "after"}) + replacement = orchestrator._agent(agent.id) + assert replacement is not agent + assert rr.agent_reasoning_profile(replacement) == profile + assert result["reasoning_profile"] == profile.to_dict() + + +def test_agent_patch_supports_explicit_profile_updates_and_persistence() -> None: + """Typed, mapping, removal, invalid, and persistence branches stay explicit.""" + rr.install_reasoning_control(EdgeAgent, EdgeClient, EdgeOrchestrator, EdgePolicy) + agent = EdgeAgent("profile_patch_agent", "before") + rr.configure_agent_reasoning(agent, common_profile()) + orchestrator = EdgeOrchestrator([agent]) + + class Store: + """Capture profile-aware re-saves after the core replaces an agent.""" + + def __init__(self) -> None: + self.saved: list[EdgeAgent] = [] + + def save(self, item: EdgeAgent) -> None: + """Record one saved replacement.""" + self.saved.append(item) + + store = Store() + orchestrator._pool_store = store + medium = ReasoningProfile( + supported_levels=("low", "medium"), + default_level="medium", + maximum_level="medium", + ) + mapped = orchestrator.patch_agent( + "default", agent.id, {"model": "mapped", "reasoning_profile": medium.to_dict()} + ) + assert mapped["reasoning_profile"] == medium.to_dict() + assert store.saved[-1] is orchestrator._agent(agent.id) + + typed = common_profile() + result = orchestrator.patch_agent( + "default", agent.id, {"model": "typed", "reasoning_profile": typed} + ) + assert result["reasoning_profile"] == typed.to_dict() + + removed = orchestrator.patch_agent( + "default", agent.id, {"model": "plain", "reasoning_profile": None} + ) + assert "reasoning_profile" not in removed + assert rr.agent_reasoning_profile(orchestrator._agent(agent.id)) is None + + with pytest.raises(TypeError, match="reasoning_profile patch"): + orchestrator.patch_agent( + "default", agent.id, {"reasoning_profile": "invalid"} + ) diff --git a/tests/test_reasoning_runtime_coverage_retry.py b/tests/test_reasoning_runtime_coverage_retry.py new file mode 100644 index 000000000..99b91240b --- /dev/null +++ b/tests/test_reasoning_runtime_coverage_retry.py @@ -0,0 +1,228 @@ +"""Coverage-focused edge tests for reasoning-runtime integration hooks.""" + +from __future__ import annotations + +from dataclasses import dataclass +import json +from typing import Any, Iterator, Mapping + +import pytest + +import contextual_orchestrator.reasoning_runtime as rr +from contextual_orchestrator.reasoning_control import ( + ReasoningDecision, + ReasoningPolicy, + ReasoningProfile, +) +from reasoning_fakes import ( + FakeAgent, + FakeClient, + FakeOrchestrator, + FakePolicy, + common_profile, + make_orchestrator, +) + +def test_retry_without_verifier_or_synthesizer_uses_escalated_worker() -> None: + orchestrator = make_orchestrator() + worker = orchestrator._select_agent("", "worker") + result = { + "answer": "41", + "verification": {"accepted": False}, + "trace": [ + { + "id": 0, + "role": "worker", + "agent_id": worker.id, + "subtask": "calculate", + "access": [], + "output": "41", + "reasoning": { + "decision": ReasoningDecision( + "low", "adaptive", "worker", 0, ("default",) + ).to_dict() + }, + } + ], + } + rr._retry_rejected_worker_once(orchestrator, result, "calculate") + assert result["answer"] == "42" + assert result["reasoning_escalation"]["accepted_after_retry"] is False + + +def test_retry_handles_none_usage_for_worker_verifier_and_synthesizer() -> None: + class NoUsageClient(FakeClient): + """Fake client that never exposes usage after a call.""" + + def take_usage(self) -> None: + return None + + agents = [ + FakeAgent("thinker_none", "m", ("thinker",)), + FakeAgent("worker_none", "m", ("worker",)), + FakeAgent("verifier_none", "m", ("verifier",)), + FakeAgent("synth_none", "m", ("synthesizer",)), + ] + for agent in agents: + rr.configure_agent_reasoning(agent, common_profile()) + orchestrator = FakeOrchestrator(agents, client=NoUsageClient(), reasoning_policy=ReasoningPolicy()) + result = orchestrator.conduct([{"role": "user", "content": "calculate"}]) + assert result["answer"] == "final 42" + worker = next(row for row in result["trace"] if row["role"] == "worker") + assert worker["reasoning"]["reasoning_tokens"] is None + + +def test_batch_rewriter_handles_blank_unknown_and_nonobject_rows() -> None: + profile = common_profile() + decisions = {"known": ReasoningDecision("medium", "x", "worker", 0, ("x",))} + raw = ( + "\n" + + json.dumps({"custom_id": "known", "body": {"model": "x"}}) + + "\n" + + json.dumps({"custom_id": 3, "body": {"model": "x"}}) + + "\n" + + json.dumps({"custom_id": "missing", "body": "not-object"}) + ).encode() + rows = [json.loads(line) for line in rr._rewrite_batch_payload(raw, decisions, profile).decode().splitlines()] + assert rows[0]["body"]["reasoning_effort"] == "medium" + assert "reasoning_effort" not in rows[1]["body"] + assert rows[2]["body"] == "not-object" + + +def test_installed_agent_policy_and_init_edge_branches() -> None: + with pytest.raises((TypeError, KeyError)): + FakeAgent.from_dict([]) # type: ignore[arg-type] + plain = FakeAgent("no_profile", "model") + assert "reasoning_profile" not in plain.to_config() + assert "reasoning_control" not in FakePolicy().as_dict() + + typed = FakeOrchestrator([plain], reasoning_policy=ReasoningPolicy(strategy="disabled")) + assert rr.orchestrator_reasoning_policy(typed).strategy == "disabled" + mapped = FakeOrchestrator( + [plain], + reasoning_policy={"strategy": "fixed", "fixed_level": "low", "max_escalations": 0}, + ) + assert rr.orchestrator_reasoning_policy(mapped).strategy == "fixed" + with pytest.raises(TypeError, match="reasoning_policy must"): + FakeOrchestrator([plain], reasoning_policy="bad") + + +def test_stream_send_planner_and_model_judge_wrappers() -> None: + orchestrator = make_orchestrator() + worker = orchestrator._select_agent("", "worker") + decision = ReasoningDecision("medium", "test", "worker", 0, ("test",)) + with rr._decision_scope(decision): + assert list(orchestrator.client._stream_send(worker, {"model": "m"})) == ["stream"] + assert orchestrator.client.sent[-1]["reasoning_effort"] == "medium" + assert orchestrator._plan_generated("plan") == "plan" + judged = orchestrator._model_judge_verification("42", {"reason": "base"}) + assert "reason" in judged + + +def test_batch_upload_without_profile_or_decisions_passes_through() -> None: + client = FakeClient() + plain = FakeAgent("plain_batch", "model") + body = json.dumps({"custom_id": "x", "body": {"model": "m"}}).encode() + client._batch_upload(plain, body) + assert client.sent[-1] == {"model": "m"} + + +def test_invoke_event_usage_loop_handles_no_matching_event() -> None: + orchestrator = make_orchestrator() + worker = orchestrator._select_agent("", "worker") + token = rr._EVENT_CAPTURE.set( + [ + { + "agent_id": "other", + "role": "other", + "profile": common_profile(), + "decision": ReasoningDecision("low", "x", "other", 0, ("x",)), + "usage": None, + } + ] + ) + try: + orchestrator._invoke( + worker, + [{"role": "user", "content": "x"}], + text="x", + role="worker", + ) + events = rr._EVENT_CAPTURE.get() + assert events is not None and events[0]["usage"] is None + finally: + rr._EVENT_CAPTURE.reset(token) + + +def test_ablation_empty_and_nontrace_dispatch_edges() -> None: + orchestrator = make_orchestrator() + with pytest.raises(ValueError, match="at least one prompt"): + orchestrator.run_reasoning_ablation([]) + + original = orchestrator._dispatch + orchestrator._dispatch = lambda _messages, _mode: {"verification": {"accepted": False}, "trace": "bad"} + try: + report = orchestrator.run_reasoning_ablation(["x"], levels=("low",)) + finally: + orchestrator._dispatch = original + assert report["cells"][0]["total_tokens"] == 0 + + +def test_capture_batch_and_workflow_ignore_nonlist_traces() -> None: + orchestrator = make_orchestrator() + records = rr._capture_batch( + orchestrator, + lambda _self, _prompts: [{"trace": "bad"}], + ["x"], + ) + assert records[0]["reasoning_control"]["strategy"] == "adaptive" + + + + +def test_retry_refreshes_downstream_reasoning_usage_evidence() -> None: + """Recomputed verifier and synthesizer traces must expose current usage.""" + class RetryUsageClient(FakeClient): + """Assign distinct token counts to first and second downstream calls.""" + + def __init__(self) -> None: + super().__init__() + self.role_calls: dict[str, int] = {} + + def _send(self, agent: FakeAgent, payload: dict[str, Any]) -> str: + """Delegate output behavior, then stamp role-specific call evidence.""" + output = super()._send(agent, payload) + system = " ".join( + item.get("content", "") + for item in payload.get("messages", []) + if item.get("role") == "system" + ) + role = next( + (name for name in ("verifier", "synthesizer") if f"Role: {name}" in system), + "", + ) + if role: + self.role_calls[role] = self.role_calls.get(role, 0) + 1 + value = (100 if role == "verifier" else 200) + self.role_calls[role] + self._usage = { + "total_tokens": value + 10, + "completion_tokens_details": {"reasoning_tokens": value}, + } + return output + + agents = [ + FakeAgent("thinker_usage", "m", ("thinker",)), + FakeAgent("worker_usage", "m", ("worker",)), + FakeAgent("verifier_usage", "m", ("verifier",)), + FakeAgent("synth_usage", "m", ("synthesizer",)), + ] + for agent in agents: + rr.configure_agent_reasoning(agent, common_profile()) + orchestrator = FakeOrchestrator( + agents, client=RetryUsageClient(), reasoning_policy=ReasoningPolicy() + ) + result = orchestrator.conduct([{"role": "user", "content": "calculate"}]) + verifier = next(row for row in result["trace"] if row["role"] == "verifier") + synthesizer = next(row for row in result["trace"] if row["role"] == "synthesizer") + assert verifier["reasoning"]["reasoning_tokens"] == 102 + assert synthesizer["reasoning"]["reasoning_tokens"] == 202 diff --git a/tests/test_reasoning_workload.py b/tests/test_reasoning_workload.py new file mode 100644 index 000000000..0cd80a75a --- /dev/null +++ b/tests/test_reasoning_workload.py @@ -0,0 +1,230 @@ +"""Behavioral tests for graph-aware test-time reasoning allocation.""" + +from __future__ import annotations + +import pytest + +from contextual_orchestrator.reasoning_control import ( + ReasoningDecision, + ReasoningPolicy, + ReasoningProfile, + ReasoningWorkload, + WorkflowReasoningCursor, + select_reasoning_decision, + workload_for_trace_row, +) +from contextual_orchestrator.reasoning_runtime import ( + current_reasoning_workload, + reasoning_workload_override, +) + +from reasoning_fakes import make_orchestrator + + +def _profile() -> ReasoningProfile: + """Return the four-level model capability used by structural tests.""" + return ReasoningProfile( + supported_levels=("minimal", "low", "medium", "high"), + default_level="low", + maximum_level="high", + ) + + +def _messages(prior: str) -> list[dict[str, str]]: + """Build the repository's bounded accessed-work prompt shape.""" + return [ + { + "role": "user", + "content": ( + "Original task:\nSummarize this note.\n\n" + f"Accessed prior work:\n{prior}\n\nSubtask:\nContinue." + ), + } + ] + + +def test_workload_round_trips_without_a_latency_control() -> None: + workload = ReasoningWorkload(3, 5, 3, 4, 3) + assert ReasoningWorkload.from_mapping(workload.to_dict()) == workload + assert "latency" not in workload.to_dict() + + +@pytest.mark.parametrize( + "value", + [ + {"unknown_key": 1}, + {"workflow_step_index": True}, + {"workflow_step_count": 0}, + {"decomposition_count": 0}, + {"workflow_step_index": -1}, + {"workflow_step_index": 1, "workflow_step_count": 1}, + {"workflow_step_index": 0, "recursion_depth": 1}, + {"workflow_step_index": 0, "accessible_step_count": 1}, + {"workflow_step_count": 1, "decomposition_count": 2}, + ], +) +def test_workload_rejects_ambiguous_or_impossible_topology(value: dict[str, object]) -> None: + with pytest.raises(ValueError): + ReasoningWorkload.from_mapping(value) + + +def test_cursor_tracks_workflow_position_recursion_and_access_fan_in() -> None: + cursor = WorkflowReasoningCursor(4) + first = cursor.observe(_messages("(none)")) + second = cursor.observe(_messages("Step 0: plan")) + third = cursor.observe(_messages("Step 0: plan\nStep 1: work")) + fourth = cursor.observe(_messages("Step 0: plan\nStep 1: work\nStep 2: verified")) + + assert first == ReasoningWorkload(0, 4, 0, 4, 0) + assert second == ReasoningWorkload(1, 4, 1, 4, 1) + assert third == ReasoningWorkload(2, 4, 2, 4, 2) + assert fourth == ReasoningWorkload(3, 4, 3, 4, 3) + assert cursor.observe(_messages("unused")) is None + + +def test_cursor_accepts_generated_plan_size_before_execution_only() -> None: + cursor = WorkflowReasoningCursor(4) + cursor.set_plan_size(5) + assert cursor.observe(_messages("(none)")) == ReasoningWorkload(0, 5, 0, 5, 0) + with pytest.raises(RuntimeError, match="cannot change"): + cursor.set_plan_size(3) + with pytest.raises(ValueError, match="positive integer"): + WorkflowReasoningCursor(0) + with pytest.raises(ValueError, match="positive integer"): + WorkflowReasoningCursor(2, decomposition_count=0) + with pytest.raises(ValueError, match="cannot exceed"): + WorkflowReasoningCursor(2, decomposition_count=3) + with pytest.raises(ValueError, match="positive integer"): + WorkflowReasoningCursor(2).set_plan_size(False) + + +def test_cursor_infers_template_access_when_fake_prior_outputs_have_no_step_labels() -> None: + cursor = WorkflowReasoningCursor(4) + assert cursor.observe(_messages("(none)")) == ReasoningWorkload(0, 4, 0, 4, 0) + assert cursor.observe(_messages("unlabelled plan output")) == ReasoningWorkload( + 1, + 4, + 1, + 4, + 1, + ) + direct = WorkflowReasoningCursor(1) + assert direct.observe([{"role": "user", "content": "plain direct request"}]) == ReasoningWorkload() + + +def test_structural_signals_raise_compute_without_a_speed_objective() -> None: + direct = select_reasoning_decision( + _profile(), + ReasoningPolicy(), + "Summarize this note.", + "worker", + workload=ReasoningWorkload(), + ) + deep = select_reasoning_decision( + _profile(), + ReasoningPolicy(), + "Summarize this note.", + "worker", + workload={ + "workflow_step_index": 3, + "workflow_step_count": 5, + "recursion_depth": 3, + "decomposition_count": 4, + "accessible_step_count": 3, + }, + ) + assert direct is not None and direct.level == "low" + assert deep is not None and deep.level == "high" + assert deep.workload == ReasoningWorkload(3, 5, 3, 4, 3) + assert { + "decomposed_workflow", + "recursive_workflow_depth", + "access_list_fan_in", + "late_workflow_integration", + }.issubset(deep.factors) + assert all("latency" not in factor and "speed" not in factor for factor in deep.factors) + + +def test_fixed_policy_records_structure_but_does_not_adapt_the_fixed_level() -> None: + workload = ReasoningWorkload(2, 4, 2, 4, 2) + decision = select_reasoning_decision( + _profile(), + ReasoningPolicy(strategy="fixed", fixed_level="medium", max_escalations=0), + "complex task", + "verifier", + workload=workload, + ) + assert decision == ReasoningDecision( + "medium", + "fixed_policy", + "verifier", + 0, + ("fixed_policy",), + workload=workload, + ) + assert decision.to_dict()["workload"] == workload.to_dict() + with pytest.raises(ValueError, match="workload must be"): + select_reasoning_decision( + _profile(), + ReasoningPolicy(), + "task", + "worker", + workload="invalid", + ) + + +def test_trace_reconstruction_uses_actual_access_lists() -> None: + trace = [ + {"id": 0, "access": []}, + {"id": 1, "access": [0]}, + {"id": 2, "access": [0, 1]}, + {"id": 3, "access": [0, 1, 2]}, + ] + assert workload_for_trace_row(trace[3], trace) == ReasoningWorkload(3, 4, 3, 4, 3) + with pytest.raises(ValueError, match="at least one"): + workload_for_trace_row({"id": 0}, []) + with pytest.raises(ValueError, match="non-negative integer"): + workload_for_trace_row({"id": "three"}, trace) + + +def test_workload_override_is_request_local_and_type_safe() -> None: + workload = ReasoningWorkload(1, 4, 1, 4, 1) + assert current_reasoning_workload() is None + with reasoning_workload_override(workload): + assert current_reasoning_workload() == workload + assert current_reasoning_workload() is None + with pytest.raises(TypeError, match="ReasoningWorkload"): + with reasoning_workload_override("invalid"): + pass + + +def test_conduct_trace_records_role_specific_graph_workload() -> None: + orchestrator = make_orchestrator() + result = orchestrator.conduct( + [{"role": "user", "content": "Summarize this note."}] + ) + workloads = [ + row["reasoning"]["decision"]["workload"] + for row in result["trace"] + ] + assert workloads == [ + ReasoningWorkload(0, 4, 0, 4, 0).to_dict(), + ReasoningWorkload(1, 4, 1, 4, 1).to_dict(), + ReasoningWorkload(2, 4, 2, 4, 2).to_dict(), + ReasoningWorkload(3, 4, 3, 4, 3).to_dict(), + ] + escalation = result["reasoning_escalation"] + assert escalation["from_level"] == "low" + assert escalation["to_level"] == "medium" + assert result["trace"][1]["reasoning"]["decision"]["level"] == "medium" + assert result["trace"][2]["reasoning"]["decision"]["level"] == "high" + + +def test_route_trace_remains_a_single_step_workload() -> None: + orchestrator = make_orchestrator() + result = orchestrator.route_once( + [{"role": "user", "content": "Summarize this note."}] + ) + decision = result["trace"][0]["reasoning"]["decision"] + assert decision["level"] == "low" + assert decision["workload"] == ReasoningWorkload().to_dict() diff --git a/tests/test_reasoning_workload_coverage.py b/tests/test_reasoning_workload_coverage.py new file mode 100644 index 000000000..6a8d95f66 --- /dev/null +++ b/tests/test_reasoning_workload_coverage.py @@ -0,0 +1,67 @@ +"""Focused branch coverage for structural reasoning workload failures.""" + +from __future__ import annotations + +import pytest + +from contextual_orchestrator._reasoning_workflow import _retry_rejected_worker_once +from contextual_orchestrator.reasoning_control import ( + ReasoningDecision, + ReasoningWorkload, + workload_for_trace_row, +) + +from reasoning_fakes import make_orchestrator + + +def test_workload_from_mapping_rejects_non_object() -> None: + """A JSON array cannot masquerade as named workflow topology evidence.""" + with pytest.raises(ValueError, match="must be an object"): + ReasoningWorkload.from_mapping([]) # type: ignore[arg-type] + + +def test_workload_trace_target_must_belong_to_workflow() -> None: + """A target row outside the observed trace cannot receive invented depth.""" + with pytest.raises(ValueError, match="outside the workflow"): + workload_for_trace_row({"id": 2}, [{"id": 0, "access": []}]) + + +def test_reasoning_decision_rejects_untyped_workload() -> None: + """Decision evidence accepts only a validated ReasoningWorkload value.""" + with pytest.raises(ValueError, match="ReasoningWorkload or None"): + ReasoningDecision( + "low", + "adaptive_policy", + "worker", + 0, + ("profile_default",), + workload="invalid", # type: ignore[arg-type] + ) + + +def test_retry_stops_when_trace_agent_no_longer_exists() -> None: + """A stale trace identity cannot trigger recomputation against another agent.""" + orchestrator = make_orchestrator() + result = { + "verification": {"accepted": False}, + "trace": [ + { + "id": 1, + "role": "worker", + "agent_id": "missing_agent", + "access": [0], + "reasoning": { + "decision": { + "level": "low", + "source": "adaptive_policy", + "complexity_score": 0, + "factors": ["profile_default"], + "escalation_index": 0, + "workload": ReasoningWorkload(1, 2, 1, 2, 1).to_dict(), + } + }, + } + ], + } + _retry_rejected_worker_once(orchestrator, result, "task") + assert "reasoning_escalation" not in result