fix(passthrough): fail over on single-tool-call-limit provider errors

contextual_orchestrator/orchestrator.py

@@ -95,6 +95,7 @@
 _PROVIDER_TOOL_DESCRIPTION_LIMIT_MESSAGE = (
     "each tool.function.description must be at most 1024 characters"
 )
+_SINGLE_TOOL_CALL_LIMIT_MESSAGE = "this model only supports single tool-calls at once"
 DEFAULT_PROVIDER_PROBE_TIMEOUT = 5.0
 MODEL_CAPABILITIES = frozenset(
     {"text", "image", "video", "speech", "transcription", "embedding", "rerank", "audio"}
@@ -764,6 +765,29 @@ def _is_oversized_tool_description_error(error: urllib.error.HTTPError) -> bool:
     )
 
 
+def _is_single_tool_call_limit_error(error: urllib.error.HTTPError) -> bool:
+    """Recognize a model that rejects a request making more than one tool call.
+
+    Some NVIDIA NIM-hosted models (observed: a vision-capable Llama variant)
+    reject any turn with more than one tool call, wrapping the sentence in a
+    longer agent-prefixed message under the generic ``invalid_request_error``
+    code rather than a distinct error code -- unlike the tool-description
+    limit above, the message text is the only reliable signal here.
+    """
+    if error.code != 400:
+        return False
+    payload = _http_error_payload(error)
+    details = payload.get("error") if isinstance(payload, dict) else None
+    message = (
+        details.get("message")
+        if isinstance(details, dict)
+        else details
+        if isinstance(details, str)
+        else None
+    )
+    return isinstance(message, str) and _SINGLE_TOOL_CALL_LIMIT_MESSAGE in message.casefold()
+
+
 def _provider_tool_execution_stopped(agent: ModelAgent) -> ToolFallbackStoppedError:
     """Convert the provider's terminal tool-stop contract to the public safe error."""
     decision = classify_tool_failure(
@@ -1252,6 +1276,11 @@ def _is_passthrough_failover_error(exc: BaseException) -> bool:
             and _is_provider_tool_description_limit_error(current)
         ):
             return True
+        if (
+            isinstance(current, urllib.error.HTTPError)
+            and _is_single_tool_call_limit_error(current)
+        ):
+            return True
         if isinstance(current, socket.gaierror) and current.errno == socket.EAI_AGAIN:
             return True
         if current.__cause__ is not None:
@@ -1263,6 +1292,38 @@ def _is_passthrough_failover_error(exc: BaseException) -> bool:
     return False
 
 
+def _is_capability_mismatch_failover_error(exc: BaseException) -> bool:
+    """Recognize a structural capability mismatch, not a reliability failure.
+
+    A model rejecting a request shape it can never support (too many tool
+    descriptions, more than one tool call per turn) says nothing about that
+    model's health for a differently-shaped future request -- unlike an
+    oversized-payload rejection (already exempted via
+    _is_request_too_large_error), this failure is not size-dependent, but the
+    same principle applies: it must not trip the circuit breaker or count as
+    a failed stability observation for the model or its group (Codex Review,
+    ContextualWisdomLab/contextual-orchestrator#986).
+    """
+    current: BaseException | None = exc
+    seen: set[int] = set()
+    for _ in range(_PROVIDER_ERROR_CHAIN_LIMIT):
+        if current is None or id(current) in seen:
+            return False
+        seen.add(id(current))
+        if isinstance(current, urllib.error.HTTPError) and (
+            _is_provider_tool_description_limit_error(current)
+            or _is_single_tool_call_limit_error(current)
+        ):
+            return True
+        if current.__cause__ is not None:
+            current = current.__cause__
+        elif current.__suppress_context__:
+            return False
+        else:
+            current = current.__context__
+    return False
+
+
 class ModelClient:
     """Small chat-completions client with retry, backoff, and mock support."""
 
@@ -3801,9 +3862,10 @@ def proxy_completion(
                     every_failure_was_request_too_large
                     and request_too_large
                 )
-                if not request_too_large:
+                capability_mismatch = _is_capability_mismatch_failover_error(exc)
+                if not (request_too_large or capability_mismatch):
                     self._record_failure(candidate.id)
-                if candidate.group_name and not request_too_large:
+                if candidate.group_name and not (request_too_large or capability_mismatch):
                     self._group_router.observe_failure(candidate.id)
                 continue
             self._record_success(candidate.id)

tests/test_passthrough_provider_failover.py

@@ -786,6 +786,121 @@ def test_virtual_passthrough_fails_over_on_provider_tool_description_limit() ->
     ]
 
 
+def test_virtual_passthrough_fails_over_on_single_tool_call_limit() -> None:
+    """A model that rejects multi-tool-call turns advances to the next provider.
+
+    Reproduces the exact NVIDIA NIM litellm error shape observed live (Strix
+    scan, ContextualWisdomLab/naruon#1486, 2026-09-01): a generic
+    ``invalid_request_error`` code (not ``invalid_tools``) with the model's
+    own capability-limit sentence embedded in a longer, agent-prefixed
+    message. Recognizing this is what stops that single-tool-call-only model
+    from ending an entire scan instead of the orchestrator just moving on to
+    a capability-matched agent.
+    """
+    failure = _http_error(
+        400,
+        {
+            "error": {
+                "code": "invalid_request_error",
+                "message": (
+                    "Model 'meta/llama-3.2-11b-vision-instruct' via agent "
+                    "'nvidia_nim_meta_llama_3_2_11b_vision_instruct': This "
+                    "model only supports single tool-calls at once! This "
+                    "model only supports single tool-calls at once!. Adjust "
+                    "the request parameters and retry."
+                ),
+            }
+        },
+    )
+    client = SequencedProxyClient(
+        {
+            "primary_agent": failure,
+            "fallback_agent": {"model": "fallback-model"},
+        }
+    )
+    orchestrator = _build(client)
+    orchestrator.agents = [
+        replace(agent, tags=(*agent.tags, "cost:free")) for agent in orchestrator.agents
+    ]
+
+    result = orchestrator.proxy_completion(
+        {
+            "model": TaskOrchestrator.FREE_MODEL,
+            "messages": [{"role": "user", "content": "use two tools at once"}],
+            "tools": [
+                {"type": "function", "function": {"name": "inspect", "description": "x"}},
+                {"type": "function", "function": {"name": "scan", "description": "y"}},
+            ],
+        }
+    )
+
+    assert result["model"] == "fallback-model"
+    assert [agent_id for agent_id, _ in client.calls] == [
+        "primary_agent",
+        "fallback_agent",
+    ]
+
+
+@pytest.mark.parametrize(
+    "failure",
+    [
+        _http_error(
+            400,
+            {
+                "error": {
+                    "code": "invalid_tools",
+                    "message": "each tool.function.description must be at most 1024 characters",
+                }
+            },
+        ),
+        _http_error(
+            400,
+            {
+                "error": {
+                    "code": "invalid_request_error",
+                    "message": "This model only supports single tool-calls at once!",
+                }
+            },
+        ),
+    ],
+    ids=["provider_tool_description_limit", "single_tool_call_limit"],
+)
+def test_capability_mismatch_failover_does_not_penalize_the_model(
+    failure: urllib.error.HTTPError,
+) -> None:
+    """A structural capability mismatch must not trip the circuit breaker.
+
+    Codex Review (ContextualWisdomLab/contextual-orchestrator#986): a model
+    rejecting a request shape it can never support (too many tool
+    descriptions, more than one tool call per turn) says nothing about that
+    model's health for a differently-shaped future request. Unlike a
+    reliability failure, this must not count as a failed stability
+    observation -- otherwise an ordinary, differently-shaped future request
+    to the same model could be wrongly deprioritized or circuit-broken.
+    """
+    client = SequencedProxyClient(
+        {
+            "primary_agent": failure,
+            "fallback_agent": {"model": "fallback-model"},
+        }
+    )
+    orchestrator = _build(client)
+
+    result = orchestrator.proxy_completion(
+        {
+            "model": "contextual-orchestrator",
+            "messages": [{"role": "user", "content": "use two tools at once"}],
+            "tools": [
+                {"type": "function", "function": {"name": "inspect", "description": "x"}},
+                {"type": "function", "function": {"name": "scan", "description": "y"}},
+            ],
+        }
+    )
+
+    assert result["model"] == "fallback-model"
+    assert orchestrator._circuit.get("primary_agent") in (None, {"failures": 0.0, "opened_at": 0.0})
+
+
 @pytest.mark.parametrize(
     "provider_error",
     [