feat: pluggable LLM provider interface for judge

fleet/__init__.py

@@ -71,6 +71,9 @@
 # Import judge data classes
 from .judge import Rubric, Criterion, File, Image, JudgeResult
 
+# Import LLM provider interface
+from .llm_provider import LLMProvider, FleetProvider, ExternalProvider
+
 # Create a module-level env attribute for convenient access
 from . import env
 from . import global_client as _global_client
@@ -99,6 +102,10 @@
     "File",
     "Image",
     "JudgeResult",
+    # LLM Providers
+    "LLMProvider",
+    "FleetProvider",
+    "ExternalProvider",
     # Exceptions
     "FleetError",
     "FleetAPIError",

fleet/_async/client.py

@@ -55,6 +55,7 @@
 if TYPE_CHECKING:
     from .verifiers import AsyncVerifierFunction
     from .judge import AsyncJudge
+    from ..llm_provider import LLMProvider
 
 
 def _json_default(x: Any) -> Any:
@@ -340,12 +341,19 @@ def message_count(self) -> int:
 
 
 class AsyncEnv(EnvironmentBase):
-    def __init__(self, client: Optional[AsyncWrapper], **kwargs):
+    def __init__(
+        self,
+        client: Optional[AsyncWrapper],
+        *,
+        llm_provider: Optional["LLMProvider"] = None,
+        **kwargs,
+    ):
         super().__init__(**kwargs)
         self._client = client
         self._apps: Dict[str, AsyncInstanceClient] = {}
         self._instance: Optional[AsyncInstanceClient] = None
         self._judge: Optional["AsyncJudge"] = None
+        self._llm_provider = llm_provider
 
     @property
     def instance(self) -> AsyncInstanceClient:
@@ -423,13 +431,18 @@ def mcp(self) -> AsyncMCPResource:
 
     @property
     def judge(self) -> "AsyncJudge":
-        """LLM-as-judge grading via orchestrator API."""
+        """LLM-as-judge grading.
+
+        Routes through Fleet orchestrator by default. Set ``llm_provider``
+        on the environment to route to an external provider instead.
+        """
         if self._judge is None:
             from .judge import AsyncJudge
 
             self._judge = AsyncJudge(
-                client=self._load_client,
+                client=self._client,
                 instance_id=self.instance_id,
+                llm_provider=self._llm_provider,
             )
         return self._judge
 

fleet/_async/judge.py

@@ -1,6 +1,9 @@
 """Fleet SDK Judge - Async version.
 
 Provides env.judge.grade() for async verifier scripts.
+
+By default routes through the Fleet orchestrator. Pass an ``llm_provider``
+to route calls to an external LLM endpoint instead.
 """
 
 from typing import Dict, List, Optional, Union, TYPE_CHECKING
@@ -23,6 +26,7 @@
 
 if TYPE_CHECKING:
     from .base import AsyncWrapper
+    from ..llm_provider import LLMProvider
 
 # Re-export data classes so `from fleet._async.judge import ...` works
 __all__ = [
@@ -36,14 +40,24 @@
 
 
 class AsyncJudge:
-    """LLM-as-judge grading — calls orchestrator API, not environment API.
+    """LLM-as-judge grading (async).
+
+    Accessed as ``env.judge`` on AsyncEnv instances.
 
-    Accessed as env.judge on AsyncEnv instances.
+    By default routes through the Fleet orchestrator API. Pass an
+    ``llm_provider`` to route calls to an external LLM endpoint instead.
     """
 
-    def __init__(self, client: "AsyncWrapper", instance_id: str):
+    def __init__(
+        self,
+        client: Optional["AsyncWrapper"],
+        instance_id: str,
+        *,
+        llm_provider: Optional["LLMProvider"] = None,
+    ):
         self._client = client
         self._instance_id = instance_id
+        self._llm_provider = llm_provider
 
     async def grade(
         self,
@@ -63,7 +77,11 @@ async def grade(
         collect: Optional[Dict[str, List[str]]] = None,
         task_id: Optional[str] = None,
     ) -> JudgeResult:
-        """Grade a submission using LLM-as-judge via the orchestrator API.
+        """Grade a submission using LLM-as-judge.
+
+        Routes through the Fleet orchestrator by default. If an
+        ``llm_provider`` was set at construction time, calls the external
+        provider directly instead.
 
         Returns a JudgeResult (float subclass with .details, .criteria, .feedback)
         that can be returned directly from a verifier function.
@@ -84,6 +102,14 @@ async def grade(
             collect: File patterns for orchestrator to collect (agentic mode).
             task_id: Optional task ID for tracking.
         """
+        # Fold reference_claims into context
+        effective_context = context
+        if reference_claims is not None:
+            if effective_context:
+                effective_context = f"{effective_context}\n\n## Reference Claims\n{reference_claims}"
+            else:
+                effective_context = f"## Reference Claims\n{reference_claims}"
+
         # Resolve Image.from_env images asynchronously before building request
         resolved_images = images
         if images and not agentic:
@@ -129,14 +155,40 @@ async def grade(
                 else:
                     resolved_files[label] = f
 
+        _print_judge_call_start(rubric, resolved_images, agentic, model, files=resolved_files)
+
+        if self._llm_provider is not None:
+            # Route through pluggable LLM provider
+            from ..llm_provider import GradeRequest
+
+            request = GradeRequest(
+                rubric=rubric,
+                submission=submission,
+                ground_truth=ground_truth,
+                problem=problem,
+                context=effective_context,
+                conversation=conversation,
+                images=resolved_images,
+                files=resolved_files,
+                model=model,
+                provider=provider,
+                agentic=agentic,
+                collect=collect,
+                task_id=task_id,
+                instance_id=self._instance_id,
+            )
+            grade_response = await self._llm_provider.agrade(request)
+            return _parse_grade_response(grade_response.to_dict())
+
+        # Default: route through Fleet orchestrator
         body = _build_grade_request(
             self._instance_id,
             rubric,
             submission,
             ground_truth=ground_truth,
             problem=problem,
-            context=context,
-            reference_claims=reference_claims,
+            context=effective_context,
+            reference_claims=None,  # already folded into context
             conversation=conversation,
             images=resolved_images,
             files=resolved_files,
@@ -147,6 +199,5 @@ async def grade(
             task_id=task_id,
         )
 
-        _print_judge_call_start(rubric, resolved_images, agentic, model, files=resolved_files)
         response = await self._client.request("POST", "/v1/judge/grade", json=body)
         return _parse_grade_response(response.json())

fleet/client.py

@@ -60,6 +60,7 @@
 if TYPE_CHECKING:
     from .verifiers import SyncVerifierFunction
     from .judge import SyncJudge
+    from .llm_provider import LLMProvider
 
 
 def _json_default(x: Any) -> Any:
@@ -344,12 +345,19 @@ def message_count(self) -> int:
 
 
 class SyncEnv(EnvironmentBase):
-    def __init__(self, client: Optional[SyncWrapper], **kwargs):
+    def __init__(
+        self,
+        client: Optional[SyncWrapper],
+        *,
+        llm_provider: Optional["LLMProvider"] = None,
+        **kwargs,
+    ):
         super().__init__(**kwargs)
         self._client = client
         self._apps: Dict[str, InstanceClient] = {}
         self._instance: Optional[InstanceClient] = None
         self._judge: Optional["SyncJudge"] = None
+        self._llm_provider = llm_provider
         self._manager_url_override: Optional[str] = None  # For URL mode
 
     @property
@@ -435,13 +443,18 @@ def mcp(self) -> SyncMCPResource:
 
     @property
     def judge(self) -> "SyncJudge":
-        """LLM-as-judge grading via orchestrator API."""
+        """LLM-as-judge grading.
+
+        Routes through Fleet orchestrator by default. Set ``llm_provider``
+        on the environment to route to an external provider instead.
+        """
         if self._judge is None:
             from .judge import SyncJudge
 
             self._judge = SyncJudge(
-                client=self._load_client,
+                client=self._client,
                 instance_id=self.instance_id,
+                llm_provider=self._llm_provider,
             )
         return self._judge
 

fleet/judge.py

@@ -1,10 +1,21 @@
-"""Fleet SDK Judge - LLM-as-Judge grading via orchestrator API.
+"""Fleet SDK Judge - LLM-as-Judge grading.
 
 Provides env.judge.grade() for verifier scripts to grade submissions
 using LLM judges without managing API keys, HTTP calls, or response parsing.
 
-All LLM calls happen server-side on the orchestrator — the SDK just sends
-the rubric, submission, and artifacts, and gets back a score.
+By default, LLM calls route through the Fleet orchestrator. For on-prem
+or external deployments, pass an ``llm_provider`` to route calls to
+any OpenAI-compatible endpoint (OpenRouter, Anthropic, local models, etc.)::
+
+    from fleet.llm_provider import ExternalProvider
+
+    provider = ExternalProvider(
+        api_key="sk-or-...",
+        base_url="https://openrouter.ai/api/v1",
+        model="anthropic/claude-sonnet-4",
+    )
+    judge = SyncJudge(client=None, instance_id="local", llm_provider=provider)
+    result = judge.grade(rubric, submission)
 """
 
 import base64
@@ -16,6 +27,7 @@
 
 if TYPE_CHECKING:
     from .base import SyncWrapper
+    from .llm_provider import LLMProvider
 
 logger = logging.getLogger(__name__)
 
@@ -943,14 +955,33 @@ def _print_judge_result(data: dict) -> None:
 
 
 class SyncJudge:
-    """LLM-as-judge grading — calls orchestrator API, not environment API.
+    """LLM-as-judge grading.
+
+    Accessed as ``env.judge`` on SyncEnv instances.
+
+    By default routes through the Fleet orchestrator API. Pass an
+    ``llm_provider`` to route calls to an external LLM endpoint instead::
+
+        from fleet.llm_provider import ExternalProvider
 
-    Accessed as env.judge on SyncEnv instances.
+        provider = ExternalProvider(
+            api_key="sk-or-...",
+            base_url="https://openrouter.ai/api/v1",
+            model="anthropic/claude-sonnet-4",
+        )
+        judge = SyncJudge(client=None, instance_id="local", llm_provider=provider)
     """
 
-    def __init__(self, client: "SyncWrapper", instance_id: str):
+    def __init__(
+        self,
+        client: Optional["SyncWrapper"],
+        instance_id: str,
+        *,
+        llm_provider: Optional["LLMProvider"] = None,
+    ):
         self._client = client
         self._instance_id = instance_id
+        self._llm_provider = llm_provider
 
     def grade(
         self,
@@ -970,7 +1001,11 @@ def grade(
         collect: Optional[Dict[str, List[str]]] = None,
         task_id: Optional[str] = None,
     ) -> JudgeResult:
-        """Grade a submission using LLM-as-judge via the orchestrator API.
+        """Grade a submission using LLM-as-judge.
+
+        Routes through the Fleet orchestrator by default. If an
+        ``llm_provider`` was set at construction time, calls the external
+        provider directly instead.
 
         Returns a JudgeResult (float subclass with .details, .criteria, .feedback)
         that can be returned directly from a verifier function.
@@ -991,14 +1026,48 @@ def grade(
             collect: File patterns for orchestrator to collect (agentic mode).
             task_id: Optional task ID for tracking.
         """
+        # Fold reference_claims into context (shared logic regardless of provider)
+        effective_context = context
+        if reference_claims is not None:
+            if effective_context:
+                effective_context = f"{effective_context}\n\n## Reference Claims\n{reference_claims}"
+            else:
+                effective_context = f"## Reference Claims\n{reference_claims}"
+
+        _print_judge_call_start(rubric, images, agentic, model, files=files)
+
+        if self._llm_provider is not None:
+            # Route through pluggable LLM provider
+            from .llm_provider import GradeRequest
+
+            request = GradeRequest(
+                rubric=rubric,
+                submission=submission,
+                ground_truth=ground_truth,
+                problem=problem,
+                context=effective_context,
+                conversation=conversation,
+                images=images,
+                files=files,
+                model=model,
+                provider=provider,
+                agentic=agentic,
+                collect=collect,
+                task_id=task_id,
+                instance_id=self._instance_id,
+            )
+            grade_response = self._llm_provider.grade(request)
+            return _parse_grade_response(grade_response.to_dict())
+
+        # Default: route through Fleet orchestrator
         body = _build_grade_request(
             self._instance_id,
             rubric,
             submission,
             ground_truth=ground_truth,
             problem=problem,
-            context=context,
-            reference_claims=reference_claims,
+            context=effective_context,
+            reference_claims=None,  # already folded into context
             conversation=conversation,
             images=images,
             files=files,
@@ -1009,6 +1078,5 @@ def grade(
             task_id=task_id,
         )
 
-        _print_judge_call_start(rubric, images, agentic, model, files=files)
         response = self._client.request("POST", "/v1/judge/grade", json=body)
         return _parse_grade_response(response.json())