Skip to content

Commit aac34fe

Browse files
committed
Stop silently dropping builtin tools on the chat dialect
The openai_chat tools loop had no BuiltinTool branch: a builtin in Request.tools vanished from the wire with no error. New typed compat knob builtin_tools ('reject' default, 'groq' mapping): reject raises UnsupportedFeatureError everywhere unproven; the groq preset maps web_search -> browser_search and code_execution -> code_interpreter, both verified live as server-executed (executed_tools trace stays in provider_data per MAP-1). OpenRouter probe showed servers may silently ignore unknown tool types, which is why passthrough is not an option. Named builtin forcing on the dialect still raises; plain required flows through (Groq wire-accepts it). See lm15-contract changes/2026-09-01-chat-dialect-builtin-tools.md. Signed-off-by: Maxime Rivest <mrive052@gmail.com>
1 parent 6b29065 commit aac34fe

3 files changed

Lines changed: 151 additions & 0 deletions

File tree

‎lm15/compat.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,14 @@ class ResolvedOpenAIResponsesCompat:
217217
"qwen_chat_template",
218218
"zai",
219219
]
220+
# BuiltinTool policy for the chat dialect. The base Chat Completions wire
221+
# carries function/custom tools ONLY (doc: chat--create.md), and some
222+
# compat servers silently IGNORE unknown tool types (OpenRouter, verified
223+
# live 2026-09-01) — so "reject" (raise) is the only safe default.
224+
# "groq" maps canonical builtin names onto Groq's server-executed tool
225+
# types (browser_search / code_interpreter, both verified live
226+
# 2026-09-01).
227+
OpenAIChatBuiltinTools = Literal["auto", "reject", "groq"]
220228

221229

222230
@dataclass(frozen=True, slots=True)
@@ -237,6 +245,7 @@ class OpenAIChatCompat:
237245
thinking_replay: OpenAIChatThinkingReplay | None = None
238246
assistant_reasoning_content: OpenAIChatAssistantReasoningContent | None = None
239247
strict_tools: OpenAIStrictTools | None = None
248+
builtin_tools: OpenAIChatBuiltinTools | None = None
240249
cache_control: OpenAICacheControl | None = None
241250
routing: JsonObject | None = None
242251
extensions: JsonObject | None = None
@@ -259,6 +268,7 @@ def __post_init__(self) -> None:
259268
"assistant_reasoning_content",
260269
)
261270
_check_literal_or_none(self.strict_tools, OpenAIStrictTools, "strict_tools")
271+
_check_literal_or_none(self.builtin_tools, OpenAIChatBuiltinTools, "builtin_tools")
262272
_check_literal_or_none(self.cache_control, OpenAICacheControl, "cache_control")
263273
_check_json_object_or_none(self.routing, "routing")
264274
_check_json_object_or_none(self.extensions, "extensions")
@@ -298,6 +308,7 @@ def preset(cls, name: str) -> "OpenAIChatCompat":
298308
thinking_format="reasoning_effort",
299309
tool_result_name="omit",
300310
strict_tools="omit",
311+
builtin_tools="groq",
301312
cache_control="none",
302313
)
303314

@@ -409,6 +420,7 @@ class ResolvedOpenAIChatCompat:
409420
thinking_replay: Literal["native", "as_text", "omit"] = "omit"
410421
assistant_reasoning_content: Literal["include_empty", "omit"] = "omit"
411422
strict_tools: Literal["include", "omit"] = "omit"
423+
builtin_tools: Literal["reject", "groq"] = "reject"
412424
cache_control: Literal["none", "openai", "anthropic"] = "openai"
413425
routing: JsonObject | None = None
414426
extensions: JsonObject | None = None
@@ -424,6 +436,7 @@ class ResolvedOpenAIChatCompat:
424436
"thinking_replay": "omit",
425437
"assistant_reasoning_content": "omit",
426438
"strict_tools": "omit",
439+
"builtin_tools": "reject",
427440
"cache_control": "openai",
428441
}
429442

‎lm15/providers/openai_chat.py‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,15 @@
5858

5959
_DEFAULT_BASE_URL = "https://api.openai.com/v1"
6060

61+
# Canonical builtin tool name → Groq server-executed tool type (compat
62+
# builtin_tools="groq"; both verified live 2026-09-01: Groq runs them
63+
# server-side and reports the trace in message.executed_tools, which
64+
# stays in provider_data per MAP-1).
65+
_GROQ_BUILTIN_MAP: dict[str, str] = {
66+
"web_search": "browser_search",
67+
"code_execution": "code_interpreter",
68+
}
69+
6170
_FINISH_REASON_MAP: dict[str, str] = {
6271
"stop": "stop",
6372
"length": "length",
@@ -277,6 +286,36 @@ def _build_messages(self, request: Request, compat: ResolvedOpenAIChatCompat) ->
277286
messages.append({"role": role, "content": content})
278287
return messages
279288

289+
def _builtin_tool_payload(self, tool: BuiltinTool, compat: ResolvedOpenAIChatCompat) -> dict[str, Any]:
290+
"""Map a BuiltinTool for the chat dialect, or raise.
291+
292+
The base Chat Completions wire carries function/custom tools only,
293+
and some compat servers silently IGNORE unknown tool types
294+
(OpenRouter returned 200 with no search, verified live 2026-09-01)
295+
— passthrough would fabricate wire shapes and silent no-ops, so
296+
every unproven target raises.
297+
"""
298+
if compat.builtin_tools == "groq":
299+
wire_type = _GROQ_BUILTIN_MAP.get(tool.name)
300+
if wire_type is None:
301+
raise UnsupportedFeatureError(
302+
f"{self.provider}: builtin tool {tool.name!r} has no Groq wire "
303+
f"mapping — supported: {sorted(_GROQ_BUILTIN_MAP)}",
304+
provider=self.provider,
305+
)
306+
entry: dict[str, Any] = {"type": wire_type}
307+
if tool.config:
308+
entry.update(tool.config)
309+
return entry
310+
raise UnsupportedFeatureError(
311+
f"{self.provider}: builtin tool {tool.name!r} is not supported on this "
312+
"server — the Chat Completions wire carries function tools only, and "
313+
"unproven servers may silently ignore unknown tool types. Use "
314+
"compat='groq' for Groq's server-executed tools, or the OpenAI "
315+
"Responses / Anthropic / Gemini providers",
316+
provider=self.provider,
317+
)
318+
280319
def _tool_choice_payload(self, request: Request) -> Any:
281320
tc = request.config.tool_choice
282321
if tc is None:
@@ -347,6 +386,8 @@ def _payload(self, request: Request, stream: bool) -> dict[str, Any]:
347386
if compat.strict_tools == "include":
348387
function_payload["strict"] = False
349388
tools_wire.append({"type": "function", "function": function_payload})
389+
elif isinstance(tool, BuiltinTool):
390+
tools_wire.append(self._builtin_tool_payload(tool, compat))
350391
if tools_wire:
351392
payload["tools"] = tools_wire
352393
tool_choice = self._tool_choice_payload(request)

‎tests/test_chat_builtin_tools.py‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
"""BuiltinTool policy on the Chat Completions dialect (2026-09-01).
2+
3+
The base chat wire carries function/custom tools only, and unproven
4+
compat servers may silently IGNORE unknown tool types (OpenRouter,
5+
verified live 2026-09-01: 200 OK, no search, no error). The dialect
6+
therefore raises by default and maps only where a live capture proves
7+
server-side execution — Groq's browser_search / code_interpreter
8+
(bodies in curl-fixtures/chat-builtin-tools-2026-09-01/).
9+
"""
10+
from __future__ import annotations
11+
12+
import pytest
13+
14+
from lm15 import BuiltinTool, FunctionTool, Message, Request, UnsupportedFeatureError
15+
from lm15.compat import OpenAIChatCompat, resolve_openai_chat_compat
16+
from lm15.providers.openai_chat import OpenAIChatLM
17+
from lm15.testing import FakeTransport
18+
19+
20+
def req(tools) -> Request:
21+
return Request(model="m", messages=(Message.user("hi"),), tools=tuple(tools))
22+
23+
24+
def lm(**kwargs) -> OpenAIChatLM:
25+
return OpenAIChatLM(api_key="k", transport=FakeTransport([]), **kwargs)
26+
27+
28+
def test_default_rejects_builtin_tools() -> None:
29+
with pytest.raises(UnsupportedFeatureError, match="builtin tool 'web_search'"):
30+
lm()._payload(req([BuiltinTool(name="web_search")]), stream=False)
31+
32+
33+
def test_openrouter_preset_rejects() -> None:
34+
# OpenRouter accepts the request and silently ignores unknown tool
35+
# types (live 2026-09-01) — the client-side raise is the protection.
36+
with pytest.raises(UnsupportedFeatureError, match="not supported"):
37+
lm(compat="openrouter")._payload(req([BuiltinTool(name="web_search")]), stream=False)
38+
39+
40+
def test_groq_maps_builtin_tools() -> None:
41+
body = lm(compat="groq")._payload(
42+
req([BuiltinTool(name="web_search"), BuiltinTool(name="code_execution")]), stream=False
43+
)
44+
assert body["tools"] == [{"type": "browser_search"}, {"type": "code_interpreter"}]
45+
46+
47+
def test_groq_builtin_config_merges() -> None:
48+
body = lm(compat="groq")._payload(
49+
req([BuiltinTool(name="web_search", config={"search_settings": {"include_domains": ["arxiv.org"]}})]),
50+
stream=False,
51+
)
52+
assert body["tools"] == [
53+
{"type": "browser_search", "search_settings": {"include_domains": ["arxiv.org"]}}
54+
]
55+
56+
57+
def test_groq_unmapped_builtin_name_raises() -> None:
58+
with pytest.raises(UnsupportedFeatureError, match="no Groq wire mapping"):
59+
lm(compat="groq")._payload(req([BuiltinTool(name="computer_use")]), stream=False)
60+
61+
62+
def test_groq_mixed_function_and_builtin() -> None:
63+
body = lm(compat="groq")._payload(
64+
req([FunctionTool(name="add", parameters={"type": "object", "properties": {}}),
65+
BuiltinTool(name="web_search")]),
66+
stream=False,
67+
)
68+
assert [t["type"] for t in body["tools"]] == ["function", "browser_search"]
69+
70+
71+
def test_builtin_tools_knob_resolution() -> None:
72+
assert resolve_openai_chat_compat(OpenAIChatCompat()).builtin_tools == "reject"
73+
assert resolve_openai_chat_compat(OpenAIChatCompat(builtin_tools="auto")).builtin_tools == "reject"
74+
assert resolve_openai_chat_compat(OpenAIChatCompat.preset("groq")).builtin_tools == "groq"
75+
with pytest.raises(ValueError):
76+
OpenAIChatCompat(builtin_tools="banana") # type: ignore[arg-type]
77+
78+
79+
def test_named_builtin_forcing_still_raises_on_groq() -> None:
80+
# Offering is proven; NAMED forcing has no documented Groq wire form.
81+
# Generic mode="required" (no allowlist) still flows through as the
82+
# plain "required" string — Groq accepts it at the wire.
83+
from lm15 import Config, ToolChoice
84+
85+
r = Request(
86+
model="m", messages=(Message.user("hi"),),
87+
tools=(BuiltinTool(name="web_search"),),
88+
config=Config(tool_choice=ToolChoice(mode="required", allowed=("web_search",))),
89+
)
90+
with pytest.raises(UnsupportedFeatureError, match="cannot force builtin"):
91+
lm(compat="groq")._payload(r, stream=False)
92+
r2 = Request(
93+
model="m", messages=(Message.user("hi"),),
94+
tools=(BuiltinTool(name="web_search"),),
95+
config=Config(tool_choice=ToolChoice(mode="required")),
96+
)
97+
assert lm(compat="groq")._payload(r2, stream=False)["tool_choice"] == "required"

0 commit comments

Comments
 (0)