Skip to content

Commit 648c659

Browse files
committed
Fix core tool-call streaming, tool-submit context, and retry behavior
1 parent c5bc34b commit 648c659

5 files changed

Lines changed: 181 additions & 18 deletions

File tree

‎lm15/middleware.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
from dataclasses import dataclass, field
66
from typing import Any
77

8+
from .errors import RateLimitError, ServerError, TimeoutError, TransportError
89
from .types import LMRequest, LMResponse, StreamEvent
910

1011
CompleteFn = Callable[[LMRequest], LMResponse]
@@ -60,12 +61,14 @@ def middleware(req: LMRequest, nxt: CompleteFn) -> LMResponse:
6061

6162

6263
def with_retries(max_retries: int = 2, sleep_base: float = 0.2) -> CompleteMiddleware:
64+
transient = (RateLimitError, TimeoutError, ServerError, TransportError)
65+
6366
def middleware(req: LMRequest, nxt: CompleteFn) -> LMResponse:
6467
last: Exception | None = None
6568
for i in range(max_retries + 1):
6669
try:
6770
return nxt(req)
68-
except Exception as e: # pragma: no cover - generic retry wrapper
71+
except transient as e:
6972
last = e
7073
if i == max_retries:
7174
raise

‎lm15/model.py‎

Lines changed: 47 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
from __future__ import annotations
22

33
import inspect
4+
import time
45
from dataclasses import dataclass
56
from typing import Any, Callable
67

78
from .capabilities import resolve_provider
89
from .client import UniversalLM
10+
from .errors import RateLimitError, ServerError, TimeoutError, TransportError
911
from .stream import Stream
1012
from .types import Config, FileUploadRequest, LMRequest, LMResponse, Message, Part, Tool
1113

@@ -318,7 +320,26 @@ def submit_tools(self, results: dict[str, Any], *, provider: str | None = None)
318320
if not parts:
319321
raise ValueError("no tool results matched pending calls")
320322

321-
return self(messages=[Message(role="tool", parts=tuple(parts))], provider=provider)
323+
tool_message = Message(role="tool", parts=tuple(parts))
324+
follow_messages = tuple(self._conversation) + (tool_message,)
325+
request, _, _, _ = self._build_request(
326+
prompt=None,
327+
messages=list(follow_messages),
328+
tools=None,
329+
reasoning=None,
330+
prefill=None,
331+
output=None,
332+
prompt_caching=None,
333+
temperature=None,
334+
max_tokens=None,
335+
top_p=None,
336+
stop=None,
337+
)
338+
resp = self._complete_with_cache(request, provider=provider or self.provider)
339+
self._pending_tool_calls = resp.tool_calls
340+
self.history.append(HistoryEntry(request=request, response=resp))
341+
self._conversation = list(request.messages) + [resp.message]
342+
return resp
322343

323344
def _build_request(
324345
self,
@@ -409,14 +430,32 @@ def _normalize_tools(self, tools: list[Tool | Callable[..., Any] | str]) -> tupl
409430
return tuple(out), registry
410431

411432
def _complete_with_cache(self, request: LMRequest, *, provider: str | None) -> LMResponse:
433+
cache_key: str | None = None
412434
if self._local_cache is not None:
413-
key = str((provider, request))
414-
if key in self._local_cache:
415-
return self._local_cache[key]
416-
resp = self._lm.complete(request, provider=provider)
417-
self._local_cache[key] = resp
418-
return resp
419-
return self._lm.complete(request, provider=provider)
435+
cache_key = str((provider, request))
436+
cached = self._local_cache.get(cache_key)
437+
if cached is not None:
438+
return cached
439+
440+
attempts = max(int(self.retries), 0) + 1
441+
last: Exception | None = None
442+
for i in range(attempts):
443+
try:
444+
resp = self._lm.complete(request, provider=provider)
445+
if self._local_cache is not None and cache_key is not None:
446+
self._local_cache[cache_key] = resp
447+
return resp
448+
except Exception as exc:
449+
last = exc
450+
if not self._is_retryable_error(exc) or i >= attempts - 1:
451+
raise
452+
time.sleep(0.2 * (2**i))
453+
454+
raise RuntimeError("unreachable") from last
455+
456+
@staticmethod
457+
def _is_retryable_error(exc: Exception) -> bool:
458+
return isinstance(exc, (RateLimitError, TimeoutError, ServerError, TransportError))
420459

421460
@staticmethod
422461
def _invoke_tool(fn: Callable[..., Any], payload: dict[str, Any]) -> Any:

‎lm15/stream.py‎

Lines changed: 24 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,8 @@ def __init__(
3636
self._started_model: str | None = None
3737
self._text_parts: list[str] = []
3838
self._thinking_parts: list[str] = []
39-
self._tool_input: dict[int, str] = {}
39+
self._tool_call_raw: dict[int, str] = {}
40+
self._tool_call_meta: dict[int, dict[str, str | dict]] = {}
4041

4142
def __iter__(self):
4243
return self
@@ -66,10 +67,17 @@ def __next__(self) -> StreamChunk:
6667
if dtype == "tool_call":
6768
idx = event.part_index or 0
6869
chunk = delta.get("input", "")
69-
agg = self._tool_input.get(idx, "") + chunk
70-
self._tool_input[idx] = agg
70+
chunk_s = chunk if isinstance(chunk, str) else json.dumps(chunk)
71+
agg = self._tool_call_raw.get(idx, "") + chunk_s
72+
self._tool_call_raw[idx] = agg
7173
parsed = self._parse_json_best_effort(agg)
72-
return StreamChunk(type="tool_call", input=parsed)
74+
meta = self._tool_call_meta.setdefault(idx, {})
75+
if delta.get("id"):
76+
meta["id"] = str(delta["id"])
77+
if delta.get("name"):
78+
meta["name"] = str(delta["name"])
79+
meta["input"] = parsed
80+
return StreamChunk(type="tool_call", input=parsed, name=meta.get("name"))
7381
if dtype == "audio":
7482
return StreamChunk(type="audio", audio=Part.audio(data=delta.get("data", "")))
7583
continue
@@ -86,10 +94,12 @@ def __next__(self) -> StreamChunk:
8694
if delta.type == "tool_call":
8795
idx = event.part_index or 0
8896
chunk = delta.input or ""
89-
agg = self._tool_input.get(idx, "") + chunk
90-
self._tool_input[idx] = agg
97+
agg = self._tool_call_raw.get(idx, "") + chunk
98+
self._tool_call_raw[idx] = agg
9199
parsed = self._parse_json_best_effort(agg)
92-
return StreamChunk(type="tool_call", input=parsed)
100+
meta = self._tool_call_meta.setdefault(idx, {})
101+
meta["input"] = parsed
102+
return StreamChunk(type="tool_call", input=parsed, name=meta.get("name"))
93103
if delta.type == "audio":
94104
return StreamChunk(type="audio", audio=Part.audio(data=delta.data or ""))
95105
continue
@@ -137,6 +147,13 @@ def _materialize_response(self, end_event: StreamEvent | None) -> LMResponse:
137147
parts.append(Part.thinking("".join(self._thinking_parts)))
138148
if self._text_parts:
139149
parts.append(Part.text_part("".join(self._text_parts)))
150+
for idx in sorted(self._tool_call_meta):
151+
meta = self._tool_call_meta[idx]
152+
tc_input = meta.get("input")
153+
payload = tc_input if isinstance(tc_input, dict) else {}
154+
tc_id = str(meta.get("id") or f"tool_call_{idx}")
155+
tc_name = str(meta.get("name") or "tool")
156+
parts.append(Part.tool_call(tc_id, tc_name, payload))
140157
if not parts:
141158
parts = [Part.text_part("")]
142159

‎tests/test_api_v2.py‎

Lines changed: 76 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
from lm15.features import EndpointSupport, ProviderManifest
1313
from lm15.model import Model
1414
from lm15.protocols import Capabilities
15-
from lm15.types import FileUploadRequest, FileUploadResponse, LMRequest, LMResponse, Message, Part, StreamEvent, Tool, Usage
15+
from lm15.types import FileUploadRequest, FileUploadResponse, LMRequest, LMResponse, Message, Part, PartDelta, StreamEvent, Tool, Usage
1616

1717

1818
class FakeAdapter:
@@ -127,6 +127,81 @@ def test_stream_error_raises_typed_provider_error(self):
127127
list(stream_obj.text)
128128
self.assertIn("provider_code=rate_limit_error", str(ctx.exception))
129129

130+
def test_stream_tool_call_materializes_in_response(self):
131+
class ToolStreamAdapter(FakeAdapter):
132+
def stream(self, request: LMRequest):
133+
yield StreamEvent(type="start", id="s1", model=request.model)
134+
yield StreamEvent(type="delta", part_index=0, delta=PartDelta(type="tool_call", input='{"city"'))
135+
yield StreamEvent(type="delta", part_index=0, delta=PartDelta(type="tool_call", input=':"Montreal"}'))
136+
yield StreamEvent(type="end", finish_reason="tool_call", usage=Usage(total_tokens=4))
137+
138+
lm = UniversalLM()
139+
lm.register(ToolStreamAdapter())
140+
m = Model(lm=lm, model="gpt-4.1-mini", provider="openai")
141+
142+
stream_obj = m.stream("weather")
143+
list(stream_obj)
144+
resp = stream_obj.response
145+
self.assertEqual(resp.finish_reason, "tool_call")
146+
self.assertEqual(len(resp.tool_calls), 1)
147+
self.assertEqual(resp.tool_calls[0].input, {"city": "Montreal"})
148+
149+
def test_submit_tools_preserves_conversation_context(self):
150+
class StrictToolAdapter:
151+
provider = "openai"
152+
capabilities = Capabilities()
153+
supports = EndpointSupport(complete=True)
154+
manifest = ProviderManifest(provider="openai", supports=supports)
155+
156+
def complete(self, request: LMRequest) -> LMResponse:
157+
last = request.messages[-1]
158+
if last.role == "user":
159+
return LMResponse(
160+
id="r1",
161+
model=request.model,
162+
message=Message(role="assistant", parts=(Part.tool_call("call_1", "get_weather", {"city": "Montreal"}),)),
163+
finish_reason="tool_call",
164+
usage=Usage(),
165+
)
166+
prev = request.messages[-2] if len(request.messages) >= 2 else None
167+
has_prev_tool_call = bool(prev and prev.role == "assistant" and any(p.type == "tool_call" for p in prev.parts))
168+
text = "ok" if has_prev_tool_call else "missing_prev_tool_call"
169+
return LMResponse(id="r2", model=request.model, message=Message.assistant(text), finish_reason="stop", usage=Usage())
170+
171+
lm = UniversalLM()
172+
lm.register(StrictToolAdapter())
173+
m = Model(lm=lm, model="gpt-4.1-mini", provider="openai")
174+
175+
first = m("weather", tools=[Tool(name="get_weather")])
176+
self.assertEqual(first.finish_reason, "tool_call")
177+
out = m.submit_tools({"call_1": "22C"})
178+
self.assertEqual(out.text, "ok")
179+
180+
def test_model_retries_transient_errors(self):
181+
class FlakyAdapter:
182+
provider = "openai"
183+
capabilities = Capabilities()
184+
supports = EndpointSupport(complete=True)
185+
manifest = ProviderManifest(provider="openai", supports=supports)
186+
187+
def __init__(self) -> None:
188+
self.calls = 0
189+
190+
def complete(self, request: LMRequest) -> LMResponse:
191+
self.calls += 1
192+
if self.calls < 3:
193+
raise RateLimitError("retry me")
194+
return LMResponse(id="ok", model=request.model, message=Message.assistant("ok"), finish_reason="stop", usage=Usage())
195+
196+
lm = UniversalLM()
197+
adapter = FlakyAdapter()
198+
lm.register(adapter)
199+
200+
m = Model(lm=lm, model="gpt-4.1-mini", provider="openai", retries=2)
201+
resp = m("hello")
202+
self.assertEqual(resp.text, "ok")
203+
self.assertEqual(adapter.calls, 3)
204+
130205

131206
if __name__ == "__main__":
132207
unittest.main()

‎tests/test_client_middleware.py‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,9 @@
77
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
88

99
from lm15.client import UniversalLM
10+
from lm15.errors import InvalidRequestError, RateLimitError
1011
from lm15.features import EndpointSupport, ProviderManifest
11-
from lm15.middleware import with_cache, with_history
12+
from lm15.middleware import with_cache, with_history, with_retries
1213
from lm15.types import LMRequest, LMResponse, Message, Part, Usage
1314

1415

@@ -53,6 +54,34 @@ def test_history_and_cache(self):
5354
self.assertEqual(adapter.calls, 1)
5455
self.assertEqual(len(hist), 1)
5556

57+
def test_retries_only_transient_errors(self):
58+
class ErrorAdapter(EchoAdapter):
59+
def __init__(self, error: Exception):
60+
super().__init__()
61+
self._error = error
62+
63+
def complete(self, request: LMRequest) -> LMResponse:
64+
self.calls += 1
65+
raise self._error
66+
67+
req = LMRequest(model="echo-model", messages=(Message.user("hi"),))
68+
69+
lm_non_transient = UniversalLM()
70+
a1 = ErrorAdapter(InvalidRequestError("bad input"))
71+
lm_non_transient.register(a1)
72+
lm_non_transient.middleware.add(with_retries(max_retries=2, sleep_base=0))
73+
with self.assertRaises(InvalidRequestError):
74+
lm_non_transient.complete(req, provider="echo")
75+
self.assertEqual(a1.calls, 1)
76+
77+
lm_transient = UniversalLM()
78+
a2 = ErrorAdapter(RateLimitError("slow down"))
79+
lm_transient.register(a2)
80+
lm_transient.middleware.add(with_retries(max_retries=2, sleep_base=0))
81+
with self.assertRaises(RateLimitError):
82+
lm_transient.complete(req, provider="echo")
83+
self.assertEqual(a2.calls, 3)
84+
5685

5786
if __name__ == "__main__":
5887
unittest.main()

0 commit comments

Comments
 (0)