diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py index 4fd49a35417..0a758e1a2bb 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -239,11 +239,18 @@ def _process_event(self, event: Any) -> None: if usage is not None: input_tokens = getattr(usage, "input_tokens", 0) or 0 output_tokens = getattr(usage, "output_tokens", 0) or 0 - cache_creation_tokens = getattr(usage, "input_tokens_details", None) # type: ignore[assignment] - cache_read_tokens = getattr(usage, "output_tokens_details", None) # type: ignore[assignment] - # Prefer direct cache fields if present - cache_creation_tokens = int(getattr(usage, "cache_creation_input_tokens", 0) or 0) - cache_read_tokens = int(getattr(usage, "cache_read_input_tokens", 0) or 0) + cache_creation_tokens = int( + getattr(usage, "cache_creation_input_tokens", 0) or 0 + ) + cache_read_tokens = int( + getattr(usage, "cache_read_input_tokens", 0) or 0 + ) + if not cache_read_tokens: + details = getattr(usage, "input_tokens_details", None) + if details is not None: + cache_read_tokens = int( + getattr(details, "cached_tokens", 0) or 0 + ) # Check if tool_use was in the output to override stop_reason if response_obj is not None: diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index 172e54de98e..b88a3245479 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -473,6 +473,17 @@ def translate_response( output_tokens=output_tokens, ) + cache_creation = int(getattr(raw_usage, "cache_creation_input_tokens", 0) or 0) + if cache_creation: + anthropic_usage["cache_creation_input_tokens"] = cache_creation + + cache_read = int(getattr(raw_usage, "cache_read_input_tokens", 0) or 0) + if not cache_read: + input_tokens_details = getattr(raw_usage, "input_tokens_details", None) + cache_read = int(getattr(input_tokens_details, "cached_tokens", 0) or 0) + if cache_read: + anthropic_usage["cache_read_input_tokens"] = cache_read + return AnthropicMessagesResponse( id=response.id, type="message", diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 6b1ca3564e3..04db27c7123 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -2030,6 +2030,51 @@ def _transform_chat_completion_usage_to_responses_usage( if output_details_dict: response_usage.output_tokens_details = OutputTokensDetails(**output_details_dict) + # Preserve Anthropic cache creation fields for downstream billing extraction. + # These are not part of the OpenAI ResponseAPIUsage schema but + # BaseLiteLLMOpenAIResponseObject has extra="allow", so setattr ensures + # they survive serialization and can be extracted by the inference proxy + # before the lossy api.Usage unmarshal. + if ( + hasattr(usage, "cache_creation_input_tokens") + and usage.cache_creation_input_tokens + ): + setattr( + response_usage, + "cache_creation_input_tokens", + usage.cache_creation_input_tokens, + ) + if hasattr(usage, "cache_read_input_tokens") and usage.cache_read_input_tokens: + setattr( + response_usage, "cache_read_input_tokens", usage.cache_read_input_tokens + ) + if ( + hasattr(usage, "prompt_tokens_details") + and usage.prompt_tokens_details is not None + ): + ptd = usage.prompt_tokens_details + if ( + hasattr(ptd, "cache_creation_token_details") + and ptd.cache_creation_token_details is not None + ): + cache_creation_dict: Dict[str, int] = {} + if ( + ptd.cache_creation_token_details.ephemeral_5m_input_tokens + is not None + ): + cache_creation_dict["ephemeral_5m_input_tokens"] = ( + ptd.cache_creation_token_details.ephemeral_5m_input_tokens + ) + if ( + ptd.cache_creation_token_details.ephemeral_1h_input_tokens + is not None + ): + cache_creation_dict["ephemeral_1h_input_tokens"] = ( + ptd.cache_creation_token_details.ephemeral_1h_input_tokens + ) + if cache_creation_dict: + setattr(response_usage, "cache_creation", cache_creation_dict) + return response_usage @staticmethod diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py index 606ff39b35e..65d3ba24342 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -961,6 +961,54 @@ def test_usage_mapped_correctly(self): assert result["usage"]["input_tokens"] == 200 assert result["usage"]["output_tokens"] == 75 + def test_should_populate_cache_creation_input_tokens(self): + """cache_creation_input_tokens extra on usage is mapped to AnthropicUsage.""" + response = _make_mock_response(output=[_make_output_message(["OK"])]) + response.usage.cache_creation_input_tokens = 1900 + response.usage.cache_read_input_tokens = 0 + response.usage.input_tokens_details = None + + result: Any = _ADAPTER.translate_response(response) + assert result["usage"]["cache_creation_input_tokens"] == 1900 + assert "cache_read_input_tokens" not in result["usage"] + + def test_should_populate_cache_read_input_tokens(self): + """cache_read_input_tokens extra on usage is mapped when present.""" + response = _make_mock_response(output=[_make_output_message(["OK"])]) + response.usage.cache_creation_input_tokens = 0 + response.usage.cache_read_input_tokens = 1900 + response.usage.input_tokens_details = None + + result: Any = _ADAPTER.translate_response(response) + assert result["usage"]["cache_read_input_tokens"] == 1900 + assert "cache_creation_input_tokens" not in result["usage"] + + def test_should_fall_back_to_input_tokens_details_cached_tokens(self): + """ + When cache_read_input_tokens is 0, fall back to + input_tokens_details.cached_tokens (OpenAI-style field). + """ + response = _make_mock_response(output=[_make_output_message(["OK"])]) + response.usage.cache_creation_input_tokens = 0 + response.usage.cache_read_input_tokens = 0 + details = MagicMock() + details.cached_tokens = 1900 + response.usage.input_tokens_details = details + + result: Any = _ADAPTER.translate_response(response) + assert result["usage"]["cache_read_input_tokens"] == 1900 + + def test_should_not_add_cache_keys_when_no_cache_activity(self): + """No cache keys on AnthropicUsage when there's no cache activity.""" + response = _make_mock_response(output=[_make_output_message(["OK"])]) + response.usage.cache_creation_input_tokens = 0 + response.usage.cache_read_input_tokens = 0 + response.usage.input_tokens_details = None + + result: Any = _ADAPTER.translate_response(response) + assert "cache_creation_input_tokens" not in result["usage"] + assert "cache_read_input_tokens" not in result["usage"] + def test_model_and_id_preserved(self): """Model and response ID from the Responses API are forwarded.""" response = _make_mock_response( diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_streaming_iterator.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_streaming_iterator.py new file mode 100644 index 00000000000..b64448f0969 --- /dev/null +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_streaming_iterator.py @@ -0,0 +1,120 @@ +""" +Tests for AnthropicResponsesStreamWrapper cache token usage mapping. + +Bug: streaming_iterator.py assigned input_tokens_details (an object) to +cache_creation_tokens, and output_tokens_details (wrong field entirely) to +cache_read_tokens. Both caused streaming responses to report zero cache tokens. +""" + +import os +import sys +from unittest.mock import MagicMock + +sys.path.insert(0, os.path.abspath("../../../../../../..")) + +from litellm.llms.anthropic.experimental_pass_through.responses_adapters.streaming_iterator import ( + AnthropicResponsesStreamWrapper, +) + + +def _make_completed_event(usage: MagicMock, status: str = "completed") -> MagicMock: + response_obj = MagicMock() + response_obj.status = status + response_obj.usage = usage + response_obj.output = [] + + event = MagicMock() + event.type = "response.completed" + event.response = response_obj + return event + + +def _get_message_delta_usage(wrapper: AnthropicResponsesStreamWrapper) -> dict: + for chunk in wrapper._chunk_queue: + if chunk.get("type") == "message_delta": + return chunk["usage"] + raise AssertionError("no message_delta chunk was queued") + + +class TestStreamingCacheTokenMapping: + """response.completed -> message_delta usage cache token mapping.""" + + def test_should_read_cache_creation_tokens_as_int_not_object(self): + """cache_creation_input_tokens must be an integer count, not input_tokens_details.""" + usage = MagicMock() + usage.input_tokens = 1936 + usage.output_tokens = 246 + usage.cache_creation_input_tokens = 1900 + usage.cache_read_input_tokens = 0 + usage.input_tokens_details = MagicMock(cached_tokens=0) + + wrapper = AnthropicResponsesStreamWrapper(responses_stream=iter([]), model="m") + wrapper._process_event(_make_completed_event(usage)) + + usage_delta = _get_message_delta_usage(wrapper) + assert usage_delta["cache_creation_input_tokens"] == 1900 + assert "cache_read_input_tokens" not in usage_delta + + def test_should_read_cache_read_tokens_from_correct_field(self): + """cache_read_input_tokens must come from cache_read_input_tokens, not output_tokens_details.""" + usage = MagicMock() + usage.input_tokens = 1936 + usage.output_tokens = 246 + usage.cache_creation_input_tokens = 0 + usage.cache_read_input_tokens = 1900 + usage.input_tokens_details = MagicMock(cached_tokens=0) + + wrapper = AnthropicResponsesStreamWrapper(responses_stream=iter([]), model="m") + wrapper._process_event(_make_completed_event(usage)) + + usage_delta = _get_message_delta_usage(wrapper) + assert usage_delta["cache_read_input_tokens"] == 1900 + assert "cache_creation_input_tokens" not in usage_delta + + def test_should_fall_back_to_input_tokens_details_cached_tokens(self): + """When cache_read_input_tokens is 0, fall back to input_tokens_details.cached_tokens.""" + usage = MagicMock() + usage.input_tokens = 1936 + usage.output_tokens = 246 + usage.cache_creation_input_tokens = 0 + usage.cache_read_input_tokens = 0 + usage.input_tokens_details = MagicMock(cached_tokens=1900) + + wrapper = AnthropicResponsesStreamWrapper(responses_stream=iter([]), model="m") + wrapper._process_event(_make_completed_event(usage)) + + usage_delta = _get_message_delta_usage(wrapper) + assert usage_delta["cache_read_input_tokens"] == 1900 + + def test_should_not_add_cache_keys_when_no_cache_activity(self): + """No cache_* keys when there's no cache activity.""" + usage = MagicMock() + usage.input_tokens = 100 + usage.output_tokens = 50 + usage.cache_creation_input_tokens = 0 + usage.cache_read_input_tokens = 0 + usage.input_tokens_details = MagicMock(cached_tokens=0) + + wrapper = AnthropicResponsesStreamWrapper(responses_stream=iter([]), model="m") + wrapper._process_event(_make_completed_event(usage)) + + usage_delta = _get_message_delta_usage(wrapper) + assert "cache_creation_input_tokens" not in usage_delta + assert "cache_read_input_tokens" not in usage_delta + assert usage_delta["input_tokens"] == 100 + assert usage_delta["output_tokens"] == 50 + + def test_should_handle_input_tokens_details_none(self): + """input_tokens_details=None should not raise when falling back.""" + usage = MagicMock() + usage.input_tokens = 100 + usage.output_tokens = 50 + usage.cache_creation_input_tokens = 0 + usage.cache_read_input_tokens = 0 + usage.input_tokens_details = None + + wrapper = AnthropicResponsesStreamWrapper(responses_stream=iter([]), model="m") + wrapper._process_event(_make_completed_event(usage)) + + usage_delta = _get_message_delta_usage(wrapper) + assert "cache_read_input_tokens" not in usage_delta diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py index d8e3f495ced..dd2d052f679 100644 --- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -16,6 +16,7 @@ ChatCompletionToolMessage, ) from litellm.types.utils import ( + CacheCreationTokenDetails, ChatCompletionMessageToolCall, Choices, CompletionTokensDetailsWrapper, @@ -1016,6 +1017,27 @@ def test_none_text_blocks_filtered_out(self): assert result[0]["text"] == "valid text" assert result[1]["text"] == "another valid" + def test_cache_control_preserved_on_content_block(self): + """ + Test that a `cache_control` field on an input content block survives + the Responses API -> Chat Completion content conversion, so the + downstream Anthropic adapter can seed its prompt cache. + """ + content = [ + { + "type": "input_text", + "text": "", + "cache_control": {"type": "ephemeral"}, + }, + {"type": "input_text", "text": "no cache control here"}, + ] + result = LiteLLMCompletionResponsesConfig._transform_responses_api_content_to_chat_completion_content( + content + ) + assert len(result) == 2 + assert result[0]["cache_control"] == {"type": "ephemeral"} + assert "cache_control" not in result[1] + class TestToolTransformation: """Test cases for tool transformation from Responses API to Chat Completion format""" @@ -2063,6 +2085,113 @@ def test_reasoning_tokens_explicit_zero_preserved(self): assert response_usage.output_tokens_details is not None assert response_usage.output_tokens_details.reasoning_tokens == 0 + def test_transform_usage_preserves_anthropic_cache_creation_fields(self): + """ + Test that Anthropic cache creation fields (cache_creation_input_tokens, + cache_read_input_tokens, cache_creation.ephemeral_5m/1h_input_tokens) survive + the Usage -> ResponseAPIUsage conversion as extras, so the inference proxy + can bill cache creation on /v1/responses. + """ + usage = Usage( + prompt_tokens=1936, + completion_tokens=246, + total_tokens=2182, + cache_creation_input_tokens=1900, + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=0, + cache_creation_token_details=CacheCreationTokenDetails( + ephemeral_5m_input_tokens=1900, + ephemeral_1h_input_tokens=0, + ), + ), + ) + + chat_completion_response = ModelResponse( + id="test-response-id", + created=1234567890, + model="anthropic-claude-4.6-sonnet", + object="chat.completion", + usage=usage, + choices=[ + Choices( + finish_reason="stop", + index=0, + message=Message(content="Key terms are...", role="assistant"), + ) + ], + ) + + response_usage = LiteLLMCompletionResponsesConfig._transform_chat_completion_usage_to_responses_usage( + chat_completion_response=chat_completion_response + ) + + assert getattr(response_usage, "cache_creation_input_tokens", None) == 1900 + assert getattr(response_usage, "cache_creation", None) == { + "ephemeral_5m_input_tokens": 1900, + "ephemeral_1h_input_tokens": 0, + } + + def test_transform_usage_preserves_anthropic_cache_read_tokens(self): + """A cache-hit turn should surface cache_read_input_tokens as an extra.""" + usage = Usage( + prompt_tokens=1936, + completion_tokens=246, + total_tokens=2182, + cache_read_input_tokens=1900, + ) + + chat_completion_response = ModelResponse( + id="test-response-id", + created=1234567890, + model="anthropic-claude-4.6-sonnet", + object="chat.completion", + usage=usage, + choices=[ + Choices( + finish_reason="stop", + index=0, + message=Message(content="Key terms are...", role="assistant"), + ) + ], + ) + + response_usage = LiteLLMCompletionResponsesConfig._transform_chat_completion_usage_to_responses_usage( + chat_completion_response=chat_completion_response + ) + + assert getattr(response_usage, "cache_read_input_tokens", None) == 1900 + + def test_transform_usage_no_cache_fields_when_absent(self): + """No cache extras should be set when the model didn't report any caching.""" + usage = Usage( + prompt_tokens=9, + completion_tokens=27, + total_tokens=36, + ) + + chat_completion_response = ModelResponse( + id="test-response-id", + created=1234567890, + model="gpt-4o", + object="chat.completion", + usage=usage, + choices=[ + Choices( + finish_reason="stop", + index=0, + message=Message(content="Hello!", role="assistant"), + ) + ], + ) + + response_usage = LiteLLMCompletionResponsesConfig._transform_chat_completion_usage_to_responses_usage( + chat_completion_response=chat_completion_response + ) + + assert not hasattr(response_usage, "cache_creation_input_tokens") + assert not hasattr(response_usage, "cache_read_input_tokens") + assert not hasattr(response_usage, "cache_creation") + class TestStreamingIDConsistency: """Test cases for consistent IDs across streaming events (issue #14962)"""