From 39730f74711e1574c7733cbc6f0a7d5833f7b073 Mon Sep 17 00:00:00 2001 From: Ansh Rohilla Date: Sat, 17 Oct 2026 17:41:31 +0530 Subject: [PATCH] fix(anthropic): record streamed tool calls and avoid corrupting response text Streamed Anthropic tool_use responses accumulated input_json_delta chunks into the same content_blocks list as text, causing gen_ai.response.text to be polluted with raw tool JSON and gen_ai.response.tool_calls to never be set. Track content blocks by index and event type across content_block_start, content_block_delta, and content_block_stop, parse accumulated tool JSON deltas into tool input on completion, and forward structured blocks to _set_output_data. Fixes getsentry/sentry-python#8050 --- sentry_sdk/integrations/anthropic.py | 160 ++++++- .../integrations/anthropic/test_anthropic.py | 391 +++++++++++++++--- 2 files changed, 492 insertions(+), 59 deletions(-) diff --git a/sentry_sdk/integrations/anthropic.py b/sentry_sdk/integrations/anthropic.py index 4b04bc1cfc..57473b37cf 100644 --- a/sentry_sdk/integrations/anthropic.py +++ b/sentry_sdk/integrations/anthropic.py @@ -323,24 +323,139 @@ def _collect_ai_data( event: "MessageStreamEvent", model: "str | None", usage: "_RecordedUsage", - content_blocks: "list[str]", + content_blocks: "list[Any]", response_id: "str | None" = None, finish_reason: "str | None" = None, -) -> "tuple[str | None, _RecordedUsage, list[str], str | None, str | None]": +) -> "tuple[str | None, _RecordedUsage, list[Any], str | None, str | None]": """ Collect model information, token usage, and collect content blocks from the AI streaming response. """ with capture_internal_exceptions(): if hasattr(event, "type"): if event.type == "content_block_start": - pass + index = getattr(event, "index", len(content_blocks)) + if index is None: + index = len(content_blocks) + while len(content_blocks) <= index: + content_blocks.append({}) + + block: dict[str, Any] = {} + if hasattr(event, "content_block"): + cb = event.content_block + block_type = getattr(cb, "type", None) + if isinstance(cb, dict): + block_type = cb.get("type", block_type) + + if block_type == "tool_use": + block = { + "type": "tool_use", + "id": ( + getattr(cb, "id", None) + if not isinstance(cb, dict) + else cb.get("id") + ), + "name": ( + getattr(cb, "name", None) + if not isinstance(cb, dict) + else cb.get("name") + ), + "input": ( + ( + getattr(cb, "input", {}) + if not isinstance(cb, dict) + else cb.get("input", {}) + ) + or {} + ), + "_partial_json": "", + } + elif block_type == "text": + block = { + "type": "text", + "text": ( + getattr(cb, "text", "") + if not isinstance(cb, dict) + else cb.get("text", "") + ), + } + elif hasattr(cb, "to_dict"): + block = cb.to_dict() + elif hasattr(cb, "model_dump"): + block = cb.model_dump() + elif isinstance(cb, dict): + block = dict(cb) + else: + block = {"type": getattr(cb, "type", "text")} + if hasattr(cb, "id"): + block["id"] = cb.id + if hasattr(cb, "name"): + block["name"] = cb.name + if hasattr(cb, "input"): + block["input"] = cb.input + if hasattr(cb, "text"): + block["text"] = cb.text + + if block.get("type") == "tool_use": + block.setdefault("input", {}) + block["_partial_json"] = "" + elif block.get("type") == "text": + block.setdefault("text", "") + + content_blocks[index] = block + elif event.type == "content_block_delta": - if hasattr(event.delta, "text"): - content_blocks.append(event.delta.text) - elif hasattr(event.delta, "partial_json"): - content_blocks.append(event.delta.partial_json) + index = getattr(event, "index", 0) + if index is None: + index = 0 + while len(content_blocks) <= index: + content_blocks.append({}) + + block = content_blocks[index] + if isinstance(block, str): + block = {"type": "text", "text": block} + content_blocks[index] = block + elif not isinstance(block, dict): + block = {} + content_blocks[index] = block + + if hasattr(event.delta, "partial_json"): + if not block: + block.update( + { + "type": "tool_use", + "id": None, + "name": None, + "input": {}, + "_partial_json": "", + } + ) + elif "_partial_json" not in block: + block["_partial_json"] = "" + block["_partial_json"] += event.delta.partial_json + + elif hasattr(event.delta, "text"): + if not block: + block.update( + { + "type": "text", + "text": "", + } + ) + elif "text" not in block: + block["text"] = "" + block["text"] += event.delta.text + elif event.type == "content_block_stop": - pass + index = getattr(event, "index", None) + if index is not None and 0 <= index < len(content_blocks): + block = content_blocks[index] + if isinstance(block, dict) and "_partial_json" in block: + raw_json = block.pop("_partial_json") + if raw_json: + try: + block["input"] = json.loads(raw_json) + except Exception: + block["input"] = raw_json # Token counting logic mirrors anthropic SDK, which also extracts already accumulated tokens. # https://github.com/anthropics/anthropic-sdk-python/blob/9c485f6966e10ae0ea9eabb3a921d2ea8145a25b/src/anthropic/lib/streaming/_messages.py#L433-L518 @@ -979,7 +1094,7 @@ def _set_streaming_output_data( integration: "AnthropicIntegration", model: "Optional[str]", usage: "_RecordedUsage", - content_blocks: "list[str]", + content_blocks: "list[Any]", response_id: "Optional[str]", finish_reason: "Optional[str]", ) -> None: @@ -994,6 +1109,31 @@ def _set_streaming_output_data( + (usage.cache_write_input_tokens or 0) ) + normalized_blocks: list[dict[str, Any]] = [] + text_chunks: list[str] = [] + + def flush_text_chunks() -> None: + if text_chunks: + normalized_blocks.append({"type": "text", "text": "".join(text_chunks)}) + text_chunks.clear() + + for block in content_blocks: + if isinstance(block, dict) and "type" in block: + flush_text_chunks() + block_copy = dict(block) + if "_partial_json" in block_copy: + raw_json = block_copy.pop("_partial_json") + if raw_json: + try: + block_copy["input"] = json.loads(raw_json) + except Exception: + block_copy["input"] = raw_json + normalized_blocks.append(block_copy) + elif isinstance(block, str): + text_chunks.append(block) + + flush_text_chunks() + _set_output_data( span=span, integration=integration, @@ -1002,7 +1142,7 @@ def _set_streaming_output_data( output_tokens=usage.output_tokens, cache_read_input_tokens=usage.cache_read_input_tokens, cache_write_input_tokens=usage.cache_write_input_tokens, - content_blocks=[{"text": "".join(content_blocks), "type": "text"}], + content_blocks=normalized_blocks, response_id=response_id, finish_reason=finish_reason, ) diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index b3835c4b72..210b1e32b1 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -4657,13 +4657,21 @@ def test_streaming_create_message_with_input_json_delta( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - == '{"location": "San Francisco, CA"}' - ) + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert json.loads( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS] + ) == [ + { + "id": "toolu_0", + "input": {"location": "San Francisco, CA"}, + "name": "get_weather", + "type": "tool_use", + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -4704,13 +4712,21 @@ def test_streaming_create_message_with_input_json_delta( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - == '{"location": "San Francisco, CA"}' - ) + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert json.loads( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS] + ) == [ + { + "id": "toolu_0", + "input": {"location": "San Francisco, CA"}, + "name": "get_weather", + "type": "tool_use", + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -4752,13 +4768,19 @@ def test_streaming_create_message_with_input_json_delta( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] - == '{"location": "San Francisco, CA"}' - ) + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert json.loads(span["data"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS]) == [ + { + "id": "toolu_0", + "input": {"location": "San Francisco, CA"}, + "name": "get_weather", + "type": "tool_use", + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -4910,13 +4932,21 @@ def test_stream_messages_with_input_json_delta( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - == '{"location": "San Francisco, CA"}' - ) + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert json.loads( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS] + ) == [ + { + "id": "toolu_0", + "input": {"location": "San Francisco, CA"}, + "name": "get_weather", + "type": "tool_use", + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -4957,13 +4987,21 @@ def test_stream_messages_with_input_json_delta( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - == '{"location": "San Francisco, CA"}' - ) + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert json.loads( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS] + ) == [ + { + "id": "toolu_0", + "input": {"location": "San Francisco, CA"}, + "name": "get_weather", + "type": "tool_use", + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -5004,13 +5042,19 @@ def test_stream_messages_with_input_json_delta( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] - == '{"location": "San Francisco, CA"}' - ) + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert json.loads(span["data"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS]) == [ + { + "id": "toolu_0", + "input": {"location": "San Francisco, CA"}, + "name": "get_weather", + "type": "tool_use", + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -5018,6 +5062,146 @@ def test_stream_messages_with_input_json_delta( assert span["data"][SPANDATA.GEN_AI_RESPONSE_STREAMING] is True +@pytest.mark.skipif( + ANTHROPIC_VERSION < (0, 27), + reason="Versions <0.27.0 do not include InputJSONDelta.", +) +@pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) +def test_stream_messages_with_text_and_tool_call( + sentry_init, + capture_events, + capture_items, + get_model_response, + server_side_event_chunks, + stream_gen_ai_spans, +): + client = Anthropic(api_key="z") + + response = get_model_response( + server_side_event_chunks( + [ + MessageStartEvent( + message=Message( + id="msg_0", + content=[], + model="claude-3-5-sonnet-20240620", + role="assistant", + stop_reason=None, + stop_sequence=None, + type="message", + usage=Usage(input_tokens=100, output_tokens=10), + ), + type="message_start", + ), + ContentBlockStartEvent( + type="content_block_start", + index=0, + content_block=TextBlock(type="text", text=""), + ), + ContentBlockDeltaEvent( + delta=TextDelta( + text="Let me check the weather.", type="text_delta" + ), + index=0, + type="content_block_delta", + ), + ContentBlockStopEvent(type="content_block_stop", index=0), + ContentBlockStartEvent( + type="content_block_start", + index=1, + content_block=ToolUseBlock( + id="toolu_0", input={}, name="get_weather", type="tool_use" + ), + ), + ContentBlockDeltaEvent( + delta=InputJSONDelta( + partial_json='{"location": "San Francisco, CA"}', + type="input_json_delta", + ), + index=1, + type="content_block_delta", + ), + ContentBlockStopEvent(type="content_block_stop", index=1), + MessageDeltaEvent( + delta=Delta(stop_reason="tool_use", stop_sequence=None), + usage=MessageDeltaUsage(output_tokens=30), + type="message_delta", + ), + ] + ) + ) + + sentry_init( + integrations=[AnthropicIntegration(include_prompts=True)], + disabled_integrations=[StdlibIntegration], + traces_sample_rate=1.0, + send_default_pii=True, + stream_gen_ai_spans=stream_gen_ai_spans, + ) + + messages = [{"role": "user", "content": "What is the weather?"}] + + if stream_gen_ai_spans: + items = capture_items("transaction", "span") + + with mock.patch.object( + client._client, + "send", + return_value=response, + ) as _, start_transaction(name="anthropic"), client.messages.stream( + max_tokens=1024, + messages=messages, + model="model", + ) as stream: + for _ in stream: + pass + + sentry_sdk.flush() + spans = [item.payload for item in items if item.type == "span"] + assert len(spans) == 1 + (span,) = spans + assert ( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] + == "Let me check the weather." + ) + assert json.loads(span["attributes"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS]) == [ + { + "id": "toolu_0", + "input": {"location": "San Francisco, CA"}, + "name": "get_weather", + "type": "tool_use", + } + ] + else: + events = capture_events() + + with mock.patch.object( + client._client, + "send", + return_value=response, + ) as _, start_transaction(name="anthropic"), client.messages.stream( + max_tokens=1024, + messages=messages, + model="model", + ) as stream: + for _ in stream: + pass + + assert len(events) == 1 + (span,) = events[0]["spans"] + assert ( + span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] == "Let me check the weather." + ) + assert json.loads(span["data"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS]) == [ + { + "id": "toolu_0", + "input": {"location": "San Francisco, CA"}, + "name": "get_weather", + "type": "tool_use", + } + ] + + @pytest.mark.parametrize("span_streaming", [True, False]) @pytest.mark.parametrize("stream_gen_ai_spans", [True, False]) @pytest.mark.asyncio @@ -5169,14 +5353,22 @@ async def test_streaming_create_message_with_input_json_delta_async( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - == '{"location": "San Francisco, CA"}' - ) + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert json.loads( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS] + ) == [ + { + "id": "toolu_0", + "input": {"location": "San Francisco, CA"}, + "name": "get_weather", + "type": "tool_use", + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -5217,14 +5409,22 @@ async def test_streaming_create_message_with_input_json_delta_async( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - == '{"location": "San Francisco, CA"}' - ) + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert json.loads( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS] + ) == [ + { + "id": "toolu_0", + "input": {"location": "San Francisco, CA"}, + "name": "get_weather", + "type": "tool_use", + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -5265,14 +5465,20 @@ async def test_streaming_create_message_with_input_json_delta_async( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] - == '{"location": "San Francisco, CA"}' - ) + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert json.loads(span["data"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS]) == [ + { + "id": "toolu_0", + "input": {"location": "San Francisco, CA"}, + "name": "get_weather", + "type": "tool_use", + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -5432,13 +5638,21 @@ async def test_stream_message_with_input_json_delta_async( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - == '{"location": "San Francisco, CA"}' - ) + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert json.loads( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS] + ) == [ + { + "id": "toolu_0", + "input": {"location": "San Francisco, CA"}, + "name": "get_weather", + "type": "tool_use", + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -5480,13 +5694,21 @@ async def test_stream_message_with_input_json_delta_async( span["attributes"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert ( - span["attributes"][SPANDATA.GEN_AI_RESPONSE_TEXT] - == '{"location": "San Francisco, CA"}' - ) + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert json.loads( + span["attributes"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS] + ) == [ + { + "id": "toolu_0", + "input": {"location": "San Francisco, CA"}, + "name": "get_weather", + "type": "tool_use", + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["attributes"] assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["attributes"] + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in span["attributes"] assert span["attributes"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["attributes"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -5528,14 +5750,20 @@ async def test_stream_message_with_input_json_delta_async( span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES] == '[{"role": "user", "content": "What is the weather like in San Francisco?"}]' ) - assert ( - span["data"][SPANDATA.GEN_AI_RESPONSE_TEXT] - == '{"location": "San Francisco, CA"}' - ) + assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert json.loads(span["data"][SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS]) == [ + { + "id": "toolu_0", + "input": {"location": "San Francisco, CA"}, + "name": "get_weather", + "type": "tool_use", + } + ] else: assert SPANDATA.GEN_AI_REQUEST_MESSAGES not in span["data"] assert SPANDATA.GEN_AI_RESPONSE_TEXT not in span["data"] + assert SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS not in span["data"] assert span["data"][SPANDATA.GEN_AI_USAGE_INPUT_TOKENS] == 366 assert span["data"][SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS] == 41 @@ -6023,11 +6251,76 @@ def test_collect_ai_data_with_input_json_delta(span_streaming): assert model is None assert new_usage.input_tokens == usage.input_tokens assert new_usage.output_tokens == usage.output_tokens - assert new_content_blocks == ["test"] + assert new_content_blocks == [ + { + "type": "tool_use", + "id": None, + "name": None, + "input": {}, + "_partial_json": "test", + } + ] assert response_id is None assert finish_reason is None +@pytest.mark.skipif( + ANTHROPIC_VERSION < (0, 27), + reason="Versions <0.27.0 do not include InputJSONDelta.", +) +def test_collect_ai_data_tool_use_full_lifecycle(): + usage = _RecordedUsage() + content_blocks = [] + + start_event = ContentBlockStartEvent( + type="content_block_start", + index=0, + content_block=ToolUseBlock( + id="toolu_0", input={}, name="get_weather", type="tool_use" + ), + ) + _collect_ai_data(start_event, None, usage, content_blocks) + assert content_blocks == [ + { + "id": "toolu_0", + "name": "get_weather", + "type": "tool_use", + "input": {}, + "_partial_json": "", + } + ] + + delta_event = ContentBlockDeltaEvent( + type="content_block_delta", + index=0, + delta=InputJSONDelta( + partial_json='{"location": "San Francisco, CA"}', + type="input_json_delta", + ), + ) + _collect_ai_data(delta_event, None, usage, content_blocks) + assert content_blocks == [ + { + "id": "toolu_0", + "name": "get_weather", + "type": "tool_use", + "input": {}, + "_partial_json": '{"location": "San Francisco, CA"}', + } + ] + + stop_event = ContentBlockStopEvent(type="content_block_stop", index=0) + _collect_ai_data(stop_event, None, usage, content_blocks) + assert content_blocks == [ + { + "id": "toolu_0", + "name": "get_weather", + "type": "tool_use", + "input": {"location": "San Francisco, CA"}, + } + ] + + @pytest.mark.skipif( ANTHROPIC_VERSION < (0, 27), reason="Versions <0.27.0 do not include InputJSONDelta.",