From bcd9741ad738784f1a90e1bca87dc709713cb0b3 Mon Sep 17 00:00:00 2001 From: Clay <5866903+jcd1991@users.noreply.github.com> Date: Sat, 21 Nov 2026 19:25:42 -0500 Subject: [PATCH] fix(langchain): prefer response model metadata --- langfuse/langchain/CallbackHandler.py | 15 +++++++++++ tests/unit/test_langchain.py | 38 +++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/langfuse/langchain/CallbackHandler.py b/langfuse/langchain/CallbackHandler.py index dac76f4fc..8ca5997c6 100644 --- a/langfuse/langchain/CallbackHandler.py +++ b/langfuse/langchain/CallbackHandler.py @@ -1829,6 +1829,21 @@ def _parse_model(response: LLMResult) -> Any: llm_model = response.llm_output[key] break + for generation in reversed(response.generations): + for generation_chunk in reversed(generation): + generation_info = getattr(generation_chunk, "generation_info", None) + if isinstance(generation_info, dict): + for key in llm_model_keys: + if generation_info.get(key): + return generation_info[key] + + message = getattr(generation_chunk, "message", None) + response_metadata = getattr(message, "response_metadata", None) + if isinstance(response_metadata, dict): + for key in llm_model_keys: + if response_metadata.get(key): + return response_metadata[key] + return llm_model diff --git a/tests/unit/test_langchain.py b/tests/unit/test_langchain.py index c55eaa3d2..f6a8e7e22 100644 --- a/tests/unit/test_langchain.py +++ b/tests/unit/test_langchain.py @@ -172,6 +172,44 @@ def test_llm_callback_exports_generation_span(langfuse_memory_client, get_span): ) +@pytest.mark.parametrize("model_source", ["generation_info", "response_metadata"]) +def test_chat_model_callback_uses_response_model_name( + langfuse_memory_client, get_span, model_source +): + generation_info = ( + {"model_name": "gpt-4o-2024-08-06"} if model_source == "generation_info" else {} + ) + response_metadata = ( + {"model_name": "gpt-4o-2024-08-06"} + if model_source == "response_metadata" + else {} + ) + response = ChatResult( + generations=[ + ChatGeneration( + message=AIMessage( + content="bonjour", response_metadata=response_metadata + ), + generation_info=generation_info, + ) + ], + llm_output={"model_name": "azure.gpt-4o"}, + ) + + with patch.object(ChatOpenAI, "_generate", return_value=response): + ChatOpenAI(api_key="test", model="azure.gpt-4o").invoke( + "hello", config={"callbacks": [CallbackHandler()]} + ) + + langfuse_memory_client.flush() + span = get_span("ChatOpenAI") + + assert ( + span.attributes[LangfuseOtelSpanAttributes.OBSERVATION_MODEL] + == "gpt-4o-2024-08-06" + ) + + def test_lcel_chain_exports_intermediate_chain_spans( langfuse_memory_client, get_span, find_spans ):