From 6a16726b466f8db0c7a7c25694ad38dce261ddc2 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Thu, 29 Oct 2026 12:32:04 -0400 Subject: [PATCH 1/5] feat(mcp): add mcp server instrumentation for mcp 1.x and 2.x MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit > 🤖 **This text was AI-generated.** ## Problem MCP servers built with the Python SDK produced no Braintrust spans. A tool call that ran an LLM or other traced work showed up as disconnected root spans with no record of the MCP request around them. ``` (before) anthropic.messages.create <- orphan root, no MCP context ``` ## Solution Add `MCPIntegration` and `wrap_mcp_server(server, capture_content=True)`, following the Braintrust MCP server spec. Each incoming request or notification gets one span covering receipt, validation, the handler, and sending the response. Nested work becomes its child: ``` (after) tools/call ask_llm type=tool └─ anthropic.messages.create type=llm ``` Automatic setup traces every MCP server in the process: ```python from braintrust.integrations.mcp import MCPIntegration MCPIntegration.setup() ``` Or wrap a single server manually: ```python from mcp.server.fastmcp import FastMCP # or MCPServer, or fastmcp.FastMCP from braintrust.integrations.mcp import wrap_mcp_server server = wrap_mcp_server(FastMCP("demo")) ``` Supported: `mcp>=1.28.1,<3` (low-level `Server`, bundled `FastMCP` on 1.x, `MCPServer` on 2.x) and standalone `fastmcp` 2.x–4.x, including `fastmcp-slim`. Spans include: - names like `tools/call `, `prompts/get `, `resources/read` - arguments as `input` and the MCP result as `output` (content capture is on by default; `capture_content=False` keeps only names, timing, metadata, and error classification) - spec metadata: `mcp.method.name`, `jsonrpc.request.id`, `mcp.session.id`, protocol version, client/server identity, and HTTP network fields - `error.type = "tool_error"` for `isError` results; JSON-RPC caller errors record `rpc.response.status_code` without failing the span - `mcp.cancelled` for cancelled requests, and an error for requests left pending when the connection closes - inline images, audio, and resource blobs as attachments - outgoing notifications as children of the request that sent them - parentage from `traceparent`/`baggage` in `params._meta`; the ambient span never becomes the parent MCP 1.x and 2.x share all span shaping in `tracing.py`. Only the lifecycle hooks differ: ``` 1.x ServerSession streams ──► read: start span (before validation) write: record result, end after send Server._handle_message ─► span is current during the handler 2.x ServerRunner._on_request/_on_notify ──► start span, run handler JSONRPCDispatcher._handle_request ────► end after the response write ``` Payloads are logged as the dicts MCP already produces. They are copied only where `_meta` must be dropped or media replaced, and all logging happens once, after the response is sent. Tests use real MCP and fastmcp clients and servers in-process, over streamable HTTP, and over stdio to a real subprocess. Cassettes record only the nested Anthropic call. New nox sessions: `test_mcp`, `test_fastmcp`, `test_fastmcp_slim`. ## Limitations and next steps - `network.transport: "pipe"` is not set for stdio. Neither MCP version exposes the transport type to the server session. - On 2.x single-exchange HTTP and in-process direct dispatch, spans end when the handler returns rather than after the HTTP write. - OTel `mcp.server.operation.duration` metrics are not emitted. - Client instrumentation and connected traces (PR 2) and `auto_instrument()` wiring plus docs (PR 3) follow in separate changes. Refs #1023 Co-Authored-By: Claude Opus 5.5 --- py/noxfile.py | 29 + py/pyproject.toml | 23 + py/src/braintrust/integrations/__init__.py | 3 + .../braintrust/integrations/mcp/__init__.py | 62 ++ .../braintrust/integrations/mcp/_server_v1.py | 278 +++++++++ .../braintrust/integrations/mcp/_server_v2.py | 197 +++++++ .../integrations/mcp/_test_harness.py | 293 ++++++++++ .../mcp_test_http_tool_with_model_call.yaml | 109 ++++ .../1.28.1/mcp_test_stdio_subprocess.yaml | 109 ++++ .../mcp_test_http_tool_with_model_call.yaml | 109 ++++ .../1.30.0/mcp_test_stdio_subprocess.yaml | 109 ++++ .../mcp_test_http_tool_with_model_call.yaml | 109 ++++ .../2.0.0/mcp_test_stdio_subprocess.yaml | 109 ++++ ...astmcp_test_http_tool_with_model_call.yaml | 109 ++++ .../2.14.7/fastmcp_test_stdio_subprocess.yaml | 109 ++++ ...astmcp_test_http_tool_with_model_call.yaml | 109 ++++ .../3.0.0/fastmcp_test_stdio_subprocess.yaml | 109 ++++ ...astmcp_test_http_tool_with_model_call.yaml | 109 ++++ .../3.4.11/fastmcp_test_stdio_subprocess.yaml | 109 ++++ ...astmcp_test_http_tool_with_model_call.yaml | 109 ++++ .../4.0.0/fastmcp_test_stdio_subprocess.yaml | 109 ++++ ...astmcp_test_http_tool_with_model_call.yaml | 109 ++++ .../latest/fastmcp_test_stdio_subprocess.yaml | 109 ++++ .../mcp_test_http_tool_with_model_call.yaml | 109 ++++ .../latest/mcp_test_stdio_subprocess.yaml | 109 ++++ .../integrations/mcp/integration.py | 38 ++ .../braintrust/integrations/mcp/patchers.py | 106 ++++ .../braintrust/integrations/mcp/test_mcp.py | 459 +++++++++++++++ py/src/braintrust/integrations/mcp/tracing.py | 530 ++++++++++++++++++ 29 files changed, 3980 insertions(+) create mode 100644 py/src/braintrust/integrations/mcp/__init__.py create mode 100644 py/src/braintrust/integrations/mcp/_server_v1.py create mode 100644 py/src/braintrust/integrations/mcp/_server_v2.py create mode 100644 py/src/braintrust/integrations/mcp/_test_harness.py create mode 100644 py/src/braintrust/integrations/mcp/cassettes/1.28.1/mcp_test_http_tool_with_model_call.yaml create mode 100644 py/src/braintrust/integrations/mcp/cassettes/1.28.1/mcp_test_stdio_subprocess.yaml create mode 100644 py/src/braintrust/integrations/mcp/cassettes/1.30.0/mcp_test_http_tool_with_model_call.yaml create mode 100644 py/src/braintrust/integrations/mcp/cassettes/1.30.0/mcp_test_stdio_subprocess.yaml create mode 100644 py/src/braintrust/integrations/mcp/cassettes/2.0.0/mcp_test_http_tool_with_model_call.yaml create mode 100644 py/src/braintrust/integrations/mcp/cassettes/2.0.0/mcp_test_stdio_subprocess.yaml create mode 100644 py/src/braintrust/integrations/mcp/cassettes/2.14.7/fastmcp_test_http_tool_with_model_call.yaml create mode 100644 py/src/braintrust/integrations/mcp/cassettes/2.14.7/fastmcp_test_stdio_subprocess.yaml create mode 100644 py/src/braintrust/integrations/mcp/cassettes/3.0.0/fastmcp_test_http_tool_with_model_call.yaml create mode 100644 py/src/braintrust/integrations/mcp/cassettes/3.0.0/fastmcp_test_stdio_subprocess.yaml create mode 100644 py/src/braintrust/integrations/mcp/cassettes/3.4.11/fastmcp_test_http_tool_with_model_call.yaml create mode 100644 py/src/braintrust/integrations/mcp/cassettes/3.4.11/fastmcp_test_stdio_subprocess.yaml create mode 100644 py/src/braintrust/integrations/mcp/cassettes/4.0.0/fastmcp_test_http_tool_with_model_call.yaml create mode 100644 py/src/braintrust/integrations/mcp/cassettes/4.0.0/fastmcp_test_stdio_subprocess.yaml create mode 100644 py/src/braintrust/integrations/mcp/cassettes/latest/fastmcp_test_http_tool_with_model_call.yaml create mode 100644 py/src/braintrust/integrations/mcp/cassettes/latest/fastmcp_test_stdio_subprocess.yaml create mode 100644 py/src/braintrust/integrations/mcp/cassettes/latest/mcp_test_http_tool_with_model_call.yaml create mode 100644 py/src/braintrust/integrations/mcp/cassettes/latest/mcp_test_stdio_subprocess.yaml create mode 100644 py/src/braintrust/integrations/mcp/integration.py create mode 100644 py/src/braintrust/integrations/mcp/patchers.py create mode 100644 py/src/braintrust/integrations/mcp/test_mcp.py create mode 100644 py/src/braintrust/integrations/mcp/tracing.py diff --git a/py/noxfile.py b/py/noxfile.py index 1b2a322e9..1be445e26 100644 --- a/py/noxfile.py +++ b/py/noxfile.py @@ -274,6 +274,10 @@ def _pinned_python_version(): OPENAI_VERSIONS = _get_matrix_versions("openai") OPENAI_ENDPOINT_VERSIONS = (OPENAI_VERSIONS[0], OPENAI_VERSIONS[-1]) +MCP_VERSIONS = _get_matrix_versions("mcp") +FASTMCP_VERSIONS = _get_matrix_versions("fastmcp") +FASTMCP_SLIM_VERSIONS = _get_matrix_versions("fastmcp-slim") + def _register_matrix_sessions(specs): """Register the routine provider sessions from compact data records.""" @@ -331,6 +335,31 @@ def run(session, version, spec=spec): "dependencies": [("instructor", "$version")], "tests": f"{INTEGRATION_DIR}/instructor/test_instructor.py", }, + { + "name": "test_mcp", + "doc": "MCP server instrumentation on the bundled FastMCP (1.x) / MCPServer (2.x).", + "versions": MCP_VERSIONS, + # Tools call Anthropic to cover nested model spans. + "dependencies": [("mcp", "$version"), ("anthropic", LATEST)], + "tests": f"{INTEGRATION_DIR}/mcp/test_mcp.py", + "env": {"BRAINTRUST_MCP_TEST_FLAVOR": "mcp"}, + }, + { + "name": "test_fastmcp", + "doc": "MCP server instrumentation on standalone FastMCP.", + "versions": FASTMCP_VERSIONS, + "dependencies": [("fastmcp", "$version"), ("anthropic", LATEST)], + "tests": f"{INTEGRATION_DIR}/mcp/test_mcp.py", + "env": {"BRAINTRUST_MCP_TEST_FLAVOR": "fastmcp"}, + }, + { + "name": "test_fastmcp_slim", + "doc": "MCP server instrumentation on fastmcp-slim without the fastmcp meta-package.", + "versions": FASTMCP_SLIM_VERSIONS, + "dependencies": [("fastmcp-slim", "$version"), ("anthropic", LATEST)], + "tests": f"{INTEGRATION_DIR}/mcp/test_mcp.py", + "env": {"BRAINTRUST_MCP_TEST_FLAVOR": "fastmcp"}, + }, { "name": "test_openai", "versions": OPENAI_VERSIONS, diff --git a/py/pyproject.toml b/py/pyproject.toml index bb8352889..6cf4c2439 100644 --- a/py/pyproject.toml +++ b/py/pyproject.toml @@ -490,6 +490,26 @@ latest = "livekit-agents==1.8.9" latest = "pipecat-ai==1.14.0" "1.3.0" = "pipecat-ai==1.3.0" +[tool.braintrust.matrix.mcp] +latest = "mcp==2.3.0" +"2.0.0" = "mcp==2.0.0" +"1.30.0" = "mcp==1.30.0" +"1.28.1" = "mcp==1.28.1" + +# Standalone FastMCP. 2.x and 3.x run on mcp 1.x; 4.x runs on mcp 2.x. Since +# 3.4 the implementation ships in fastmcp-slim and fastmcp only pins it. +[tool.braintrust.matrix.fastmcp] +latest = "fastmcp==4.1.3" +"4.0.0" = "fastmcp==4.0.0" +"3.4.11" = "fastmcp==3.4.11" +"3.0.0" = "fastmcp==3.0.0" +"2.14.7" = "fastmcp==2.14.7" + +# Same versions as fastmcp, so the sessions share cassette directories. +[tool.braintrust.matrix.fastmcp-slim] +latest = "fastmcp-slim[client,server]==4.1.3" +"3.4.11" = "fastmcp-slim[client,server]==3.4.11" + [tool.braintrust.matrix.claude-agent-sdk] latest = "claude-agent-sdk==0.2.174" # The integration still supports >=0.1.10, but 0.1.9-0.1.17 were deleted from @@ -680,6 +700,7 @@ langchain = ["langchain-core", "deepagents"] litellm = ["litellm"] livekit_agents = ["livekit-agents"] llamaindex = ["llama-index-core"] +mcp = ["mcp", "fastmcp", "fastmcp-slim"] mistral = ["mistralai"] openai = ["openai"] openai_agents = ["openai-agents"] @@ -713,6 +734,8 @@ instructor = "instructor" litellm = "litellm" livekit-agents = "livekit.agents" mistralai = "mistralai" +mcp = "mcp" +fastmcp = "fastmcp" huggingface-hub = "huggingface_hub" openai = "openai" openai-agents = "agents" diff --git a/py/src/braintrust/integrations/__init__.py b/py/src/braintrust/integrations/__init__.py index 38acb690f..c9e81b75a 100644 --- a/py/src/braintrust/integrations/__init__.py +++ b/py/src/braintrust/integrations/__init__.py @@ -25,6 +25,7 @@ "LiteLLMIntegration": "litellm", "LiveKitAgentsIntegration": "livekit_agents", "LlamaIndexIntegration": "llamaindex", + "MCPIntegration": "mcp", "MistralIntegration": "mistral", "OpenAIIntegration": "openai", "OpenAIAgentsIntegration": "openai_agents", @@ -59,6 +60,7 @@ from .litellm import LiteLLMIntegration as LiteLLMIntegration from .livekit_agents import LiveKitAgentsIntegration as LiveKitAgentsIntegration from .llamaindex import LlamaIndexIntegration as LlamaIndexIntegration + from .mcp import MCPIntegration as MCPIntegration from .mistral import MistralIntegration as MistralIntegration from .openai import OpenAIIntegration as OpenAIIntegration from .openai_agents import OpenAIAgentsIntegration as OpenAIAgentsIntegration @@ -92,6 +94,7 @@ "LiveKitAgentsIntegration", "LangChainIntegration", "LlamaIndexIntegration", + "MCPIntegration", "MistralIntegration", "OpenAIIntegration", "OpenAIAgentsIntegration", diff --git a/py/src/braintrust/integrations/mcp/__init__.py b/py/src/braintrust/integrations/mcp/__init__.py new file mode 100644 index 000000000..01132bcff --- /dev/null +++ b/py/src/braintrust/integrations/mcp/__init__.py @@ -0,0 +1,62 @@ +"""Braintrust instrumentation for Model Context Protocol (MCP) servers.""" + +import logging +from typing import Any, TypeVar + +from braintrust.integrations.versioning import version_satisfies + +from .integration import MCPIntegration +from .patchers import ServerRunPatcher +from .tracing import SERVER_CONFIG_ATTR, MCPServerConfig + + +logger = logging.getLogger(__name__) + +ServerT = TypeVar("ServerT") + + +def _resolve_lowlevel_server(server: Any) -> Any | None: + try: + from mcp.server.lowlevel.server import Server + except ImportError: + return None + if isinstance(server, Server): + return server + # Bundled FastMCP (1.x), MCPServer (2.x), and standalone fastmcp.FastMCP + # all delegate protocol handling to a low-level server. + for attr in ("_lowlevel_server", "_mcp_server"): + candidate = getattr(server, attr, None) + if isinstance(candidate, Server): + return candidate + return None + + +def wrap_mcp_server(server: ServerT, capture_content: bool = True) -> ServerT: + """Trace incoming MCP requests to *server*, from receipt through response sending. + + Accepts a low-level ``mcp.server.lowlevel.Server``, the bundled + ``FastMCP`` (MCP 1.x) or ``MCPServer`` (MCP 2.x), or a standalone + ``fastmcp.FastMCP`` server, and returns the same object. Calling it again + updates the settings without adding duplicate spans. + + :param server: The MCP server to trace. + :param capture_content: Log tool and prompt arguments, results, resource + contents, and notification parameters (default). When False, spans keep + names, timing, allowlisted metadata, and error classification only. + """ + lowlevel = _resolve_lowlevel_server(server) + if lowlevel is None: + logger.warning("Unsupported MCP server %r. Not wrapping.", type(server).__name__) + return server + if not MCPIntegration.install_patchers(): + logger.warning("Unsupported mcp version. Not wrapping.") + return server + setattr(lowlevel, SERVER_CONFIG_ATTR, MCPServerConfig(capture_content=capture_content)) + import mcp + + if version_satisfies(MCPIntegration.detect_version(mcp), ServerRunPatcher.version_spec): + ServerRunPatcher.wrap_target(lowlevel) + return server + + +__all__ = ["MCPIntegration", "wrap_mcp_server"] diff --git a/py/src/braintrust/integrations/mcp/_server_v1.py b/py/src/braintrust/integrations/mcp/_server_v1.py new file mode 100644 index 000000000..82634a41b --- /dev/null +++ b/py/src/braintrust/integrations/mcp/_server_v1.py @@ -0,0 +1,278 @@ +"""Server request lifecycle hooks for MCP 1.x. + +MCP 1.x validates requests inside ``BaseSession._receive_loop`` and writes +validation errors straight to the session's write stream, so handler-level +hooks cannot see receipt, validation failures, or response sending. Instead, +each traced ``ServerSession`` gets thin observers around its read stream, +write stream, and incoming-message queue: + +- the read observer starts an operation when the transport delivers a parsed + request or notification, before validation; +- the incoming-queue observer links a validated notification to its operation; +- ``Server._handle_message`` makes the operation current while handlers run; +- the write observer records the result or error and ends the operation once + the transport accepted the response. + +The observers only read messages; they never change what MCP sends or receives. +""" + +import contextvars +import functools +import weakref +from typing import Any + +from .tracing import ( + SERVER_CONFIG_ATTR, + ConnectionClosedError, + MCPServerConfig, + ServerOperation, + activate, + call_arg, + contain, + end_outgoing_notification, + global_server_config, + http_session_id, + http_transport_metadata, + related_operation, + run_operation, + start_outgoing_notification, + start_server_operation, +) + + +# Set while a wrapped ``Server.run`` constructs its session. +_pending_server_config: contextvars.ContextVar[MCPServerConfig | None] = contextvars.ContextVar( + "braintrust_mcp_pending_server_config", default=None +) + +_session_traces: "weakref.WeakKeyDictionary[Any, _SessionTrace]" = weakref.WeakKeyDictionary() + + +@functools.cache +def _mcp() -> Any: + """MCP 1.x types, imported on first use so this module loads without mcp.""" + from mcp import types + from mcp.shared.session import RequestResponder # pylint: disable=import-error,no-name-in-module # MCP 1.x + + return types, RequestResponder + + +class _SessionTrace: + """Pending operations for one server session (one MCP connection).""" + + def __init__(self, session: Any, config: MCPServerConfig): + self._session_ref = weakref.ref(session) + self.config = config + self.pending: dict[Any, ServerOperation] = {} + self.forwarded_notifications: dict[int, ServerOperation] = {} + self.inbound_notification: ServerOperation | None = None + self.protocol_version: str | None = None + + def _session_attr(self, name: str) -> Any: + session = self._session_ref() + return getattr(session, name, None) if session is not None else None + + def settle_inbound(self) -> None: + """End a notification the receive loop handled without forwarding it.""" + operation, self.inbound_notification = self.inbound_notification, None + if operation is not None: + operation.finish() + + def on_receive(self, item: Any) -> None: + types, _ = _mcp() + message = getattr(getattr(item, "message", None), "root", None) + is_request = isinstance(message, types.JSONRPCRequest) + if not is_request and not isinstance(message, types.JSONRPCNotification): + # Responses to server-initiated requests are not operations. + return + request_context = getattr(item.metadata, "request_context", None) + params = message.params + operation = start_server_operation( + method=message.method, + params=params, + request_id=message.id if is_request else None, + config=self.config, + session_id=http_session_id(request_context), + protocol_version=self.protocol_version, + client_info=getattr(self._session_attr("_client_params"), "clientInfo", None), + transport_metadata=http_transport_metadata(request_context), + jsonrpc_version=message.jsonrpc, + ) + if operation is None: + return + if is_request: + self.pending[message.id] = operation + return + if message.method == "notifications/cancelled" and isinstance(params, dict): + # The session cancels only requests that are still in flight. + target_id = params.get("requestId") + target = self.pending.get(target_id) + if target is not None and target_id in (self._session_attr("_in_flight") or {}): + target.mark_cancelled() + self.inbound_notification = operation + + def on_forward(self, message: Any) -> None: + types, _ = _mcp() + if self.inbound_notification is not None and isinstance(message, types.ClientNotification): + self.forwarded_notifications[id(message)] = self.inbound_notification + self.inbound_notification = None + + def on_send(self, item: Any) -> tuple[ServerOperation | None, Any]: + types, _ = _mcp() + message = getattr(getattr(item, "message", None), "root", None) + if isinstance(message, types.JSONRPCResponse): + operation = self.pending.pop(message.id, None) + if operation is not None: + version = message.result.get("protocolVersion") if operation.method == "initialize" else None + if isinstance(version, str): + self.protocol_version = version + operation.record_result(message.result) + return operation, None + if isinstance(message, types.JSONRPCError): + operation = self.pending.pop(message.id, None) + if operation is not None: + operation.record_error_response(message.error.code, message.error.message) + return operation, None + if isinstance(message, types.JSONRPCNotification): + related_id = getattr(item.metadata, "related_request_id", None) + span = start_outgoing_notification( + method=message.method, + params=message.params, + config=self.config, + related=related_operation(related_id) or self.pending.get(related_id), + ) + return None, span + return None, None + + def close(self) -> None: + operations = [*self.pending.values(), *self.forwarded_notifications.values()] + if self.inbound_notification is not None: + operations.append(self.inbound_notification) + self.pending.clear() + self.forwarded_notifications.clear() + self.inbound_notification = None + for operation in operations: + operation.finish(ConnectionClosedError()) + + +class _StreamObserver: + """Delegates to an anyio memory stream; subclasses observe its messages.""" + + def __init__(self, stream: Any, trace: _SessionTrace): + self._stream = stream + self._trace = trace + + def __getattr__(self, name: str) -> Any: + return getattr(self._stream, name) + + async def __aenter__(self) -> "_StreamObserver": + await self._stream.__aenter__() + return self + + async def __aexit__(self, *exc_info: Any) -> Any: + return await self._stream.__aexit__(*exc_info) + + +class _ObservedReadStream(_StreamObserver): + async def __aexit__(self, *exc_info: Any) -> Any: + try: + return await self._stream.__aexit__(*exc_info) + finally: + contain(self._trace.close) + + def __aiter__(self) -> "_ObservedReadStream": + return self + + async def __anext__(self) -> Any: + contain(self._trace.settle_inbound) + try: + item = await self._stream.__anext__() + except BaseException: + contain(self._trace.close) + raise + contain(self._trace.on_receive, item) + return item + + async def receive(self) -> Any: + return await self.__anext__() + + +class _ObservedWriteStream(_StreamObserver): + async def send(self, item: Any) -> None: + operation, notification_span = contain(self._trace.on_send, item) or (None, None) + + def settle(error: BaseException | None = None) -> None: + if operation is not None: + operation.finish(error) + if notification_span is not None: + end_outgoing_notification(notification_span, error) + + try: + await self._stream.send(item) + except BaseException as error: + settle(error) + raise + settle() + + +class _ObservedIncomingWriter(_StreamObserver): + async def send(self, item: Any) -> None: + contain(self._trace.on_forward, item) + await self._stream.send(item) + + +def _install_session_observers(session: Any, config: MCPServerConfig) -> None: + if session in _session_traces: + return + streams = ("_read_stream", "_write_stream", "_incoming_message_stream_writer") + if not all(hasattr(session, name) for name in streams): + return + trace = _SessionTrace(session, config) + session._read_stream = _ObservedReadStream(session._read_stream, trace) + session._write_stream = _ObservedWriteStream(session._write_stream, trace) + session._incoming_message_stream_writer = _ObservedIncomingWriter(session._incoming_message_stream_writer, trace) + _session_traces[session] = trace + + +def server_session_init_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any: + result = wrapped(*args, **kwargs) + config = _pending_server_config.get() or global_server_config() + if config is not None: + # A session consumes the config of the run that created it, so a + # server started from inside a handler is not traced implicitly. + _pending_server_config.set(None) + contain(_install_session_observers, instance, config) + return result + + +async def server_handle_message_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any: + message = call_arg(args, kwargs, 0, "message") + session = call_arg(args, kwargs, 1, "session") + trace = _session_traces.get(session) if session is not None else None + if trace is None: + return await wrapped(*args, **kwargs) + if isinstance(message, _mcp()[1]): + operation = trace.pending.get(message.request_id) + if operation is None: + return await wrapped(*args, **kwargs) + # The write observer ends the operation once the response is sent. + with activate(operation): + return await wrapped(*args, **kwargs) + operation = trace.forwarded_notifications.pop(id(message), None) + if operation is None: + return await wrapped(*args, **kwargs) + return await run_operation(operation, wrapped, args, kwargs) + + +async def server_run_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any: + """Per-server ``run`` wrapper installed by ``wrap_mcp_server`` on MCP 1.x. + + Per instance rather than on ``Server.run``, because fastmcp 2/3 override + ``run`` without calling it. + """ + config = getattr(instance, SERVER_CONFIG_ATTR, None) + token = _pending_server_config.set(config if isinstance(config, MCPServerConfig) else None) + try: + return await wrapped(*args, **kwargs) + finally: + _pending_server_config.reset(token) diff --git a/py/src/braintrust/integrations/mcp/_server_v2.py b/py/src/braintrust/integrations/mcp/_server_v2.py new file mode 100644 index 000000000..c99b1806f --- /dev/null +++ b/py/src/braintrust/integrations/mcp/_server_v2.py @@ -0,0 +1,197 @@ +"""Server request lifecycle hooks for MCP 2.x. + +Every MCP 2.x server path (stdio, streamable HTTP in both protocol eras, and +in-process clients) dispatches through ``ServerRunner._on_request`` and +``ServerRunner._on_notify``, which validate, dispatch, run the handler, and +shape the wire result. Operations start there. + +When a ``JSONRPCDispatcher`` owns the request, it writes the response after the +runner returns. Its ``_handle_request`` hook keeps the operation open through +the response write, observes the written error code, and detects requests the +peer cancelled (which the dispatcher settles without a response). Without a +dispatcher (single-exchange HTTP, in-process direct dispatch) the operation +ends when the runner returns. +""" + +import contextvars +import weakref +from dataclasses import dataclass +from typing import Any + +import anyio + +from .tracing import ( + MCPServerConfig, + ServerOperation, + activate, + call_arg, + contain, + end_outgoing_notification, + http_session_id, + http_transport_metadata, + related_operation, + run_operation, + server_config, + start_outgoing_notification, + start_server_operation, +) + + +@dataclass +class _DispatchSlot: + """Links a dispatcher-owned request to the operation the runner starts.""" + + dispatcher: Any + operation: ServerOperation | None = None + written: bool = False + + +_dispatch_slot: contextvars.ContextVar[_DispatchSlot | None] = contextvars.ContextVar( + "braintrust_mcp_dispatch_slot", default=None +) + +# Dispatchers that serve a traced server, for outgoing notifications. +_server_dispatchers: "weakref.WeakKeyDictionary[Any, MCPServerConfig]" = weakref.WeakKeyDictionary() + + +def _cancel_requested(dctx: Any) -> bool: + event = getattr(dctx, "cancel_requested", None) + try: + return bool(event is not None and event.is_set()) + except Exception: + return False + + +def _start(runner: Any, args: Any, kwargs: Any, config: MCPServerConfig, notification: bool) -> ServerOperation | None: + dctx = call_arg(args, kwargs, 0, "dctx") + connection = getattr(runner, "connection", None) + request_context = getattr(getattr(dctx, "message_metadata", None), "request_context", None) + return start_server_operation( + method=call_arg(args, kwargs, 1, "method"), + params=call_arg(args, kwargs, 2, "params"), + request_id=None if notification else getattr(dctx, "request_id", None), + config=config, + session_id=getattr(connection, "session_id", None) or http_session_id(request_context), + protocol_version=getattr(connection, "protocol_version", None), + client_info=getattr(getattr(connection, "client_params", None), "client_info", None), + transport_metadata=http_transport_metadata(request_context), + ) + + +def _record_handler_error(operation: ServerOperation, dctx: Any, error: BaseException) -> None: + if isinstance(error, anyio.get_cancelled_exc_class()) and _cancel_requested(dctx): + operation.mark_cancelled() + return + error_data = None + if isinstance(error, Exception): + from mcp.shared.jsonrpc_dispatcher import handler_exception_to_error_data + + error_data = contain(handler_exception_to_error_data, error) + if error_data is not None: + operation.record_error_response(error_data.code, error_data.message) + else: + operation.record_exception(error) + + +async def runner_on_request_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any: + slot = _dispatch_slot.get() + if slot is not None and slot.operation is not None: + # Another layer already traces this request. + return await wrapped(*args, **kwargs) + config = server_config(getattr(instance, "server", None)) + operation = contain(_start, instance, args, kwargs, config, False) if config is not None else None + if operation is None: + return await wrapped(*args, **kwargs) + dctx = call_arg(args, kwargs, 0, "dctx") + dispatcher_owned = slot is not None + if dispatcher_owned: + slot.operation = operation + if _server_dispatchers.get(slot.dispatcher) is not config: + _server_dispatchers[slot.dispatcher] = config + try: + with activate(operation): + result = await wrapped(*args, **kwargs) + except BaseException as error: + _record_handler_error(operation, dctx, error) + if not dispatcher_owned: + operation.finish() + raise + operation.record_result(result) + if not dispatcher_owned: + if _cancel_requested(dctx): + operation.mark_cancelled() + operation.finish() + return result + + +async def runner_on_notify_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any: + config = server_config(getattr(instance, "server", None)) + operation = contain(_start, instance, args, kwargs, config, True) if config is not None else None + if operation is None: + return await wrapped(*args, **kwargs) + return await run_operation(operation, wrapped, args, kwargs) + + +async def dispatcher_handle_request_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any: + slot = _DispatchSlot(dispatcher=instance) + token = _dispatch_slot.set(slot) + error = None + try: + return await wrapped(*args, **kwargs) + except BaseException as caught: + error = caught + raise + finally: + _dispatch_slot.reset(token) + operation = slot.operation + if operation is not None: + # A request the peer cancelled settles without a response. + if not slot.written and _cancel_requested(call_arg(args, kwargs, 1, "dctx")): + operation.mark_cancelled() + operation.finish(error) + + +def _slot_for(request_id: Any) -> _DispatchSlot | None: + slot = _dispatch_slot.get() + if slot is None or slot.operation is None or slot.operation.request_id != request_id: + return None + return slot + + +async def dispatcher_write_result_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any: + slot = _slot_for(call_arg(args, kwargs, 0, "request_id")) + if slot is not None: + slot.written = True + return await wrapped(*args, **kwargs) + + +async def dispatcher_write_error_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any: + slot = _slot_for(call_arg(args, kwargs, 0, "request_id")) + if slot is not None: + slot.written = True + error = call_arg(args, kwargs, 1, "error") + slot.operation.record_error_response(getattr(error, "code", None), getattr(error, "message", None)) + return await wrapped(*args, **kwargs) + + +async def dispatcher_notify_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any: + config = _server_dispatchers.get(instance) + span = None + if config is not None: + span = contain( + lambda: start_outgoing_notification( + method=call_arg(args, kwargs, 0, "method"), + params=call_arg(args, kwargs, 1, "params"), + config=config, + related=related_operation(kwargs.get("_related_request_id")), + ) + ) + if span is None: + return await wrapped(*args, **kwargs) + try: + result = await wrapped(*args, **kwargs) + except BaseException as error: + end_outgoing_notification(span, error) + raise + end_outgoing_notification(span) + return result diff --git a/py/src/braintrust/integrations/mcp/_test_harness.py b/py/src/braintrust/integrations/mcp/_test_harness.py new file mode 100644 index 000000000..5a991fcb5 --- /dev/null +++ b/py/src/braintrust/integrations/mcp/_test_harness.py @@ -0,0 +1,293 @@ +"""Shared MCP server/client harness for the MCP integration tests. + +Builds the same demo server on the bundled server abstraction (``FastMCP`` on +MCP 1.x, ``MCPServer`` on MCP 2.x) or on standalone ``fastmcp``, and connects +real MCP clients to it in-process, over streamable HTTP (uvicorn on +localhost), or over stdio (a real server subprocess running this module). + +Run as ``python _test_harness.py stdio`` to serve the demo server over stdio; +the subprocess records its spans with the memory logger, replays nested model +calls from a cassette, and writes the spans to ``BRAINTRUST_MCP_TEST_SPANS_FILE`` +on exit. +""" + +import contextlib +import importlib.util +import inspect +import json +import os +import sys +from collections.abc import AsyncIterator +from typing import Any + +import anyio +import mcp +from braintrust.integrations.mcp import MCPIntegration +from braintrust.integrations.mcp import _resolve_lowlevel_server as lowlevel_server +from braintrust.integrations.mcp.patchers import _MCP_V2 +from braintrust.integrations.versioning import version_satisfies + + +FLAVOR = os.environ.get("BRAINTRUST_MCP_TEST_FLAVOR") or ( + "fastmcp" if importlib.util.find_spec("fastmcp") is not None else "mcp" +) +MCP_V2 = version_satisfies(MCPIntegration.detect_version(mcp), _MCP_V2) +FASTMCP_MAJOR = int(importlib.import_module("fastmcp").__version__.split(".")[0]) if FLAVOR == "fastmcp" else 0 +LLM_MODEL = "claude-haiku-4-5-20251001" +PNG_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==" +SPANS_FILE_ENV = "BRAINTRUST_MCP_TEST_SPANS_FILE" +CASSETTE_ENV = "BRAINTRUST_MCP_TEST_CASSETTE" +RECORD_MODE_ENV = "BRAINTRUST_MCP_TEST_RECORD_MODE" +WRAP_ENV = "BRAINTRUST_MCP_TEST_WRAP" + +# Set by the ``wait_for_cancel`` tool so tests can cancel a running request. +cancel_tool_started: anyio.Event | None = None +cancel_tool_request_id: Any = None + + +def _server_classes() -> tuple[Any, Any]: + if FLAVOR == "fastmcp": + from fastmcp import Context, FastMCP + + return FastMCP, Context + if MCP_V2: + from mcp.server.mcpserver import Context, MCPServer + + return MCPServer, Context + from mcp.server.fastmcp import Context, FastMCP # pylint: disable=no-name-in-module # MCP 1.x + + return FastMCP, Context + + +def build_server(name: str = "braintrust-mcp-test") -> Any: + from braintrust.logger import start_span + + server_class, context_class = _server_classes() + server = server_class(name) + + @server.tool() + async def add(a: int, b: int) -> int: + """Add two integers.""" + with start_span(name="nested-work") as span: + span.log(input={"a": a, "b": b}) + return a + b + + @server.tool() + async def fail() -> str: + """Always fails.""" + raise ValueError("tool exploded") + + @server.tool() + async def image() -> Any: + """Return an inline image.""" + return [mcp.types.ImageContent.model_validate({"type": "image", "data": PNG_BASE64, "mimeType": "image/png"})] + + async def notify(ctx: context_class) -> str: # type: ignore[valid-type] + """Send a log notification while running.""" + await ctx.info("working on it") + return "done" + + server.tool()(notify) + + async def wait_for_cancel(ctx: context_class) -> str: # type: ignore[valid-type] + """Block until cancelled.""" + global cancel_tool_request_id + cancel_tool_request_id = ctx.request_id + if cancel_tool_started is not None: + cancel_tool_started.set() + await anyio.sleep(30) + return "not cancelled" + + server.tool()(wait_for_cancel) + + @server.tool() + async def ask_llm(question: str) -> str: + """Answer with a model call made inside the tool.""" + import anthropic + from braintrust.integrations.anthropic import wrap_anthropic + + client = wrap_anthropic(anthropic.AsyncAnthropic()) + response = await client.messages.create( + model=LLM_MODEL, + max_tokens=32, + messages=[{"role": "user", "content": question}], + ) + return response.content[0].text + + @server.prompt() + def greet(name: str) -> str: + """Greet someone.""" + return f"Say hello to {name}." + + @server.resource("resource://greeting") + def greeting() -> str: + return "Hello from a resource" + + @server.resource("resource://pixel.png", mime_type="image/png") + def pixel() -> bytes: + import base64 + + return base64.b64decode(PNG_BASE64) + + return server + + +def http_app(server: Any) -> Any: + if FLAVOR == "fastmcp": + return server.http_app() + return server.streamable_http_app() + + +class Client: + """Uniform calls over the real MCP or fastmcp client for each flavor.""" + + def __init__(self, client: Any): + self.raw = client + + async def call_tool(self, name: str, arguments: dict[str, Any] | None = None, meta: Any = None) -> Any: + kwargs: dict[str, Any] = {} + if meta is not None: + kwargs["meta"] = meta + if FLAVOR == "fastmcp": + kwargs["raise_on_error"] = False + return await self.raw.call_tool(name, arguments or {}, **kwargs) + + async def get_prompt(self, name: str, arguments: dict[str, Any]) -> Any: + return await self.raw.get_prompt(name, arguments) + + async def read_resource(self, uri: str) -> Any: + if FLAVOR == "mcp" and not MCP_V2: + from pydantic import AnyUrl + + return await self.raw.read_resource(AnyUrl(uri)) + return await self.raw.read_resource(uri) + + async def list_tools(self) -> Any: + return await self.raw.list_tools() + + +@contextlib.asynccontextmanager +async def connect_in_memory(server: Any, mode: str | None = None) -> AsyncIterator[Client]: + if FLAVOR == "fastmcp": + from fastmcp import Client as FastMCPClient + + async with FastMCPClient(server) as client: + yield Client(client) + elif MCP_V2: + from mcp import Client as MCPClient + + async with MCPClient(server, mode=mode or "legacy", log_level="info") as client: + yield Client(client) + else: + from mcp.shared.memory import ( # pylint: disable=no-name-in-module # MCP 1.x + create_connected_server_and_client_session, + ) + + async with create_connected_server_and_client_session(lowlevel_server(server)) as session: + yield Client(session) + + +@contextlib.asynccontextmanager +async def connect_http(url: str) -> AsyncIterator[Client]: + if FLAVOR == "fastmcp": + from fastmcp import Client as FastMCPClient + + async with FastMCPClient(url) as client: + yield Client(client) + elif MCP_V2: + from mcp import Client as MCPClient + + async with MCPClient(url, log_level="info") as client: + yield Client(client) + else: + from mcp import ClientSession + from mcp.client.streamable_http import streamablehttp_client # pylint: disable=no-name-in-module # MCP 1.x + + async with streamablehttp_client(url) as (read, write, _): + async with ClientSession(read, write) as session: + await session.initialize() + yield Client(session) + + +@contextlib.asynccontextmanager +async def connect_stdio(env: dict[str, str]) -> AsyncIterator[Client]: + args = [os.path.abspath(__file__), "stdio"] + full_env = {**os.environ, **env} + if FLAVOR == "fastmcp": + from fastmcp import Client as FastMCPClient + from fastmcp.client.transports import StdioTransport + + transport = StdioTransport(command=sys.executable, args=args, env=full_env, keep_alive=False) + async with FastMCPClient(transport) as client: + yield Client(client) + return + from mcp import StdioServerParameters + from mcp.client.stdio import stdio_client + + params = StdioServerParameters(command=sys.executable, args=args, env=full_env) + if MCP_V2: + from mcp import Client as MCPClient + + async with MCPClient(stdio_client(params), log_level="info") as client: + yield Client(client) + else: + from mcp import ClientSession + + async with stdio_client(params) as (read, write): + async with ClientSession(read, write) as session: + await session.initialize() + yield Client(session) + + +@contextlib.asynccontextmanager +async def serve_http(app: Any) -> AsyncIterator[str]: + """Serve *app* on an ephemeral localhost port in the current event loop. + + Same loop and thread as the test, so the server's spans reach the test's + memory logger. + """ + import uvicorn + + config = uvicorn.Config(app, host="127.0.0.1", port=0, log_level="warning", lifespan="on") + server = uvicorn.Server(config) + async with anyio.create_task_group() as tg: + tg.start_soon(server.serve) + with anyio.fail_after(10): + while not server.started: + await anyio.sleep(0.01) + port = server.servers[0].sockets[0].getsockname()[1] + try: + yield f"http://127.0.0.1:{port}/mcp" + finally: + server.should_exit = True + + +async def _serve_stdio() -> None: + from braintrust.conftest import get_vcr_config + from braintrust.integrations.mcp import wrap_mcp_server + from braintrust.integrations.test_utils import autoinstrument_test_context + + vcr_config = get_vcr_config() + record_mode = os.environ.get(RECORD_MODE_ENV) + if record_mode: + vcr_config["record_mode"] = record_mode + with autoinstrument_test_context( + os.environ[CASSETTE_ENV], integration="mcp", use_vcr=record_mode != "disabled", vcr_config=vcr_config + ) as memory_logger: + server = build_server() + if os.environ.get(WRAP_ENV) == "setup": + MCPIntegration.setup() + else: + wrap_mcp_server(server) + run = server.run_stdio_async + if "show_banner" in inspect.signature(run).parameters: + await run(show_banner=False) + else: + await run() + spans = memory_logger.pop() + with open(os.environ[SPANS_FILE_ENV], "w") as f: + json.dump(spans, f, default=repr) + + +if __name__ == "__main__" and sys.argv[1:] == ["stdio"]: + anyio.run(_serve_stdio) diff --git a/py/src/braintrust/integrations/mcp/cassettes/1.28.1/mcp_test_http_tool_with_model_call.yaml b/py/src/braintrust/integrations/mcp/cassettes/1.28.1/mcp_test_http_tool_with_model_call.yaml new file mode 100644 index 000000000..caed6040f --- /dev/null +++ b/py/src/braintrust/integrations/mcp/cassettes/1.28.1/mcp_test_http_tool_with_model_call.yaml @@ -0,0 +1,109 @@ +interactions: +- request: + body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single + word: pong"}],"model":"claude-haiku-4-5-20251001"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + anthropic-version: + - '2026-10-21' + connection: + - keep-alive + content-length: + - '127' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - AsyncAnthropic/Python 2.3.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.3.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWpEYvfW2MdeQfjBGcUu","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' + headers: + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '10000000' + anthropic-ratelimit-input-tokens-remaining: + - '10000000' + anthropic-ratelimit-input-tokens-reset: + - '2026-10-29T16:06:31Z' + anthropic-ratelimit-output-tokens-limit: + - '2000000' + anthropic-ratelimit-output-tokens-remaining: + - '2000000' + anthropic-ratelimit-output-tokens-reset: + - '2026-10-29T16:06:31Z' + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-10-29T16:06:31Z' + anthropic-ratelimit-tokens-limit: + - '12000000' + anthropic-ratelimit-tokens-remaining: + - '12000000' + anthropic-ratelimit-tokens-reset: + - '2026-10-29T16:06:31Z' + anthropic-workspace-id: + - wrkspc_018bFG1FgNVWqx5pthtdGTzS + cf-cache-status: + - DYNAMIC + cf-ray: + - a5236d8cdb6cac00-YYZ + connection: + - keep-alive + content-length: + - '515' + content-security-policy: + - default-src 'none'; frame-ancestors 'none' + content-type: + - application/json + date: + - Thu, 29 Oct 2026 16:06:31 GMT + request-id: + - req_011CgWpEYhXgYf1SWaPSg3WD + server: + - cloudflare + server-timing: + - x-originResponse;dur=323 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + traceresponse: + - 00-bbf31ac2c3eee3dca8a2e91220d987c4-b530afaf227747d2-01 + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-robots-tag: + - none + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/mcp/cassettes/1.28.1/mcp_test_stdio_subprocess.yaml b/py/src/braintrust/integrations/mcp/cassettes/1.28.1/mcp_test_stdio_subprocess.yaml new file mode 100644 index 000000000..2c22b2ac3 --- /dev/null +++ b/py/src/braintrust/integrations/mcp/cassettes/1.28.1/mcp_test_stdio_subprocess.yaml @@ -0,0 +1,109 @@ +interactions: +- request: + body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single + word: pong"}],"model":"claude-haiku-4-5-20251001"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + anthropic-version: + - '2026-10-21' + connection: + - keep-alive + content-length: + - '127' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - AsyncAnthropic/Python 2.3.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.3.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWpEg5HdeGux7bVRTNWm","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' + headers: + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '10000000' + anthropic-ratelimit-input-tokens-remaining: + - '10000000' + anthropic-ratelimit-input-tokens-reset: + - '2026-10-29T16:06:33Z' + anthropic-ratelimit-output-tokens-limit: + - '2000000' + anthropic-ratelimit-output-tokens-remaining: + - '2000000' + anthropic-ratelimit-output-tokens-reset: + - '2026-10-29T16:06:33Z' + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-10-29T16:06:32Z' + anthropic-ratelimit-tokens-limit: + - '12000000' + anthropic-ratelimit-tokens-remaining: + - '12000000' + anthropic-ratelimit-tokens-reset: + - '2026-10-29T16:06:33Z' + anthropic-workspace-id: + - wrkspc_018bFG1FgNVWqx5pthtdGTzS + cf-cache-status: + - DYNAMIC + cf-ray: + - a5236d975d0a9113-YYZ + connection: + - keep-alive + content-length: + - '515' + content-security-policy: + - default-src 'none'; frame-ancestors 'none' + content-type: + - application/json + date: + - Thu, 29 Oct 2026 16:06:33 GMT + request-id: + - req_011CgWpEfrPTmFwTM4BB9ube + server: + - cloudflare + server-timing: + - x-originResponse;dur=281 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + traceresponse: + - 00-8607b05eae3059c98503eff51e44fe52-741bcedf17686783-01 + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-robots-tag: + - none + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/mcp/cassettes/1.30.0/mcp_test_http_tool_with_model_call.yaml b/py/src/braintrust/integrations/mcp/cassettes/1.30.0/mcp_test_http_tool_with_model_call.yaml new file mode 100644 index 000000000..8dc544455 --- /dev/null +++ b/py/src/braintrust/integrations/mcp/cassettes/1.30.0/mcp_test_http_tool_with_model_call.yaml @@ -0,0 +1,109 @@ +interactions: +- request: + body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single + word: pong"}],"model":"claude-haiku-4-5-20251001"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + anthropic-version: + - '2026-10-21' + connection: + - keep-alive + content-length: + - '127' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - AsyncAnthropic/Python 2.3.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.3.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWpCkfnyaPeQ544GcEMy","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' + headers: + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '10000000' + anthropic-ratelimit-input-tokens-remaining: + - '10000000' + anthropic-ratelimit-input-tokens-reset: + - '2026-10-29T16:06:07Z' + anthropic-ratelimit-output-tokens-limit: + - '2000000' + anthropic-ratelimit-output-tokens-remaining: + - '2000000' + anthropic-ratelimit-output-tokens-reset: + - '2026-10-29T16:06:07Z' + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-10-29T16:06:06Z' + anthropic-ratelimit-tokens-limit: + - '12000000' + anthropic-ratelimit-tokens-remaining: + - '12000000' + anthropic-ratelimit-tokens-reset: + - '2026-10-29T16:06:07Z' + anthropic-workspace-id: + - wrkspc_018bFG1FgNVWqx5pthtdGTzS + cf-cache-status: + - DYNAMIC + cf-ray: + - a5236cf3ecc6abb8-YYZ + connection: + - keep-alive + content-length: + - '515' + content-security-policy: + - default-src 'none'; frame-ancestors 'none' + content-type: + - application/json + date: + - Thu, 29 Oct 2026 16:06:07 GMT + request-id: + - req_011CgWpCk3bMDgkGqBL2QuHH + server: + - cloudflare + server-timing: + - x-originResponse;dur=394 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + traceresponse: + - 00-cd32ea7dcaf28d98c608743508dfa3aa-6001f83e29308b74-01 + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-robots-tag: + - none + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/mcp/cassettes/1.30.0/mcp_test_stdio_subprocess.yaml b/py/src/braintrust/integrations/mcp/cassettes/1.30.0/mcp_test_stdio_subprocess.yaml new file mode 100644 index 000000000..8f93cd9f2 --- /dev/null +++ b/py/src/braintrust/integrations/mcp/cassettes/1.30.0/mcp_test_stdio_subprocess.yaml @@ -0,0 +1,109 @@ +interactions: +- request: + body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single + word: pong"}],"model":"claude-haiku-4-5-20251001"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + anthropic-version: + - '2026-10-21' + connection: + - keep-alive + content-length: + - '127' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - AsyncAnthropic/Python 2.3.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.3.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWpDEVd1a7GErXVSQG6t","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' + headers: + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '10000000' + anthropic-ratelimit-input-tokens-remaining: + - '10000000' + anthropic-ratelimit-input-tokens-reset: + - '2026-10-29T16:06:13Z' + anthropic-ratelimit-output-tokens-limit: + - '2000000' + anthropic-ratelimit-output-tokens-remaining: + - '2000000' + anthropic-ratelimit-output-tokens-reset: + - '2026-10-29T16:06:13Z' + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-10-29T16:06:13Z' + anthropic-ratelimit-tokens-limit: + - '12000000' + anthropic-ratelimit-tokens-remaining: + - '12000000' + anthropic-ratelimit-tokens-reset: + - '2026-10-29T16:06:13Z' + anthropic-workspace-id: + - wrkspc_018bFG1FgNVWqx5pthtdGTzS + cf-cache-status: + - DYNAMIC + cf-ray: + - a5236d1d0ab5a211-YYZ + connection: + - keep-alive + content-length: + - '515' + content-security-policy: + - default-src 'none'; frame-ancestors 'none' + content-type: + - application/json + date: + - Thu, 29 Oct 2026 16:06:13 GMT + request-id: + - req_011CgWpDE92vER7E8z75SEkG + server: + - cloudflare + server-timing: + - x-originResponse;dur=366 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + traceresponse: + - 00-238cadab9b4caea6197202118bcbdcd5-d2024a2f60b280b7-01 + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-robots-tag: + - none + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/mcp/cassettes/2.0.0/mcp_test_http_tool_with_model_call.yaml b/py/src/braintrust/integrations/mcp/cassettes/2.0.0/mcp_test_http_tool_with_model_call.yaml new file mode 100644 index 000000000..d26976c9d --- /dev/null +++ b/py/src/braintrust/integrations/mcp/cassettes/2.0.0/mcp_test_http_tool_with_model_call.yaml @@ -0,0 +1,109 @@ +interactions: +- request: + body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single + word: pong"}],"model":"claude-haiku-4-5-20251001"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + anthropic-version: + - '2026-10-21' + connection: + - keep-alive + content-length: + - '127' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - AsyncAnthropic/Python 2.3.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.3.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWpAA1v4Lozz5WsTFbzH","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' + headers: + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '10000000' + anthropic-ratelimit-input-tokens-remaining: + - '10000000' + anthropic-ratelimit-input-tokens-reset: + - '2026-10-29T16:05:31Z' + anthropic-ratelimit-output-tokens-limit: + - '2000000' + anthropic-ratelimit-output-tokens-remaining: + - '2000000' + anthropic-ratelimit-output-tokens-reset: + - '2026-10-29T16:05:31Z' + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-10-29T16:05:31Z' + anthropic-ratelimit-tokens-limit: + - '12000000' + anthropic-ratelimit-tokens-remaining: + - '12000000' + anthropic-ratelimit-tokens-reset: + - '2026-10-29T16:05:31Z' + anthropic-workspace-id: + - wrkspc_018bFG1FgNVWqx5pthtdGTzS + cf-cache-status: + - DYNAMIC + cf-ray: + - a5236c18481eac52-YYZ + connection: + - keep-alive + content-length: + - '515' + content-security-policy: + - default-src 'none'; frame-ancestors 'none' + content-type: + - application/json + date: + - Thu, 29 Oct 2026 16:05:31 GMT + request-id: + - req_011CgWpA9ozqrCTvUTbvz9uK + server: + - cloudflare + server-timing: + - x-originResponse;dur=317 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + traceresponse: + - 00-e049acfb5e2207d1ec88abdb6442fdd1-13de36ce6aa95537-01 + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-robots-tag: + - none + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/mcp/cassettes/2.0.0/mcp_test_stdio_subprocess.yaml b/py/src/braintrust/integrations/mcp/cassettes/2.0.0/mcp_test_stdio_subprocess.yaml new file mode 100644 index 000000000..c3f09639a --- /dev/null +++ b/py/src/braintrust/integrations/mcp/cassettes/2.0.0/mcp_test_stdio_subprocess.yaml @@ -0,0 +1,109 @@ +interactions: +- request: + body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single + word: pong"}],"model":"claude-haiku-4-5-20251001"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + anthropic-version: + - '2026-10-21' + connection: + - keep-alive + content-length: + - '127' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - AsyncAnthropic/Python 2.3.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.3.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWpAT9oHqhsAqAY28czq","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' + headers: + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '10000000' + anthropic-ratelimit-input-tokens-remaining: + - '10000000' + anthropic-ratelimit-input-tokens-reset: + - '2026-10-29T16:05:35Z' + anthropic-ratelimit-output-tokens-limit: + - '2000000' + anthropic-ratelimit-output-tokens-remaining: + - '2000000' + anthropic-ratelimit-output-tokens-reset: + - '2026-10-29T16:05:35Z' + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-10-29T16:05:35Z' + anthropic-ratelimit-tokens-limit: + - '12000000' + anthropic-ratelimit-tokens-remaining: + - '12000000' + anthropic-ratelimit-tokens-reset: + - '2026-10-29T16:05:35Z' + anthropic-workspace-id: + - wrkspc_018bFG1FgNVWqx5pthtdGTzS + cf-cache-status: + - DYNAMIC + cf-ray: + - a5236c3148ba1185-YYZ + connection: + - keep-alive + content-length: + - '515' + content-security-policy: + - default-src 'none'; frame-ancestors 'none' + content-type: + - application/json + date: + - Thu, 29 Oct 2026 16:05:35 GMT + request-id: + - req_011CgWpASvQJA9Nbjy2AkNkC + server: + - cloudflare + server-timing: + - x-originResponse;dur=314 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + traceresponse: + - 00-facc1caf1a40f68f551d20b87fbca8af-438d4a1e35cbdbcc-01 + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-robots-tag: + - none + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/mcp/cassettes/2.14.7/fastmcp_test_http_tool_with_model_call.yaml b/py/src/braintrust/integrations/mcp/cassettes/2.14.7/fastmcp_test_http_tool_with_model_call.yaml new file mode 100644 index 000000000..cc4829f1e --- /dev/null +++ b/py/src/braintrust/integrations/mcp/cassettes/2.14.7/fastmcp_test_http_tool_with_model_call.yaml @@ -0,0 +1,109 @@ +interactions: +- request: + body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single + word: pong"}],"model":"claude-haiku-4-5-20251001"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + anthropic-version: + - '2026-10-21' + connection: + - keep-alive + content-length: + - '127' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - AsyncAnthropic/Python 2.3.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.3.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWpKZp8UQJfXMSYzr6Gg","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' + headers: + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '10000000' + anthropic-ratelimit-input-tokens-remaining: + - '10000000' + anthropic-ratelimit-input-tokens-reset: + - '2026-10-29T16:07:39Z' + anthropic-ratelimit-output-tokens-limit: + - '2000000' + anthropic-ratelimit-output-tokens-remaining: + - '2000000' + anthropic-ratelimit-output-tokens-reset: + - '2026-10-29T16:07:39Z' + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-10-29T16:07:39Z' + anthropic-ratelimit-tokens-limit: + - '12000000' + anthropic-ratelimit-tokens-remaining: + - '12000000' + anthropic-ratelimit-tokens-reset: + - '2026-10-29T16:07:39Z' + anthropic-workspace-id: + - wrkspc_018bFG1FgNVWqx5pthtdGTzS + cf-cache-status: + - DYNAMIC + cf-ray: + - a5236f363bf3ac5a-YYZ + connection: + - keep-alive + content-length: + - '515' + content-security-policy: + - default-src 'none'; frame-ancestors 'none' + content-type: + - application/json + date: + - Thu, 29 Oct 2026 16:07:39 GMT + request-id: + - req_011CgWpKZcEP4jmHuJsH8JKc + server: + - cloudflare + server-timing: + - x-originResponse;dur=305 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + traceresponse: + - 00-c29eaa7c8730e83732d4fb344c2349a9-f8a9e32d2e94d041-01 + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-robots-tag: + - none + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/mcp/cassettes/2.14.7/fastmcp_test_stdio_subprocess.yaml b/py/src/braintrust/integrations/mcp/cassettes/2.14.7/fastmcp_test_stdio_subprocess.yaml new file mode 100644 index 000000000..9c88e938c --- /dev/null +++ b/py/src/braintrust/integrations/mcp/cassettes/2.14.7/fastmcp_test_stdio_subprocess.yaml @@ -0,0 +1,109 @@ +interactions: +- request: + body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single + word: pong"}],"model":"claude-haiku-4-5-20251001"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + anthropic-version: + - '2026-10-21' + connection: + - keep-alive + content-length: + - '127' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - AsyncAnthropic/Python 2.3.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.3.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWpKgmbDkYJo1xkq4KN5","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' + headers: + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '10000000' + anthropic-ratelimit-input-tokens-remaining: + - '10000000' + anthropic-ratelimit-input-tokens-reset: + - '2026-10-29T16:07:41Z' + anthropic-ratelimit-output-tokens-limit: + - '2000000' + anthropic-ratelimit-output-tokens-remaining: + - '2000000' + anthropic-ratelimit-output-tokens-reset: + - '2026-10-29T16:07:41Z' + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-10-29T16:07:40Z' + anthropic-ratelimit-tokens-limit: + - '12000000' + anthropic-ratelimit-tokens-remaining: + - '12000000' + anthropic-ratelimit-tokens-reset: + - '2026-10-29T16:07:41Z' + anthropic-workspace-id: + - wrkspc_018bFG1FgNVWqx5pthtdGTzS + cf-cache-status: + - DYNAMIC + cf-ray: + - a5236f405d62c85f-YYZ + connection: + - keep-alive + content-length: + - '515' + content-security-policy: + - default-src 'none'; frame-ancestors 'none' + content-type: + - application/json + date: + - Thu, 29 Oct 2026 16:07:41 GMT + request-id: + - req_011CgWpKgYT4YXV8RA9hL9Qs + server: + - cloudflare + server-timing: + - x-originResponse;dur=302 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + traceresponse: + - 00-fe34676afb80ed923e5884ec20140a90-5e4f928193693034-01 + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-robots-tag: + - none + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/mcp/cassettes/3.0.0/fastmcp_test_http_tool_with_model_call.yaml b/py/src/braintrust/integrations/mcp/cassettes/3.0.0/fastmcp_test_http_tool_with_model_call.yaml new file mode 100644 index 000000000..561ee088a --- /dev/null +++ b/py/src/braintrust/integrations/mcp/cassettes/3.0.0/fastmcp_test_http_tool_with_model_call.yaml @@ -0,0 +1,109 @@ +interactions: +- request: + body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single + word: pong"}],"model":"claude-haiku-4-5-20251001"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + anthropic-version: + - '2026-10-21' + connection: + - keep-alive + content-length: + - '127' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - AsyncAnthropic/Python 2.3.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.3.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWpJfUg12QGjHwxu31Cr","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' + headers: + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '10000000' + anthropic-ratelimit-input-tokens-remaining: + - '10000000' + anthropic-ratelimit-input-tokens-reset: + - '2026-10-29T16:07:27Z' + anthropic-ratelimit-output-tokens-limit: + - '2000000' + anthropic-ratelimit-output-tokens-remaining: + - '2000000' + anthropic-ratelimit-output-tokens-reset: + - '2026-10-29T16:07:27Z' + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-10-29T16:07:27Z' + anthropic-ratelimit-tokens-limit: + - '12000000' + anthropic-ratelimit-tokens-remaining: + - '12000000' + anthropic-ratelimit-tokens-reset: + - '2026-10-29T16:07:27Z' + anthropic-workspace-id: + - wrkspc_018bFG1FgNVWqx5pthtdGTzS + cf-cache-status: + - DYNAMIC + cf-ray: + - a5236ee9ab414a29-YYZ + connection: + - keep-alive + content-length: + - '515' + content-security-policy: + - default-src 'none'; frame-ancestors 'none' + content-type: + - application/json + date: + - Thu, 29 Oct 2026 16:07:27 GMT + request-id: + - req_011CgWpJfFY8771sq9EbfV5F + server: + - cloudflare + server-timing: + - x-originResponse;dur=328 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + traceresponse: + - 00-d73fe9d11caeb0c51e14a43dc557f349-ce39e9976f9e72fe-01 + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-robots-tag: + - none + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/mcp/cassettes/3.0.0/fastmcp_test_stdio_subprocess.yaml b/py/src/braintrust/integrations/mcp/cassettes/3.0.0/fastmcp_test_stdio_subprocess.yaml new file mode 100644 index 000000000..7afb0f366 --- /dev/null +++ b/py/src/braintrust/integrations/mcp/cassettes/3.0.0/fastmcp_test_stdio_subprocess.yaml @@ -0,0 +1,109 @@ +interactions: +- request: + body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single + word: pong"}],"model":"claude-haiku-4-5-20251001"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + anthropic-version: + - '2026-10-21' + connection: + - keep-alive + content-length: + - '127' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - AsyncAnthropic/Python 2.3.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.3.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWpJnjjb1Z1BZbuT991x","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' + headers: + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '10000000' + anthropic-ratelimit-input-tokens-remaining: + - '10000000' + anthropic-ratelimit-input-tokens-reset: + - '2026-10-29T16:07:28Z' + anthropic-ratelimit-output-tokens-limit: + - '2000000' + anthropic-ratelimit-output-tokens-remaining: + - '2000000' + anthropic-ratelimit-output-tokens-reset: + - '2026-10-29T16:07:28Z' + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-10-29T16:07:28Z' + anthropic-ratelimit-tokens-limit: + - '12000000' + anthropic-ratelimit-tokens-remaining: + - '12000000' + anthropic-ratelimit-tokens-reset: + - '2026-10-29T16:07:28Z' + anthropic-workspace-id: + - wrkspc_018bFG1FgNVWqx5pthtdGTzS + cf-cache-status: + - DYNAMIC + cf-ray: + - a5236ef448e439f6-YYZ + connection: + - keep-alive + content-length: + - '515' + content-security-policy: + - default-src 'none'; frame-ancestors 'none' + content-type: + - application/json + date: + - Thu, 29 Oct 2026 16:07:29 GMT + request-id: + - req_011CgWpJnX6fDyCtvkeQ1JRF + server: + - cloudflare + server-timing: + - x-originResponse;dur=294 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + traceresponse: + - 00-c0cef9901d6af20019e2daf70eae19d2-f64c5616a94f6ab7-01 + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-robots-tag: + - none + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/mcp/cassettes/3.4.11/fastmcp_test_http_tool_with_model_call.yaml b/py/src/braintrust/integrations/mcp/cassettes/3.4.11/fastmcp_test_http_tool_with_model_call.yaml new file mode 100644 index 000000000..73efba51b --- /dev/null +++ b/py/src/braintrust/integrations/mcp/cassettes/3.4.11/fastmcp_test_http_tool_with_model_call.yaml @@ -0,0 +1,109 @@ +interactions: +- request: + body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single + word: pong"}],"model":"claude-haiku-4-5-20251001"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + anthropic-version: + - '2026-10-21' + connection: + - keep-alive + content-length: + - '127' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - AsyncAnthropic/Python 2.3.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.3.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWpHn9zVTS3rudzRsFTH","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' + headers: + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '10000000' + anthropic-ratelimit-input-tokens-remaining: + - '10000000' + anthropic-ratelimit-input-tokens-reset: + - '2026-10-29T16:07:15Z' + anthropic-ratelimit-output-tokens-limit: + - '2000000' + anthropic-ratelimit-output-tokens-remaining: + - '2000000' + anthropic-ratelimit-output-tokens-reset: + - '2026-10-29T16:07:15Z' + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-10-29T16:07:15Z' + anthropic-ratelimit-tokens-limit: + - '12000000' + anthropic-ratelimit-tokens-remaining: + - '12000000' + anthropic-ratelimit-tokens-reset: + - '2026-10-29T16:07:15Z' + anthropic-workspace-id: + - wrkspc_018bFG1FgNVWqx5pthtdGTzS + cf-cache-status: + - DYNAMIC + cf-ray: + - a5236e9e984bac40-YYZ + connection: + - keep-alive + content-length: + - '515' + content-security-policy: + - default-src 'none'; frame-ancestors 'none' + content-type: + - application/json + date: + - Thu, 29 Oct 2026 16:07:15 GMT + request-id: + - req_011CgWpHmvM3iYSzuJJCY4oh + server: + - cloudflare + server-timing: + - x-originResponse;dur=312 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + traceresponse: + - 00-73d1d292de1b013959c84868fedf97a6-cc7be964e0d8369c-01 + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-robots-tag: + - none + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/mcp/cassettes/3.4.11/fastmcp_test_stdio_subprocess.yaml b/py/src/braintrust/integrations/mcp/cassettes/3.4.11/fastmcp_test_stdio_subprocess.yaml new file mode 100644 index 000000000..01d4966c8 --- /dev/null +++ b/py/src/braintrust/integrations/mcp/cassettes/3.4.11/fastmcp_test_stdio_subprocess.yaml @@ -0,0 +1,109 @@ +interactions: +- request: + body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single + word: pong"}],"model":"claude-haiku-4-5-20251001"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + anthropic-version: + - '2026-10-21' + connection: + - keep-alive + content-length: + - '127' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - AsyncAnthropic/Python 2.3.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.3.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWpHu7vsexu25xg5hsgb","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' + headers: + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '10000000' + anthropic-ratelimit-input-tokens-remaining: + - '10000000' + anthropic-ratelimit-input-tokens-reset: + - '2026-10-29T16:07:16Z' + anthropic-ratelimit-output-tokens-limit: + - '2000000' + anthropic-ratelimit-output-tokens-remaining: + - '2000000' + anthropic-ratelimit-output-tokens-reset: + - '2026-10-29T16:07:16Z' + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-10-29T16:07:16Z' + anthropic-ratelimit-tokens-limit: + - '12000000' + anthropic-ratelimit-tokens-remaining: + - '12000000' + anthropic-ratelimit-tokens-reset: + - '2026-10-29T16:07:16Z' + anthropic-workspace-id: + - wrkspc_018bFG1FgNVWqx5pthtdGTzS + cf-cache-status: + - DYNAMIC + cf-ray: + - a5236ea8ce7b982f-YYZ + connection: + - keep-alive + content-length: + - '515' + content-security-policy: + - default-src 'none'; frame-ancestors 'none' + content-type: + - application/json + date: + - Thu, 29 Oct 2026 16:07:16 GMT + request-id: + - req_011CgWpHtw2ZurrrJfCFe6Sg + server: + - cloudflare + server-timing: + - x-originResponse;dur=306 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + traceresponse: + - 00-b1cd01e23e0c29dcea98179c73501387-4546a5a4c8cfcab8-01 + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-robots-tag: + - none + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/mcp/cassettes/4.0.0/fastmcp_test_http_tool_with_model_call.yaml b/py/src/braintrust/integrations/mcp/cassettes/4.0.0/fastmcp_test_http_tool_with_model_call.yaml new file mode 100644 index 000000000..7c15b352e --- /dev/null +++ b/py/src/braintrust/integrations/mcp/cassettes/4.0.0/fastmcp_test_http_tool_with_model_call.yaml @@ -0,0 +1,109 @@ +interactions: +- request: + body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single + word: pong"}],"model":"claude-haiku-4-5-20251001"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + anthropic-version: + - '2026-10-21' + connection: + - keep-alive + content-length: + - '127' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - AsyncAnthropic/Python 2.3.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.3.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWpGugtqXaugZD4zFL22","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' + headers: + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '10000000' + anthropic-ratelimit-input-tokens-remaining: + - '10000000' + anthropic-ratelimit-input-tokens-reset: + - '2026-10-29T16:07:03Z' + anthropic-ratelimit-output-tokens-limit: + - '2000000' + anthropic-ratelimit-output-tokens-remaining: + - '2000000' + anthropic-ratelimit-output-tokens-reset: + - '2026-10-29T16:07:03Z' + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-10-29T16:07:03Z' + anthropic-ratelimit-tokens-limit: + - '12000000' + anthropic-ratelimit-tokens-remaining: + - '12000000' + anthropic-ratelimit-tokens-reset: + - '2026-10-29T16:07:03Z' + anthropic-workspace-id: + - wrkspc_018bFG1FgNVWqx5pthtdGTzS + cf-cache-status: + - DYNAMIC + cf-ray: + - a5236e54cecd1707-YYZ + connection: + - keep-alive + content-length: + - '515' + content-security-policy: + - default-src 'none'; frame-ancestors 'none' + content-type: + - application/json + date: + - Thu, 29 Oct 2026 16:07:03 GMT + request-id: + - req_011CgWpGuUzE3uCduPxGG3aa + server: + - cloudflare + server-timing: + - x-originResponse;dur=338 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + traceresponse: + - 00-ecf9ec6919820925d253367c66baf5b5-d70540e4375e624a-01 + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-robots-tag: + - none + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/mcp/cassettes/4.0.0/fastmcp_test_stdio_subprocess.yaml b/py/src/braintrust/integrations/mcp/cassettes/4.0.0/fastmcp_test_stdio_subprocess.yaml new file mode 100644 index 000000000..99f0a8d1a --- /dev/null +++ b/py/src/braintrust/integrations/mcp/cassettes/4.0.0/fastmcp_test_stdio_subprocess.yaml @@ -0,0 +1,109 @@ +interactions: +- request: + body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single + word: pong"}],"model":"claude-haiku-4-5-20251001"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + anthropic-version: + - '2026-10-21' + connection: + - keep-alive + content-length: + - '127' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - AsyncAnthropic/Python 2.3.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.3.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWpH31Aphoziz7Q1NMnQ","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' + headers: + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '10000000' + anthropic-ratelimit-input-tokens-remaining: + - '10000000' + anthropic-ratelimit-input-tokens-reset: + - '2026-10-29T16:07:05Z' + anthropic-ratelimit-output-tokens-limit: + - '2000000' + anthropic-ratelimit-output-tokens-remaining: + - '2000000' + anthropic-ratelimit-output-tokens-reset: + - '2026-10-29T16:07:05Z' + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-10-29T16:07:05Z' + anthropic-ratelimit-tokens-limit: + - '12000000' + anthropic-ratelimit-tokens-remaining: + - '12000000' + anthropic-ratelimit-tokens-reset: + - '2026-10-29T16:07:05Z' + anthropic-workspace-id: + - wrkspc_018bFG1FgNVWqx5pthtdGTzS + cf-cache-status: + - DYNAMIC + cf-ray: + - a5236e5f8bfaa21c-YYZ + connection: + - keep-alive + content-length: + - '515' + content-security-policy: + - default-src 'none'; frame-ancestors 'none' + content-type: + - application/json + date: + - Thu, 29 Oct 2026 16:07:05 GMT + request-id: + - req_011CgWpH2nn3JAxMNhPByxNd + server: + - cloudflare + server-timing: + - x-originResponse;dur=315 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + traceresponse: + - 00-0612c2fbe2de42ebb20966e9f7d2f41c-4a7bb08db02cfbde-01 + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-robots-tag: + - none + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/mcp/cassettes/latest/fastmcp_test_http_tool_with_model_call.yaml b/py/src/braintrust/integrations/mcp/cassettes/latest/fastmcp_test_http_tool_with_model_call.yaml new file mode 100644 index 000000000..52b0ff8f3 --- /dev/null +++ b/py/src/braintrust/integrations/mcp/cassettes/latest/fastmcp_test_http_tool_with_model_call.yaml @@ -0,0 +1,109 @@ +interactions: +- request: + body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single + word: pong"}],"model":"claude-haiku-4-5-20251001"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + anthropic-version: + - '2026-10-21' + connection: + - keep-alive + content-length: + - '127' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - AsyncAnthropic/Python 2.3.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.3.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWpFgkjba6aaK3VW3gHk","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' + headers: + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '10000000' + anthropic-ratelimit-input-tokens-remaining: + - '10000000' + anthropic-ratelimit-input-tokens-reset: + - '2026-10-29T16:06:46Z' + anthropic-ratelimit-output-tokens-limit: + - '2000000' + anthropic-ratelimit-output-tokens-remaining: + - '2000000' + anthropic-ratelimit-output-tokens-reset: + - '2026-10-29T16:06:46Z' + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-10-29T16:06:46Z' + anthropic-ratelimit-tokens-limit: + - '12000000' + anthropic-ratelimit-tokens-remaining: + - '12000000' + anthropic-ratelimit-tokens-reset: + - '2026-10-29T16:06:46Z' + anthropic-workspace-id: + - wrkspc_018bFG1FgNVWqx5pthtdGTzS + cf-cache-status: + - DYNAMIC + cf-ray: + - a5236ded2968aadf-YYZ + connection: + - keep-alive + content-length: + - '515' + content-security-policy: + - default-src 'none'; frame-ancestors 'none' + content-type: + - application/json + date: + - Thu, 29 Oct 2026 16:06:46 GMT + request-id: + - req_011CgWpFgVc5XVaMGqBbqqrv + server: + - cloudflare + server-timing: + - x-originResponse;dur=326 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + traceresponse: + - 00-bfe330cba19666216e5011913133963b-ba7d70d6d23cef60-01 + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-robots-tag: + - none + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/mcp/cassettes/latest/fastmcp_test_stdio_subprocess.yaml b/py/src/braintrust/integrations/mcp/cassettes/latest/fastmcp_test_stdio_subprocess.yaml new file mode 100644 index 000000000..486f582b9 --- /dev/null +++ b/py/src/braintrust/integrations/mcp/cassettes/latest/fastmcp_test_stdio_subprocess.yaml @@ -0,0 +1,109 @@ +interactions: +- request: + body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single + word: pong"}],"model":"claude-haiku-4-5-20251001"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + anthropic-version: + - '2026-10-21' + connection: + - keep-alive + content-length: + - '127' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - AsyncAnthropic/Python 2.3.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.3.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWpFvQmJqfbXe7Runacm","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' + headers: + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '10000000' + anthropic-ratelimit-input-tokens-remaining: + - '10000000' + anthropic-ratelimit-input-tokens-reset: + - '2026-10-29T16:06:50Z' + anthropic-ratelimit-output-tokens-limit: + - '2000000' + anthropic-ratelimit-output-tokens-remaining: + - '2000000' + anthropic-ratelimit-output-tokens-reset: + - '2026-10-29T16:06:50Z' + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-10-29T16:06:49Z' + anthropic-ratelimit-tokens-limit: + - '12000000' + anthropic-ratelimit-tokens-remaining: + - '12000000' + anthropic-ratelimit-tokens-reset: + - '2026-10-29T16:06:50Z' + anthropic-workspace-id: + - wrkspc_018bFG1FgNVWqx5pthtdGTzS + cf-cache-status: + - DYNAMIC + cf-ray: + - a5236e011c33b687-YYZ + connection: + - keep-alive + content-length: + - '515' + content-security-policy: + - default-src 'none'; frame-ancestors 'none' + content-type: + - application/json + date: + - Thu, 29 Oct 2026 16:06:50 GMT + request-id: + - req_011CgWpFvB8HjL4aJGoPPfFW + server: + - cloudflare + server-timing: + - x-originResponse;dur=297 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + traceresponse: + - 00-0c3bd3c8086da65d7dfc75c6afadde32-13c6671007cd8f8d-01 + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-robots-tag: + - none + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/mcp/cassettes/latest/mcp_test_http_tool_with_model_call.yaml b/py/src/braintrust/integrations/mcp/cassettes/latest/mcp_test_http_tool_with_model_call.yaml new file mode 100644 index 000000000..d7061aa36 --- /dev/null +++ b/py/src/braintrust/integrations/mcp/cassettes/latest/mcp_test_http_tool_with_model_call.yaml @@ -0,0 +1,109 @@ +interactions: +- request: + body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single + word: pong"}],"model":"claude-haiku-4-5-20251001"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + anthropic-version: + - '2026-10-21' + connection: + - keep-alive + content-length: + - '127' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - AsyncAnthropic/Python 2.3.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.3.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWp8bLbyPqkQSDooZowN","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' + headers: + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '10000000' + anthropic-ratelimit-input-tokens-remaining: + - '10000000' + anthropic-ratelimit-input-tokens-reset: + - '2026-10-29T16:05:10Z' + anthropic-ratelimit-output-tokens-limit: + - '2000000' + anthropic-ratelimit-output-tokens-remaining: + - '2000000' + anthropic-ratelimit-output-tokens-reset: + - '2026-10-29T16:05:10Z' + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-10-29T16:05:10Z' + anthropic-ratelimit-tokens-limit: + - '12000000' + anthropic-ratelimit-tokens-remaining: + - '12000000' + anthropic-ratelimit-tokens-reset: + - '2026-10-29T16:05:10Z' + anthropic-workspace-id: + - wrkspc_018bFG1FgNVWqx5pthtdGTzS + cf-cache-status: + - DYNAMIC + cf-ray: + - a5236b93adc54fcb-YYZ + connection: + - keep-alive + content-length: + - '515' + content-security-policy: + - default-src 'none'; frame-ancestors 'none' + content-type: + - application/json + date: + - Thu, 29 Oct 2026 16:05:10 GMT + request-id: + - req_011CgWp8b3FGkeZhLjn4QZKe + server: + - cloudflare + server-timing: + - x-originResponse;dur=313 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + traceresponse: + - 00-774cab9cef25d192b9b61ccc847c919a-b4cfb5c5bc1e9202-01 + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-robots-tag: + - none + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/mcp/cassettes/latest/mcp_test_stdio_subprocess.yaml b/py/src/braintrust/integrations/mcp/cassettes/latest/mcp_test_stdio_subprocess.yaml new file mode 100644 index 000000000..356321569 --- /dev/null +++ b/py/src/braintrust/integrations/mcp/cassettes/latest/mcp_test_stdio_subprocess.yaml @@ -0,0 +1,109 @@ +interactions: +- request: + body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single + word: pong"}],"model":"claude-haiku-4-5-20251001"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + anthropic-version: + - '2026-10-21' + connection: + - keep-alive + content-length: + - '127' + content-type: + - application/json + host: + - api.anthropic.com + user-agent: + - AsyncAnthropic/Python 2.3.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - async:asyncio + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 2.3.0 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.14.7 + x-stainless-timeout: + - '600' + method: POST + uri: https://api.anthropic.com/v1/messages + response: + body: + string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWp8j1UjE54Q6n7Sz1aQ","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' + headers: + anthropic-organization-id: + - 27796668-7351-40ac-acc4-024aee8995a5 + anthropic-ratelimit-input-tokens-limit: + - '10000000' + anthropic-ratelimit-input-tokens-remaining: + - '10000000' + anthropic-ratelimit-input-tokens-reset: + - '2026-10-29T16:05:12Z' + anthropic-ratelimit-output-tokens-limit: + - '2000000' + anthropic-ratelimit-output-tokens-remaining: + - '2000000' + anthropic-ratelimit-output-tokens-reset: + - '2026-10-29T16:05:12Z' + anthropic-ratelimit-requests-limit: + - '20000' + anthropic-ratelimit-requests-remaining: + - '19999' + anthropic-ratelimit-requests-reset: + - '2026-10-29T16:05:12Z' + anthropic-ratelimit-tokens-limit: + - '12000000' + anthropic-ratelimit-tokens-remaining: + - '12000000' + anthropic-ratelimit-tokens-reset: + - '2026-10-29T16:05:12Z' + anthropic-workspace-id: + - wrkspc_018bFG1FgNVWqx5pthtdGTzS + cf-cache-status: + - DYNAMIC + cf-ray: + - a5236b9efbb11341-YYZ + connection: + - keep-alive + content-length: + - '515' + content-security-policy: + - default-src 'none'; frame-ancestors 'none' + content-type: + - application/json + date: + - Thu, 29 Oct 2026 16:05:12 GMT + request-id: + - req_011CgWp8inb4wREanqJMLkAq + server: + - cloudflare + server-timing: + - x-originResponse;dur=334 + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + traceresponse: + - 00-d1974e70a014c44c4e19a91f2f43eb8b-89d2ac1c34f4e67f-01 + transfer-encoding: + - chunked + vary: + - Accept-Encoding + x-robots-tag: + - none + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/mcp/integration.py b/py/src/braintrust/integrations/mcp/integration.py new file mode 100644 index 000000000..d1afec346 --- /dev/null +++ b/py/src/braintrust/integrations/mcp/integration.py @@ -0,0 +1,38 @@ +from typing import Any + +from braintrust.integrations.base import BaseIntegration + +from .patchers import SERVER_PATCHERS +from .tracing import MCPServerConfig, set_global_server_config + + +class MCPIntegration(BaseIntegration): + """Braintrust instrumentation for MCP servers on mcp>=1.28.1,<3. + + Covers low-level servers and the server abstractions built on them: the + bundled ``FastMCP`` (1.x) / ``MCPServer`` (2.x) and standalone ``fastmcp``. + """ + + name = "mcp" + import_names = ("mcp",) + distribution_names = ("mcp",) + min_version = "1.28.1" + patchers = SERVER_PATCHERS + + @classmethod + def setup(cls, *, target: Any | None = None, capture_content: bool = True) -> bool: + """Trace every MCP server in the process. + + :param capture_content: Log arguments, results, and notification + parameters. When False, spans keep names, timing, allowlisted + metadata, and error classification only. + """ + if not cls.install_patchers(target=target): + return False + set_global_server_config(MCPServerConfig(capture_content=capture_content)) + return True + + @classmethod + def install_patchers(cls, *, target: Any | None = None) -> bool: + """Install the class-level hooks without tracing any server by itself.""" + return super().setup(target=target) diff --git a/py/src/braintrust/integrations/mcp/patchers.py b/py/src/braintrust/integrations/mcp/patchers.py new file mode 100644 index 000000000..2e60f3521 --- /dev/null +++ b/py/src/braintrust/integrations/mcp/patchers.py @@ -0,0 +1,106 @@ +from braintrust.integrations.base import FunctionWrapperPatcher + +from . import _server_v1, _server_v2 + + +_MCP_V1 = "<2.0.0a0" +_MCP_V2 = ">=2.0.0a0,<3" + + +# -- MCP 1.x ------------------------------------------------------------------ + + +class ServerSessionInitPatcher(FunctionWrapperPatcher): + """Observe traced sessions' streams from receipt through response sending.""" + + name = "mcp.server.v1.session_init" + version_spec = _MCP_V1 + target_module = "mcp.server.session" + target_path = "ServerSession.__init__" + wrapper = _server_v1.server_session_init_wrapper + + +class ServerHandleMessagePatcher(FunctionWrapperPatcher): + """Make the operation span current while request and notification handlers run.""" + + name = "mcp.server.v1.handle_message" + version_spec = _MCP_V1 + target_module = "mcp.server.lowlevel.server" + target_path = "Server._handle_message" + wrapper = _server_v1.server_handle_message_wrapper + + +class ServerRunPatcher(FunctionWrapperPatcher): + """Per-server ``run`` hook used by ``wrap_mcp_server`` (never applied globally).""" + + name = "mcp.server.v1.run" + version_spec = _MCP_V1 + target_path = "run" + wrapper = _server_v1.server_run_wrapper + + +# -- MCP 2.x ------------------------------------------------------------------ + + +class ServerRunnerRequestPatcher(FunctionWrapperPatcher): + name = "mcp.server.v2.runner_on_request" + version_spec = _MCP_V2 + target_module = "mcp.server.runner" + target_path = "ServerRunner._on_request" + wrapper = _server_v2.runner_on_request_wrapper + + +class ServerRunnerNotifyPatcher(FunctionWrapperPatcher): + name = "mcp.server.v2.runner_on_notify" + version_spec = _MCP_V2 + target_module = "mcp.server.runner" + target_path = "ServerRunner._on_notify" + wrapper = _server_v2.runner_on_notify_wrapper + + +class DispatcherHandleRequestPatcher(FunctionWrapperPatcher): + """Keep dispatcher-owned operations open until the response write settles.""" + + name = "mcp.server.v2.dispatcher_handle_request" + version_spec = _MCP_V2 + target_module = "mcp.shared.jsonrpc_dispatcher" + target_path = "JSONRPCDispatcher._handle_request" + wrapper = _server_v2.dispatcher_handle_request_wrapper + + +class DispatcherWriteResultPatcher(FunctionWrapperPatcher): + name = "mcp.server.v2.dispatcher_write_result" + version_spec = _MCP_V2 + target_module = "mcp.shared.jsonrpc_dispatcher" + target_path = "JSONRPCDispatcher._write_result" + wrapper = _server_v2.dispatcher_write_result_wrapper + + +class DispatcherWriteErrorPatcher(FunctionWrapperPatcher): + name = "mcp.server.v2.dispatcher_write_error" + version_spec = _MCP_V2 + target_module = "mcp.shared.jsonrpc_dispatcher" + target_path = "JSONRPCDispatcher._write_error" + wrapper = _server_v2.dispatcher_write_error_wrapper + + +class DispatcherNotifyPatcher(FunctionWrapperPatcher): + """Trace notifications sent by dispatchers that serve a traced server.""" + + name = "mcp.server.v2.dispatcher_notify" + version_spec = _MCP_V2 + target_module = "mcp.shared.jsonrpc_dispatcher" + target_path = "JSONRPCDispatcher.notify" + wrapper = _server_v2.dispatcher_notify_wrapper + + +SERVER_PATCHERS = ( + ServerSessionInitPatcher, + ServerHandleMessagePatcher, + ServerRunnerRequestPatcher, + ServerRunnerNotifyPatcher, + DispatcherHandleRequestPatcher, + DispatcherWriteResultPatcher, + DispatcherWriteErrorPatcher, + DispatcherNotifyPatcher, +) diff --git a/py/src/braintrust/integrations/mcp/test_mcp.py b/py/src/braintrust/integrations/mcp/test_mcp.py new file mode 100644 index 000000000..f7ebe9876 --- /dev/null +++ b/py/src/braintrust/integrations/mcp/test_mcp.py @@ -0,0 +1,459 @@ +"""MCP server instrumentation tests. + +The same tests run against the bundled server abstraction (``test_mcp`` +sessions: ``FastMCP`` on MCP 1.x, ``MCPServer`` on MCP 2.x) and standalone +``fastmcp`` (``test_fastmcp`` sessions). MCP traffic is always real: clients +talk to servers in-process, over streamable HTTP on localhost, or over stdio +to a server subprocess. Model calls made inside tools replay from cassettes. +""" + +import asyncio +import json + +import anyio +import pytest +from braintrust import Attachment, logger +from braintrust.conftest import get_vcr_config +from braintrust.integrations.mcp import MCPIntegration, wrap_mcp_server +from braintrust.integrations.mcp import _test_harness as harness +from braintrust.integrations.mcp.tracing import set_global_server_config +from braintrust.logger import start_span +from braintrust.test_helpers import find_span_by_name, init_test_logger + + +PROJECT_NAME = "test-mcp" +FLAVOR = harness.FLAVOR +INSTRUMENTATION = "mcp-auto" + + +@pytest.fixture +def memory_logger(): + init_test_logger(PROJECT_NAME) + with logger._internal_with_memory_background_logger() as bgl: + yield bgl + + +@pytest.fixture(autouse=True) +def reset_global_server_config(): + yield + set_global_server_config(None) + + +@pytest.fixture(scope="module") +def vcr_config(): + # MCP traffic to the localhost test server is real; only model calls replay. + return {**get_vcr_config(), "ignore_localhost": True} + + +def _mcp_spans(spans): + return [span for span in spans if "mcp.method.name" in (span.get("metadata") or {})] + + +def _assert_origin(span): + assert span["context"]["span_origin"]["instrumentation"]["name"] == INSTRUMENTATION + + +@pytest.mark.asyncio +async def test_request_spans(memory_logger): + server = wrap_mcp_server(harness.build_server()) + + async with harness.connect_in_memory(server) as client: + await client.list_tools() + await client.call_tool("add", {"a": 2, "b": 3}) + await client.call_tool("fail") + + spans = memory_logger.pop() + span = find_span_by_name(spans, "tools/call add") + assert span["span_attributes"]["type"] == "tool" + assert span["input"] == {"a": 2, "b": 3} + assert span["output"]["content"] == [{"type": "text", "text": "5"}] + assert span["output"]["isError"] is False + metadata = span["metadata"] + assert metadata["mcp.method.name"] == "tools/call" + assert metadata["gen_ai.tool.name"] == "add" + assert metadata["gen_ai.operation.name"] == "execute_tool" + assert isinstance(metadata["jsonrpc.request.id"], str) + assert metadata["mcp.protocol.version"] + assert metadata["mcp.client.name"] + assert "error.type" not in metadata + _assert_origin(span) + + nested = find_span_by_name(spans, "nested-work") + assert nested["span_parents"] == [span["span_id"]] + assert nested["root_span_id"] == span["root_span_id"] + # User spans do not inherit the integration's provenance. + assert nested["context"]["span_origin"]["instrumentation"]["name"] != INSTRUMENTATION + + # A returned tool error fails the span without a JSON-RPC status code. + failed = find_span_by_name(spans, "tools/call fail") + assert failed["metadata"]["error.type"] == "tool_error" + assert "rpc.response.status_code" not in failed["metadata"] + assert failed["output"]["isError"] is True + assert failed["error"] == failed["output"]["content"][0]["text"] + + # Initialization and discovery requests get method-named function spans + # without payloads. + discovery = [ + s + for s in _mcp_spans(spans) + if s["metadata"]["mcp.method.name"] in ("initialize", "server/discover", "tools/list") + ] + assert {s["metadata"]["mcp.method.name"] for s in discovery} & {"initialize", "server/discover"} + assert "tools/list" in {s["span_attributes"]["name"] for s in discovery} + for span in discovery: + assert span["span_attributes"]["type"] == "function" + assert span.get("input") is None + assert span.get("output") is None + + +def _missing_resource_status_code(): + if harness.MCP_V2: + return "-32602" + if FLAVOR == "fastmcp" and harness.FASTMCP_MAJOR >= 3: + return "-32002" + # MCP 1.x FastMCP and fastmcp 2 report unknown resources as handler failures. + return "0" + + +@pytest.mark.asyncio +async def test_prompt_and_resource_spans(memory_logger): + server = wrap_mcp_server(harness.build_server()) + + async with harness.connect_in_memory(server) as client: + await client.get_prompt("greet", {"name": "Ada"}) + await client.read_resource("resource://greeting") + with pytest.raises(Exception): + await client.read_resource("resource://missing") + + spans = memory_logger.pop() + prompt = find_span_by_name(spans, "prompts/get greet") + assert prompt["span_attributes"]["type"] == "function" + assert prompt["input"] == {"name": "Ada"} + assert prompt["output"]["messages"][0]["role"] == "user" + assert prompt["output"]["messages"][0]["content"] == {"type": "text", "text": "Say hello to Ada."} + assert prompt["metadata"]["gen_ai.prompt.name"] == "greet" + assert "gen_ai.operation.name" not in prompt["metadata"] + + reads = {s["input"]["uri"]: s for s in spans if s["span_attributes"]["name"] == "resources/read"} + resource = reads["resource://greeting"] + assert resource["span_attributes"]["type"] == "function" + assert resource["metadata"]["mcp.resource.uri"] == "resource://greeting" + assert resource["output"]["contents"][0]["text"] == "Hello from a resource" + # Resource URIs stay out of span names. + assert all("resource://" not in s["span_attributes"]["name"] for s in spans) + + missing = reads["resource://missing"] + code = _missing_resource_status_code() + assert missing["metadata"]["rpc.response.status_code"] == code + if code == "0": + assert missing["metadata"]["error.type"] == "0" + assert missing["error"] + else: + # Caller errors do not mark the server span as failed. + assert "error.type" not in missing["metadata"] + assert not missing.get("error") + assert missing.get("output") is None + + +@pytest.mark.asyncio +async def test_inline_media_becomes_attachments(memory_logger): + server = wrap_mcp_server(harness.build_server()) + + async with harness.connect_in_memory(server) as client: + result = await client.call_tool("image") + await client.read_resource("resource://pixel.png") + + # The application still sees the original base64 data. + assert result.content[0].data == harness.PNG_BASE64 + + spans = memory_logger.pop() + image_block = find_span_by_name(spans, "tools/call image")["output"]["content"][0] + assert image_block["type"] == "image" + assert image_block["mimeType"] == "image/png" + assert isinstance(image_block["data"], Attachment) + assert image_block["data"].reference["content_type"] == "image/png" + + resource = find_span_by_name(spans, "resources/read")["output"]["contents"][0] + assert resource["uri"] == "resource://pixel.png" + assert isinstance(resource["blob"], Attachment) + assert resource["blob"].reference["filename"] == "pixel.png" + + +@pytest.mark.asyncio +async def test_capture_content_disabled(memory_logger): + server = wrap_mcp_server(harness.build_server(), capture_content=False) + + async with harness.connect_in_memory(server) as client: + await client.call_tool("add", {"a": 1, "b": 1}) + await client.call_tool("fail") + await client.read_resource("resource://greeting") + + spans = memory_logger.pop() + for span in _mcp_spans(spans): + assert span.get("input") is None, span["span_attributes"]["name"] + assert span.get("output") is None, span["span_attributes"]["name"] + failed = find_span_by_name(spans, "tools/call fail") + assert failed["error"] == "Tool returned an error result" + assert failed["metadata"]["error.type"] == "tool_error" + # Identity, timing, and allowlisted metadata stay available. + assert find_span_by_name(spans, "resources/read")["metadata"]["mcp.resource.uri"] == "resource://greeting" + + +@pytest.mark.asyncio +async def test_outgoing_notification_is_child_of_request(memory_logger): + server = wrap_mcp_server(harness.build_server()) + + async with harness.connect_in_memory(server) as client: + await client.call_tool("notify") + + spans = memory_logger.pop() + tool = find_span_by_name(spans, "tools/call notify") + notification = find_span_by_name(spans, "notifications/message") + assert notification["span_parents"] == [tool["span_id"]] + assert notification["span_attributes"]["type"] == "function" + assert notification["metadata"]["mcp.method.name"] == "notifications/message" + assert notification["output"]["level"] == "info" + assert "jsonrpc.request.id" not in notification["metadata"] + _assert_origin(notification) + + +@pytest.mark.asyncio +async def test_incoming_trace_context_sets_parent(memory_logger): + server = wrap_mcp_server(harness.build_server()) + with start_span(name="remote-caller") as remote: + carrier = remote.inject({}) + + async with harness.connect_in_memory(server) as client: + await client.call_tool("add", {"a": 1, "b": 2}, meta=dict(carrier)) + # Invalid context, under an ambient caller span. + with start_span(name="test-caller") as caller: + await client.call_tool("add", {"a": 1, "b": 2}, meta={"traceparent": "00-zz-bad-01"}) + + spans = memory_logger.pop() + linked, fresh = [s for s in spans if s["span_attributes"]["name"] == "tools/call add"] + assert linked["root_span_id"] == remote.root_span_id + assert linked["span_parents"] == [remote.span_id] + # Propagation fields are never copied into the span. + assert not any(key in linked["metadata"] for key in ("traceparent", "baggage", "tracestate", "_meta")) + # Invalid context starts a new trace and does not break the request; the + # ambient span is never the parent. + assert fresh["root_span_id"] not in (remote.root_span_id, caller.root_span_id) + assert not fresh.get("span_parents") + + +@pytest.mark.asyncio +async def test_concurrent_requests_keep_separate_contexts(memory_logger): + server = wrap_mcp_server(harness.build_server()) + + async with harness.connect_in_memory(server) as client: + await asyncio.gather(*(client.call_tool("add", {"a": i, "b": i}) for i in range(5))) + + spans = memory_logger.pop() + tools = {s["span_id"]: s for s in spans if s["span_attributes"]["name"] == "tools/call add"} + assert len(tools) == 5 + assert len({s["metadata"]["jsonrpc.request.id"] for s in tools.values()}) == 5 + for nested in (s for s in spans if s["span_attributes"]["name"] == "nested-work"): + (parent_id,) = nested["span_parents"] + assert tools[parent_id]["input"] == nested["input"] + + +@pytest.mark.asyncio +async def test_repeated_instrumentation_does_not_duplicate_spans(memory_logger): + server = harness.build_server() + wrap_mcp_server(server) + wrap_mcp_server(server) + wrap_mcp_server(harness.lowlevel_server(server)) + assert MCPIntegration.setup() + assert MCPIntegration.setup() + + async with harness.connect_in_memory(server) as client: + await client.call_tool("add", {"a": 1, "b": 2}) + await client.call_tool("add", {"a": 1, "b": 2}) + + spans = memory_logger.pop() + tool_spans = [s for s in spans if s["span_attributes"]["name"] == "tools/call add"] + # Separate requests have distinct spans, and no request is traced twice. + assert len(tool_spans) == 2 + assert len({s["span_id"] for s in tool_spans}) == 2 + + +@pytest.mark.asyncio +async def test_only_wrapped_servers_are_traced_until_setup(memory_logger): + wrap_mcp_server(harness.build_server("other")) + server = harness.build_server() + + async with harness.connect_in_memory(server) as client: + await client.call_tool("add", {"a": 1, "b": 2}) + spans = memory_logger.pop() + assert not _mcp_spans(spans) + assert [s["span_attributes"]["name"] for s in spans] == ["nested-work"] + + assert MCPIntegration.setup() + async with harness.connect_in_memory(server) as client: + await client.call_tool("add", {"a": 1, "b": 2}) + _assert_origin(find_span_by_name(memory_logger.pop(), "tools/call add")) + + +@pytest.mark.asyncio +async def test_cancelled_request(memory_logger): + server = wrap_mcp_server(harness.build_server()) + harness.cancel_tool_started = anyio.Event() + + async with harness.connect_in_memory(server) as client: + if not harness.MCP_V2: + # MCP 1.x clients do not send notifications/cancelled on their own. + from mcp import types + + session = client.raw if FLAVOR == "mcp" else client.raw.session + + async def cancel_when_started(): + await harness.cancel_tool_started.wait() + await session.send_notification( + types.ClientNotification( # pylint: disable=not-callable # MCP 1.x root model + types.CancelledNotification( + params=types.CancelledNotificationParams(requestId=int(harness.cancel_tool_request_id)) + ) + ) + ) + + async with anyio.create_task_group() as tg: + tg.start_soon(cancel_when_started) + with pytest.raises(Exception): + await client.call_tool("wait_for_cancel") + else: + # The client sends notifications/cancelled when its call is cancelled. + with anyio.move_on_after(10): + async with anyio.create_task_group() as tg: + tg.start_soon(client.call_tool, "wait_for_cancel") + await harness.cancel_tool_started.wait() + tg.cancel_scope.cancel() + await client.call_tool("add", {"a": 1, "b": 1}) + + spans = memory_logger.pop() + cancelled = find_span_by_name(spans, "tools/call wait_for_cancel") + assert cancelled["metadata"]["mcp.cancelled"] is True + assert "error.type" not in cancelled["metadata"] + assert not cancelled.get("error") + assert cancelled.get("output") is None + + notification = find_span_by_name(spans, "notifications/cancelled") + assert notification["span_id"] != cancelled["span_id"] + assert "mcp.cancelled" not in notification["metadata"] + assert "reason" not in (notification.get("input") or {}) + # Requests after the cancellation are unaffected. + assert "mcp.cancelled" not in find_span_by_name(spans, "tools/call add")["metadata"] + + +@pytest.mark.asyncio +async def test_disconnect_ends_pending_request_with_error(memory_logger): + server = wrap_mcp_server(harness.build_server()) + harness.cancel_tool_started = anyio.Event() + + async def call(client): + with pytest.raises(BaseException): + await client.call_tool("wait_for_cancel") + + with anyio.move_on_after(10): + async with anyio.create_task_group() as tg: + async with harness.connect_in_memory(server) as client: + tg.start_soon(call, client) + await harness.cancel_tool_started.wait() + # The connection closed under the pending request. + tg.cancel_scope.cancel() + + spans = memory_logger.pop() + span = find_span_by_name(spans, "tools/call wait_for_cancel") + assert span["error"] + assert span["metadata"]["error.type"] + assert "mcp.cancelled" not in span["metadata"] + + +@pytest.mark.vcr(cassette_name=f"{FLAVOR}_test_http_tool_with_model_call") +@pytest.mark.asyncio +async def test_http_tool_with_model_call(memory_logger): + server = wrap_mcp_server(harness.build_server()) + + async with harness.serve_http(harness.http_app(server)) as url: + async with harness.connect_http(url) as client: + await client.call_tool("ask_llm", {"question": "Reply with the single word: pong"}) + + spans = memory_logger.pop() + tool = find_span_by_name(spans, "tools/call ask_llm") + llm = find_span_by_name(spans, "anthropic.messages.create") + assert llm["span_parents"] == [tool["span_id"]] + assert llm["span_attributes"]["type"] == "llm" + assert llm["metrics"]["tokens"] > 0 + # MCP spans never invent model fields. + assert "model" not in tool["metadata"] + assert "tokens" not in tool["metrics"] + assert tool["output"]["content"][0]["type"] == "text" + + metadata = tool["metadata"] + assert metadata["network.protocol.name"] == "http" + assert metadata["network.protocol.version"] == "1.1" + assert metadata["network.transport"] == "tcp" + assert metadata["client.address"] == "127.0.0.1" + assert isinstance(metadata["client.port"], int) + if "mcp.session.id" in metadata: + assert isinstance(metadata["mcp.session.id"], str) + _assert_origin(tool) + + +@pytest.mark.parametrize("wrap", ["wrap", "setup"]) +@pytest.mark.asyncio +async def test_stdio_subprocess(tmp_path, request, wrap): + spans_file = tmp_path / "spans.json" + record_mode = request.config.getoption("--vcr-record", default=None) + if request.config.getoption("--disable-vcr", default=False): + record_mode = "disabled" + env = { + harness.SPANS_FILE_ENV: str(spans_file), + harness.CASSETTE_ENV: f"{FLAVOR}_test_stdio_subprocess", + harness.WRAP_ENV: wrap, + } + if record_mode: + env[harness.RECORD_MODE_ENV] = record_mode + + async with harness.connect_stdio(env) as client: + await client.call_tool("add", {"a": 2, "b": 2}) + await client.call_tool("ask_llm", {"question": "Reply with the single word: pong"}) + await client.call_tool("notify") + + with anyio.fail_after(10): + while not spans_file.exists(): + await anyio.sleep(0.05) + spans = json.loads(spans_file.read_text()) + + add = find_span_by_name(spans, "tools/call add") + assert add["input"] == {"a": 2, "b": 2} + assert find_span_by_name(spans, "nested-work")["span_parents"] == [add["span_id"]] + ask = find_span_by_name(spans, "tools/call ask_llm") + assert find_span_by_name(spans, "anthropic.messages.create")["span_parents"] == [ask["span_id"]] + assert find_span_by_name(spans, "notifications/message")["span_parents"] == [ + find_span_by_name(spans, "tools/call notify")["span_id"] + ] + for span in _mcp_spans(spans): + assert "network.protocol.name" not in span["metadata"] + _assert_origin(span) + initialized = [s for s in spans if s["span_attributes"]["name"] == "notifications/initialized"] + if initialized: + assert "jsonrpc.request.id" not in initialized[0]["metadata"] + + +@pytest.mark.skipif(FLAVOR != "mcp" or not harness.MCP_V2, reason="MCP 2.x in-process modern protocol path") +@pytest.mark.asyncio +async def test_modern_in_process_requests(memory_logger): + # Modern in-process clients dispatch directly, without a JSON-RPC + # dispatcher, so operations end when the runner returns. + server = wrap_mcp_server(harness.build_server()) + + async with harness.connect_in_memory(server, mode="auto") as client: + await client.call_tool("add", {"a": 1, "b": 2}) + await client.call_tool("fail") + + spans = memory_logger.pop() + add = find_span_by_name(spans, "tools/call add") + assert find_span_by_name(spans, "nested-work")["span_parents"] == [add["span_id"]] + assert add["metadata"]["mcp.protocol.version"] >= "2026" + assert add["metadata"]["mcp.server.name"] == "braintrust-mcp-test" + assert find_span_by_name(spans, "tools/call fail")["metadata"]["error.type"] == "tool_error" diff --git a/py/src/braintrust/integrations/mcp/tracing.py b/py/src/braintrust/integrations/mcp/tracing.py new file mode 100644 index 000000000..170e29b57 --- /dev/null +++ b/py/src/braintrust/integrations/mcp/tracing.py @@ -0,0 +1,530 @@ +"""Version-independent span shaping for MCP server operations. + +The version-specific lifecycle hooks (``_server_v1`` / ``_server_v2``) observe +when a request arrives, which result or error is sent, and when sending +finishes. They feed plain JSON-RPC values (method, params dict, result dict, +error code) into the helpers here, so span names, payloads, metadata, and +error classification are identical across MCP 1.x and 2.x. +""" + +import contextlib +import contextvars +import logging +from collections.abc import Iterator, Mapping +from dataclasses import dataclass +from typing import Any +from urllib.parse import urlsplit + +from braintrust import logger as _bt_logger +from braintrust.context import BraintrustContextManager +from braintrust.integrations.utils import _resolved_attachment_from_base64 +from braintrust.logger import NOOP_SPAN, Span, extract_trace_context, start_span +from braintrust.span_types import SpanTypeAttribute + + +logger = logging.getLogger(__name__) + +_INSTRUMENTATION = "mcp-auto" +_INTERNAL = {"instrumentation": _INSTRUMENTATION} + +# OTel MCP server conventions classify these JSON-RPC errors as caller errors. +_CALLER_ERROR_CODES = frozenset({-32700, -32600, -32601, -32602, -32002}) +_CONTENT_METHODS = frozenset({"tools/call", "prompts/get", "resources/read"}) +# Result keys whose items are protocol content (may carry ``_meta`` or inline media). +_CONTENT_KEYS = ("content", "contents", "messages") +_RESOURCE_URI_METHODS = frozenset( + {"resources/read", "resources/subscribe", "resources/unsubscribe", "notifications/resources/updated"} +) +_TRACE_CONTEXT_KEYS = ("traceparent", "tracestate", "baggage") +_PROTOCOL_VERSION_META_KEY = "io.modelcontextprotocol/protocolVersion" +_CLIENT_INFO_META_KEY = "io.modelcontextprotocol/clientInfo" +_SERVER_INFO_META_KEY = "io.modelcontextprotocol/serverInfo" +_RELATED_TASK_META_KEY = "io.modelcontextprotocol/related-task" +_TOOL_ERROR_FALLBACK = "Tool returned an error result" +_CONNECTION_CLOSED_MESSAGE = "MCP connection closed before the response completed" +_MAX_CONTENT_DEPTH = 128 +_DROP = object() + + +@dataclass(frozen=True) +class MCPServerConfig: + """Per-server tracing settings.""" + + capture_content: bool = True + + +# Per-server config set by ``wrap_mcp_server`` on the low-level server. +SERVER_CONFIG_ATTR = "__braintrust_mcp_server_config__" + +_global_server_config: MCPServerConfig | None = None + + +def set_global_server_config(config: MCPServerConfig | None) -> None: + global _global_server_config + _global_server_config = config + + +def global_server_config() -> MCPServerConfig | None: + return _global_server_config + + +def server_config(server: Any) -> MCPServerConfig | None: + """The config ``wrap_mcp_server`` set on *server*, else the global one.""" + config = getattr(server, SERVER_CONFIG_ATTR, None) + return config if isinstance(config, MCPServerConfig) else _global_server_config + + +class ConnectionClosedError(ConnectionError): + """Recorded when a connection ends while an operation is still pending.""" + + def __init__(self) -> None: + super().__init__(_CONNECTION_CLOSED_MESSAGE) + + +def contain(callback: Any, *args: Any) -> Any: + """Run instrumentation code without letting its failures reach MCP.""" + try: + return callback(*args) + except Exception: + logger.debug("MCP instrumentation failed", exc_info=True) + return None + + +def call_arg(args: Any, kwargs: Any, index: int, name: str) -> Any: + return args[index] if len(args) > index else kwargs.get(name) + + +@dataclass +class ServerOperation: + """Telemetry state for one incoming request or notification. + + The outcome is collected here and logged once, when the operation ends + after the response was sent. Only the span is shared with the + application, through the current-span context. + """ + + span: Span + method: str + request_id: str | int | None + capture_content: bool + ended: bool = False + cancelled: bool = False + result: Mapping[str, Any] | None = None + status_code: str | None = None + failure: Any = None + error_type: str | None = None + + def record_result(self, result: Any) -> None: + """Keep the JSON-RPC result; it is shaped when the operation ends.""" + if not self.ended and isinstance(result, Mapping): + self.result = result + + def record_error_response(self, code: Any, message: Any) -> None: + """Classify a JSON-RPC error response.""" + if self.ended: + return + self.status_code = str(code) + if self.cancelled: + return + # The response classifies the outcome, replacing a handler exception + # the library already converted into this error. + self.failure = None + self.error_type = None + if isinstance(code, int) and code in _CALLER_ERROR_CODES: + return + self.failure = f"MCP error {code}: {message}" if isinstance(message, str) and message else f"MCP error {code}" + self.error_type = str(code) + + def record_exception(self, error: BaseException) -> None: + """Record a failure that may not be followed by an error response.""" + if not self.ended and self.failure is None: + self.failure = error + self.error_type = type(error).__name__ + + def mark_cancelled(self) -> None: + if not self.ended: + self.cancelled = True + + def finish(self, error: BaseException | None = None) -> None: + """Log the outcome and end the span once; later calls are ignored.""" + if self.ended: + return + self.ended = True + event: dict[str, Any] = {} + metadata: dict[str, Any] = {} + result = self.result + if result is not None: + metadata.update(protocol_meta_metadata(result.get("_meta"), "response")) + if self.method == "initialize": + if isinstance(result.get("protocolVersion"), str): + metadata["mcp.protocol.version"] = result["protocolVersion"] + metadata.update(identity_metadata(result.get("serverInfo"), "server")) + if self.capture_content and self.method in _CONTENT_METHODS: + event["output"] = sanitize_result(result) + if self.method == "tools/call" and result.get("isError") is True and self.failure is None: + self.failure = tool_error_message(result, self.capture_content) + self.error_type = "tool_error" + if self.status_code is not None: + metadata["rpc.response.status_code"] = self.status_code + if error is not None and self.failure is None: + self.failure = error + self.error_type = type(error).__name__ + if self.cancelled: + metadata["mcp.cancelled"] = True + elif self.failure is not None: + event["error"] = self.failure + metadata["error.type"] = self.error_type or "_OTHER" + if metadata: + event["metadata"] = metadata + if event: + self.span.log(**event) + self.span.end() + + +_active_operation: contextvars.ContextVar[ServerOperation | None] = contextvars.ContextVar( + "braintrust_mcp_server_operation", default=None +) + + +@contextlib.contextmanager +def activate(operation: ServerOperation) -> Iterator[None]: + """Make *operation* the current MCP operation and Braintrust span.""" + operation_token = _active_operation.set(operation) + span_token = _bt_logger._state.context_manager.set_current_span(operation.span) + try: + yield + finally: + _bt_logger._state.context_manager.unset_current_span(span_token) + _active_operation.reset(operation_token) + + +async def run_operation(operation: ServerOperation, wrapped: Any, args: Any, kwargs: Any) -> Any: + """Run *wrapped* as *operation* and end the operation when it returns.""" + try: + with activate(operation): + result = await wrapped(*args, **kwargs) + except BaseException as error: + operation.finish(error) + raise + operation.finish() + return result + + +def related_operation(request_id: Any) -> ServerOperation | None: + """The running request an outgoing notification belongs to. + + Notifications are sent from the request's handler, where it is the active + operation. Libraries may re-expose the ID as a string (``ctx.request_id``). + """ + operation = _active_operation.get() + if operation is None or operation.ended or request_id is None: + return None + if operation.request_id == request_id or str(operation.request_id) == str(request_id): + return operation + return None + + +def _meta_of(params: Mapping[str, Any]) -> Mapping[str, Any]: + meta = params.get("_meta") + return meta if isinstance(meta, Mapping) else {} + + +def _is_task_message(method: str, params: Mapping[str, Any], meta: Mapping[str, Any]) -> bool: + """Task submission and polling have a separate lifecycle and are not traced.""" + return ( + method.startswith("tasks/") or params.get("task") is not None or meta.get(_RELATED_TASK_META_KEY) is not None + ) + + +def start_server_operation( + *, + method: str, + params: Any, + request_id: Any, + config: MCPServerConfig, + session_id: str | None = None, + protocol_version: str | None = None, + client_info: Any = None, + transport_metadata: Mapping[str, Any] | None = None, + jsonrpc_version: Any = "2.0", +) -> ServerOperation | None: + """Start the span for one incoming request (or notification when *request_id* is None).""" + if not isinstance(method, str) or (request_id is not None and not _is_request_id(request_id)): + return None + params = params if isinstance(params, Mapping) else {} + meta = _meta_of(params) + if _is_task_message(method, params, meta): + return None + is_notification = request_id is None + is_tool_call = method == "tools/call" and not is_notification + target = params.get("name") if method in ("tools/call", "prompts/get") and not is_notification else None + target = target if isinstance(target, str) and target else None + + metadata: dict[str, Any] = {"mcp.method.name": method} + if not is_notification: + metadata["jsonrpc.request.id"] = str(request_id) + if session_id: + metadata["mcp.session.id"] = session_id + if is_tool_call: + metadata["gen_ai.operation.name"] = "execute_tool" + if target: + metadata["gen_ai.tool.name"] = target + if method == "prompts/get" and target: + metadata["gen_ai.prompt.name"] = target + if method in _RESOURCE_URI_METHODS and isinstance(params.get("uri"), str): + metadata["mcp.resource.uri"] = params["uri"] + if protocol_version: + metadata["mcp.protocol.version"] = protocol_version + if isinstance(jsonrpc_version, str) and jsonrpc_version != "2.0": + metadata["jsonrpc.protocol.version"] = jsonrpc_version + if transport_metadata: + metadata.update(transport_metadata) + metadata.update(identity_metadata(client_info, "client")) + if method == "initialize": + metadata.update(identity_metadata(params.get("clientInfo"), "client")) + if isinstance(params.get("protocolVersion"), str): + metadata["mcp.protocol.version"] = params["protocolVersion"] + metadata.update(protocol_meta_metadata(meta, "request")) + + event: dict[str, Any] = {"metadata": metadata} + if config.capture_content: + payload = _operation_input(method, params, is_notification) + if payload is not None: + event["input"] = payload + + span = start_detached_span( + name=f"{method} {target}" if target else method, + type=SpanTypeAttribute.TOOL if is_tool_call else SpanTypeAttribute.FUNCTION, + parent=extract_trace_context(_trace_carrier(meta)), + **event, + ) + return ServerOperation(span=span, method=method, request_id=request_id, capture_content=config.capture_content) + + +def start_outgoing_notification( + *, method: Any, params: Any, config: MCPServerConfig, related: ServerOperation | None +) -> Span | None: + """Start the span for a notification the server sends.""" + if not isinstance(method, str): + return None + params = params if isinstance(params, Mapping) else {} + meta = _meta_of(params) + if _is_task_message(method, params, meta): + return None + metadata: dict[str, Any] = {"mcp.method.name": method} + if method in _RESOURCE_URI_METHODS and isinstance(params.get("uri"), str): + metadata["mcp.resource.uri"] = params["uri"] + event: dict[str, Any] = {"metadata": metadata} + if config.capture_content: + payload = notification_content(method, params) + if payload: + event["output"] = payload + if related is not None and not related.ended: + return related.span.start_span(name=method, type=SpanTypeAttribute.FUNCTION, internal=_INTERNAL, **event) + return start_detached_span( + name=method, + type=SpanTypeAttribute.FUNCTION, + parent=extract_trace_context(_trace_carrier(meta)), + **event, + ) + + +def end_outgoing_notification(span: Span, error: BaseException | None = None) -> None: + if error is not None: + span.log(error=error, metadata={"error.type": type(error).__name__}) + span.end() + + +def start_detached_span(*, parent: Any, **kwargs: Any) -> Span: + """Start a span whose parent is only the propagated MCP context. + + One transport request can carry several MCP operations, so the ambient + context (an HTTP server span, or the caller's span for in-process servers) + must not become the parent. Without propagated context this starts a new + trace under the active logger or experiment. + """ + state = _bt_logger._state + context_manager = state.context_manager + otel_context = None + if isinstance(context_manager, BraintrustContextManager): + span_token = context_manager.set_current_span(None) + else: + from opentelemetry import context as otel_context + + span_token = otel_context.attach(otel_context.Context()) + parent_token = state.current_parent.set(None) + try: + return start_span(parent=parent, set_current=False, internal=_INTERNAL, **kwargs) + except Exception: + logger.debug("Failed to start MCP span", exc_info=True) + return NOOP_SPAN + finally: + state.current_parent.reset(parent_token) + if otel_context is not None: + otel_context.detach(span_token) + else: + context_manager.unset_current_span(span_token) + + +def _trace_carrier(meta: Mapping[str, Any]) -> dict[str, str]: + return {key: meta[key] for key in _TRACE_CONTEXT_KEYS if isinstance(meta.get(key), str)} + + +def _is_request_id(value: Any) -> bool: + return isinstance(value, (str, int)) and not isinstance(value, bool) + + +def _operation_input(method: str, params: Mapping[str, Any], is_notification: bool) -> Any: + if is_notification: + return notification_content(method, params) or None + if method in ("tools/call", "prompts/get"): + # Arguments are application data; log them as received. + return params.get("arguments") + if method == "resources/read" and "uri" in params: + return {"uri": params["uri"]} + return None + + +def notification_content(method: str, params: Mapping[str, Any]) -> Mapping[str, Any]: + """Notification parameters without ``_meta`` or a free-form cancellation reason.""" + omit = ("_meta", "reason") if method == "notifications/cancelled" else ("_meta",) + if not any(key in params for key in omit): + return params + return {key: value for key, value in params.items() if key not in omit} + + +def identity_metadata(info: Any, side: str) -> dict[str, str]: + """Translate self-reported client or server identity.""" + if info is None: + return {} + result: dict[str, str] = {} + for key in ("name", "title", "version"): + value = info.get(key) if isinstance(info, Mapping) else getattr(info, key, None) + if isinstance(value, str): + result[f"mcp.{side}.{key}"] = value + return result + + +def protocol_meta_metadata(meta: Any, direction: str) -> dict[str, str]: + """Translate the allowlisted ``_meta`` keys; everything else is omitted.""" + if not isinstance(meta, Mapping): + return {} + metadata: dict[str, str] = {} + if direction == "request": + if isinstance(meta.get(_PROTOCOL_VERSION_META_KEY), str): + metadata["mcp.protocol.version"] = meta[_PROTOCOL_VERSION_META_KEY] + metadata.update(identity_metadata(meta.get(_CLIENT_INFO_META_KEY), "client")) + else: + metadata.update(identity_metadata(meta.get(_SERVER_INFO_META_KEY), "server")) + return metadata + + +def http_transport_metadata(request: Any) -> dict[str, Any]: + """Read network attributes from an ASGI request attached to a message.""" + scope = getattr(request, "scope", None) + if not isinstance(scope, Mapping) or scope.get("type") != "http": + return {} + metadata: dict[str, Any] = {"network.protocol.name": "http"} + http_version = scope.get("http_version") + if isinstance(http_version, str): + metadata["network.protocol.version"] = http_version + metadata["network.transport"] = "quic" if http_version.startswith("3") else "tcp" + client = scope.get("client") + if isinstance(client, (tuple, list)) and len(client) == 2 and isinstance(client[0], str): + metadata["client.address"] = client[0] + if isinstance(client[1], int): + metadata["client.port"] = client[1] + return metadata + + +def http_session_id(request: Any) -> str | None: + headers = getattr(request, "headers", None) + try: + value = headers.get("mcp-session-id") if headers is not None else None + except Exception: + return None + return value if isinstance(value, str) and value else None + + +def tool_error_message(result: Mapping[str, Any], capture_content: bool) -> str: + if capture_content and isinstance(result.get("content"), list): + texts = [ + block["text"] + for block in result["content"] + if isinstance(block, Mapping) and block.get("type") == "text" and isinstance(block.get("text"), str) + ] + if texts: + return "\n".join(texts) + return _TOOL_ERROR_FALLBACK + + +def sanitize_result(result: Mapping[str, Any]) -> Mapping[str, Any]: + """Prepare a tool, prompt, or resource result for logging without serializing it. + + Only the protocol content lists are walked; application data such as + ``structuredContent`` is passed through untouched. Braintrust serializes the + rest when the span is sent. + """ + changed: dict[str, Any] = {} + for key in _CONTENT_KEYS: + if key in result: + value = sanitize_content(result[key]) + if value is not result[key]: + changed[key] = value + if not changed and "_meta" not in result: + return result + return {key: changed.get(key, value) for key, value in result.items() if key != "_meta"} + + +def sanitize_content(value: Any, _depth: int = 0) -> Any: + """Drop ``_meta`` and replace inline base64 media with attachments in place. + + Copy-on-write: containers that need no change are returned as-is, and the + original value is never mutated. + """ + if _depth >= _MAX_CONTENT_DEPTH: + return value + if isinstance(value, list): + items = None + for index, item in enumerate(value): + new_item = sanitize_content(item, _depth + 1) + if new_item is not item and items is None: + items = value[:index] + if items is not None: + items.append(new_item) + return value if items is None else items + if not isinstance(value, Mapping): + return value + + media_key = None + if value.get("type") in ("image", "audio") and isinstance(value.get("data"), str): + media_key = "data" + elif isinstance(value.get("uri"), str) and isinstance(value.get("blob"), str): + media_key = "blob" + + copy: dict[str, Any] | None = None + for index, (key, item) in enumerate(value.items()): + if key == "_meta": + new_item: Any = _DROP + elif key == media_key: + new_item = _to_attachment(item, value.get("mimeType"), value.get("uri")) + else: + new_item = sanitize_content(item, _depth + 1) + if new_item is not item and copy is None: + copy = dict(list(value.items())[:index]) + if copy is not None and new_item is not _DROP: + copy[key] = new_item + return value if copy is None else copy + + +def _to_attachment(data: str, mime_type: Any, uri: Any) -> Any: + content_type = mime_type if isinstance(mime_type, str) and mime_type else "application/octet-stream" + filename = None + if isinstance(uri, str): + parts = urlsplit(uri) + filename = f"{parts.netloc}{parts.path}".rsplit("/", 1)[-1] or None + resolved = _resolved_attachment_from_base64( + data, content_type, filename=filename, prefix=content_type.split("/")[0] + ) + # Values that cannot be decoded are logged unchanged. + return resolved.attachment if resolved is not None else data From 46d74b4ffa188ffe77a1bb691fc0222672f49848 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Thu, 29 Oct 2026 15:06:09 -0400 Subject: [PATCH 2/5] fix(mcp): record raised 1.x handler errors and skip fastmcp 2 on 3.15 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit > 🤖 **This text was AI-generated.** ## Problem With MCP 1.x and `raise_exceptions=True`, a request handler exception propagates instead of becoming an error response. The server span never saw it: the request stayed pending until the session closed, so the span was labeled with the wrong error. ``` prompts/get broken error.type = ConnectionClosedError <- real error lost ``` Separately, `test_fastmcp(2.14.7)` failed on the Python 3.15 Windows CI job. fastmcp 2.x depends on `lupa`, which has no cp315 wheels, and its source build fails on Windows. ## Solution `Server._handle_message`'s wrapper now records the exception and ends the span before re-raising, if no response has been sent for that request: ``` prompts/get broken error.type = ValueError error = "...prompt exploded" ``` `nox_session_constraints.py` caps fastmcp releases below 3.0.0 at Python < 3.15. The constraints file can't express a Windows-only skip, so 2.14.7 also skips on 3.15 for Linux and macOS; it is still tested on 3.10–3.14. Addresses review feedback on #1024. Refs #1023 Co-Authored-By: Claude Opus 5.5 --- py/nox_session_constraints.py | 2 ++ .../braintrust/integrations/mcp/_server_v1.py | 9 ++++++++- .../integrations/mcp/_test_harness.py | 13 +++++++++++-- py/src/braintrust/integrations/mcp/test_mcp.py | 17 +++++++++++++++++ 4 files changed, 38 insertions(+), 3 deletions(-) diff --git a/py/nox_session_constraints.py b/py/nox_session_constraints.py index f6482d302..8c88c1fb3 100644 --- a/py/nox_session_constraints.py +++ b/py/nox_session_constraints.py @@ -30,6 +30,8 @@ "test_langchain": {"max_python_exclusive": "3.15"}, # ormsgpack "test_openrouter": {"versions": {"latest": {"max_python_exclusive": "3.15"}}}, # pydantic-core 2.41 "test_openai_ddtrace": {"max_python_exclusive": "3.15"}, # ddtrace requires Python < 3.15 + # fastmcp 2.x depends on lupa, which has no cp315 wheels and does not build on Windows. + "test_fastmcp": {"versions": {"not_latest_below": {"version": "3.0.0", "max_python_exclusive": "3.15"}}}, } diff --git a/py/src/braintrust/integrations/mcp/_server_v1.py b/py/src/braintrust/integrations/mcp/_server_v1.py index 82634a41b..99ab5fa0b 100644 --- a/py/src/braintrust/integrations/mcp/_server_v1.py +++ b/py/src/braintrust/integrations/mcp/_server_v1.py @@ -257,7 +257,14 @@ async def server_handle_message_wrapper(wrapped: Any, instance: Any, args: Any, return await wrapped(*args, **kwargs) # The write observer ends the operation once the response is sent. with activate(operation): - return await wrapped(*args, **kwargs) + try: + return await wrapped(*args, **kwargs) + except Exception as error: + # ``raise_exceptions=True`` propagates the handler error + # instead of sending a response. + if trace.pending.pop(message.request_id, None) is operation: + operation.finish(error) + raise operation = trace.forwarded_notifications.pop(id(message), None) if operation is None: return await wrapped(*args, **kwargs) diff --git a/py/src/braintrust/integrations/mcp/_test_harness.py b/py/src/braintrust/integrations/mcp/_test_harness.py index 5a991fcb5..e56c76084 100644 --- a/py/src/braintrust/integrations/mcp/_test_harness.py +++ b/py/src/braintrust/integrations/mcp/_test_harness.py @@ -119,6 +119,11 @@ def greet(name: str) -> str: """Greet someone.""" return f"Say hello to {name}." + @server.prompt() + def broken() -> str: + """Always fails to render.""" + raise RuntimeError("prompt exploded") + @server.resource("resource://greeting") def greeting() -> str: return "Hello from a resource" @@ -167,7 +172,9 @@ async def list_tools(self) -> Any: @contextlib.asynccontextmanager -async def connect_in_memory(server: Any, mode: str | None = None) -> AsyncIterator[Client]: +async def connect_in_memory( + server: Any, mode: str | None = None, raise_exceptions: bool = False +) -> AsyncIterator[Client]: if FLAVOR == "fastmcp": from fastmcp import Client as FastMCPClient @@ -183,7 +190,9 @@ async def connect_in_memory(server: Any, mode: str | None = None) -> AsyncIterat create_connected_server_and_client_session, ) - async with create_connected_server_and_client_session(lowlevel_server(server)) as session: + async with create_connected_server_and_client_session( + lowlevel_server(server), raise_exceptions=raise_exceptions + ) as session: yield Client(session) diff --git a/py/src/braintrust/integrations/mcp/test_mcp.py b/py/src/braintrust/integrations/mcp/test_mcp.py index f7ebe9876..43d09a706 100644 --- a/py/src/braintrust/integrations/mcp/test_mcp.py +++ b/py/src/braintrust/integrations/mcp/test_mcp.py @@ -368,6 +368,23 @@ async def call(client): assert "mcp.cancelled" not in span["metadata"] +@pytest.mark.skipif(FLAVOR != "mcp" or harness.MCP_V2, reason="MCP 1.x re-raises handler exceptions") +@pytest.mark.asyncio +async def test_raised_handler_exception_is_recorded(memory_logger): + # With raise_exceptions=True, MCP 1.x propagates a handler exception + # instead of sending an error response. + server = wrap_mcp_server(harness.build_server()) + + with pytest.raises(BaseException): + async with harness.connect_in_memory(server, raise_exceptions=True) as client: + await client.get_prompt("broken", {}) + + span = find_span_by_name(memory_logger.pop(), "prompts/get broken") + assert span["metadata"]["error.type"] == "ValueError" + assert "prompt exploded" in str(span["error"]) + assert "rpc.response.status_code" not in span["metadata"] + + @pytest.mark.vcr(cassette_name=f"{FLAVOR}_test_http_tool_with_model_call") @pytest.mark.asyncio async def test_http_tool_with_model_call(memory_logger): From 6e1c7bcc306532c6a7915066cceba855e7381106 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Thu, 29 Oct 2026 15:20:26 -0400 Subject: [PATCH 3/5] fix(mcp): apply re-wrapped settings to open mcp 1.x connections MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit > 🤖 **This text was AI-generated.** ## Problem On MCP 1.x, each connection kept the tracing settings it was opened with. Re-wrapping a server while a client was connected did not reach that client: ```python server = wrap_mcp_server(FastMCP("demo")) # ... client connects and calls a tool: input/output captured ... wrap_mcp_server(server, capture_content=False) # ... same client calls again: input/output still captured ``` MCP 2.x was not affected because it reads the server's settings on every request. ## Solution A 1.x connection now keeps a weak reference to its server and reads the server's current settings for each incoming message and outgoing notification, matching 2.x: ``` before: session ──► config (frozen at connect) after: session ──► weakref(server) ──► server's current config ``` Connections created while only `MCPIntegration.setup()` is active keep following the global setting, and a server started from inside another server's handler is still not attributed to the outer server. Addresses review feedback on #1024. Refs #1023 Co-Authored-By: Claude Opus 5.5 --- .../braintrust/integrations/mcp/_server_v1.py | 48 +++++++++++-------- .../braintrust/integrations/mcp/test_mcp.py | 15 ++++++ 2 files changed, 44 insertions(+), 19 deletions(-) diff --git a/py/src/braintrust/integrations/mcp/_server_v1.py b/py/src/braintrust/integrations/mcp/_server_v1.py index 99ab5fa0b..0d28a1080 100644 --- a/py/src/braintrust/integrations/mcp/_server_v1.py +++ b/py/src/braintrust/integrations/mcp/_server_v1.py @@ -35,14 +35,15 @@ http_transport_metadata, related_operation, run_operation, + server_config, start_outgoing_notification, start_server_operation, ) -# Set while a wrapped ``Server.run`` constructs its session. -_pending_server_config: contextvars.ContextVar[MCPServerConfig | None] = contextvars.ContextVar( - "braintrust_mcp_pending_server_config", default=None +# The wrapped server whose ``run`` is constructing a session. +_pending_server: contextvars.ContextVar["weakref.ref[Any] | None"] = contextvars.ContextVar( + "braintrust_mcp_pending_server", default=None ) _session_traces: "weakref.WeakKeyDictionary[Any, _SessionTrace]" = weakref.WeakKeyDictionary() @@ -60,14 +61,19 @@ def _mcp() -> Any: class _SessionTrace: """Pending operations for one server session (one MCP connection).""" - def __init__(self, session: Any, config: MCPServerConfig): + def __init__(self, session: Any, server_ref: "weakref.ref[Any] | None"): self._session_ref = weakref.ref(session) - self.config = config + self._server_ref = server_ref self.pending: dict[Any, ServerOperation] = {} self.forwarded_notifications: dict[int, ServerOperation] = {} self.inbound_notification: ServerOperation | None = None self.protocol_version: str | None = None + @property + def config(self) -> MCPServerConfig | None: + """The server's current settings, so re-wrapping applies to open connections.""" + return server_config(self._server_ref() if self._server_ref is not None else None) + def _session_attr(self, name: str) -> Any: session = self._session_ref() return getattr(session, name, None) if session is not None else None @@ -85,13 +91,16 @@ def on_receive(self, item: Any) -> None: if not is_request and not isinstance(message, types.JSONRPCNotification): # Responses to server-initiated requests are not operations. return + config = self.config + if config is None: + return request_context = getattr(item.metadata, "request_context", None) params = message.params operation = start_server_operation( method=message.method, params=params, request_id=message.id if is_request else None, - config=self.config, + config=config, session_id=http_session_id(request_context), protocol_version=self.protocol_version, client_info=getattr(self._session_attr("_client_params"), "clientInfo", None), @@ -133,12 +142,13 @@ def on_send(self, item: Any) -> tuple[ServerOperation | None, Any]: if operation is not None: operation.record_error_response(message.error.code, message.error.message) return operation, None - if isinstance(message, types.JSONRPCNotification): + config = self.config + if isinstance(message, types.JSONRPCNotification) and config is not None: related_id = getattr(item.metadata, "related_request_id", None) span = start_outgoing_notification( method=message.method, params=message.params, - config=self.config, + config=config, related=related_operation(related_id) or self.pending.get(related_id), ) return None, span @@ -221,13 +231,13 @@ async def send(self, item: Any) -> None: await self._stream.send(item) -def _install_session_observers(session: Any, config: MCPServerConfig) -> None: +def _install_session_observers(session: Any, server_ref: "weakref.ref[Any] | None") -> None: if session in _session_traces: return streams = ("_read_stream", "_write_stream", "_incoming_message_stream_writer") if not all(hasattr(session, name) for name in streams): return - trace = _SessionTrace(session, config) + trace = _SessionTrace(session, server_ref) session._read_stream = _ObservedReadStream(session._read_stream, trace) session._write_stream = _ObservedWriteStream(session._write_stream, trace) session._incoming_message_stream_writer = _ObservedIncomingWriter(session._incoming_message_stream_writer, trace) @@ -236,12 +246,12 @@ def _install_session_observers(session: Any, config: MCPServerConfig) -> None: def server_session_init_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any: result = wrapped(*args, **kwargs) - config = _pending_server_config.get() or global_server_config() - if config is not None: - # A session consumes the config of the run that created it, so a - # server started from inside a handler is not traced implicitly. - _pending_server_config.set(None) - contain(_install_session_observers, instance, config) + server_ref = _pending_server.get() + if server_ref is not None or global_server_config() is not None: + # A session belongs to the run that created it, so a server started + # from inside a handler is not attributed to the outer server. + _pending_server.set(None) + contain(_install_session_observers, instance, server_ref) return result @@ -277,9 +287,9 @@ async def server_run_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any Per instance rather than on ``Server.run``, because fastmcp 2/3 override ``run`` without calling it. """ - config = getattr(instance, SERVER_CONFIG_ATTR, None) - token = _pending_server_config.set(config if isinstance(config, MCPServerConfig) else None) + is_wrapped = isinstance(getattr(instance, SERVER_CONFIG_ATTR, None), MCPServerConfig) + token = _pending_server.set(weakref.ref(instance) if is_wrapped else None) try: return await wrapped(*args, **kwargs) finally: - _pending_server_config.reset(token) + _pending_server.reset(token) diff --git a/py/src/braintrust/integrations/mcp/test_mcp.py b/py/src/braintrust/integrations/mcp/test_mcp.py index 43d09a706..54d31833a 100644 --- a/py/src/braintrust/integrations/mcp/test_mcp.py +++ b/py/src/braintrust/integrations/mcp/test_mcp.py @@ -199,6 +199,21 @@ async def test_capture_content_disabled(memory_logger): assert find_span_by_name(spans, "resources/read")["metadata"]["mcp.resource.uri"] == "resource://greeting" +@pytest.mark.asyncio +async def test_rewrap_updates_open_connections(memory_logger): + server = wrap_mcp_server(harness.build_server()) + + async with harness.connect_in_memory(server) as client: + await client.call_tool("add", {"a": 1, "b": 1}) + wrap_mcp_server(server, capture_content=False) + await client.call_tool("add", {"a": 2, "b": 2}) + + captured, redacted = [s for s in memory_logger.pop() if s["span_attributes"]["name"] == "tools/call add"] + assert captured["input"] == {"a": 1, "b": 1} + assert redacted.get("input") is None + assert redacted.get("output") is None + + @pytest.mark.asyncio async def test_outgoing_notification_is_child_of_request(memory_logger): server = wrap_mcp_server(harness.build_server()) From f9d719bfdec5df206cd1afbe3d25d5bed3556d03 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Thu, 29 Oct 2026 16:39:00 -0400 Subject: [PATCH 4/5] fix(mcp): redact error text without content capture and refresh 2.x settings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit > 🤖 **This text was AI-generated.** ## Problem Two ways content still reached spans after `capture_content=False`: 1. JSON-RPC error messages and exception messages were copied into the span's `error`. They can echo application data: ``` error = "MCP error 0: Error rendering prompt broken: prompt exploded" ``` 2. On MCP 2.x, outgoing notifications used a cached copy of the settings from the connection's last request. Re-wrapping the server only took effect once another request arrived, so a notification sent in between logged its payload. ## Solution 1. With content capture off, error responses log only their code (`MCP error 0`) and exceptions log only their type name. `error.type` and `rpc.response.status_code` are unchanged. 2. Each 2.x dispatcher now keeps a weak reference to the server it serves, and notifications read that server's current settings: ``` before: dispatcher ──► config (from the last request) after: dispatcher ──► weakref(server) ──► server's current config ``` Addresses review feedback on #1024. Refs #1023 Co-Authored-By: Claude Opus 5.5 --- .../braintrust/integrations/mcp/_server_v2.py | 14 +++++++++----- .../integrations/mcp/_test_harness.py | 10 ++++++++++ py/src/braintrust/integrations/mcp/test_mcp.py | 17 +++++++++++++++++ py/src/braintrust/integrations/mcp/tracing.py | 8 ++++++-- 4 files changed, 42 insertions(+), 7 deletions(-) diff --git a/py/src/braintrust/integrations/mcp/_server_v2.py b/py/src/braintrust/integrations/mcp/_server_v2.py index c99b1806f..aabfabc93 100644 --- a/py/src/braintrust/integrations/mcp/_server_v2.py +++ b/py/src/braintrust/integrations/mcp/_server_v2.py @@ -50,8 +50,8 @@ class _DispatchSlot: "braintrust_mcp_dispatch_slot", default=None ) -# Dispatchers that serve a traced server, for outgoing notifications. -_server_dispatchers: "weakref.WeakKeyDictionary[Any, MCPServerConfig]" = weakref.WeakKeyDictionary() +# The server each dispatcher serves, so outgoing notifications use its current settings. +_server_dispatchers: "weakref.WeakKeyDictionary[Any, weakref.ref[Any]]" = weakref.WeakKeyDictionary() def _cancel_requested(dctx: Any) -> bool: @@ -106,8 +106,10 @@ async def runner_on_request_wrapper(wrapped: Any, instance: Any, args: Any, kwar dispatcher_owned = slot is not None if dispatcher_owned: slot.operation = operation - if _server_dispatchers.get(slot.dispatcher) is not config: - _server_dispatchers[slot.dispatcher] = config + server = getattr(instance, "server", None) + server_ref = _server_dispatchers.get(slot.dispatcher) + if server is not None and (server_ref is None or server_ref() is not server): + _server_dispatchers[slot.dispatcher] = weakref.ref(server) try: with activate(operation): result = await wrapped(*args, **kwargs) @@ -175,7 +177,9 @@ async def dispatcher_write_error_wrapper(wrapped: Any, instance: Any, args: Any, async def dispatcher_notify_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any: - config = _server_dispatchers.get(instance) + server_ref = _server_dispatchers.get(instance) + server = server_ref() if server_ref is not None else None + config = server_config(server) if server is not None else None span = None if config is not None: span = contain( diff --git a/py/src/braintrust/integrations/mcp/_test_harness.py b/py/src/braintrust/integrations/mcp/_test_harness.py index e56c76084..8a4e1452f 100644 --- a/py/src/braintrust/integrations/mcp/_test_harness.py +++ b/py/src/braintrust/integrations/mcp/_test_harness.py @@ -89,6 +89,16 @@ async def notify(ctx: context_class) -> str: # type: ignore[valid-type] server.tool()(notify) + async def notify_after_disabling_capture(ctx: context_class) -> str: # type: ignore[valid-type] + """Turn content capture off mid-request, then send a notification.""" + from braintrust.integrations.mcp import wrap_mcp_server + + wrap_mcp_server(server, capture_content=False) + await ctx.info("sensitive progress detail") + return "done" + + server.tool()(notify_after_disabling_capture) + async def wait_for_cancel(ctx: context_class) -> str: # type: ignore[valid-type] """Block until cancelled.""" global cancel_tool_request_id diff --git a/py/src/braintrust/integrations/mcp/test_mcp.py b/py/src/braintrust/integrations/mcp/test_mcp.py index 54d31833a..9a9b57ef0 100644 --- a/py/src/braintrust/integrations/mcp/test_mcp.py +++ b/py/src/braintrust/integrations/mcp/test_mcp.py @@ -187,6 +187,8 @@ async def test_capture_content_disabled(memory_logger): await client.call_tool("add", {"a": 1, "b": 1}) await client.call_tool("fail") await client.read_resource("resource://greeting") + with pytest.raises(Exception): + await client.get_prompt("broken", {}) spans = memory_logger.pop() for span in _mcp_spans(spans): @@ -195,6 +197,10 @@ async def test_capture_content_disabled(memory_logger): failed = find_span_by_name(spans, "tools/call fail") assert failed["error"] == "Tool returned an error result" assert failed["metadata"]["error.type"] == "tool_error" + # JSON-RPC and exception messages can echo application data. + broken = find_span_by_name(spans, "prompts/get broken") + assert broken["metadata"]["error.type"] + assert "exploded" not in str(broken["error"]) # Identity, timing, and allowlisted metadata stay available. assert find_span_by_name(spans, "resources/read")["metadata"]["mcp.resource.uri"] == "resource://greeting" @@ -214,6 +220,17 @@ async def test_rewrap_updates_open_connections(memory_logger): assert redacted.get("output") is None +@pytest.mark.asyncio +async def test_rewrap_applies_to_notifications_sent_mid_request(memory_logger): + server = wrap_mcp_server(harness.build_server()) + + async with harness.connect_in_memory(server) as client: + await client.call_tool("notify_after_disabling_capture") + + notification = find_span_by_name(memory_logger.pop(), "notifications/message") + assert notification.get("output") is None + + @pytest.mark.asyncio async def test_outgoing_notification_is_child_of_request(memory_logger): server = wrap_mcp_server(harness.build_server()) diff --git a/py/src/braintrust/integrations/mcp/tracing.py b/py/src/braintrust/integrations/mcp/tracing.py index 170e29b57..471508f78 100644 --- a/py/src/braintrust/integrations/mcp/tracing.py +++ b/py/src/braintrust/integrations/mcp/tracing.py @@ -132,7 +132,9 @@ def record_error_response(self, code: Any, message: Any) -> None: self.error_type = None if isinstance(code, int) and code in _CALLER_ERROR_CODES: return - self.failure = f"MCP error {code}: {message}" if isinstance(message, str) and message else f"MCP error {code}" + # Error messages can echo application data. + detail = message if self.capture_content and isinstance(message, str) and message else None + self.failure = f"MCP error {code}: {detail}" if detail else f"MCP error {code}" self.error_type = str(code) def record_exception(self, error: BaseException) -> None: @@ -172,7 +174,9 @@ def finish(self, error: BaseException | None = None) -> None: if self.cancelled: metadata["mcp.cancelled"] = True elif self.failure is not None: - event["error"] = self.failure + # Without content capture, exceptions keep only their type. + redact = not self.capture_content and isinstance(self.failure, BaseException) + event["error"] = type(self.failure).__name__ if redact else self.failure metadata["error.type"] = self.error_type or "_OTHER" if metadata: event["metadata"] = metadata From 580f4430dbc8ee7819442a88b45d8e6813ee86e7 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Thu, 29 Oct 2026 17:05:26 -0400 Subject: [PATCH 5/5] test(mcp): drop cassettes and the nested anthropic call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit > 🤖 **This text was AI-generated.** ## Problem The MCP tests checked in 18 cassettes (one pair per mcp/fastmcp version), but none of them recorded MCP traffic. They only replayed an Anthropic call made inside a test tool, to show a nested LLM span under the MCP tool span: ``` tools/call ask_llm └─ anthropic.messages.create <- the only thing the cassettes replayed ``` That cost an `anthropic` install in every MCP session, an API key to re-record, and a new cassette pair for every version added to the matrix. ## Solution Remove the `ask_llm` tool, the cassettes, the VCR setup in the stdio subprocess, the `anthropic` session dependency, and the MCP entry in `[tool.braintrust.cassette-dirs]`. Nesting is still covered on every transport by the manual span the `add` tool creates: ``` tools/call add └─ nested-work <- in-process, streamable HTTP, and stdio ``` MCP traffic was never recorded and stays real: clients talk to servers in-process, over streamable HTTP on localhost, and over stdio to a real subprocess. ## Limitations There is no longer a check that another integration's spans (for example `wrap_anthropic`) nest under MCP tool spans. This also departs from #1023's request for cassette-backed tests, since the MCP tests no longer need any recorded traffic. Refs #1023 Co-Authored-By: Claude Opus 5.5 --- py/noxfile.py | 7 +- py/pyproject.toml | 2 - .../integrations/mcp/_test_harness.py | 37 ++---- .../mcp_test_http_tool_with_model_call.yaml | 109 ------------------ .../1.28.1/mcp_test_stdio_subprocess.yaml | 109 ------------------ .../mcp_test_http_tool_with_model_call.yaml | 109 ------------------ .../1.30.0/mcp_test_stdio_subprocess.yaml | 109 ------------------ .../mcp_test_http_tool_with_model_call.yaml | 109 ------------------ .../2.0.0/mcp_test_stdio_subprocess.yaml | 109 ------------------ ...astmcp_test_http_tool_with_model_call.yaml | 109 ------------------ .../2.14.7/fastmcp_test_stdio_subprocess.yaml | 109 ------------------ ...astmcp_test_http_tool_with_model_call.yaml | 109 ------------------ .../3.0.0/fastmcp_test_stdio_subprocess.yaml | 109 ------------------ ...astmcp_test_http_tool_with_model_call.yaml | 109 ------------------ .../3.4.11/fastmcp_test_stdio_subprocess.yaml | 109 ------------------ ...astmcp_test_http_tool_with_model_call.yaml | 109 ------------------ .../4.0.0/fastmcp_test_stdio_subprocess.yaml | 109 ------------------ ...astmcp_test_http_tool_with_model_call.yaml | 109 ------------------ .../latest/fastmcp_test_stdio_subprocess.yaml | 109 ------------------ .../mcp_test_http_tool_with_model_call.yaml | 109 ------------------ .../latest/mcp_test_stdio_subprocess.yaml | 109 ------------------ .../braintrust/integrations/mcp/test_mcp.py | 39 ++----- 22 files changed, 18 insertions(+), 2029 deletions(-) delete mode 100644 py/src/braintrust/integrations/mcp/cassettes/1.28.1/mcp_test_http_tool_with_model_call.yaml delete mode 100644 py/src/braintrust/integrations/mcp/cassettes/1.28.1/mcp_test_stdio_subprocess.yaml delete mode 100644 py/src/braintrust/integrations/mcp/cassettes/1.30.0/mcp_test_http_tool_with_model_call.yaml delete mode 100644 py/src/braintrust/integrations/mcp/cassettes/1.30.0/mcp_test_stdio_subprocess.yaml delete mode 100644 py/src/braintrust/integrations/mcp/cassettes/2.0.0/mcp_test_http_tool_with_model_call.yaml delete mode 100644 py/src/braintrust/integrations/mcp/cassettes/2.0.0/mcp_test_stdio_subprocess.yaml delete mode 100644 py/src/braintrust/integrations/mcp/cassettes/2.14.7/fastmcp_test_http_tool_with_model_call.yaml delete mode 100644 py/src/braintrust/integrations/mcp/cassettes/2.14.7/fastmcp_test_stdio_subprocess.yaml delete mode 100644 py/src/braintrust/integrations/mcp/cassettes/3.0.0/fastmcp_test_http_tool_with_model_call.yaml delete mode 100644 py/src/braintrust/integrations/mcp/cassettes/3.0.0/fastmcp_test_stdio_subprocess.yaml delete mode 100644 py/src/braintrust/integrations/mcp/cassettes/3.4.11/fastmcp_test_http_tool_with_model_call.yaml delete mode 100644 py/src/braintrust/integrations/mcp/cassettes/3.4.11/fastmcp_test_stdio_subprocess.yaml delete mode 100644 py/src/braintrust/integrations/mcp/cassettes/4.0.0/fastmcp_test_http_tool_with_model_call.yaml delete mode 100644 py/src/braintrust/integrations/mcp/cassettes/4.0.0/fastmcp_test_stdio_subprocess.yaml delete mode 100644 py/src/braintrust/integrations/mcp/cassettes/latest/fastmcp_test_http_tool_with_model_call.yaml delete mode 100644 py/src/braintrust/integrations/mcp/cassettes/latest/fastmcp_test_stdio_subprocess.yaml delete mode 100644 py/src/braintrust/integrations/mcp/cassettes/latest/mcp_test_http_tool_with_model_call.yaml delete mode 100644 py/src/braintrust/integrations/mcp/cassettes/latest/mcp_test_stdio_subprocess.yaml diff --git a/py/noxfile.py b/py/noxfile.py index 1be445e26..062c028ef 100644 --- a/py/noxfile.py +++ b/py/noxfile.py @@ -339,8 +339,7 @@ def run(session, version, spec=spec): "name": "test_mcp", "doc": "MCP server instrumentation on the bundled FastMCP (1.x) / MCPServer (2.x).", "versions": MCP_VERSIONS, - # Tools call Anthropic to cover nested model spans. - "dependencies": [("mcp", "$version"), ("anthropic", LATEST)], + "dependencies": [("mcp", "$version")], "tests": f"{INTEGRATION_DIR}/mcp/test_mcp.py", "env": {"BRAINTRUST_MCP_TEST_FLAVOR": "mcp"}, }, @@ -348,7 +347,7 @@ def run(session, version, spec=spec): "name": "test_fastmcp", "doc": "MCP server instrumentation on standalone FastMCP.", "versions": FASTMCP_VERSIONS, - "dependencies": [("fastmcp", "$version"), ("anthropic", LATEST)], + "dependencies": [("fastmcp", "$version")], "tests": f"{INTEGRATION_DIR}/mcp/test_mcp.py", "env": {"BRAINTRUST_MCP_TEST_FLAVOR": "fastmcp"}, }, @@ -356,7 +355,7 @@ def run(session, version, spec=spec): "name": "test_fastmcp_slim", "doc": "MCP server instrumentation on fastmcp-slim without the fastmcp meta-package.", "versions": FASTMCP_SLIM_VERSIONS, - "dependencies": [("fastmcp-slim", "$version"), ("anthropic", LATEST)], + "dependencies": [("fastmcp-slim", "$version")], "tests": f"{INTEGRATION_DIR}/mcp/test_mcp.py", "env": {"BRAINTRUST_MCP_TEST_FLAVOR": "fastmcp"}, }, diff --git a/py/pyproject.toml b/py/pyproject.toml index 6cf4c2439..11911885a 100644 --- a/py/pyproject.toml +++ b/py/pyproject.toml @@ -505,7 +505,6 @@ latest = "fastmcp==4.1.3" "3.0.0" = "fastmcp==3.0.0" "2.14.7" = "fastmcp==2.14.7" -# Same versions as fastmcp, so the sessions share cassette directories. [tool.braintrust.matrix.fastmcp-slim] latest = "fastmcp-slim[client,server]==4.1.3" "3.4.11" = "fastmcp-slim[client,server]==3.4.11" @@ -700,7 +699,6 @@ langchain = ["langchain-core", "deepagents"] litellm = ["litellm"] livekit_agents = ["livekit-agents"] llamaindex = ["llama-index-core"] -mcp = ["mcp", "fastmcp", "fastmcp-slim"] mistral = ["mistralai"] openai = ["openai"] openai_agents = ["openai-agents"] diff --git a/py/src/braintrust/integrations/mcp/_test_harness.py b/py/src/braintrust/integrations/mcp/_test_harness.py index 8a4e1452f..1d8252efe 100644 --- a/py/src/braintrust/integrations/mcp/_test_harness.py +++ b/py/src/braintrust/integrations/mcp/_test_harness.py @@ -6,9 +6,8 @@ localhost), or over stdio (a real server subprocess running this module). Run as ``python _test_harness.py stdio`` to serve the demo server over stdio; -the subprocess records its spans with the memory logger, replays nested model -calls from a cassette, and writes the spans to ``BRAINTRUST_MCP_TEST_SPANS_FILE`` -on exit. +the subprocess records its spans with the memory logger and writes them to +``BRAINTRUST_MCP_TEST_SPANS_FILE`` on exit. """ import contextlib @@ -33,11 +32,8 @@ ) MCP_V2 = version_satisfies(MCPIntegration.detect_version(mcp), _MCP_V2) FASTMCP_MAJOR = int(importlib.import_module("fastmcp").__version__.split(".")[0]) if FLAVOR == "fastmcp" else 0 -LLM_MODEL = "claude-haiku-4-5-20251001" PNG_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==" SPANS_FILE_ENV = "BRAINTRUST_MCP_TEST_SPANS_FILE" -CASSETTE_ENV = "BRAINTRUST_MCP_TEST_CASSETTE" -RECORD_MODE_ENV = "BRAINTRUST_MCP_TEST_RECORD_MODE" WRAP_ENV = "BRAINTRUST_MCP_TEST_WRAP" # Set by the ``wait_for_cancel`` tool so tests can cancel a running request. @@ -110,20 +106,6 @@ async def wait_for_cancel(ctx: context_class) -> str: # type: ignore[valid-type server.tool()(wait_for_cancel) - @server.tool() - async def ask_llm(question: str) -> str: - """Answer with a model call made inside the tool.""" - import anthropic - from braintrust.integrations.anthropic import wrap_anthropic - - client = wrap_anthropic(anthropic.AsyncAnthropic()) - response = await client.messages.create( - model=LLM_MODEL, - max_tokens=32, - messages=[{"role": "user", "content": question}], - ) - return response.content[0].text - @server.prompt() def greet(name: str) -> str: """Greet someone.""" @@ -282,17 +264,12 @@ async def serve_http(app: Any) -> AsyncIterator[str]: async def _serve_stdio() -> None: - from braintrust.conftest import get_vcr_config + from braintrust import logger from braintrust.integrations.mcp import wrap_mcp_server - from braintrust.integrations.test_utils import autoinstrument_test_context - - vcr_config = get_vcr_config() - record_mode = os.environ.get(RECORD_MODE_ENV) - if record_mode: - vcr_config["record_mode"] = record_mode - with autoinstrument_test_context( - os.environ[CASSETTE_ENV], integration="mcp", use_vcr=record_mode != "disabled", vcr_config=vcr_config - ) as memory_logger: + from braintrust.test_helpers import init_test_logger + + init_test_logger("test-mcp-stdio") + with logger._internal_with_memory_background_logger() as memory_logger: server = build_server() if os.environ.get(WRAP_ENV) == "setup": MCPIntegration.setup() diff --git a/py/src/braintrust/integrations/mcp/cassettes/1.28.1/mcp_test_http_tool_with_model_call.yaml b/py/src/braintrust/integrations/mcp/cassettes/1.28.1/mcp_test_http_tool_with_model_call.yaml deleted file mode 100644 index caed6040f..000000000 --- a/py/src/braintrust/integrations/mcp/cassettes/1.28.1/mcp_test_http_tool_with_model_call.yaml +++ /dev/null @@ -1,109 +0,0 @@ -interactions: -- request: - body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single - word: pong"}],"model":"claude-haiku-4-5-20251001"}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate, zstd - anthropic-version: - - '2026-10-21' - connection: - - keep-alive - content-length: - - '127' - content-type: - - application/json - host: - - api.anthropic.com - user-agent: - - AsyncAnthropic/Python 2.3.0 - x-stainless-arch: - - arm64 - x-stainless-async: - - async:asyncio - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 2.3.0 - x-stainless-read-timeout: - - '600' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.14.7 - x-stainless-timeout: - - '600' - method: POST - uri: https://api.anthropic.com/v1/messages - response: - body: - string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWpEYvfW2MdeQfjBGcUu","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' - headers: - anthropic-organization-id: - - 27796668-7351-40ac-acc4-024aee8995a5 - anthropic-ratelimit-input-tokens-limit: - - '10000000' - anthropic-ratelimit-input-tokens-remaining: - - '10000000' - anthropic-ratelimit-input-tokens-reset: - - '2026-10-29T16:06:31Z' - anthropic-ratelimit-output-tokens-limit: - - '2000000' - anthropic-ratelimit-output-tokens-remaining: - - '2000000' - anthropic-ratelimit-output-tokens-reset: - - '2026-10-29T16:06:31Z' - anthropic-ratelimit-requests-limit: - - '20000' - anthropic-ratelimit-requests-remaining: - - '19999' - anthropic-ratelimit-requests-reset: - - '2026-10-29T16:06:31Z' - anthropic-ratelimit-tokens-limit: - - '12000000' - anthropic-ratelimit-tokens-remaining: - - '12000000' - anthropic-ratelimit-tokens-reset: - - '2026-10-29T16:06:31Z' - anthropic-workspace-id: - - wrkspc_018bFG1FgNVWqx5pthtdGTzS - cf-cache-status: - - DYNAMIC - cf-ray: - - a5236d8cdb6cac00-YYZ - connection: - - keep-alive - content-length: - - '515' - content-security-policy: - - default-src 'none'; frame-ancestors 'none' - content-type: - - application/json - date: - - Thu, 29 Oct 2026 16:06:31 GMT - request-id: - - req_011CgWpEYhXgYf1SWaPSg3WD - server: - - cloudflare - server-timing: - - x-originResponse;dur=323 - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - traceresponse: - - 00-bbf31ac2c3eee3dca8a2e91220d987c4-b530afaf227747d2-01 - transfer-encoding: - - chunked - vary: - - Accept-Encoding - x-robots-tag: - - none - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/mcp/cassettes/1.28.1/mcp_test_stdio_subprocess.yaml b/py/src/braintrust/integrations/mcp/cassettes/1.28.1/mcp_test_stdio_subprocess.yaml deleted file mode 100644 index 2c22b2ac3..000000000 --- a/py/src/braintrust/integrations/mcp/cassettes/1.28.1/mcp_test_stdio_subprocess.yaml +++ /dev/null @@ -1,109 +0,0 @@ -interactions: -- request: - body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single - word: pong"}],"model":"claude-haiku-4-5-20251001"}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate, zstd - anthropic-version: - - '2026-10-21' - connection: - - keep-alive - content-length: - - '127' - content-type: - - application/json - host: - - api.anthropic.com - user-agent: - - AsyncAnthropic/Python 2.3.0 - x-stainless-arch: - - arm64 - x-stainless-async: - - async:asyncio - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 2.3.0 - x-stainless-read-timeout: - - '600' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.14.7 - x-stainless-timeout: - - '600' - method: POST - uri: https://api.anthropic.com/v1/messages - response: - body: - string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWpEg5HdeGux7bVRTNWm","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' - headers: - anthropic-organization-id: - - 27796668-7351-40ac-acc4-024aee8995a5 - anthropic-ratelimit-input-tokens-limit: - - '10000000' - anthropic-ratelimit-input-tokens-remaining: - - '10000000' - anthropic-ratelimit-input-tokens-reset: - - '2026-10-29T16:06:33Z' - anthropic-ratelimit-output-tokens-limit: - - '2000000' - anthropic-ratelimit-output-tokens-remaining: - - '2000000' - anthropic-ratelimit-output-tokens-reset: - - '2026-10-29T16:06:33Z' - anthropic-ratelimit-requests-limit: - - '20000' - anthropic-ratelimit-requests-remaining: - - '19999' - anthropic-ratelimit-requests-reset: - - '2026-10-29T16:06:32Z' - anthropic-ratelimit-tokens-limit: - - '12000000' - anthropic-ratelimit-tokens-remaining: - - '12000000' - anthropic-ratelimit-tokens-reset: - - '2026-10-29T16:06:33Z' - anthropic-workspace-id: - - wrkspc_018bFG1FgNVWqx5pthtdGTzS - cf-cache-status: - - DYNAMIC - cf-ray: - - a5236d975d0a9113-YYZ - connection: - - keep-alive - content-length: - - '515' - content-security-policy: - - default-src 'none'; frame-ancestors 'none' - content-type: - - application/json - date: - - Thu, 29 Oct 2026 16:06:33 GMT - request-id: - - req_011CgWpEfrPTmFwTM4BB9ube - server: - - cloudflare - server-timing: - - x-originResponse;dur=281 - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - traceresponse: - - 00-8607b05eae3059c98503eff51e44fe52-741bcedf17686783-01 - transfer-encoding: - - chunked - vary: - - Accept-Encoding - x-robots-tag: - - none - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/mcp/cassettes/1.30.0/mcp_test_http_tool_with_model_call.yaml b/py/src/braintrust/integrations/mcp/cassettes/1.30.0/mcp_test_http_tool_with_model_call.yaml deleted file mode 100644 index 8dc544455..000000000 --- a/py/src/braintrust/integrations/mcp/cassettes/1.30.0/mcp_test_http_tool_with_model_call.yaml +++ /dev/null @@ -1,109 +0,0 @@ -interactions: -- request: - body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single - word: pong"}],"model":"claude-haiku-4-5-20251001"}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate, zstd - anthropic-version: - - '2026-10-21' - connection: - - keep-alive - content-length: - - '127' - content-type: - - application/json - host: - - api.anthropic.com - user-agent: - - AsyncAnthropic/Python 2.3.0 - x-stainless-arch: - - arm64 - x-stainless-async: - - async:asyncio - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 2.3.0 - x-stainless-read-timeout: - - '600' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.14.7 - x-stainless-timeout: - - '600' - method: POST - uri: https://api.anthropic.com/v1/messages - response: - body: - string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWpCkfnyaPeQ544GcEMy","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' - headers: - anthropic-organization-id: - - 27796668-7351-40ac-acc4-024aee8995a5 - anthropic-ratelimit-input-tokens-limit: - - '10000000' - anthropic-ratelimit-input-tokens-remaining: - - '10000000' - anthropic-ratelimit-input-tokens-reset: - - '2026-10-29T16:06:07Z' - anthropic-ratelimit-output-tokens-limit: - - '2000000' - anthropic-ratelimit-output-tokens-remaining: - - '2000000' - anthropic-ratelimit-output-tokens-reset: - - '2026-10-29T16:06:07Z' - anthropic-ratelimit-requests-limit: - - '20000' - anthropic-ratelimit-requests-remaining: - - '19999' - anthropic-ratelimit-requests-reset: - - '2026-10-29T16:06:06Z' - anthropic-ratelimit-tokens-limit: - - '12000000' - anthropic-ratelimit-tokens-remaining: - - '12000000' - anthropic-ratelimit-tokens-reset: - - '2026-10-29T16:06:07Z' - anthropic-workspace-id: - - wrkspc_018bFG1FgNVWqx5pthtdGTzS - cf-cache-status: - - DYNAMIC - cf-ray: - - a5236cf3ecc6abb8-YYZ - connection: - - keep-alive - content-length: - - '515' - content-security-policy: - - default-src 'none'; frame-ancestors 'none' - content-type: - - application/json - date: - - Thu, 29 Oct 2026 16:06:07 GMT - request-id: - - req_011CgWpCk3bMDgkGqBL2QuHH - server: - - cloudflare - server-timing: - - x-originResponse;dur=394 - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - traceresponse: - - 00-cd32ea7dcaf28d98c608743508dfa3aa-6001f83e29308b74-01 - transfer-encoding: - - chunked - vary: - - Accept-Encoding - x-robots-tag: - - none - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/mcp/cassettes/1.30.0/mcp_test_stdio_subprocess.yaml b/py/src/braintrust/integrations/mcp/cassettes/1.30.0/mcp_test_stdio_subprocess.yaml deleted file mode 100644 index 8f93cd9f2..000000000 --- a/py/src/braintrust/integrations/mcp/cassettes/1.30.0/mcp_test_stdio_subprocess.yaml +++ /dev/null @@ -1,109 +0,0 @@ -interactions: -- request: - body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single - word: pong"}],"model":"claude-haiku-4-5-20251001"}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate, zstd - anthropic-version: - - '2026-10-21' - connection: - - keep-alive - content-length: - - '127' - content-type: - - application/json - host: - - api.anthropic.com - user-agent: - - AsyncAnthropic/Python 2.3.0 - x-stainless-arch: - - arm64 - x-stainless-async: - - async:asyncio - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 2.3.0 - x-stainless-read-timeout: - - '600' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.14.7 - x-stainless-timeout: - - '600' - method: POST - uri: https://api.anthropic.com/v1/messages - response: - body: - string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWpDEVd1a7GErXVSQG6t","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' - headers: - anthropic-organization-id: - - 27796668-7351-40ac-acc4-024aee8995a5 - anthropic-ratelimit-input-tokens-limit: - - '10000000' - anthropic-ratelimit-input-tokens-remaining: - - '10000000' - anthropic-ratelimit-input-tokens-reset: - - '2026-10-29T16:06:13Z' - anthropic-ratelimit-output-tokens-limit: - - '2000000' - anthropic-ratelimit-output-tokens-remaining: - - '2000000' - anthropic-ratelimit-output-tokens-reset: - - '2026-10-29T16:06:13Z' - anthropic-ratelimit-requests-limit: - - '20000' - anthropic-ratelimit-requests-remaining: - - '19999' - anthropic-ratelimit-requests-reset: - - '2026-10-29T16:06:13Z' - anthropic-ratelimit-tokens-limit: - - '12000000' - anthropic-ratelimit-tokens-remaining: - - '12000000' - anthropic-ratelimit-tokens-reset: - - '2026-10-29T16:06:13Z' - anthropic-workspace-id: - - wrkspc_018bFG1FgNVWqx5pthtdGTzS - cf-cache-status: - - DYNAMIC - cf-ray: - - a5236d1d0ab5a211-YYZ - connection: - - keep-alive - content-length: - - '515' - content-security-policy: - - default-src 'none'; frame-ancestors 'none' - content-type: - - application/json - date: - - Thu, 29 Oct 2026 16:06:13 GMT - request-id: - - req_011CgWpDE92vER7E8z75SEkG - server: - - cloudflare - server-timing: - - x-originResponse;dur=366 - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - traceresponse: - - 00-238cadab9b4caea6197202118bcbdcd5-d2024a2f60b280b7-01 - transfer-encoding: - - chunked - vary: - - Accept-Encoding - x-robots-tag: - - none - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/mcp/cassettes/2.0.0/mcp_test_http_tool_with_model_call.yaml b/py/src/braintrust/integrations/mcp/cassettes/2.0.0/mcp_test_http_tool_with_model_call.yaml deleted file mode 100644 index d26976c9d..000000000 --- a/py/src/braintrust/integrations/mcp/cassettes/2.0.0/mcp_test_http_tool_with_model_call.yaml +++ /dev/null @@ -1,109 +0,0 @@ -interactions: -- request: - body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single - word: pong"}],"model":"claude-haiku-4-5-20251001"}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate, zstd - anthropic-version: - - '2026-10-21' - connection: - - keep-alive - content-length: - - '127' - content-type: - - application/json - host: - - api.anthropic.com - user-agent: - - AsyncAnthropic/Python 2.3.0 - x-stainless-arch: - - arm64 - x-stainless-async: - - async:asyncio - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 2.3.0 - x-stainless-read-timeout: - - '600' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.14.7 - x-stainless-timeout: - - '600' - method: POST - uri: https://api.anthropic.com/v1/messages - response: - body: - string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWpAA1v4Lozz5WsTFbzH","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' - headers: - anthropic-organization-id: - - 27796668-7351-40ac-acc4-024aee8995a5 - anthropic-ratelimit-input-tokens-limit: - - '10000000' - anthropic-ratelimit-input-tokens-remaining: - - '10000000' - anthropic-ratelimit-input-tokens-reset: - - '2026-10-29T16:05:31Z' - anthropic-ratelimit-output-tokens-limit: - - '2000000' - anthropic-ratelimit-output-tokens-remaining: - - '2000000' - anthropic-ratelimit-output-tokens-reset: - - '2026-10-29T16:05:31Z' - anthropic-ratelimit-requests-limit: - - '20000' - anthropic-ratelimit-requests-remaining: - - '19999' - anthropic-ratelimit-requests-reset: - - '2026-10-29T16:05:31Z' - anthropic-ratelimit-tokens-limit: - - '12000000' - anthropic-ratelimit-tokens-remaining: - - '12000000' - anthropic-ratelimit-tokens-reset: - - '2026-10-29T16:05:31Z' - anthropic-workspace-id: - - wrkspc_018bFG1FgNVWqx5pthtdGTzS - cf-cache-status: - - DYNAMIC - cf-ray: - - a5236c18481eac52-YYZ - connection: - - keep-alive - content-length: - - '515' - content-security-policy: - - default-src 'none'; frame-ancestors 'none' - content-type: - - application/json - date: - - Thu, 29 Oct 2026 16:05:31 GMT - request-id: - - req_011CgWpA9ozqrCTvUTbvz9uK - server: - - cloudflare - server-timing: - - x-originResponse;dur=317 - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - traceresponse: - - 00-e049acfb5e2207d1ec88abdb6442fdd1-13de36ce6aa95537-01 - transfer-encoding: - - chunked - vary: - - Accept-Encoding - x-robots-tag: - - none - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/mcp/cassettes/2.0.0/mcp_test_stdio_subprocess.yaml b/py/src/braintrust/integrations/mcp/cassettes/2.0.0/mcp_test_stdio_subprocess.yaml deleted file mode 100644 index c3f09639a..000000000 --- a/py/src/braintrust/integrations/mcp/cassettes/2.0.0/mcp_test_stdio_subprocess.yaml +++ /dev/null @@ -1,109 +0,0 @@ -interactions: -- request: - body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single - word: pong"}],"model":"claude-haiku-4-5-20251001"}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate, zstd - anthropic-version: - - '2026-10-21' - connection: - - keep-alive - content-length: - - '127' - content-type: - - application/json - host: - - api.anthropic.com - user-agent: - - AsyncAnthropic/Python 2.3.0 - x-stainless-arch: - - arm64 - x-stainless-async: - - async:asyncio - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 2.3.0 - x-stainless-read-timeout: - - '600' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.14.7 - x-stainless-timeout: - - '600' - method: POST - uri: https://api.anthropic.com/v1/messages - response: - body: - string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWpAT9oHqhsAqAY28czq","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' - headers: - anthropic-organization-id: - - 27796668-7351-40ac-acc4-024aee8995a5 - anthropic-ratelimit-input-tokens-limit: - - '10000000' - anthropic-ratelimit-input-tokens-remaining: - - '10000000' - anthropic-ratelimit-input-tokens-reset: - - '2026-10-29T16:05:35Z' - anthropic-ratelimit-output-tokens-limit: - - '2000000' - anthropic-ratelimit-output-tokens-remaining: - - '2000000' - anthropic-ratelimit-output-tokens-reset: - - '2026-10-29T16:05:35Z' - anthropic-ratelimit-requests-limit: - - '20000' - anthropic-ratelimit-requests-remaining: - - '19999' - anthropic-ratelimit-requests-reset: - - '2026-10-29T16:05:35Z' - anthropic-ratelimit-tokens-limit: - - '12000000' - anthropic-ratelimit-tokens-remaining: - - '12000000' - anthropic-ratelimit-tokens-reset: - - '2026-10-29T16:05:35Z' - anthropic-workspace-id: - - wrkspc_018bFG1FgNVWqx5pthtdGTzS - cf-cache-status: - - DYNAMIC - cf-ray: - - a5236c3148ba1185-YYZ - connection: - - keep-alive - content-length: - - '515' - content-security-policy: - - default-src 'none'; frame-ancestors 'none' - content-type: - - application/json - date: - - Thu, 29 Oct 2026 16:05:35 GMT - request-id: - - req_011CgWpASvQJA9Nbjy2AkNkC - server: - - cloudflare - server-timing: - - x-originResponse;dur=314 - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - traceresponse: - - 00-facc1caf1a40f68f551d20b87fbca8af-438d4a1e35cbdbcc-01 - transfer-encoding: - - chunked - vary: - - Accept-Encoding - x-robots-tag: - - none - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/mcp/cassettes/2.14.7/fastmcp_test_http_tool_with_model_call.yaml b/py/src/braintrust/integrations/mcp/cassettes/2.14.7/fastmcp_test_http_tool_with_model_call.yaml deleted file mode 100644 index cc4829f1e..000000000 --- a/py/src/braintrust/integrations/mcp/cassettes/2.14.7/fastmcp_test_http_tool_with_model_call.yaml +++ /dev/null @@ -1,109 +0,0 @@ -interactions: -- request: - body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single - word: pong"}],"model":"claude-haiku-4-5-20251001"}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate, zstd - anthropic-version: - - '2026-10-21' - connection: - - keep-alive - content-length: - - '127' - content-type: - - application/json - host: - - api.anthropic.com - user-agent: - - AsyncAnthropic/Python 2.3.0 - x-stainless-arch: - - arm64 - x-stainless-async: - - async:asyncio - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 2.3.0 - x-stainless-read-timeout: - - '600' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.14.7 - x-stainless-timeout: - - '600' - method: POST - uri: https://api.anthropic.com/v1/messages - response: - body: - string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWpKZp8UQJfXMSYzr6Gg","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' - headers: - anthropic-organization-id: - - 27796668-7351-40ac-acc4-024aee8995a5 - anthropic-ratelimit-input-tokens-limit: - - '10000000' - anthropic-ratelimit-input-tokens-remaining: - - '10000000' - anthropic-ratelimit-input-tokens-reset: - - '2026-10-29T16:07:39Z' - anthropic-ratelimit-output-tokens-limit: - - '2000000' - anthropic-ratelimit-output-tokens-remaining: - - '2000000' - anthropic-ratelimit-output-tokens-reset: - - '2026-10-29T16:07:39Z' - anthropic-ratelimit-requests-limit: - - '20000' - anthropic-ratelimit-requests-remaining: - - '19999' - anthropic-ratelimit-requests-reset: - - '2026-10-29T16:07:39Z' - anthropic-ratelimit-tokens-limit: - - '12000000' - anthropic-ratelimit-tokens-remaining: - - '12000000' - anthropic-ratelimit-tokens-reset: - - '2026-10-29T16:07:39Z' - anthropic-workspace-id: - - wrkspc_018bFG1FgNVWqx5pthtdGTzS - cf-cache-status: - - DYNAMIC - cf-ray: - - a5236f363bf3ac5a-YYZ - connection: - - keep-alive - content-length: - - '515' - content-security-policy: - - default-src 'none'; frame-ancestors 'none' - content-type: - - application/json - date: - - Thu, 29 Oct 2026 16:07:39 GMT - request-id: - - req_011CgWpKZcEP4jmHuJsH8JKc - server: - - cloudflare - server-timing: - - x-originResponse;dur=305 - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - traceresponse: - - 00-c29eaa7c8730e83732d4fb344c2349a9-f8a9e32d2e94d041-01 - transfer-encoding: - - chunked - vary: - - Accept-Encoding - x-robots-tag: - - none - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/mcp/cassettes/2.14.7/fastmcp_test_stdio_subprocess.yaml b/py/src/braintrust/integrations/mcp/cassettes/2.14.7/fastmcp_test_stdio_subprocess.yaml deleted file mode 100644 index 9c88e938c..000000000 --- a/py/src/braintrust/integrations/mcp/cassettes/2.14.7/fastmcp_test_stdio_subprocess.yaml +++ /dev/null @@ -1,109 +0,0 @@ -interactions: -- request: - body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single - word: pong"}],"model":"claude-haiku-4-5-20251001"}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate, zstd - anthropic-version: - - '2026-10-21' - connection: - - keep-alive - content-length: - - '127' - content-type: - - application/json - host: - - api.anthropic.com - user-agent: - - AsyncAnthropic/Python 2.3.0 - x-stainless-arch: - - arm64 - x-stainless-async: - - async:asyncio - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 2.3.0 - x-stainless-read-timeout: - - '600' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.14.7 - x-stainless-timeout: - - '600' - method: POST - uri: https://api.anthropic.com/v1/messages - response: - body: - string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWpKgmbDkYJo1xkq4KN5","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' - headers: - anthropic-organization-id: - - 27796668-7351-40ac-acc4-024aee8995a5 - anthropic-ratelimit-input-tokens-limit: - - '10000000' - anthropic-ratelimit-input-tokens-remaining: - - '10000000' - anthropic-ratelimit-input-tokens-reset: - - '2026-10-29T16:07:41Z' - anthropic-ratelimit-output-tokens-limit: - - '2000000' - anthropic-ratelimit-output-tokens-remaining: - - '2000000' - anthropic-ratelimit-output-tokens-reset: - - '2026-10-29T16:07:41Z' - anthropic-ratelimit-requests-limit: - - '20000' - anthropic-ratelimit-requests-remaining: - - '19999' - anthropic-ratelimit-requests-reset: - - '2026-10-29T16:07:40Z' - anthropic-ratelimit-tokens-limit: - - '12000000' - anthropic-ratelimit-tokens-remaining: - - '12000000' - anthropic-ratelimit-tokens-reset: - - '2026-10-29T16:07:41Z' - anthropic-workspace-id: - - wrkspc_018bFG1FgNVWqx5pthtdGTzS - cf-cache-status: - - DYNAMIC - cf-ray: - - a5236f405d62c85f-YYZ - connection: - - keep-alive - content-length: - - '515' - content-security-policy: - - default-src 'none'; frame-ancestors 'none' - content-type: - - application/json - date: - - Thu, 29 Oct 2026 16:07:41 GMT - request-id: - - req_011CgWpKgYT4YXV8RA9hL9Qs - server: - - cloudflare - server-timing: - - x-originResponse;dur=302 - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - traceresponse: - - 00-fe34676afb80ed923e5884ec20140a90-5e4f928193693034-01 - transfer-encoding: - - chunked - vary: - - Accept-Encoding - x-robots-tag: - - none - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/mcp/cassettes/3.0.0/fastmcp_test_http_tool_with_model_call.yaml b/py/src/braintrust/integrations/mcp/cassettes/3.0.0/fastmcp_test_http_tool_with_model_call.yaml deleted file mode 100644 index 561ee088a..000000000 --- a/py/src/braintrust/integrations/mcp/cassettes/3.0.0/fastmcp_test_http_tool_with_model_call.yaml +++ /dev/null @@ -1,109 +0,0 @@ -interactions: -- request: - body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single - word: pong"}],"model":"claude-haiku-4-5-20251001"}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate, zstd - anthropic-version: - - '2026-10-21' - connection: - - keep-alive - content-length: - - '127' - content-type: - - application/json - host: - - api.anthropic.com - user-agent: - - AsyncAnthropic/Python 2.3.0 - x-stainless-arch: - - arm64 - x-stainless-async: - - async:asyncio - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 2.3.0 - x-stainless-read-timeout: - - '600' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.14.7 - x-stainless-timeout: - - '600' - method: POST - uri: https://api.anthropic.com/v1/messages - response: - body: - string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWpJfUg12QGjHwxu31Cr","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' - headers: - anthropic-organization-id: - - 27796668-7351-40ac-acc4-024aee8995a5 - anthropic-ratelimit-input-tokens-limit: - - '10000000' - anthropic-ratelimit-input-tokens-remaining: - - '10000000' - anthropic-ratelimit-input-tokens-reset: - - '2026-10-29T16:07:27Z' - anthropic-ratelimit-output-tokens-limit: - - '2000000' - anthropic-ratelimit-output-tokens-remaining: - - '2000000' - anthropic-ratelimit-output-tokens-reset: - - '2026-10-29T16:07:27Z' - anthropic-ratelimit-requests-limit: - - '20000' - anthropic-ratelimit-requests-remaining: - - '19999' - anthropic-ratelimit-requests-reset: - - '2026-10-29T16:07:27Z' - anthropic-ratelimit-tokens-limit: - - '12000000' - anthropic-ratelimit-tokens-remaining: - - '12000000' - anthropic-ratelimit-tokens-reset: - - '2026-10-29T16:07:27Z' - anthropic-workspace-id: - - wrkspc_018bFG1FgNVWqx5pthtdGTzS - cf-cache-status: - - DYNAMIC - cf-ray: - - a5236ee9ab414a29-YYZ - connection: - - keep-alive - content-length: - - '515' - content-security-policy: - - default-src 'none'; frame-ancestors 'none' - content-type: - - application/json - date: - - Thu, 29 Oct 2026 16:07:27 GMT - request-id: - - req_011CgWpJfFY8771sq9EbfV5F - server: - - cloudflare - server-timing: - - x-originResponse;dur=328 - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - traceresponse: - - 00-d73fe9d11caeb0c51e14a43dc557f349-ce39e9976f9e72fe-01 - transfer-encoding: - - chunked - vary: - - Accept-Encoding - x-robots-tag: - - none - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/mcp/cassettes/3.0.0/fastmcp_test_stdio_subprocess.yaml b/py/src/braintrust/integrations/mcp/cassettes/3.0.0/fastmcp_test_stdio_subprocess.yaml deleted file mode 100644 index 7afb0f366..000000000 --- a/py/src/braintrust/integrations/mcp/cassettes/3.0.0/fastmcp_test_stdio_subprocess.yaml +++ /dev/null @@ -1,109 +0,0 @@ -interactions: -- request: - body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single - word: pong"}],"model":"claude-haiku-4-5-20251001"}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate, zstd - anthropic-version: - - '2026-10-21' - connection: - - keep-alive - content-length: - - '127' - content-type: - - application/json - host: - - api.anthropic.com - user-agent: - - AsyncAnthropic/Python 2.3.0 - x-stainless-arch: - - arm64 - x-stainless-async: - - async:asyncio - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 2.3.0 - x-stainless-read-timeout: - - '600' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.14.7 - x-stainless-timeout: - - '600' - method: POST - uri: https://api.anthropic.com/v1/messages - response: - body: - string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWpJnjjb1Z1BZbuT991x","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' - headers: - anthropic-organization-id: - - 27796668-7351-40ac-acc4-024aee8995a5 - anthropic-ratelimit-input-tokens-limit: - - '10000000' - anthropic-ratelimit-input-tokens-remaining: - - '10000000' - anthropic-ratelimit-input-tokens-reset: - - '2026-10-29T16:07:28Z' - anthropic-ratelimit-output-tokens-limit: - - '2000000' - anthropic-ratelimit-output-tokens-remaining: - - '2000000' - anthropic-ratelimit-output-tokens-reset: - - '2026-10-29T16:07:28Z' - anthropic-ratelimit-requests-limit: - - '20000' - anthropic-ratelimit-requests-remaining: - - '19999' - anthropic-ratelimit-requests-reset: - - '2026-10-29T16:07:28Z' - anthropic-ratelimit-tokens-limit: - - '12000000' - anthropic-ratelimit-tokens-remaining: - - '12000000' - anthropic-ratelimit-tokens-reset: - - '2026-10-29T16:07:28Z' - anthropic-workspace-id: - - wrkspc_018bFG1FgNVWqx5pthtdGTzS - cf-cache-status: - - DYNAMIC - cf-ray: - - a5236ef448e439f6-YYZ - connection: - - keep-alive - content-length: - - '515' - content-security-policy: - - default-src 'none'; frame-ancestors 'none' - content-type: - - application/json - date: - - Thu, 29 Oct 2026 16:07:29 GMT - request-id: - - req_011CgWpJnX6fDyCtvkeQ1JRF - server: - - cloudflare - server-timing: - - x-originResponse;dur=294 - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - traceresponse: - - 00-c0cef9901d6af20019e2daf70eae19d2-f64c5616a94f6ab7-01 - transfer-encoding: - - chunked - vary: - - Accept-Encoding - x-robots-tag: - - none - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/mcp/cassettes/3.4.11/fastmcp_test_http_tool_with_model_call.yaml b/py/src/braintrust/integrations/mcp/cassettes/3.4.11/fastmcp_test_http_tool_with_model_call.yaml deleted file mode 100644 index 73efba51b..000000000 --- a/py/src/braintrust/integrations/mcp/cassettes/3.4.11/fastmcp_test_http_tool_with_model_call.yaml +++ /dev/null @@ -1,109 +0,0 @@ -interactions: -- request: - body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single - word: pong"}],"model":"claude-haiku-4-5-20251001"}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate, zstd - anthropic-version: - - '2026-10-21' - connection: - - keep-alive - content-length: - - '127' - content-type: - - application/json - host: - - api.anthropic.com - user-agent: - - AsyncAnthropic/Python 2.3.0 - x-stainless-arch: - - arm64 - x-stainless-async: - - async:asyncio - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 2.3.0 - x-stainless-read-timeout: - - '600' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.14.7 - x-stainless-timeout: - - '600' - method: POST - uri: https://api.anthropic.com/v1/messages - response: - body: - string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWpHn9zVTS3rudzRsFTH","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' - headers: - anthropic-organization-id: - - 27796668-7351-40ac-acc4-024aee8995a5 - anthropic-ratelimit-input-tokens-limit: - - '10000000' - anthropic-ratelimit-input-tokens-remaining: - - '10000000' - anthropic-ratelimit-input-tokens-reset: - - '2026-10-29T16:07:15Z' - anthropic-ratelimit-output-tokens-limit: - - '2000000' - anthropic-ratelimit-output-tokens-remaining: - - '2000000' - anthropic-ratelimit-output-tokens-reset: - - '2026-10-29T16:07:15Z' - anthropic-ratelimit-requests-limit: - - '20000' - anthropic-ratelimit-requests-remaining: - - '19999' - anthropic-ratelimit-requests-reset: - - '2026-10-29T16:07:15Z' - anthropic-ratelimit-tokens-limit: - - '12000000' - anthropic-ratelimit-tokens-remaining: - - '12000000' - anthropic-ratelimit-tokens-reset: - - '2026-10-29T16:07:15Z' - anthropic-workspace-id: - - wrkspc_018bFG1FgNVWqx5pthtdGTzS - cf-cache-status: - - DYNAMIC - cf-ray: - - a5236e9e984bac40-YYZ - connection: - - keep-alive - content-length: - - '515' - content-security-policy: - - default-src 'none'; frame-ancestors 'none' - content-type: - - application/json - date: - - Thu, 29 Oct 2026 16:07:15 GMT - request-id: - - req_011CgWpHmvM3iYSzuJJCY4oh - server: - - cloudflare - server-timing: - - x-originResponse;dur=312 - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - traceresponse: - - 00-73d1d292de1b013959c84868fedf97a6-cc7be964e0d8369c-01 - transfer-encoding: - - chunked - vary: - - Accept-Encoding - x-robots-tag: - - none - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/mcp/cassettes/3.4.11/fastmcp_test_stdio_subprocess.yaml b/py/src/braintrust/integrations/mcp/cassettes/3.4.11/fastmcp_test_stdio_subprocess.yaml deleted file mode 100644 index 01d4966c8..000000000 --- a/py/src/braintrust/integrations/mcp/cassettes/3.4.11/fastmcp_test_stdio_subprocess.yaml +++ /dev/null @@ -1,109 +0,0 @@ -interactions: -- request: - body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single - word: pong"}],"model":"claude-haiku-4-5-20251001"}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate, zstd - anthropic-version: - - '2026-10-21' - connection: - - keep-alive - content-length: - - '127' - content-type: - - application/json - host: - - api.anthropic.com - user-agent: - - AsyncAnthropic/Python 2.3.0 - x-stainless-arch: - - arm64 - x-stainless-async: - - async:asyncio - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 2.3.0 - x-stainless-read-timeout: - - '600' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.14.7 - x-stainless-timeout: - - '600' - method: POST - uri: https://api.anthropic.com/v1/messages - response: - body: - string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWpHu7vsexu25xg5hsgb","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' - headers: - anthropic-organization-id: - - 27796668-7351-40ac-acc4-024aee8995a5 - anthropic-ratelimit-input-tokens-limit: - - '10000000' - anthropic-ratelimit-input-tokens-remaining: - - '10000000' - anthropic-ratelimit-input-tokens-reset: - - '2026-10-29T16:07:16Z' - anthropic-ratelimit-output-tokens-limit: - - '2000000' - anthropic-ratelimit-output-tokens-remaining: - - '2000000' - anthropic-ratelimit-output-tokens-reset: - - '2026-10-29T16:07:16Z' - anthropic-ratelimit-requests-limit: - - '20000' - anthropic-ratelimit-requests-remaining: - - '19999' - anthropic-ratelimit-requests-reset: - - '2026-10-29T16:07:16Z' - anthropic-ratelimit-tokens-limit: - - '12000000' - anthropic-ratelimit-tokens-remaining: - - '12000000' - anthropic-ratelimit-tokens-reset: - - '2026-10-29T16:07:16Z' - anthropic-workspace-id: - - wrkspc_018bFG1FgNVWqx5pthtdGTzS - cf-cache-status: - - DYNAMIC - cf-ray: - - a5236ea8ce7b982f-YYZ - connection: - - keep-alive - content-length: - - '515' - content-security-policy: - - default-src 'none'; frame-ancestors 'none' - content-type: - - application/json - date: - - Thu, 29 Oct 2026 16:07:16 GMT - request-id: - - req_011CgWpHtw2ZurrrJfCFe6Sg - server: - - cloudflare - server-timing: - - x-originResponse;dur=306 - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - traceresponse: - - 00-b1cd01e23e0c29dcea98179c73501387-4546a5a4c8cfcab8-01 - transfer-encoding: - - chunked - vary: - - Accept-Encoding - x-robots-tag: - - none - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/mcp/cassettes/4.0.0/fastmcp_test_http_tool_with_model_call.yaml b/py/src/braintrust/integrations/mcp/cassettes/4.0.0/fastmcp_test_http_tool_with_model_call.yaml deleted file mode 100644 index 7c15b352e..000000000 --- a/py/src/braintrust/integrations/mcp/cassettes/4.0.0/fastmcp_test_http_tool_with_model_call.yaml +++ /dev/null @@ -1,109 +0,0 @@ -interactions: -- request: - body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single - word: pong"}],"model":"claude-haiku-4-5-20251001"}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate, zstd - anthropic-version: - - '2026-10-21' - connection: - - keep-alive - content-length: - - '127' - content-type: - - application/json - host: - - api.anthropic.com - user-agent: - - AsyncAnthropic/Python 2.3.0 - x-stainless-arch: - - arm64 - x-stainless-async: - - async:asyncio - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 2.3.0 - x-stainless-read-timeout: - - '600' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.14.7 - x-stainless-timeout: - - '600' - method: POST - uri: https://api.anthropic.com/v1/messages - response: - body: - string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWpGugtqXaugZD4zFL22","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' - headers: - anthropic-organization-id: - - 27796668-7351-40ac-acc4-024aee8995a5 - anthropic-ratelimit-input-tokens-limit: - - '10000000' - anthropic-ratelimit-input-tokens-remaining: - - '10000000' - anthropic-ratelimit-input-tokens-reset: - - '2026-10-29T16:07:03Z' - anthropic-ratelimit-output-tokens-limit: - - '2000000' - anthropic-ratelimit-output-tokens-remaining: - - '2000000' - anthropic-ratelimit-output-tokens-reset: - - '2026-10-29T16:07:03Z' - anthropic-ratelimit-requests-limit: - - '20000' - anthropic-ratelimit-requests-remaining: - - '19999' - anthropic-ratelimit-requests-reset: - - '2026-10-29T16:07:03Z' - anthropic-ratelimit-tokens-limit: - - '12000000' - anthropic-ratelimit-tokens-remaining: - - '12000000' - anthropic-ratelimit-tokens-reset: - - '2026-10-29T16:07:03Z' - anthropic-workspace-id: - - wrkspc_018bFG1FgNVWqx5pthtdGTzS - cf-cache-status: - - DYNAMIC - cf-ray: - - a5236e54cecd1707-YYZ - connection: - - keep-alive - content-length: - - '515' - content-security-policy: - - default-src 'none'; frame-ancestors 'none' - content-type: - - application/json - date: - - Thu, 29 Oct 2026 16:07:03 GMT - request-id: - - req_011CgWpGuUzE3uCduPxGG3aa - server: - - cloudflare - server-timing: - - x-originResponse;dur=338 - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - traceresponse: - - 00-ecf9ec6919820925d253367c66baf5b5-d70540e4375e624a-01 - transfer-encoding: - - chunked - vary: - - Accept-Encoding - x-robots-tag: - - none - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/mcp/cassettes/4.0.0/fastmcp_test_stdio_subprocess.yaml b/py/src/braintrust/integrations/mcp/cassettes/4.0.0/fastmcp_test_stdio_subprocess.yaml deleted file mode 100644 index 99f0a8d1a..000000000 --- a/py/src/braintrust/integrations/mcp/cassettes/4.0.0/fastmcp_test_stdio_subprocess.yaml +++ /dev/null @@ -1,109 +0,0 @@ -interactions: -- request: - body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single - word: pong"}],"model":"claude-haiku-4-5-20251001"}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate, zstd - anthropic-version: - - '2026-10-21' - connection: - - keep-alive - content-length: - - '127' - content-type: - - application/json - host: - - api.anthropic.com - user-agent: - - AsyncAnthropic/Python 2.3.0 - x-stainless-arch: - - arm64 - x-stainless-async: - - async:asyncio - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 2.3.0 - x-stainless-read-timeout: - - '600' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.14.7 - x-stainless-timeout: - - '600' - method: POST - uri: https://api.anthropic.com/v1/messages - response: - body: - string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWpH31Aphoziz7Q1NMnQ","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' - headers: - anthropic-organization-id: - - 27796668-7351-40ac-acc4-024aee8995a5 - anthropic-ratelimit-input-tokens-limit: - - '10000000' - anthropic-ratelimit-input-tokens-remaining: - - '10000000' - anthropic-ratelimit-input-tokens-reset: - - '2026-10-29T16:07:05Z' - anthropic-ratelimit-output-tokens-limit: - - '2000000' - anthropic-ratelimit-output-tokens-remaining: - - '2000000' - anthropic-ratelimit-output-tokens-reset: - - '2026-10-29T16:07:05Z' - anthropic-ratelimit-requests-limit: - - '20000' - anthropic-ratelimit-requests-remaining: - - '19999' - anthropic-ratelimit-requests-reset: - - '2026-10-29T16:07:05Z' - anthropic-ratelimit-tokens-limit: - - '12000000' - anthropic-ratelimit-tokens-remaining: - - '12000000' - anthropic-ratelimit-tokens-reset: - - '2026-10-29T16:07:05Z' - anthropic-workspace-id: - - wrkspc_018bFG1FgNVWqx5pthtdGTzS - cf-cache-status: - - DYNAMIC - cf-ray: - - a5236e5f8bfaa21c-YYZ - connection: - - keep-alive - content-length: - - '515' - content-security-policy: - - default-src 'none'; frame-ancestors 'none' - content-type: - - application/json - date: - - Thu, 29 Oct 2026 16:07:05 GMT - request-id: - - req_011CgWpH2nn3JAxMNhPByxNd - server: - - cloudflare - server-timing: - - x-originResponse;dur=315 - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - traceresponse: - - 00-0612c2fbe2de42ebb20966e9f7d2f41c-4a7bb08db02cfbde-01 - transfer-encoding: - - chunked - vary: - - Accept-Encoding - x-robots-tag: - - none - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/mcp/cassettes/latest/fastmcp_test_http_tool_with_model_call.yaml b/py/src/braintrust/integrations/mcp/cassettes/latest/fastmcp_test_http_tool_with_model_call.yaml deleted file mode 100644 index 52b0ff8f3..000000000 --- a/py/src/braintrust/integrations/mcp/cassettes/latest/fastmcp_test_http_tool_with_model_call.yaml +++ /dev/null @@ -1,109 +0,0 @@ -interactions: -- request: - body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single - word: pong"}],"model":"claude-haiku-4-5-20251001"}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate, zstd - anthropic-version: - - '2026-10-21' - connection: - - keep-alive - content-length: - - '127' - content-type: - - application/json - host: - - api.anthropic.com - user-agent: - - AsyncAnthropic/Python 2.3.0 - x-stainless-arch: - - arm64 - x-stainless-async: - - async:asyncio - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 2.3.0 - x-stainless-read-timeout: - - '600' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.14.7 - x-stainless-timeout: - - '600' - method: POST - uri: https://api.anthropic.com/v1/messages - response: - body: - string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWpFgkjba6aaK3VW3gHk","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' - headers: - anthropic-organization-id: - - 27796668-7351-40ac-acc4-024aee8995a5 - anthropic-ratelimit-input-tokens-limit: - - '10000000' - anthropic-ratelimit-input-tokens-remaining: - - '10000000' - anthropic-ratelimit-input-tokens-reset: - - '2026-10-29T16:06:46Z' - anthropic-ratelimit-output-tokens-limit: - - '2000000' - anthropic-ratelimit-output-tokens-remaining: - - '2000000' - anthropic-ratelimit-output-tokens-reset: - - '2026-10-29T16:06:46Z' - anthropic-ratelimit-requests-limit: - - '20000' - anthropic-ratelimit-requests-remaining: - - '19999' - anthropic-ratelimit-requests-reset: - - '2026-10-29T16:06:46Z' - anthropic-ratelimit-tokens-limit: - - '12000000' - anthropic-ratelimit-tokens-remaining: - - '12000000' - anthropic-ratelimit-tokens-reset: - - '2026-10-29T16:06:46Z' - anthropic-workspace-id: - - wrkspc_018bFG1FgNVWqx5pthtdGTzS - cf-cache-status: - - DYNAMIC - cf-ray: - - a5236ded2968aadf-YYZ - connection: - - keep-alive - content-length: - - '515' - content-security-policy: - - default-src 'none'; frame-ancestors 'none' - content-type: - - application/json - date: - - Thu, 29 Oct 2026 16:06:46 GMT - request-id: - - req_011CgWpFgVc5XVaMGqBbqqrv - server: - - cloudflare - server-timing: - - x-originResponse;dur=326 - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - traceresponse: - - 00-bfe330cba19666216e5011913133963b-ba7d70d6d23cef60-01 - transfer-encoding: - - chunked - vary: - - Accept-Encoding - x-robots-tag: - - none - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/mcp/cassettes/latest/fastmcp_test_stdio_subprocess.yaml b/py/src/braintrust/integrations/mcp/cassettes/latest/fastmcp_test_stdio_subprocess.yaml deleted file mode 100644 index 486f582b9..000000000 --- a/py/src/braintrust/integrations/mcp/cassettes/latest/fastmcp_test_stdio_subprocess.yaml +++ /dev/null @@ -1,109 +0,0 @@ -interactions: -- request: - body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single - word: pong"}],"model":"claude-haiku-4-5-20251001"}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate, zstd - anthropic-version: - - '2026-10-21' - connection: - - keep-alive - content-length: - - '127' - content-type: - - application/json - host: - - api.anthropic.com - user-agent: - - AsyncAnthropic/Python 2.3.0 - x-stainless-arch: - - arm64 - x-stainless-async: - - async:asyncio - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 2.3.0 - x-stainless-read-timeout: - - '600' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.14.7 - x-stainless-timeout: - - '600' - method: POST - uri: https://api.anthropic.com/v1/messages - response: - body: - string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWpFvQmJqfbXe7Runacm","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' - headers: - anthropic-organization-id: - - 27796668-7351-40ac-acc4-024aee8995a5 - anthropic-ratelimit-input-tokens-limit: - - '10000000' - anthropic-ratelimit-input-tokens-remaining: - - '10000000' - anthropic-ratelimit-input-tokens-reset: - - '2026-10-29T16:06:50Z' - anthropic-ratelimit-output-tokens-limit: - - '2000000' - anthropic-ratelimit-output-tokens-remaining: - - '2000000' - anthropic-ratelimit-output-tokens-reset: - - '2026-10-29T16:06:50Z' - anthropic-ratelimit-requests-limit: - - '20000' - anthropic-ratelimit-requests-remaining: - - '19999' - anthropic-ratelimit-requests-reset: - - '2026-10-29T16:06:49Z' - anthropic-ratelimit-tokens-limit: - - '12000000' - anthropic-ratelimit-tokens-remaining: - - '12000000' - anthropic-ratelimit-tokens-reset: - - '2026-10-29T16:06:50Z' - anthropic-workspace-id: - - wrkspc_018bFG1FgNVWqx5pthtdGTzS - cf-cache-status: - - DYNAMIC - cf-ray: - - a5236e011c33b687-YYZ - connection: - - keep-alive - content-length: - - '515' - content-security-policy: - - default-src 'none'; frame-ancestors 'none' - content-type: - - application/json - date: - - Thu, 29 Oct 2026 16:06:50 GMT - request-id: - - req_011CgWpFvB8HjL4aJGoPPfFW - server: - - cloudflare - server-timing: - - x-originResponse;dur=297 - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - traceresponse: - - 00-0c3bd3c8086da65d7dfc75c6afadde32-13c6671007cd8f8d-01 - transfer-encoding: - - chunked - vary: - - Accept-Encoding - x-robots-tag: - - none - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/mcp/cassettes/latest/mcp_test_http_tool_with_model_call.yaml b/py/src/braintrust/integrations/mcp/cassettes/latest/mcp_test_http_tool_with_model_call.yaml deleted file mode 100644 index d7061aa36..000000000 --- a/py/src/braintrust/integrations/mcp/cassettes/latest/mcp_test_http_tool_with_model_call.yaml +++ /dev/null @@ -1,109 +0,0 @@ -interactions: -- request: - body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single - word: pong"}],"model":"claude-haiku-4-5-20251001"}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate, zstd - anthropic-version: - - '2026-10-21' - connection: - - keep-alive - content-length: - - '127' - content-type: - - application/json - host: - - api.anthropic.com - user-agent: - - AsyncAnthropic/Python 2.3.0 - x-stainless-arch: - - arm64 - x-stainless-async: - - async:asyncio - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 2.3.0 - x-stainless-read-timeout: - - '600' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.14.7 - x-stainless-timeout: - - '600' - method: POST - uri: https://api.anthropic.com/v1/messages - response: - body: - string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWp8bLbyPqkQSDooZowN","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' - headers: - anthropic-organization-id: - - 27796668-7351-40ac-acc4-024aee8995a5 - anthropic-ratelimit-input-tokens-limit: - - '10000000' - anthropic-ratelimit-input-tokens-remaining: - - '10000000' - anthropic-ratelimit-input-tokens-reset: - - '2026-10-29T16:05:10Z' - anthropic-ratelimit-output-tokens-limit: - - '2000000' - anthropic-ratelimit-output-tokens-remaining: - - '2000000' - anthropic-ratelimit-output-tokens-reset: - - '2026-10-29T16:05:10Z' - anthropic-ratelimit-requests-limit: - - '20000' - anthropic-ratelimit-requests-remaining: - - '19999' - anthropic-ratelimit-requests-reset: - - '2026-10-29T16:05:10Z' - anthropic-ratelimit-tokens-limit: - - '12000000' - anthropic-ratelimit-tokens-remaining: - - '12000000' - anthropic-ratelimit-tokens-reset: - - '2026-10-29T16:05:10Z' - anthropic-workspace-id: - - wrkspc_018bFG1FgNVWqx5pthtdGTzS - cf-cache-status: - - DYNAMIC - cf-ray: - - a5236b93adc54fcb-YYZ - connection: - - keep-alive - content-length: - - '515' - content-security-policy: - - default-src 'none'; frame-ancestors 'none' - content-type: - - application/json - date: - - Thu, 29 Oct 2026 16:05:10 GMT - request-id: - - req_011CgWp8b3FGkeZhLjn4QZKe - server: - - cloudflare - server-timing: - - x-originResponse;dur=313 - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - traceresponse: - - 00-774cab9cef25d192b9b61ccc847c919a-b4cfb5c5bc1e9202-01 - transfer-encoding: - - chunked - vary: - - Accept-Encoding - x-robots-tag: - - none - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/mcp/cassettes/latest/mcp_test_stdio_subprocess.yaml b/py/src/braintrust/integrations/mcp/cassettes/latest/mcp_test_stdio_subprocess.yaml deleted file mode 100644 index 356321569..000000000 --- a/py/src/braintrust/integrations/mcp/cassettes/latest/mcp_test_stdio_subprocess.yaml +++ /dev/null @@ -1,109 +0,0 @@ -interactions: -- request: - body: '{"max_tokens":32,"messages":[{"role":"user","content":"Reply with the single - word: pong"}],"model":"claude-haiku-4-5-20251001"}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate, zstd - anthropic-version: - - '2026-10-21' - connection: - - keep-alive - content-length: - - '127' - content-type: - - application/json - host: - - api.anthropic.com - user-agent: - - AsyncAnthropic/Python 2.3.0 - x-stainless-arch: - - arm64 - x-stainless-async: - - async:asyncio - x-stainless-lang: - - python - x-stainless-os: - - MacOS - x-stainless-package-version: - - 2.3.0 - x-stainless-read-timeout: - - '600' - x-stainless-retry-count: - - '0' - x-stainless-runtime: - - CPython - x-stainless-runtime-version: - - 3.14.7 - x-stainless-timeout: - - '600' - method: POST - uri: https://api.anthropic.com/v1/messages - response: - body: - string: '{"model":"claude-haiku-4-5-20251001","id":"msg_011CgWp8j1UjE54Q6n7Sz1aQ","type":"message","role":"assistant","content":[{"type":"text","text":"pong"}],"container":null,"stop_reason":"end_turn","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":15,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"cache_creation":{"ephemeral_5m_input_tokens":0,"ephemeral_1h_input_tokens":0},"output_tokens":5,"service_tier":"standard","inference_geo":"not_available","speed":"standard"},"diagnostics":null}' - headers: - anthropic-organization-id: - - 27796668-7351-40ac-acc4-024aee8995a5 - anthropic-ratelimit-input-tokens-limit: - - '10000000' - anthropic-ratelimit-input-tokens-remaining: - - '10000000' - anthropic-ratelimit-input-tokens-reset: - - '2026-10-29T16:05:12Z' - anthropic-ratelimit-output-tokens-limit: - - '2000000' - anthropic-ratelimit-output-tokens-remaining: - - '2000000' - anthropic-ratelimit-output-tokens-reset: - - '2026-10-29T16:05:12Z' - anthropic-ratelimit-requests-limit: - - '20000' - anthropic-ratelimit-requests-remaining: - - '19999' - anthropic-ratelimit-requests-reset: - - '2026-10-29T16:05:12Z' - anthropic-ratelimit-tokens-limit: - - '12000000' - anthropic-ratelimit-tokens-remaining: - - '12000000' - anthropic-ratelimit-tokens-reset: - - '2026-10-29T16:05:12Z' - anthropic-workspace-id: - - wrkspc_018bFG1FgNVWqx5pthtdGTzS - cf-cache-status: - - DYNAMIC - cf-ray: - - a5236b9efbb11341-YYZ - connection: - - keep-alive - content-length: - - '515' - content-security-policy: - - default-src 'none'; frame-ancestors 'none' - content-type: - - application/json - date: - - Thu, 29 Oct 2026 16:05:12 GMT - request-id: - - req_011CgWp8inb4wREanqJMLkAq - server: - - cloudflare - server-timing: - - x-originResponse;dur=334 - strict-transport-security: - - max-age=31536000; includeSubDomains; preload - traceresponse: - - 00-d1974e70a014c44c4e19a91f2f43eb8b-89d2ac1c34f4e67f-01 - transfer-encoding: - - chunked - vary: - - Accept-Encoding - x-robots-tag: - - none - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/mcp/test_mcp.py b/py/src/braintrust/integrations/mcp/test_mcp.py index 9a9b57ef0..c262a325c 100644 --- a/py/src/braintrust/integrations/mcp/test_mcp.py +++ b/py/src/braintrust/integrations/mcp/test_mcp.py @@ -4,7 +4,7 @@ sessions: ``FastMCP`` on MCP 1.x, ``MCPServer`` on MCP 2.x) and standalone ``fastmcp`` (``test_fastmcp`` sessions). MCP traffic is always real: clients talk to servers in-process, over streamable HTTP on localhost, or over stdio -to a server subprocess. Model calls made inside tools replay from cassettes. +to a server subprocess, so no cassettes are needed. """ import asyncio @@ -13,7 +13,6 @@ import anyio import pytest from braintrust import Attachment, logger -from braintrust.conftest import get_vcr_config from braintrust.integrations.mcp import MCPIntegration, wrap_mcp_server from braintrust.integrations.mcp import _test_harness as harness from braintrust.integrations.mcp.tracing import set_global_server_config @@ -39,12 +38,6 @@ def reset_global_server_config(): set_global_server_config(None) -@pytest.fixture(scope="module") -def vcr_config(): - # MCP traffic to the localhost test server is real; only model calls replay. - return {**get_vcr_config(), "ignore_localhost": True} - - def _mcp_spans(spans): return [span for span in spans if "mcp.method.name" in (span.get("metadata") or {})] @@ -417,25 +410,21 @@ async def test_raised_handler_exception_is_recorded(memory_logger): assert "rpc.response.status_code" not in span["metadata"] -@pytest.mark.vcr(cassette_name=f"{FLAVOR}_test_http_tool_with_model_call") @pytest.mark.asyncio -async def test_http_tool_with_model_call(memory_logger): +async def test_streamable_http(memory_logger): server = wrap_mcp_server(harness.build_server()) async with harness.serve_http(harness.http_app(server)) as url: async with harness.connect_http(url) as client: - await client.call_tool("ask_llm", {"question": "Reply with the single word: pong"}) + await client.call_tool("add", {"a": 2, "b": 3}) spans = memory_logger.pop() - tool = find_span_by_name(spans, "tools/call ask_llm") - llm = find_span_by_name(spans, "anthropic.messages.create") - assert llm["span_parents"] == [tool["span_id"]] - assert llm["span_attributes"]["type"] == "llm" - assert llm["metrics"]["tokens"] > 0 + tool = find_span_by_name(spans, "tools/call add") + assert find_span_by_name(spans, "nested-work")["span_parents"] == [tool["span_id"]] + assert tool["output"]["content"] == [{"type": "text", "text": "5"}] # MCP spans never invent model fields. assert "model" not in tool["metadata"] assert "tokens" not in tool["metrics"] - assert tool["output"]["content"][0]["type"] == "text" metadata = tool["metadata"] assert metadata["network.protocol.name"] == "http" @@ -450,22 +439,12 @@ async def test_http_tool_with_model_call(memory_logger): @pytest.mark.parametrize("wrap", ["wrap", "setup"]) @pytest.mark.asyncio -async def test_stdio_subprocess(tmp_path, request, wrap): +async def test_stdio_subprocess(tmp_path, wrap): spans_file = tmp_path / "spans.json" - record_mode = request.config.getoption("--vcr-record", default=None) - if request.config.getoption("--disable-vcr", default=False): - record_mode = "disabled" - env = { - harness.SPANS_FILE_ENV: str(spans_file), - harness.CASSETTE_ENV: f"{FLAVOR}_test_stdio_subprocess", - harness.WRAP_ENV: wrap, - } - if record_mode: - env[harness.RECORD_MODE_ENV] = record_mode + env = {harness.SPANS_FILE_ENV: str(spans_file), harness.WRAP_ENV: wrap} async with harness.connect_stdio(env) as client: await client.call_tool("add", {"a": 2, "b": 2}) - await client.call_tool("ask_llm", {"question": "Reply with the single word: pong"}) await client.call_tool("notify") with anyio.fail_after(10): @@ -476,8 +455,6 @@ async def test_stdio_subprocess(tmp_path, request, wrap): add = find_span_by_name(spans, "tools/call add") assert add["input"] == {"a": 2, "b": 2} assert find_span_by_name(spans, "nested-work")["span_parents"] == [add["span_id"]] - ask = find_span_by_name(spans, "tools/call ask_llm") - assert find_span_by_name(spans, "anthropic.messages.create")["span_parents"] == [ask["span_id"]] assert find_span_by_name(spans, "notifications/message")["span_parents"] == [ find_span_by_name(spans, "tools/call notify")["span_id"] ]