-
Notifications
You must be signed in to change notification settings - Fork 3.3k
feat: implement OTEL mcp.server.* metrics #2394
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
192b727
89f7f82
baf622e
4107e8c
b1e90b2
0418e32
e0823b3
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -40,7 +40,7 @@ dependencies = [ | |
| "pyjwt[crypto]>=2.10.1", | ||
| "typing-extensions>=4.13.0", | ||
| "typing-inspection>=0.4.1", | ||
| "opentelemetry-api>=1.28.0", | ||
| "opentelemetry-api>=1.30.0", | ||
| ] | ||
|
|
||
| [project.optional-dependencies] | ||
|
|
@@ -72,7 +72,7 @@ dev = [ | |
| "coverage[toml]>=7.10.7,<=7.13", | ||
| "pillow>=12.0", | ||
| "strict-no-cover", | ||
| "logfire>=3.0.0", | ||
| "logfire>=3.20.0", | ||
| ] | ||
|
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Bumped from 3.0.0: logfire's |
||
| docs = [ | ||
| "mkdocs>=1.6.1", | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,20 +1,35 @@ | ||
| from __future__ import annotations | ||
|
|
||
| import json | ||
| from typing import Any, cast | ||
|
|
||
| import pytest | ||
| from logfire.testing import CaptureLogfire | ||
| from opentelemetry.sdk.metrics._internal.point import MetricsData | ||
|
|
||
| from mcp import types | ||
| from mcp.client.client import Client | ||
| from mcp.server.context import ServerRequestContext | ||
| from mcp.server.lowlevel.server import Server | ||
| from mcp.server.mcpserver import MCPServer | ||
| from mcp.shared.exceptions import MCPError | ||
|
|
||
| pytestmark = pytest.mark.anyio | ||
|
|
||
|
|
||
| def _get_mcp_metrics(capfire: CaptureLogfire) -> dict[str, Any]: | ||
| """Return collected metrics whose name starts with 'mcp.', keyed by name.""" | ||
|
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
| exported = json.loads(cast(MetricsData, capfire.metrics_reader.get_metrics_data()).to_json()) | ||
| [resource_metric] = exported["resource_metrics"] | ||
| all_metrics = [metric for scope_metric in resource_metric["scope_metrics"] for metric in scope_metric["metrics"]] | ||
| return {m["name"]: m for m in all_metrics if m["name"].startswith("mcp.")} | ||
|
|
||
|
|
||
| # Logfire warns about propagated trace context by default (distributed_tracing=None). | ||
| # This is expected here since we're testing cross-boundary context propagation. | ||
| @pytest.mark.filterwarnings("ignore::RuntimeWarning") | ||
| async def test_client_and_server_spans(capfire: CaptureLogfire): | ||
| """Verify that calling a tool produces client and server spans with correct attributes.""" | ||
| async def test_client_and_server_instrumentation(capfire: CaptureLogfire): | ||
| """Verify that calling a tool produces client and server spans and metrics with correct attributes.""" | ||
| server = MCPServer("test") | ||
|
|
||
| @server.tool() | ||
|
|
@@ -42,3 +57,78 @@ def greet(name: str) -> str: | |
|
|
||
| # Server span should be in the same trace as the client span (context propagation). | ||
| assert server_span["context"]["trace_id"] == client_span["context"]["trace_id"] | ||
|
|
||
| metrics = _get_mcp_metrics(capfire) | ||
|
|
||
| assert "mcp.server.operation.duration" in metrics | ||
| assert "mcp.server.session.duration" in metrics | ||
|
|
||
| op_metric = metrics["mcp.server.operation.duration"] | ||
| assert op_metric["unit"] == "s" | ||
| op_points = op_metric["data"]["data_points"] | ||
|
|
||
| # tools/call data point | ||
| tools_call_point = next(p for p in op_points if p["attributes"]["mcp.method.name"] == "tools/call") | ||
| assert tools_call_point["attributes"]["gen_ai.tool.name"] == "greet" | ||
| assert tools_call_point["attributes"]["gen_ai.operation.name"] == "execute_tool" | ||
| assert tools_call_point["attributes"]["mcp.protocol.version"] == "2025-11-25" | ||
| assert tools_call_point["count"] == 1 | ||
| assert tools_call_point["sum"] >= 0 | ||
|
|
||
| # tools/list is also called during initialization | ||
| assert any(p["attributes"]["mcp.method.name"] == "tools/list" for p in op_points) | ||
|
|
||
| session_metric = metrics["mcp.server.session.duration"] | ||
| assert session_metric["unit"] == "s" | ||
| [session_point] = session_metric["data"]["data_points"] | ||
| assert session_point["attributes"]["mcp.protocol.version"] == "2025-11-25" | ||
| assert "error.type" not in session_point["attributes"] | ||
| assert session_point["count"] == 1 | ||
| assert session_point["sum"] >= 0 | ||
|
|
||
|
|
||
| @pytest.mark.filterwarnings("ignore::RuntimeWarning") | ||
| async def test_server_operation_error_metrics(capfire: CaptureLogfire): | ||
| """Verify that error.type and rpc.response.status_code are set when a handler raises MCPError.""" | ||
|
|
||
| async def handle_call_tool( | ||
| ctx: ServerRequestContext[Any], params: types.CallToolRequestParams | ||
| ) -> types.CallToolResult: | ||
| raise MCPError(types.INVALID_PARAMS, "bad params") | ||
|
|
||
| server = Server("test", on_call_tool=handle_call_tool) | ||
|
|
||
| async with Client(server) as client: | ||
| with pytest.raises(MCPError): | ||
| await client.call_tool("boom", {}) | ||
|
|
||
| metrics = _get_mcp_metrics(capfire) | ||
| op_points = metrics["mcp.server.operation.duration"]["data"]["data_points"] | ||
| error_point = next(p for p in op_points if p["attributes"]["mcp.method.name"] == "tools/call") | ||
| assert error_point["attributes"]["error.type"] == str(types.INVALID_PARAMS) | ||
| assert error_point["attributes"]["rpc.response.status_code"] == str(types.INVALID_PARAMS) | ||
|
|
||
|
|
||
| @pytest.mark.filterwarnings("ignore::RuntimeWarning") | ||
| async def test_server_session_error_metrics(capfire: CaptureLogfire): | ||
| """Verify that error.type is set on session duration when the session exits with an exception.""" | ||
|
|
||
| async def handle_call_tool( | ||
| ctx: ServerRequestContext[Any], params: types.CallToolRequestParams | ||
| ) -> types.CallToolResult: | ||
| raise RuntimeError("unexpected crash") | ||
|
|
||
| server = Server("test", on_call_tool=handle_call_tool) | ||
|
|
||
| # raise_exceptions=True lets the RuntimeError escape the handler and crash the session, | ||
| # simulating what happens in production when an unhandled exception exits the session block. | ||
| with pytest.raises(Exception): | ||
| async with Client(server, raise_exceptions=True) as client: | ||
| await client.call_tool("boom", {}) | ||
|
|
||
| metrics = _get_mcp_metrics(capfire) | ||
| session_points = metrics["mcp.server.session.duration"]["data"]["data_points"] | ||
| error_session_points = [p for p in session_points if "error.type" in p["attributes"]] | ||
| assert len(error_session_points) >= 1 | ||
| # anyio wraps task group exceptions in ExceptionGroup | ||
| assert error_session_points[0]["attributes"]["error.type"] in ("RuntimeError", "ExceptionGroup") | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Bumped from 1.28.0:
explicit_bucket_boundaries_advisorywas added toMeter.create_histogram()in opentelemetry-api 1.30.0. The previous minimum caused aTypeErrorat runtime when the OTel proxy meter replayed histogram creation against the real meter.