-
Notifications
You must be signed in to change notification settings - Fork 3.9k
Add an MQTT 5 transport example #3520
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: transport-grpc
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| user server-alice | ||
| topic read mcp/alice/+/requests | ||
| topic write mcp/alice/+/responses | ||
|
|
||
| user server-bob | ||
| topic read mcp/bob/+/requests | ||
| topic write mcp/bob/+/responses | ||
|
|
||
| user health | ||
| topic write health | ||
|
|
||
| user alice | ||
| topic write mcp/alice/+/requests | ||
| topic read mcp/alice/+/responses | ||
|
|
||
| user bob | ||
| topic write mcp/bob/+/requests | ||
| topic read mcp/bob/+/responses |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| listener 1883 | ||
| allow_anonymous false | ||
| use_username_as_clientid true | ||
| password_file /tmp/passwords | ||
| acl_file /mosquitto/config/mosquitto.acl | ||
| persistence false | ||
| log_dest stdout | ||
| log_type all | ||
| max_packet_size 4195328 | ||
| max_inflight_messages 32 | ||
| max_queued_messages 256 |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| services: | ||
| mqtt: | ||
| image: eclipse-mosquitto:2.0.22@sha256:212f89e1eaeb2c322d6441b64396e3346026674db8fa9c27beac293405c32b3c | ||
| ports: | ||
| - "127.0.0.1:${MQTT_TEST_PORT:-13883}:1883" | ||
| volumes: | ||
| - ./brokers/mosquitto.conf:/mosquitto/config/mosquitto.conf:ro | ||
| - ./brokers/mosquitto.acl:/mosquitto/config/mosquitto.acl:ro | ||
| entrypoint: ["/bin/sh", "-ec"] | ||
| command: | ||
| - | | ||
| mosquitto_passwd -b -c /tmp/passwords server-alice test-server-alice-password | ||
| mosquitto_passwd -b /tmp/passwords server-bob test-server-bob-password | ||
| mosquitto_passwd -b /tmp/passwords health test-health-password | ||
| mosquitto_passwd -b /tmp/passwords alice test-alice-password | ||
| mosquitto_passwd -b /tmp/passwords bob test-bob-password | ||
| chmod 644 /tmp/passwords | ||
| exec mosquitto -c /mosquitto/config/mosquitto.conf | ||
| healthcheck: | ||
| test: | ||
| [ | ||
| "CMD", | ||
| "mosquitto_pub", | ||
| "-h", | ||
| "127.0.0.1", | ||
| "-u", | ||
| "health", | ||
| "-P", | ||
| "test-health-password", | ||
| "-t", | ||
| "health", | ||
| "-m", | ||
| "", | ||
| "-q", | ||
| "2", | ||
| ] | ||
| interval: 1s | ||
| timeout: 3s | ||
| retries: 20 |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,97 @@ | ||
| """Shared live-broker checks for the reference adapters.""" | ||
|
|
||
| from collections.abc import AsyncIterator, Awaitable, Callable | ||
| from contextlib import AsyncExitStack, asynccontextmanager | ||
| from dataclasses import dataclass | ||
| from functools import partial | ||
| from typing import Any, TypeAlias | ||
| from uuid import uuid4 | ||
|
|
||
| import anyio | ||
| from mcp import Client | ||
| from mcp.server import Server, ServerRequestContext | ||
| from mcp.server.mcpserver import Context, MCPServer | ||
| from mcp.shared.transport import MessageMetadata, Transport, TransportContext | ||
| from mcp.types import CallToolRequestParams, CallToolResult, ListToolsResult, PaginatedRequestParams, TextContent, Tool | ||
|
|
||
| TransportFactory: TypeAlias = Callable[[AsyncExitStack, str, str, bool], Awaitable[Transport]] | ||
|
|
||
|
|
||
| @dataclass(kw_only=True, frozen=True) | ||
| class BrokerContext(TransportContext): | ||
| principal: str | ||
|
|
||
|
|
||
| def peer_context(metadata: MessageMetadata, *, principal: str, kind: str) -> BrokerContext: | ||
| return BrokerContext(kind=kind, can_send_request=True, principal=principal) | ||
|
|
||
|
|
||
| async def verify(factory: TransportFactory, *, kind: str, highlevel: bool, mode: str) -> None: | ||
| """Check real concurrent calls, peer metadata, both server APIs, and shared lifespan.""" | ||
| entered = {"alice": anyio.Event(), "bob": anyio.Event()} | ||
| sessions = {principal: uuid4().hex for principal in entered} | ||
| lifecycle: list[str] = [] | ||
|
|
||
| async def identity(transport: TransportContext | None) -> str: | ||
| assert isinstance(transport, BrokerContext) | ||
| principal = transport.principal | ||
| entered[principal].set() | ||
| await entered["bob" if principal == "alice" else "alice"].wait() | ||
| return principal | ||
|
|
||
| @asynccontextmanager | ||
| async def lifespan(server: Server[Any] | MCPServer[Any]) -> AsyncIterator[None]: | ||
| lifecycle.append("start") | ||
| try: | ||
| yield None | ||
| finally: | ||
| lifecycle.append("stop") | ||
|
|
||
| if highlevel: | ||
| server = MCPServer("Broker", lifespan=lifespan) | ||
|
|
||
| @server.tool() | ||
| async def identify(ctx: Context) -> str: | ||
| return await identity(ctx.transport) | ||
|
|
||
| else: | ||
|
|
||
| async def list_tools(ctx: ServerRequestContext, params: PaginatedRequestParams | None) -> ListToolsResult: | ||
| return ListToolsResult(tools=[Tool(name="identify", input_schema={"type": "object"})]) | ||
|
|
||
| async def call_tool(ctx: ServerRequestContext, params: CallToolRequestParams) -> CallToolResult: | ||
| assert params.name == "identify" | ||
| return CallToolResult(content=[TextContent(text=await identity(ctx.transport))]) | ||
|
|
||
| server = Server("Broker", lifespan=lifespan, on_list_tools=list_tools, on_call_tool=call_tool) | ||
|
|
||
| results: dict[str, str] = {} | ||
|
|
||
| async def call(client: Client, principal: str) -> None: | ||
| result = await client.call_tool("identify") | ||
| content = result.content[0] | ||
| assert isinstance(content, TextContent) | ||
| results[principal] = content.text | ||
|
|
||
| async with AsyncExitStack() as stack: | ||
| transports = { | ||
| principal: await factory(stack, principal, session, True) for principal, session in sessions.items() | ||
| } | ||
| runtime = await stack.enter_async_context(server.serve()) | ||
| clients: dict[str, Client] = {} | ||
| for principal, transport in transports.items(): | ||
| await runtime.connect( | ||
| transport, | ||
| session_id=sessions[principal], | ||
| transport_builder=partial(peer_context, principal=principal, kind=kind), | ||
| ) | ||
| client_transport = await factory(stack, principal, sessions[principal], False) | ||
| clients[principal] = await stack.enter_async_context( | ||
| Client(client_transport, mode=mode, read_timeout_seconds=5) | ||
| ) | ||
| async with anyio.create_task_group() as tg: | ||
| for principal, client in clients.items(): | ||
| tg.start_soon(call, client, principal) | ||
| assert results == {"alice": "alice", "bob": "bob"} | ||
| assert lifecycle == ["start"] | ||
| assert lifecycle == ["start", "stop"] |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| """Exercise the MQTT 5 adapter against the local Mosquitto broker.""" | ||
|
|
||
| import os | ||
| from contextlib import AsyncExitStack | ||
|
|
||
| import aiomqtt | ||
| import anyio | ||
| from mcp.shared.transport import Transport | ||
|
|
||
| from demo_common import verify | ||
| from mcp_transport_examples.mqtt import mqtt_transport | ||
|
|
||
|
|
||
| async def open_transport(stack: AsyncExitStack, principal: str, session: str, server_side: bool) -> Transport: | ||
| user = f"server-{principal}" if server_side else principal | ||
| topic = f"mcp/{principal}/{session}" | ||
| incoming, outgoing = ("requests", "responses") if server_side else ("responses", "requests") | ||
| client = await stack.enter_async_context( | ||
| aiomqtt.Client( | ||
| "127.0.0.1", | ||
| int(os.environ.get("MQTT_TEST_PORT", "13883")), | ||
| username=user, | ||
| password=f"test-{user}-password", | ||
| identifier=f"mcp-{principal}-{session}-{'server' if server_side else 'client'}", | ||
| protocol=aiomqtt.ProtocolVersion.V5, | ||
| keepalive=15, | ||
| will=aiomqtt.Will(f"{topic}/{outgoing}", payload=b"", qos=2, retain=False), | ||
| max_queued_incoming_messages=256, | ||
| ) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🟡 (optional) Users of this adapter silently lose requests or responses during a burst, and the affected call hangs until its read timeout, or forever when Extended reasoning...…Fix: never drop acknowledged messages; apply backpressure (block paho's reader or bound with an unbounded queue plus a stream-level ValueError) or fail the connection when the queue fills so the peer sees CONNECTION_CLOSED instead of a hang. The finder dismissed this because README calls it an unresolved blocker and the CI demo only makes two calls; a documented drop is still silent data loss for anyone who copies the reference example, which is the stated purpose of the package. Trace: paho's socket reader runs once per loop iteration and delivers one PUBLISH per PUBREL to aiomqtt's on_message, which does queue.put_nowait. On QueueFull aiomqtt logs a warning and returns; paho has already sent PUBREC and will send PUBCOMP, so the broker treats delivery as complete. The consumer side, mqtt.py:67-71, iterates client.messages: each item costs a create_task for queue.get, an asyncio.wait, a yield, then a zero-buffer send that must rendezvous with the dispatcher's receive, then _dispatch. That is roughly four to six loop turns per message versus one to two per incoming packet. Under a… Verification: normal; acknowledged in diff: examples/transports/README.md:105 ("The example bounds aiomqtt's incoming queue at 256 messages, but aiomqtt can drop messages when it fills... unresolved reliability blockers") and the PR description lists "queue-overflow behavior" as an open merge gate — the note is accurate about the bound but does not resolve the loss. Trigger: any burst where more than 256… |
||
| ) | ||
| return mqtt_transport(client, incoming_topic=f"{topic}/{incoming}", outgoing_topic=f"{topic}/{outgoing}") | ||
|
|
||
|
|
||
| async def main() -> None: | ||
| for highlevel in (False, True): | ||
| for mode in ("legacy", "auto", "2026-07-28"): | ||
| with anyio.fail_after(5): | ||
| await verify(open_transport, kind="mqtt", highlevel=highlevel, mode=mode) | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| anyio.run(main) | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,71 @@ | ||
| """Check remote-peer loss using Mosquitto's session-takeover behavior.""" | ||
|
|
||
| import os | ||
| from contextlib import AsyncExitStack | ||
| from uuid import uuid4 | ||
|
|
||
| import aiomqtt | ||
| import anyio | ||
| from mcp import Client, MCPError | ||
| from mcp.server.mcpserver import MCPServer | ||
| from mcp.types import CONNECTION_CLOSED | ||
|
|
||
| from demo_mqtt import open_transport | ||
|
|
||
|
|
||
| async def main() -> None: | ||
| """A broker-forced disconnect publishes the configured Last Will and settles a waiting MCP call.""" | ||
| entered = anyio.Event() | ||
| finished = anyio.Event() | ||
| server = MCPServer("peer loss") | ||
| session = uuid4().hex | ||
|
|
||
| @server.tool() | ||
| async def hold() -> str: | ||
| entered.set() | ||
| await anyio.sleep_forever() | ||
| raise NotImplementedError | ||
|
|
||
| with anyio.fail_after(5): | ||
| async with AsyncExitStack() as stack: | ||
| server_transport = await open_transport(stack, "alice", session, True) | ||
| client_transport = await open_transport(stack, "alice", session, False) | ||
| runtime = await stack.enter_async_context(server.serve()) | ||
| await runtime.connect(server_transport) | ||
| client = await stack.enter_async_context(Client(client_transport, read_timeout_seconds=None)) | ||
|
|
||
|
Comment on lines
+32
to
+36
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🔴 CI operators get a red "Check live MQTT routing and peer loss" step after merging, if aiomqtt re-raises an unexpected disconnect on client exit as aiomqtt 2.x does. The evicted server-side Extended reasoning...The disconnect check needs the server-alice connection to be killed by the broker so the Last Will fires. demo_mqtt_disconnect.py:59-66 opens a second connection with username server-alice; with use_username_as_clientid true the broker takes over the session and sends the old connection a DISCONNECT with reason 0x8E. paho invokes on_disconnect with that reason code; aiomqtt's _on_disconnect sets the client's _disconnected future to an exception because the reason is not MQTT_ERR_SUCCESS. The client's messages iterator then raises MqttError, which mqtt.py:81 catches, so the read stream ends and the pending call fails with CONNECTION_CLOSED as asserted. finished.wait() returns and the task group exits. AsyncExitStack then unwinds: MCP Client, then runtime, then the client-side aiomqtt.Client (graceful), then the server-side aiomqtt.Client entered at demo_mqtt.py:19. aiomqtt 2.x aexit checks _disconnected.done() and, when it holds an exception, re-raises it. That MqttCodeError propagates out of… Verification: normal — triggering condition: the pinned aiomqtt (uv.lock: |
||
| async def call() -> None: | ||
| try: | ||
| await client.call_tool("hold") | ||
| except MCPError as exc: | ||
| assert exc.code == CONNECTION_CLOSED | ||
| else: | ||
| raise AssertionError("Remote peer loss must fail the pending call") | ||
| finished.set() | ||
|
|
||
| async with anyio.create_task_group() as tg: | ||
| tg.start_soon(call) | ||
| await entered.wait() | ||
| async with aiomqtt.Client( | ||
| "127.0.0.1", | ||
| int(os.environ.get("MQTT_TEST_PORT", "13883")), | ||
| username="bob", | ||
| password="test-bob-password", | ||
| identifier="server-alice", | ||
| protocol=aiomqtt.ProtocolVersion.V5, | ||
| ): | ||
| assert [tool.name for tool in (await client.list_tools()).tools] == ["hold"] | ||
| assert not finished.is_set() | ||
| async with aiomqtt.Client( | ||
| "127.0.0.1", | ||
| int(os.environ.get("MQTT_TEST_PORT", "13883")), | ||
| username="server-alice", | ||
| password="test-server-alice-password", | ||
| identifier="server-alice", | ||
| protocol=aiomqtt.ProtocolVersion.V5, | ||
| ): | ||
| await finished.wait() | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| anyio.run(main) | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
P1: When more than 256 messages arrive before
mqtt_transportdrains the queue, aiomqtt can drop them silently, so MCP requests disappear and wait for client timeouts. Replace this drop-on-full configuration with an overflow path that applies backpressure or fails the connection explicitly.Prompt for AI agents