From adb149d5a928fe975b84e6818c6cb4d5a8e04975 Mon Sep 17 00:00:00 2001 From: "randomizedcoder dave.seddon.ca@gmail.com" Date: Tue, 25 Aug 2026 18:39:16 -0700 Subject: [PATCH] hp1/hp3: persistent jumbo MTU 9700 on the 25 GbE RoCE links Set enp1s0f0np0 / enp1s0f1np1 to MTU 9700 (ConnectX-4 Lx driver maxmtu is 9978; 9700 is NVIDIA's practical max for this NIC) via a systemd-udevd .link file, so it applies at device appearance independent of the imperative xdp2.testbed NIC wiring (whose `jumbo` option only hardcodes 9000). This lifts the RoCE path MTU (ibv active_mtu) from 1024 to its 4096 ceiling and raises the single-stream TCP baseline. Verified end-to-end: ping -M do -s 9672 passes 0% loss hp1<->hp3. Supports uds-rdma-proxy design 37 (large-frame throughput: a single stream now beats single-stream TCP). Co-Authored-By: Claude Opus 4.8 --- hp/hp1/configuration.nix | 2 ++ hp/hp1/roce-mtu.nix | 26 ++++++++++++++++++++++++++ hp/hp3/configuration.nix | 2 ++ hp/hp3/roce-mtu.nix | 26 ++++++++++++++++++++++++++ 4 files changed, 56 insertions(+) create mode 100644 hp/hp1/roce-mtu.nix create mode 100644 hp/hp3/roce-mtu.nix diff --git a/hp/hp1/configuration.nix b/hp/hp1/configuration.nix index 45f4592..4563f2b 100644 --- a/hp/hp1/configuration.nix +++ b/hp/hp1/configuration.nix @@ -52,6 +52,8 @@ # design 32: urp acceptor endpoint + PTP grandmaster (lab time sync) ./urp.nix ./ptp.nix + # design 37: persistent jumbo MTU (9700) on the 25 GbE RoCE links + ./roce-mtu.nix ]; # Bootloader. diff --git a/hp/hp1/roce-mtu.nix b/hp/hp1/roce-mtu.nix new file mode 100644 index 0000000..9573b89 --- /dev/null +++ b/hp/hp1/roce-mtu.nix @@ -0,0 +1,26 @@ +# hp/hp1/roce-mtu.nix +# +# Persistent jumbo MTU on the 25 GbE RoCEv2 links (uds-rdma-proxy design 37 +# throughput work). The ConnectX-4 Lx driver reports maxmtu 9978; NVIDIA's +# practical max for this NIC (MT27710) is 9700. A large L2 MTU lifts the RoCE +# path MTU (ibv active_mtu) from 1024 to its 4096 ceiling — the real RDMA win — +# and also raises the single-stream TCP baseline. +# +# Set via a systemd-udevd .link file so it applies at device appearance, +# independent of the imperative xdp2.testbed NIC wiring (whose `jumbo` option +# would only hardcode 9000). A 10-* prefix sorts ahead of the empty +# 40-.link, so this is the matching .link for these ports. +# +# Verify after `nixos-rebuild switch` + reboot: +# ip -d link show enp1s0f0np0 # mtu 9700 +# ibv_devinfo | grep active_mtu # 4096 (was 1024) +# ping -M do -s 9672 10.10.2.3 # jumbo path passes end-to-end +# If the path cannot pass 9700 (switch/peer limit), drop MTUBytes to "9000". +{ ... }: + +{ + systemd.network.links."10-roce-jumbo" = { + matchConfig.OriginalName = "enp1s0f0np0 enp1s0f1np1"; + linkConfig.MTUBytes = "9700"; + }; +} diff --git a/hp/hp3/configuration.nix b/hp/hp3/configuration.nix index b21aac6..ff1d7e9 100644 --- a/hp/hp3/configuration.nix +++ b/hp/hp3/configuration.nix @@ -41,6 +41,8 @@ # design 32: urp initiator endpoint + PTP slave (lab time sync) ./urp.nix ./ptp.nix + # design 37: persistent jumbo MTU (9700) on the 25 GbE RoCE links + ./roce-mtu.nix ]; # Bootloader. diff --git a/hp/hp3/roce-mtu.nix b/hp/hp3/roce-mtu.nix new file mode 100644 index 0000000..83c36d0 --- /dev/null +++ b/hp/hp3/roce-mtu.nix @@ -0,0 +1,26 @@ +# hp/hp3/roce-mtu.nix +# +# Persistent jumbo MTU on the 25 GbE RoCEv2 links (uds-rdma-proxy design 37 +# throughput work). The ConnectX-4 Lx driver reports maxmtu 9978; NVIDIA's +# practical max for this NIC (MT27710) is 9700. A large L2 MTU lifts the RoCE +# path MTU (ibv active_mtu) from 1024 to its 4096 ceiling — the real RDMA win — +# and also raises the single-stream TCP baseline. +# +# Set via a systemd-udevd .link file so it applies at device appearance, +# independent of the imperative xdp2.testbed NIC wiring (whose `jumbo` option +# would only hardcode 9000). A 10-* prefix sorts ahead of the empty +# 40-.link, so this is the matching .link for these ports. +# +# Verify after `nixos-rebuild switch` + reboot: +# ip -d link show enp1s0f0np0 # mtu 9700 +# ibv_devinfo | grep active_mtu # 4096 (was 1024) +# ping -M do -s 9672 10.10.2.1 # jumbo path passes end-to-end +# If the path cannot pass 9700 (switch/peer limit), drop MTUBytes to "9000". +{ ... }: + +{ + systemd.network.links."10-roce-jumbo" = { + matchConfig.OriginalName = "enp1s0f0np0 enp1s0f1np1"; + linkConfig.MTUBytes = "9700"; + }; +}