diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1024-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1024-mtp-kvoffload.yaml index 715b0c631e..3e61deb987 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1024-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-12p4d-dep8-dep16-c1024-mtp-kvoffload.yaml @@ -129,6 +129,7 @@ backend: prefill: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro + enable-metrics: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -163,6 +164,7 @@ backend: decode: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro + enable-metrics: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -206,7 +208,8 @@ benchmark: RESULT_DIR: /logs/agentic PORT: "8000" IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" - diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c8-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c8-mtp-kvoffload.yaml index dc91c255ce..2ac8d01b52 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c8-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p1d-dep8-tp4-c8-mtp-kvoffload.yaml @@ -129,6 +129,7 @@ backend: prefill: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro + enable-metrics: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -163,6 +164,7 @@ backend: decode: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro + enable-metrics: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -205,7 +207,8 @@ benchmark: RESULT_DIR: /logs/agentic PORT: "8000" IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" - diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml index 6df695b914..60d0fadda1 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-2p4d-dep8-dep16-c256-mtp-kvoffload.yaml @@ -129,6 +129,7 @@ backend: prefill: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro + enable-metrics: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -163,6 +164,7 @@ backend: decode: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro + enable-metrics: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -206,7 +208,8 @@ benchmark: RESULT_DIR: /logs/agentic PORT: "8000" IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" - diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c256-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c256-mtp-kvoffload.yaml index 107a787ba9..25bdf62713 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c256-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-4p4d-dep8-dep16-c256-mtp-kvoffload.yaml @@ -129,6 +129,7 @@ backend: prefill: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro + enable-metrics: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -163,6 +164,7 @@ backend: decode: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro + enable-metrics: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -206,7 +208,8 @@ benchmark: RESULT_DIR: /logs/agentic PORT: "8000" IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" - diff --git a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c512-mtp-kvoffload.yaml b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c512-mtp-kvoffload.yaml index 27a4e42d22..3cb306c684 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c512-mtp-kvoffload.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic/disagg-gb300-6p4d-dep8-dep16-c512-mtp-kvoffload.yaml @@ -129,6 +129,7 @@ backend: prefill: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro + enable-metrics: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -163,6 +164,7 @@ backend: decode: host: 0.0.0.0 served-model-name: deepseek-ai/DeepSeek-V4-Pro + enable-metrics: true model-path: /model/ trust-remote-code: true watchdog-timeout: 86400 @@ -206,7 +208,8 @@ benchmark: RESULT_DIR: /logs/agentic PORT: "8000" IS_MULTINODE: "true" - AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "1" + AIPERF_HTTP_X_DYNAMO_SESSION_ID_FROM_CORRELATION_ID: "true" + AIPERF_USE_DYNAMO_CONV_AWARE_ROUTING: "0" + AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "sglang:" AIPERF_DATASET_MMAP_CACHE_DIR: "/aiperf_mmap_cache" HF_HUB_CACHE: "/hf_hub_cache" - diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 7b7510790d..100aaa43fc 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5526,6 +5526,17 @@ - "Enable SGLang backend metrics on every aggregate, prefill, and decode engine, and fail before publishing a partial trace artifact if required sglang: metrics are absent." - "Record the recipes' active HiCache host-DRAM tier and, for disaggregated points, Dynamo router commit 5a638087 in nvidia-master metadata so generated artifacts no longer report kv_offloading=none, allocated_cpu_dram_gb=0, or a null router." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2477 + +- config-keys: + - dsv4-fp4-gb300-dynamo-sglang-agentic-disagg + scenario-type: + - agentic-coding + description: + - "Refresh submission with up to date AgentX harness." + - "Collect every SGLang worker's metrics through AgentX and fail when required sglang: series are absent." + - "Enable SGLang metrics on every prefill and decode engine." + - "Use supported header-based Dynamo session routing with the in-repo AIPerf build." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2521 - config-keys: - kimik3-fp4-h200-vllm-agentic diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index d955d18ebc..e90c8cad2e 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -206,13 +206,15 @@ if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5" \ recipes/sglang/qwen3.5 elif [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "dsv4" ]]; then - # DSv4 GB300 sglang agentic: NVIDIA/srt-slurm v1.0.10 has the nginx - # client_max_body_size fix (>1 MiB agentic warmup bodies), the - # session-affinity frontend, and the BenchmarkType.CUSTOM / extra_mount - # schema these recipes need. + # DSv4 GB300 SGLang agentic uses NVIDIA/srt-slurm v1.0.38. In addition to + # the nginx body-size fix, session-affinity frontend, and custom benchmark + # schema required by these recipes, this release injects every logical + # SGLang worker leader's /metrics URL into AIPERF_SERVER_METRICS_URLS. + # AgentX forwards that list to aiperf's --server-metrics argument so its + # trace artifacts include backend metrics for every engine. git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" - git checkout v1.0.10 + git checkout v1.0.38 mkdir -p recipes/sglang/deepseek-v4/agentic cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/agentic" \ recipes/sglang/deepseek-v4/agentic