[DSv4][bring-up] FP8 H100 Dynamo-vLLM multinode benchmark / DSv4 FP8 H100 Dynamo-vLLM 多节点基准测试（bring-up）

benchmarks/multi_node/srt-slurm-recipes/configs/vllm-container-deps.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+set -euo pipefail
+
+apt-get -y update && apt-get install -y --no-install-recommends --allow-change-held-packages numactl
+
+pip install msgpack
+
+if [ -f /configs/patches/vllm_numa_bind_hash_fix.py ]; then
+    python3 /configs/patches/vllm_numa_bind_hash_fix.py
+fi

benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/agg-h100-tep8pp2.yaml

@@ -0,0 +1,109 @@
+name: "dsv4-vllm-agg-h100-tep8pp2-1k1k"
+
+# DeepSeek-V4-Pro FP8 H100 AGGREGATED TP8 x PP2 recipe (2 nodes / 16 GPUs).
+# The ~1TB FP8 checkpoint does not fit one 8xH100 node (640GB). TP8 x PP2
+# shards attention/dense (/8) and layers (/2) across 16 GPUs -- the original
+# memory-correct sizing that loaded the model and reached prefill. That
+# topology's ONLY failure was the disaggregated NIXL KV-transfer stall between
+# the prefill and decode workers; running AGGREGATED (single worker, decode
+# num-worker 0) removes the P/D split entirely, so there is no NIXL transfer to
+# stall on. TP8 (not TP16) preserves the FP8 128-column block divisibility
+# (moe_intermediate 3072 / 8 = 384, divisible by 128; / 16 = 192 is not).
+# Node allocation for this framework is tp*pp/gpus_per_node = 8*2/8 = 2 nodes;
+# EP and DP-attention do NOT contribute GPUs, which is why the earlier DP32
+# aggregated attempt collapsed onto a single node.
+model:
+  path: "deepseek-v4-pro"
+  container: "vllm/vllm-openai:v0.21.0"
+  precision: "fp8"
+
+dynamo:
+  install: true
+  wheel: "1.2.0.dev20260426"
+
+setup_script: vllm-container-deps.sh
+
+slurm:
+  time_limit: "8:00:00"
+
+health_check:
+  interval_seconds: 10
+  max_attempts: 1440
+
+sbatch_directives:
+  cpus-per-task: "144"
+  mem: "0"
+  exclude: "hpc-gpu-1-0,hpc-gpu-1-1,hpc-gpu-1-4,hpc-gpu-1-5,hpc-gpu-1-7,hpc-gpu-1-8,hpc-gpu-1-13,hpc-gpu-1-16,hpc-gpu-1-19"
+
+resources:
+  gpu_type: "h100"
+  gpus_per_node: 8
+  agg_nodes: 2
+  agg_workers: 1
+  gpus_per_agg: 16
+
+frontend:
+  type: dynamo
+  enable_multiple_frontends: false
+
+backend:
+  type: vllm
+  connector: null
+  aggregated_environment:
+    NCCL_CUMEM_ENABLE: "1"
+    PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True"
+    TILELANG_CLEANUP_TEMP_FILES: "1"
+    UCX_MEMTYPE_CACHE: "n"
+    UCX_MEMTYPE_REG_WHOLE: "n"
+    UCX_NET_DEVICES: "all"
+    VLLM_SERVER_DEV_MODE: "1"
+  vllm_config:
+    aggregated:
+      served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
+      block-size: 256
+      tensor-parallel-size: 8
+      pipeline-parallel-size: 2
+      enable-expert-parallel: true
+      kv-cache-dtype: "fp8"
+      gpu-memory-utilization: 0.92
+      max-model-len: 2304
+      # max-num-seqs 256 (not 16): the running-batch cap. At 16 the engine served
+      # at most 16 sequences at once, so every concurrency >=16 queued behind a
+      # batch-16 engine -- decode throughput plateaued and the whole conc sweep
+      # collapsed to a narrow interactivity band (this was the "range too small").
+      # On-cluster prototype (job 11364) at 256: throughput scales 56->2079 tok/s
+      # from conc 1->256 (interactivity 65->8 tok/s/user) on the same 2 nodes, and
+      # high-conc levels finish far faster (batch-256 vs batch-16). Fits at 1k1k
+      # KV (256 x 2304 tokens) within the 2-node headroom at gpu-mem-util 0.92.
+      max-num-seqs: 256
+      max-num-batched-tokens: 16384
+      no-enable-prefix-caching: true
+      no-enable-flashinfer-autotune: true
+      # CUDA graphs (not eager): the eager run served correctly but decode ran
+      # at ~144ms/token (Python dispatch-bound, flat vs concurrency), so the
+      # conc sweep could not finish in 8h. On-cluster prototype (job 11256)
+      # confirmed FULL_AND_PIECEWISE graphs FIT this 2-node TP8xPP2 layout at
+      # gpu-mem-util 0.92 and drop TPOT to ~15ms (~9.4x). No extra nodes needed.
+      compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'
+      reasoning-parser: deepseek_v4
+      tokenizer-mode: deepseek_v4
+      trust-remote-code: true
+
+benchmark:
+  type: "sa-bench"
+  isl: 1024
+  osl: 1024
+  concurrencies: "1x2x4x8x16x32x64x128x256x512"
+  req_rate: "inf"
+  use_chat_template: true
+  custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer"
+
+identity:
+  model:
+    repo: "deepseek-ai/DeepSeek-V4-Pro"
+    revision: "0366e4e064385807ea86b088a5c6c878ff23343b"
+  container:
+    image: "vllm/vllm-openai:v0.21.0"
+  frameworks:
+    dynamo: "1.2.0.dev20260426"
+    vllm: "0.21.0"

benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/agg-h100-tep8pp4.yaml

@@ -0,0 +1,112 @@
+name: "dsv4-vllm-agg-h100-tep8pp4-1k1k"
+
+# DeepSeek-V4-Pro FP8 H100 AGGREGATED TP8 x PP4 recipe (4 nodes / 32 GPUs).
+# PARETO POINT (interactivity end). This is the same aggregated engine as the
+# 2-node TP8xPP2 baseline, but spread over 4 nodes: PP4 puts 1/4 of the layers
+# on each GPU (vs 1/2 at PP2), roughly halving per-GPU weight+expert memory and
+# leaving far more KV headroom. It is deliberately NOT throughput-optimal --
+# an on-cluster prototype measured ~43 tok/s/GPU at conc256 vs ~130 for
+# TP8xPP2, i.e. it is DOMINATED on the tok/s/GPU axis. We keep it because the
+# site plots a tok/s/GPU-vs-interactivity Pareto frontier: PP4's value (if any)
+# is on the INTERACTIVITY axis, and that has to be measured across the conc
+# range, not assumed. Spreading the model over 2x the GPUs can sustain a given
+# tok/s/user target at concurrencies the 2-node layout cannot hold, so PP4 may
+# contribute non-dominated points at the low-latency / high-tok-s-user corner
+# even though its peak throughput/GPU is lower. The frontier construction drops
+# it automatically wherever PP2 dominates; the point of running it is to get the
+# data. TP8 (not TP16) preserves FP8 128-column block divisibility
+# (moe_intermediate 3072 / 8 = 384, divisible by 128). Node allocation is
+# tp*pp/gpus_per_node = 8*4/8 = 4 nodes; EP and DP-attention add no GPUs.
+# Aggregated (decode num-worker 0) => no NIXL P/D transfer to stall on.
+model:
+  path: "deepseek-v4-pro"
+  container: "vllm/vllm-openai:v0.21.0"
+  precision: "fp8"
+
+dynamo:
+  install: true
+  wheel: "1.2.0.dev20260426"
+
+setup_script: vllm-container-deps.sh
+
+slurm:
+  time_limit: "8:00:00"
+
+health_check:
+  interval_seconds: 10
+  max_attempts: 1440
+
+sbatch_directives:
+  cpus-per-task: "144"
+  mem: "0"
+  exclude: "hpc-gpu-1-0,hpc-gpu-1-1,hpc-gpu-1-4,hpc-gpu-1-5,hpc-gpu-1-7,hpc-gpu-1-8,hpc-gpu-1-13,hpc-gpu-1-16,hpc-gpu-1-19"
+
+resources:
+  gpu_type: "h100"
+  gpus_per_node: 8
+  agg_nodes: 4
+  agg_workers: 1
+  gpus_per_agg: 32
+
+frontend:
+  type: dynamo
+  enable_multiple_frontends: false
+
+backend:
+  type: vllm
+  connector: null
+  aggregated_environment:
+    NCCL_CUMEM_ENABLE: "1"
+    PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True"
+    TILELANG_CLEANUP_TEMP_FILES: "1"
+    UCX_MEMTYPE_CACHE: "n"
+    UCX_MEMTYPE_REG_WHOLE: "n"
+    UCX_NET_DEVICES: "all"
+    VLLM_SERVER_DEV_MODE: "1"
+  vllm_config:
+    aggregated:
+      served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
+      block-size: 256
+      tensor-parallel-size: 8
+      pipeline-parallel-size: 4
+      enable-expert-parallel: true
+      kv-cache-dtype: "fp8"
+      gpu-memory-utilization: 0.92
+      max-model-len: 2304
+      # max-num-seqs 256, matched to the TP8xPP2 baseline so the interactivity
+      # comparison is at equal running-batch. PP4 has ~2x the KV headroom of PP2
+      # (half the layers per GPU) so it could take a larger cap, but the conc
+      # sweep tops out at 256 here, so 256 is the useful ceiling.
+      max-num-seqs: 256
+      max-num-batched-tokens: 16384
+      no-enable-prefix-caching: true
+      no-enable-flashinfer-autotune: true
+      # CUDA graphs (not eager): confirmed on-cluster that FULL_AND_PIECEWISE
+      # graphs fit this 4-node TP8xPP4 layout at gpu-mem-util 0.92.
+      compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'
+      reasoning-parser: deepseek_v4
+      tokenizer-mode: deepseek_v4
+      trust-remote-code: true
+
+benchmark:
+  type: "sa-bench"
+  isl: 1024
+  osl: 1024
+  # Weighted toward the low/mid-interactivity band (the reason PP4 is here). We
+  # stop at conc256 rather than 512: PP4 is already throughput-dominated by that
+  # point, so higher conc only adds cost in a region PP2 owns. conc 1-256 maps
+  # the interactivity curve where PP4 could hold a non-dominated corner.
+  concurrencies: "1x2x4x8x16x32x64x128x256"
+  req_rate: "inf"
+  use_chat_template: true
+  custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer"
+
+identity:
+  model:
+    repo: "deepseek-ai/DeepSeek-V4-Pro"
+    revision: "0366e4e064385807ea86b088a5c6c878ff23343b"
+  container:
+    image: "vllm/vllm-openai:v0.21.0"
+  frameworks:
+    dynamo: "1.2.0.dev20260426"
+    vllm: "0.21.0"

benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-dep32.yaml

@@ -0,0 +1,153 @@
+name: "dsv4-vllm-disagg-h100-dep32-1k1k"
+
+# DeepSeek-V4-Pro FP8 H100 DISAGGREGATED DP32 x EP32 recipe (multi_node_dep).
+# PARETO POINT (throughput end). 1 prefill worker + 1 decode worker, each
+# data-parallel-32 x expert-parallel-32 (tp1) => 4 nodes/worker => 8 GPU nodes
+# + 1 dedicated etcd/NATS infra node = 9 nodes. Data-parallel attention + full
+# expert parallelism is the throughput-optimal MoE shape (no per-token TP all-
+# reduce), and disaggregation lets prefill and decode scale independently -- the
+# recipes.vllm.ai multi_node_dep strategy for the high-concurrency corner.
+#
+# DP MUST be 32 on H100: EP must divide the 256 routed experts, and the model
+# only fits with enough ranks -- DP16 OOMs at model load (~78 GiB/GPU), DP24
+# gives EP24 which does not divide 256, DP32 gives EP32 = 8 experts/rank
+# (~52 GiB/GPU weights) and fits at gpu-mem-util 0.90. An on-cluster prototype
+# confirmed both workers load (51.98 GiB/GPU) and complete the NIXL P/D
+# handshake at DP scale. This is the fragile / expensive point of the frontier
+# (2 x ~805 GiB checkpoint from NFS => >13 min bring-up, DP32-rank shared-memory
+# sync can stall); it is included so the throughput end is measured, and it is
+# acceptable for it to fail without blocking the rest of the sweep (the PR runs
+# with fail-fast OFF).
+#
+# NixlConnector + PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True REQUIRES
+# enable-sleep-mode: true on both workers, or vLLM rejects the config at
+# startup (VllmConfig validation error).
+model:
+  path: "deepseek-v4-pro"
+  container: "vllm/vllm-openai:v0.21.0"
+  precision: "fp8"
+
+dynamo:
+  install: true
+  wheel: "1.2.0.dev20260426"
+
+setup_script: vllm-container-deps.sh
+
+slurm:
+  time_limit: "8:00:00"
+
+health_check:
+  interval_seconds: 10
+  max_attempts: 1440
+
+sbatch_directives:
+  cpus-per-task: "144"
+  mem: "0"
+  exclude: "hpc-gpu-1-0,hpc-gpu-1-1,hpc-gpu-1-4,hpc-gpu-1-5,hpc-gpu-1-7,hpc-gpu-1-8,hpc-gpu-1-13,hpc-gpu-1-16,hpc-gpu-1-19"
+
+resources:
+  gpu_type: "h100"
+  gpus_per_node: 8
+  prefill_nodes: 4
+  decode_nodes: 4
+  prefill_workers: 1
+  decode_workers: 1
+  gpus_per_prefill: 32
+  gpus_per_decode: 32
+
+infra:
+  etcd_nats_dedicated_node: true
+
+frontend:
+  type: dynamo
+  enable_multiple_frontends: false
+
+backend:
+  type: vllm
+  connector: null
+  prefill_environment:
+    VLLM_ENGINE_READY_TIMEOUT_S: "3600"
+    NCCL_CUMEM_ENABLE: "1"
+    PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True"
+    TILELANG_CLEANUP_TEMP_FILES: "1"
+    UCX_MEMTYPE_CACHE: "n"
+    UCX_MEMTYPE_REG_WHOLE: "n"
+    UCX_NET_DEVICES: "all"
+    VLLM_SERVER_DEV_MODE: "1"
+  decode_environment:
+    VLLM_ENGINE_READY_TIMEOUT_S: "3600"
+    NCCL_CUMEM_ENABLE: "1"
+    PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True"
+    TILELANG_CLEANUP_TEMP_FILES: "1"
+    UCX_MEMTYPE_CACHE: "n"
+    UCX_MEMTYPE_REG_WHOLE: "n"
+    UCX_NET_DEVICES: "all"
+    VLLM_SERVER_DEV_MODE: "1"
+  vllm_config:
+    prefill:
+      served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
+      block-size: 256
+      kv-cache-dtype: "fp8"
+      kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
+      tensor-parallel-size: 1
+      pipeline-parallel-size: 1
+      data-parallel-size: 32
+      data-parallel-hybrid-lb: true
+      data-parallel-rpc-port: 13345
+      enable-expert-parallel: true
+      enable-sleep-mode: true
+      gpu-memory-utilization: 0.90
+      max-model-len: 2304
+      max-num-seqs: 16
+      max-num-batched-tokens: 16384
+      no-enable-prefix-caching: true
+      no-enable-flashinfer-autotune: true
+      compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}'
+      reasoning-parser: deepseek_v4
+      tokenizer-mode: deepseek_v4
+      trust-remote-code: true
+    decode:
+      served-model-name: "deepseek-ai/DeepSeek-V4-Pro"
+      block-size: 256
+      kv-cache-dtype: "fp8"
+      kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'
+      tensor-parallel-size: 1
+      pipeline-parallel-size: 1
+      data-parallel-size: 32
+      data-parallel-hybrid-lb: true
+      data-parallel-rpc-port: 13345
+      enable-expert-parallel: true
+      enable-sleep-mode: true
+      gpu-memory-utilization: 0.90
+      max-model-len: 2304
+      max-num-seqs: 256
+      max-num-batched-tokens: 512
+      no-enable-prefix-caching: true
+      no-enable-flashinfer-autotune: true
+      compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'
+      reasoning-parser: deepseek_v4
+      tokenizer-mode: deepseek_v4
+      trust-remote-code: true
+
+benchmark:
+  type: "sa-bench"
+  isl: 1024
+  osl: 1024
+  # High-conc only: DEP saturates late because concurrency is split across 32 DP
+  # ranks (conc512 => 16/rank, conc2048 => 64/rank). This is the throughput
+  # anchor of the frontier -- the low/mid-interactivity band is owned by the
+  # aggregated topologies, so DP32 is only run where it can win (peak tok/s/GPU).
+  concurrencies: "512x1024x2048"
+  req_rate: "inf"
+  use_chat_template: true
+  custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer"
+
+identity:
+  model:
+    repo: "deepseek-ai/DeepSeek-V4-Pro"
+    revision: "0366e4e064385807ea86b088a5c6c878ff23343b"
+  container:
+    image: "vllm/vllm-openai:v0.21.0"
+  frameworks:
+    dynamo: "1.2.0.dev20260426"
+    vllm: "0.21.0"