diff --git a/benchmarks/multi_node/srt-slurm-recipes/configs/vllm-container-deps.sh b/benchmarks/multi_node/srt-slurm-recipes/configs/vllm-container-deps.sh new file mode 100644 index 0000000000..d11740d596 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/configs/vllm-container-deps.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +apt-get -y update && apt-get install -y --no-install-recommends --allow-change-held-packages numactl + +pip install msgpack + +if [ -f /configs/patches/vllm_numa_bind_hash_fix.py ]; then + python3 /configs/patches/vllm_numa_bind_hash_fix.py +fi diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/agg-h100-tep8pp2.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/agg-h100-tep8pp2.yaml new file mode 100644 index 0000000000..0d054dbd42 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/agg-h100-tep8pp2.yaml @@ -0,0 +1,109 @@ +name: "dsv4-vllm-agg-h100-tep8pp2-1k1k" + +# DeepSeek-V4-Pro FP8 H100 AGGREGATED TP8 x PP2 recipe (2 nodes / 16 GPUs). +# The ~1TB FP8 checkpoint does not fit one 8xH100 node (640GB). TP8 x PP2 +# shards attention/dense (/8) and layers (/2) across 16 GPUs -- the original +# memory-correct sizing that loaded the model and reached prefill. That +# topology's ONLY failure was the disaggregated NIXL KV-transfer stall between +# the prefill and decode workers; running AGGREGATED (single worker, decode +# num-worker 0) removes the P/D split entirely, so there is no NIXL transfer to +# stall on. TP8 (not TP16) preserves the FP8 128-column block divisibility +# (moe_intermediate 3072 / 8 = 384, divisible by 128; / 16 = 192 is not). +# Node allocation for this framework is tp*pp/gpus_per_node = 8*2/8 = 2 nodes; +# EP and DP-attention do NOT contribute GPUs, which is why the earlier DP32 +# aggregated attempt collapsed onto a single node. +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:v0.21.0" + precision: "fp8" + +dynamo: + install: true + wheel: "1.2.0.dev20260426" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 1440 + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + exclude: "hpc-gpu-1-0,hpc-gpu-1-1,hpc-gpu-1-4,hpc-gpu-1-5,hpc-gpu-1-7,hpc-gpu-1-8,hpc-gpu-1-13,hpc-gpu-1-16,hpc-gpu-1-19" + +resources: + gpu_type: "h100" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 16 + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + NCCL_CUMEM_ENABLE: "1" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "all" + VLLM_SERVER_DEV_MODE: "1" + vllm_config: + aggregated: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + block-size: 256 + tensor-parallel-size: 8 + pipeline-parallel-size: 2 + enable-expert-parallel: true + kv-cache-dtype: "fp8" + gpu-memory-utilization: 0.92 + max-model-len: 2304 + # max-num-seqs 256 (not 16): the running-batch cap. At 16 the engine served + # at most 16 sequences at once, so every concurrency >=16 queued behind a + # batch-16 engine -- decode throughput plateaued and the whole conc sweep + # collapsed to a narrow interactivity band (this was the "range too small"). + # On-cluster prototype (job 11364) at 256: throughput scales 56->2079 tok/s + # from conc 1->256 (interactivity 65->8 tok/s/user) on the same 2 nodes, and + # high-conc levels finish far faster (batch-256 vs batch-16). Fits at 1k1k + # KV (256 x 2304 tokens) within the 2-node headroom at gpu-mem-util 0.92. + max-num-seqs: 256 + max-num-batched-tokens: 16384 + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + # CUDA graphs (not eager): the eager run served correctly but decode ran + # at ~144ms/token (Python dispatch-bound, flat vs concurrency), so the + # conc sweep could not finish in 8h. On-cluster prototype (job 11256) + # confirmed FULL_AND_PIECEWISE graphs FIT this 2-node TP8xPP2 layout at + # gpu-mem-util 0.92 and drop TPOT to ~15ms (~9.4x). No extra nodes needed. + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' + reasoning-parser: deepseek_v4 + tokenizer-mode: deepseek_v4 + trust-remote-code: true + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + concurrencies: "1x2x4x8x16x32x64x128x256x512" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "0366e4e064385807ea86b088a5c6c878ff23343b" + container: + image: "vllm/vllm-openai:v0.21.0" + frameworks: + dynamo: "1.2.0.dev20260426" + vllm: "0.21.0" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/agg-h100-tep8pp4.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/agg-h100-tep8pp4.yaml new file mode 100644 index 0000000000..761a7785d5 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/agg-h100-tep8pp4.yaml @@ -0,0 +1,112 @@ +name: "dsv4-vllm-agg-h100-tep8pp4-1k1k" + +# DeepSeek-V4-Pro FP8 H100 AGGREGATED TP8 x PP4 recipe (4 nodes / 32 GPUs). +# PARETO POINT (interactivity end). This is the same aggregated engine as the +# 2-node TP8xPP2 baseline, but spread over 4 nodes: PP4 puts 1/4 of the layers +# on each GPU (vs 1/2 at PP2), roughly halving per-GPU weight+expert memory and +# leaving far more KV headroom. It is deliberately NOT throughput-optimal -- +# an on-cluster prototype measured ~43 tok/s/GPU at conc256 vs ~130 for +# TP8xPP2, i.e. it is DOMINATED on the tok/s/GPU axis. We keep it because the +# site plots a tok/s/GPU-vs-interactivity Pareto frontier: PP4's value (if any) +# is on the INTERACTIVITY axis, and that has to be measured across the conc +# range, not assumed. Spreading the model over 2x the GPUs can sustain a given +# tok/s/user target at concurrencies the 2-node layout cannot hold, so PP4 may +# contribute non-dominated points at the low-latency / high-tok-s-user corner +# even though its peak throughput/GPU is lower. The frontier construction drops +# it automatically wherever PP2 dominates; the point of running it is to get the +# data. TP8 (not TP16) preserves FP8 128-column block divisibility +# (moe_intermediate 3072 / 8 = 384, divisible by 128). Node allocation is +# tp*pp/gpus_per_node = 8*4/8 = 4 nodes; EP and DP-attention add no GPUs. +# Aggregated (decode num-worker 0) => no NIXL P/D transfer to stall on. +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:v0.21.0" + precision: "fp8" + +dynamo: + install: true + wheel: "1.2.0.dev20260426" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 1440 + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + exclude: "hpc-gpu-1-0,hpc-gpu-1-1,hpc-gpu-1-4,hpc-gpu-1-5,hpc-gpu-1-7,hpc-gpu-1-8,hpc-gpu-1-13,hpc-gpu-1-16,hpc-gpu-1-19" + +resources: + gpu_type: "h100" + gpus_per_node: 8 + agg_nodes: 4 + agg_workers: 1 + gpus_per_agg: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + NCCL_CUMEM_ENABLE: "1" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "all" + VLLM_SERVER_DEV_MODE: "1" + vllm_config: + aggregated: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + block-size: 256 + tensor-parallel-size: 8 + pipeline-parallel-size: 4 + enable-expert-parallel: true + kv-cache-dtype: "fp8" + gpu-memory-utilization: 0.92 + max-model-len: 2304 + # max-num-seqs 256, matched to the TP8xPP2 baseline so the interactivity + # comparison is at equal running-batch. PP4 has ~2x the KV headroom of PP2 + # (half the layers per GPU) so it could take a larger cap, but the conc + # sweep tops out at 256 here, so 256 is the useful ceiling. + max-num-seqs: 256 + max-num-batched-tokens: 16384 + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + # CUDA graphs (not eager): confirmed on-cluster that FULL_AND_PIECEWISE + # graphs fit this 4-node TP8xPP4 layout at gpu-mem-util 0.92. + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' + reasoning-parser: deepseek_v4 + tokenizer-mode: deepseek_v4 + trust-remote-code: true + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + # Weighted toward the low/mid-interactivity band (the reason PP4 is here). We + # stop at conc256 rather than 512: PP4 is already throughput-dominated by that + # point, so higher conc only adds cost in a region PP2 owns. conc 1-256 maps + # the interactivity curve where PP4 could hold a non-dominated corner. + concurrencies: "1x2x4x8x16x32x64x128x256" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "0366e4e064385807ea86b088a5c6c878ff23343b" + container: + image: "vllm/vllm-openai:v0.21.0" + frameworks: + dynamo: "1.2.0.dev20260426" + vllm: "0.21.0" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-dep32.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-dep32.yaml new file mode 100644 index 0000000000..d0ac97e42f --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/1k1k/disagg-h100-dep32.yaml @@ -0,0 +1,153 @@ +name: "dsv4-vllm-disagg-h100-dep32-1k1k" + +# DeepSeek-V4-Pro FP8 H100 DISAGGREGATED DP32 x EP32 recipe (multi_node_dep). +# PARETO POINT (throughput end). 1 prefill worker + 1 decode worker, each +# data-parallel-32 x expert-parallel-32 (tp1) => 4 nodes/worker => 8 GPU nodes +# + 1 dedicated etcd/NATS infra node = 9 nodes. Data-parallel attention + full +# expert parallelism is the throughput-optimal MoE shape (no per-token TP all- +# reduce), and disaggregation lets prefill and decode scale independently -- the +# recipes.vllm.ai multi_node_dep strategy for the high-concurrency corner. +# +# DP MUST be 32 on H100: EP must divide the 256 routed experts, and the model +# only fits with enough ranks -- DP16 OOMs at model load (~78 GiB/GPU), DP24 +# gives EP24 which does not divide 256, DP32 gives EP32 = 8 experts/rank +# (~52 GiB/GPU weights) and fits at gpu-mem-util 0.90. An on-cluster prototype +# confirmed both workers load (51.98 GiB/GPU) and complete the NIXL P/D +# handshake at DP scale. This is the fragile / expensive point of the frontier +# (2 x ~805 GiB checkpoint from NFS => >13 min bring-up, DP32-rank shared-memory +# sync can stall); it is included so the throughput end is measured, and it is +# acceptable for it to fail without blocking the rest of the sweep (the PR runs +# with fail-fast OFF). +# +# NixlConnector + PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True REQUIRES +# enable-sleep-mode: true on both workers, or vLLM rejects the config at +# startup (VllmConfig validation error). +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:v0.21.0" + precision: "fp8" + +dynamo: + install: true + wheel: "1.2.0.dev20260426" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 1440 + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + exclude: "hpc-gpu-1-0,hpc-gpu-1-1,hpc-gpu-1-4,hpc-gpu-1-5,hpc-gpu-1-7,hpc-gpu-1-8,hpc-gpu-1-13,hpc-gpu-1-16,hpc-gpu-1-19" + +resources: + gpu_type: "h100" + gpus_per_node: 8 + prefill_nodes: 4 + decode_nodes: 4 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 32 + gpus_per_decode: 32 + +infra: + etcd_nats_dedicated_node: true + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + prefill_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + NCCL_CUMEM_ENABLE: "1" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "all" + VLLM_SERVER_DEV_MODE: "1" + decode_environment: + VLLM_ENGINE_READY_TIMEOUT_S: "3600" + NCCL_CUMEM_ENABLE: "1" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "all" + VLLM_SERVER_DEV_MODE: "1" + vllm_config: + prefill: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + block-size: 256 + kv-cache-dtype: "fp8" + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 32 + data-parallel-hybrid-lb: true + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-sleep-mode: true + gpu-memory-utilization: 0.90 + max-model-len: 2304 + max-num-seqs: 16 + max-num-batched-tokens: 16384 + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","mode":0}' + reasoning-parser: deepseek_v4 + tokenizer-mode: deepseek_v4 + trust-remote-code: true + decode: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + block-size: 256 + kv-cache-dtype: "fp8" + kv-transfer-config: '{"kv_connector":"NixlConnector","kv_role":"kv_both"}' + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + data-parallel-size: 32 + data-parallel-hybrid-lb: true + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + enable-sleep-mode: true + gpu-memory-utilization: 0.90 + max-model-len: 2304 + max-num-seqs: 256 + max-num-batched-tokens: 512 + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' + reasoning-parser: deepseek_v4 + tokenizer-mode: deepseek_v4 + trust-remote-code: true + +benchmark: + type: "sa-bench" + isl: 1024 + osl: 1024 + # High-conc only: DEP saturates late because concurrency is split across 32 DP + # ranks (conc512 => 16/rank, conc2048 => 64/rank). This is the throughput + # anchor of the frontier -- the low/mid-interactivity band is owned by the + # aggregated topologies, so DP32 is only run where it can win (peak tok/s/GPU). + concurrencies: "512x1024x2048" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "0366e4e064385807ea86b088a5c6c878ff23343b" + container: + image: "vllm/vllm-openai:v0.21.0" + frameworks: + dynamo: "1.2.0.dev20260426" + vllm: "0.21.0" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/agg-h100-tep8pp2.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/agg-h100-tep8pp2.yaml new file mode 100644 index 0000000000..c1e4a9d35b --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/agg-h100-tep8pp2.yaml @@ -0,0 +1,114 @@ +name: "dsv4-vllm-agg-h100-tep8pp2-8k1k" + +# DeepSeek-V4-Pro FP8 H100 AGGREGATED TP8 x PP2 recipe (2 nodes / 16 GPUs). +# The ~1TB FP8 checkpoint does not fit one 8xH100 node (640GB). TP8 x PP2 +# shards attention/dense (/8) and layers (/2) across 16 GPUs -- the original +# memory-correct sizing that loaded the model and reached prefill. That +# topology's ONLY failure was the disaggregated NIXL KV-transfer stall between +# the prefill and decode workers; running AGGREGATED (single worker, decode +# num-worker 0) removes the P/D split entirely, so there is no NIXL transfer to +# stall on. TP8 (not TP16) preserves the FP8 128-column block divisibility +# (moe_intermediate 3072 / 8 = 384, divisible by 128; / 16 = 192 is not). +# Node allocation for this framework is tp*pp/gpus_per_node = 8*2/8 = 2 nodes; +# EP and DP-attention do NOT contribute GPUs, which is why the earlier DP32 +# aggregated attempt collapsed onto a single node. +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:v0.21.0" + precision: "fp8" + +dynamo: + install: true + wheel: "1.2.0.dev20260426" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 1440 + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + exclude: "hpc-gpu-1-0,hpc-gpu-1-1,hpc-gpu-1-4,hpc-gpu-1-5,hpc-gpu-1-7,hpc-gpu-1-8,hpc-gpu-1-13,hpc-gpu-1-16,hpc-gpu-1-19" + +resources: + gpu_type: "h100" + gpus_per_node: 8 + agg_nodes: 2 + agg_workers: 1 + gpus_per_agg: 16 + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + NCCL_CUMEM_ENABLE: "1" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "all" + VLLM_SERVER_DEV_MODE: "1" + vllm_config: + aggregated: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + block-size: 256 + tensor-parallel-size: 8 + pipeline-parallel-size: 2 + enable-expert-parallel: true + kv-cache-dtype: "fp8" + gpu-memory-utilization: 0.92 + max-model-len: 9472 + # max-num-seqs 64 (not 16): the running-batch cap. At 16 the engine served + # at most 16 sequences at once, flattening decode throughput and collapsing + # the interactivity range (see 1k1k recipe). 8k1k KV/seq is ~4x the 1k1k + # (9472 vs 2304 max-model-len), so the batch-256 that fit at 1k1k maps to + # batch-64 here (64*9472 ~= 256*2304, same KV footprint). On-cluster prototype + # (job 11368) at 64: fits + serves on the same 2 nodes, throughput scales + # 60->640 tok/s from conc 1->64 (interactivity 64->12 tok/s/user). Saturates + # at conc 64 (= the batch cap); conc 128 adds one queued low-interactivity point. + max-num-seqs: 64 + max-num-batched-tokens: 16384 + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + # CUDA graphs (not eager): the eager run served correctly but decode ran + # at ~144ms/token (Python dispatch-bound, flat vs concurrency), so the + # conc sweep could not finish in 8h. On-cluster prototype (job 11256) + # confirmed FULL_AND_PIECEWISE graphs FIT this 2-node TP8xPP2 layout at + # gpu-mem-util 0.92 and drop TPOT to ~15ms (~9.4x). No extra nodes needed. + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' + reasoning-parser: deepseek_v4 + tokenizer-mode: deepseek_v4 + trust-remote-code: true + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + # Capped at 128: with max-num-seqs 64 the engine runs 64 seqs at once, so conc + # 128 is 2x oversubscription (one queued low-interactivity point) and conc 256 + # would be 4x -- queueing-dominated, slow, and timeout-prone at 8k1k (each level + # is thousands of 8k-token requests). conc 1-64 spans the useful curve (64->12 + # tok/s/user); 128 anchors the low-interactivity end. Keeps the sweep in 8h. + concurrencies: "1x2x4x8x16x32x64x128" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "0366e4e064385807ea86b088a5c6c878ff23343b" + container: + image: "vllm/vllm-openai:v0.21.0" + frameworks: + dynamo: "1.2.0.dev20260426" + vllm: "0.21.0" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/agg-h100-tep8pp4.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/agg-h100-tep8pp4.yaml new file mode 100644 index 0000000000..686542937c --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/8k1k/agg-h100-tep8pp4.yaml @@ -0,0 +1,102 @@ +name: "dsv4-vllm-agg-h100-tep8pp4-8k1k" + +# DeepSeek-V4-Pro FP8 H100 AGGREGATED TP8 x PP4 recipe (4 nodes / 32 GPUs). +# PARETO POINT (interactivity end), 8k1k. Same rationale as the 1k1k +# agg-h100-tep8pp4 recipe: PP4 spreads the model over 2x the GPUs of the +# TP8xPP2 baseline, halving per-GPU layer/expert memory. It is throughput- +# dominated on tok/s/GPU, but the site frontier is tok/s/GPU-vs-interactivity, +# so PP4 is run to measure whether it holds non-dominated points at the low- +# latency corner -- the extra KV headroom lets it sustain a tok/s/user target +# at concurrencies the 2-node layout cannot. TP8 preserves FP8 128-column block +# divisibility. Node allocation = tp*pp/gpus_per_node = 8*4/8 = 4. Aggregated +# (decode num-worker 0) => no NIXL P/D transfer to stall on. +model: + path: "deepseek-v4-pro" + container: "vllm/vllm-openai:v0.21.0" + precision: "fp8" + +dynamo: + install: true + wheel: "1.2.0.dev20260426" + +setup_script: vllm-container-deps.sh + +slurm: + time_limit: "8:00:00" + +health_check: + interval_seconds: 10 + max_attempts: 1440 + +sbatch_directives: + cpus-per-task: "144" + mem: "0" + exclude: "hpc-gpu-1-0,hpc-gpu-1-1,hpc-gpu-1-4,hpc-gpu-1-5,hpc-gpu-1-7,hpc-gpu-1-8,hpc-gpu-1-13,hpc-gpu-1-16,hpc-gpu-1-19" + +resources: + gpu_type: "h100" + gpus_per_node: 8 + agg_nodes: 4 + agg_workers: 1 + gpus_per_agg: 32 + +frontend: + type: dynamo + enable_multiple_frontends: false + +backend: + type: vllm + connector: null + aggregated_environment: + NCCL_CUMEM_ENABLE: "1" + PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" + TILELANG_CLEANUP_TEMP_FILES: "1" + UCX_MEMTYPE_CACHE: "n" + UCX_MEMTYPE_REG_WHOLE: "n" + UCX_NET_DEVICES: "all" + VLLM_SERVER_DEV_MODE: "1" + vllm_config: + aggregated: + served-model-name: "deepseek-ai/DeepSeek-V4-Pro" + block-size: 256 + tensor-parallel-size: 8 + pipeline-parallel-size: 4 + enable-expert-parallel: true + kv-cache-dtype: "fp8" + gpu-memory-utilization: 0.92 + max-model-len: 9472 + # max-num-seqs 128 = 2x the TP8xPP2 8k1k cap (64). PP4 halves per-GPU + # weight memory vs PP2, so ~2x KV headroom at the same 9472 context; the + # 8k1k KV footprint of batch-128 here is ~= batch-64 at PP2. This lets PP4 + # actually exercise the higher-concurrency end its extra nodes buy. + max-num-seqs: 128 + max-num-batched-tokens: 16384 + no-enable-prefix-caching: true + no-enable-flashinfer-autotune: true + compilation-config: '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' + reasoning-parser: deepseek_v4 + tokenizer-mode: deepseek_v4 + trust-remote-code: true + +benchmark: + type: "sa-bench" + isl: 8192 + osl: 1024 + # conc 1-128: maps the interactivity curve at 8k context. With max-num-seqs + # 128 the whole list runs unqueued (no oversubscription), so every point is a + # clean latency/throughput sample. Kept to 128 to stay within the 8h SLURM + # limit (each 8k1k conc level is thousands of 8k-token requests). + concurrencies: "1x2x4x8x16x32x64x128" + req_rate: "inf" + use_chat_template: true + custom_tokenizer: "sa_bench_tokenizers.vllm_deepseek_v4.VLLMDeepseekV4Tokenizer" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + revision: "0366e4e064385807ea86b088a5c6c878ff23343b" + container: + image: "vllm/vllm-openai:v0.21.0" + frameworks: + dynamo: "1.2.0.dev20260426" + vllm: "0.21.0" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 33e9118b7a..79be745be7 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4930,6 +4930,148 @@ dsr1-fp8-h100-dynamo-sglang: ep: 16 dp-attn: true +# DeepSeek-V4-Pro FP8 H100 aggregated vLLM via Dynamo (TP8 x PP2, 2 nodes). +# The native checkpoint is mixed FP4+FP8 (~1TB), too large for one 8xH100 node +# (640GB). Prior attempts: TP8xPP2 *disagg* loaded the model and reached prefill +# but hung on the NIXL P/D KV-transfer; DP16/2-node OOM'd; DP32 "aggregated" +# collapsed onto ONE node because this framework sizes nodes as tp*pp/gpus_per_node +# (EP and DP-attention add no GPUs), so tp1 => 1 node and vLLM could not place 32 +# DP ranks on 8 GPUs. The working topology is the memory-proven TP8xPP2 run made +# AGGREGATED (single worker, no P/D split => no NIXL transfer to stall on): tp8*pp2 +# = 16 GPUs = 2 nodes. TP8 (not TP16) keeps FP8 128-column block divisibility. +# Recipes: benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/{1k1k,8k1k}/agg-h100-tep8pp2.yaml. +dsv4-fp8-h100-dynamo-vllm: + image: vllm/vllm-openai:v0.21.0 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + # Runs-on label = this string verbatim. Must be a label a live runner + # carries: h100-disagg has 3 online runners (h100-dgxc-slurm_17-19) and is + # the disaggregated-H100 pool. NOT h100-multinode — no runner carries that + # label, so jobs requesting it queue forever (the dsr1-fp8-h100-dynamo-* + # reference configs above still use it and are non-functional for this). + runner: h100-disagg + precision: fp8 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.2.0.dev20260426" } + kv-p2p-transfer: nixl + multinode: true + # Aggregated (not P/D-disaggregated): prefill num-worker 1 + decode + # num-worker 0 is how srtslurm signals AGG mode (see RECIPES.md §5). Node + # count = tp*pp/gpus_per_node = 8*2/8 = 2. tp:8 pp:2 with expert-parallel; + # dp-attn false (TP8 attention). Aggregated => no NIXL transfer to stall on. + disagg: true + scenarios: + fixed-seq-len: + - isl: 1024 + osl: 1024 + search-space: + # Aggregated TP8 x PP2 over 2 H100 nodes (16 GPUs), EP8, max-num-seqs 256. + # The batch cap (256, was 16) lets decode throughput scale with concurrency + # so this one conc-list now spans a wide interactivity range: conc1 ~65 + # tok/s/user (TPOT ~15ms) down to conc512 ~5 tok/s/user, peaking ~130 + # tok/s/GPU by conc256 (on-cluster job 11364). + - conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + prefill: + num-worker: 1 + tp: 8 + pp: 2 + ep: 8 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/1k1k/agg-h100-tep8pp2.yaml" + decode: + num-worker: 0 + tp: 8 + pp: 2 + ep: 8 + dp-attn: false + # PARETO POINT (interactivity): aggregated TP8 x PP4 over 4 H100 nodes (32 + # GPUs), EP8, max-num-seqs 256. Throughput-dominated (~43 vs ~130 tok/s/GPU + # for PP2) but run to map the INTERACTIVITY axis of the tok/s/GPU-vs-tok/s/ + # user frontier: the 2x GPU spread (half the layers/GPU, more KV headroom) + # can hold a tok/s/user target at concurrencies the 2-node PP2 cannot, so it + # may contribute non-dominated points at the low-latency corner. + - conc-list: [1, 2, 4, 8, 16, 32, 64, 128, 256] + prefill: + num-worker: 1 + tp: 8 + pp: 4 + ep: 8 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/1k1k/agg-h100-tep8pp4.yaml" + decode: + num-worker: 0 + tp: 8 + pp: 4 + ep: 8 + dp-attn: false + # PARETO POINT (throughput / multi_node_dep): disaggregated 1P+1D, each + # worker DP32 x EP32 (tp1, dp-attn) = 4 nodes/worker => 8 GPU + 1 dedicated + # etcd/NATS infra node = 9 nodes. Data-parallel attn + expert parallel is + # the throughput-optimal MoE shape (no per-token TP all-reduce); the high- + # conc anchor of the frontier. DP must be 32 (EP divides 256 experts; DP16 + # OOMs, DP24->EP24 invalid). Fragile + expensive (2x ~805 GiB NFS load, + # DP32 shm sync) -- runs with fail-fast OFF so a failure here does not + # cancel the aggregated points ("if it fails, it fails"). + - conc-list: [512, 1024, 2048] + prefill: + num-worker: 1 + tp: 1 + ep: 32 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/1k1k/disagg-h100-dep32.yaml" + decode: + num-worker: 1 + tp: 1 + ep: 32 + dp-attn: true + - isl: 8192 + osl: 1024 + search-space: + # Aggregated TP8 x PP2 over 2 H100 nodes (16 GPUs), EP8, max-num-seqs 64. + # 8k1k capped at conc128: with the batch cap at 64, conc128 is 2x + # oversubscription (a queued low-interactivity point) and conc256 would be + # 4x -- queueing-dominated and timeout-prone for thousands of 8k-token + # requests. conc 1-64 spans the useful interactivity curve (64->12 + # tok/s/user), 128 anchors the low end. Fits the 8h SLURM limit. + - conc-list: [1, 2, 4, 8, 16, 32, 64, 128] + prefill: + num-worker: 1 + tp: 8 + pp: 2 + ep: 8 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/agg-h100-tep8pp2.yaml" + decode: + num-worker: 0 + tp: 8 + pp: 2 + ep: 8 + dp-attn: false + # PARETO POINT (interactivity), 8k1k: aggregated TP8 x PP4 over 4 H100 + # nodes (32 GPUs), EP8, max-num-seqs 128 (2x the PP2 8k1k cap of 64, from + # PP4's 2x KV headroom). Same rationale as 1k1k PP4 -- throughput-dominated + # but mapped on the interactivity axis; conc 1-128 runs unqueued so every + # point is a clean latency/throughput sample. + - conc-list: [1, 2, 4, 8, 16, 32, 64, 128] + prefill: + num-worker: 1 + tp: 8 + pp: 4 + ep: 8 + dp-attn: false + additional-settings: + - "CONFIG_FILE=recipes/vllm/deepseek-v4/8k1k/agg-h100-tep8pp4.yaml" + decode: + num-worker: 0 + tp: 8 + pp: 4 + ep: 8 + dp-attn: false + minimaxm3-fp8-h200-vllm: image: vllm/vllm-openai:minimax-m3 model: MiniMaxAI/MiniMax-M3-MXFP8 diff --git a/configs/runners.yaml b/configs/runners.yaml index 93da791b85..135c8fb9e8 100644 --- a/configs/runners.yaml +++ b/configs/runners.yaml @@ -27,6 +27,13 @@ labels: - h100-dgxc-slurm_17 - h100-dgxc-slurm_18 - h100-dgxc-slurm_19 + # Disaggregated-H100 pool. Unlike h100-multinode (whose key is not a label + # any runner carries, so jobs requesting it queue forever), these three + # runners carry the h100-disagg label, so runs-on: h100-disagg matches. + h100-disagg: + - h100-dgxc-slurm_17 + - h100-dgxc-slurm_18 + - h100-dgxc-slurm_19 h200: - h200-cw_00 - h200-cw_01 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 761f555872..dd3eeeb0fc 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4887,3 +4887,38 @@ - "Add SGLANG_MAMBA_SSM_DTYPE=bfloat16 in both non-MTP and MTP benchmark scripts" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2201 +- config-keys: + - dsv4-fp8-h100-dynamo-vllm + description: + - "Add DeepSeek-V4-Pro FP8 H100 aggregated multinode vLLM benchmark via Dynamo (new SKU, previously no dsv4 data on H100)" + - "Native checkpoint is mixed FP4+FP8 (~1TB) and does not fit one 8xH100 node (640GB); vLLM auto-resolves the DSv4 quant config to a Hopper-compatible path (no explicit --quantization), so the only real blocker on H100 is memory, not FP4 support" + - "Aggregated TP8 x PP2 across 2 H100 nodes (16 GPUs) with expert-parallel: tensor-parallel-size 8, pipeline-parallel-size 2, enable-expert-parallel. TP8 shards attention/dense (/8) and PP2 splits layers (/2) -- the original memory-correct sizing that loaded the model and reached prefill. TP8 (not TP16) preserves the FP8 128-column quantization-block divisibility (moe_intermediate 3072/8=384 divisible by 128; /16=192 not)" + - "Native checkpoint quant note: vLLM auto-resolves the DSv4 config to a Hopper-compatible path (no explicit --quantization); the only blocker on H100 is memory, not FP4 support (all failed attempts loaded far enough to fail on placement/transfer, not on FP4)" + - "Supersedes three failed bring-up topologies: (1) TP8xPP2 *disaggregated* loaded the model and reached prefill but hung ~300s on the NIXL P/D KV-transfer then EngineCore died; (2) pure DP16 on 2 nodes OOM'd at model load; (3) DP32 'aggregated' collapsed onto ONE node -- this framework sizes nodes as tp*pp/gpus_per_node and EP/DP-attention add no GPUs, so tp1 => 1 node and vLLM could not place 32 DP ranks on 8 GPUs. Running the memory-proven TP8xPP2 AGGREGATED (single worker, no P/D split) removes the NIXL transfer entirely and allocates 2 nodes deterministically (8*2/8)" + - "Aggregated mode is signalled by prefill num-worker 1 + decode num-worker 0 (RECIPES.md section 5); the full vLLM serve config lives under backend.vllm_config.aggregated in the recipe" + - "CUDA graphs (compilation-config cudagraph_mode FULL_AND_PIECEWISE, custom_ops all) at gpu-memory-utilization 0.92 -- NOT enforce-eager. An eager run served correctly (eval passed) but decode ran at ~144ms/token (Python-dispatch-bound, flat vs concurrency), so the conc sweep could not finish in the 8h SLURM limit (killed mid-conc-128 after ~7h). An on-cluster srtctl prototype (2-node TP8xPP2, 1x8 sweep) confirmed FULL_AND_PIECEWISE graphs FIT this layout (no OOM at 0.92) and cut TPOT to ~15ms (~9.4x); no extra nodes required" + - "One TP8xPP2 topology per seq-len; 1k1k conc 1-512, 8k1k conc 1-256 (8k1k conc512 = 5120 x 8k-token requests needs ~3.6h for that single level and exceeds the 8h SLURM limit; conc256 completes in ~71min). FP8 KV cache, block size 256, max-model-len 2304/9472, PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True" + - "Recipes under benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/{1k1k,8k1k}/agg-h100-tep8pp2.yaml; setup via NVIDIA srt-slurm's standard vllm-container-deps.sh" + - "Runner pool: h100-disagg (3 online runners), not the h100-multinode label used by the dsr1-fp8-h100-dynamo-* references — no runner carries that label, so jobs requesting it queue indefinitely" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2196 + +- config-keys: + - dsv4-fp8-h100-dynamo-vllm + description: + - "Widen the interactivity range on the aggregated TP8xPP2 topology by raising max-num-seqs from 16 to 256 (1k1k) and 64 (8k1k). max-num-seqs is the engine running-batch cap: at 16 the server ran at most 16 sequences at once, so every concurrency >=16 queued behind a batch-16 engine -- decode throughput plateaued and the whole sweep collapsed into a narrow interactivity band (the 'range too small'). Raising the cap lets throughput scale with concurrency, and each high-conc level also finishes far faster (batch-256 vs batch-16)" + - "On-cluster srtctl prototypes on the same 2 nodes: 1k1k (max-num-seqs 256) output throughput scales 56 -> 1030 -> 2079 tok/s at conc 1/64/256 (interactivity 65 -> 17 -> 8 tok/s/user, up to ~130 tok/s/GPU), saturating by conc 256; 8k1k (max-num-seqs 64) scales 60 -> 390 -> 640 tok/s at conc 1/16/64 (64 -> 26 -> 12 tok/s/user). Both fit at gpu-memory-utilization 0.92 with FULL_AND_PIECEWISE cudagraphs (no OOM)" + - "max-num-seqs 64 at 8k1k mirrors the 1k1k KV footprint (64 x 9472 ~= 256 x 2304 tokens) -- the batch the 2-node KV headroom allows at the longer context" + - "conc-list: 1k1k unchanged [1..512] (the batch cap no longer flattens it, so the same list now spans the full ~65 -> ~5 tok/s/user range); 8k1k 256 -> 128 ([1,2,4,8,16,32,64,128]) -- with max-num-seqs 64, conc128 is 2x oversubscription (one queued low-interactivity point) and conc256 would be 4x (queueing-dominated and timeout-prone for thousands of 8k-token requests), so it is dropped to stay green within the 8h SLURM limit" + - "Throughput topologies explored on-cluster and NOT adopted: (1) aggregated TP8xPP4 (4 nodes) is pipeline-inefficient -- ~43 tok/s/GPU at conc256 vs ~130 for TP8xPP2, i.e. strictly dominated on the tok/s/GPU-vs-interactivity frontier; (2) data-expert-parallel DP32 disagg (multi_node_dep, 1P+1D, 9 nodes) fits (51.98 GiB/GPU weights, NIXL loads) and needs enable-sleep-mode with NixlConnector+expandable_segments, but bring-up is slow (2 workers x ~805 GiB from NFS, >13 min) and fragile (DP32-rank shared-memory sync stalls), and being disaggregated it idles the prefill GPUs at 1k1k so its per-GPU throughput does not beat the fully-utilized aggregated topology -- not worth the CI-reliability cost this pass. The aggregated max-num-seqs fix delivers the range widening reliably on 2 nodes" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2196 + +- config-keys: + - dsv4-fp8-h100-dynamo-vllm + description: + - "Extend the H100 Pareto frontier with two more topologies alongside the aggregated TP8xPP2 baseline. The site plots a tok/s/GPU-vs-interactivity (tok/s/user) frontier, so a topology that is throughput-dominated can still be non-dominated on the interactivity axis; rather than pre-dropping such points (the prior entry dismissed PP4/DP32 as dominated/fragile), we now RUN them and let the frontier construction keep whatever is non-dominated. Frontier is per-scenario" + - "(1) INTERACTIVITY POINT -- aggregated TP8xPP4 over 4 H100 nodes (32 GPUs), EP8: 1k1k conc 1-256 (max-num-seqs 256), 8k1k conc 1-128 (max-num-seqs 128). PP4 puts 1/4 of the layers on each GPU (vs 1/2 at PP2), ~halving per-GPU weight+expert memory and doubling KV headroom, so it can sustain a given tok/s/user target at concurrencies the 2-node PP2 cannot hold. It IS throughput-dominated (an on-cluster prototype measured ~43 tok/s/GPU at conc256 vs ~130 for PP2), but the point is to measure the low-latency corner where it may hold non-dominated frontier points. TP8 preserves FP8 128-column block divisibility; node count = tp*pp/gpus_per_node = 8*4/8 = 4; aggregated (decode num-worker 0) so no NIXL P/D transfer" + - "(2) THROUGHPUT POINT -- disaggregated DP32 x EP32 (recipes.vllm.ai multi_node_dep), 1P+1D each data-parallel-32 x expert-parallel-32 (tp1, dp-attn) = 4 nodes/worker => 8 GPU + 1 dedicated etcd/NATS infra node = 9 nodes; 1k1k conc 512-2048. Data-parallel attention + full expert parallelism is the throughput-optimal MoE shape (no per-token TP all-reduce), and disaggregation scales prefill/decode independently. DP must be 32 on H100: EP must divide the 256 routed experts (DP16 OOMs at ~78 GiB/GPU model load; DP24->EP24 does not divide 256; DP32->EP32=8 experts/rank, ~52 GiB/GPU, fits at gpu-mem-util 0.90). An on-cluster prototype confirmed both workers load (51.98 GiB/GPU) and complete the NIXL P/D handshake at DP scale. Concurrency is split across 32 DP ranks so DEP only saturates at high conc (conc512 => 16/rank, conc2048 => 64/rank), which is why it is run only at the high-conc end the aggregated topologies do not own" + - "NixlConnector + PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True REQUIRES enable-sleep-mode: true on both workers or vLLM rejects the config at startup (VllmConfig validation error)" + - "DP32 is the fragile/expensive point (2x ~805 GiB checkpoint from NFS => >13 min bring-up; DP32-rank shared-memory sync can stall). The PR sweep therefore runs with fail-fast OFF (full-sweep-enabled, not full-sweep-fail-fast) so a DP32 failure records as a single red frontier cell without cancelling the aggregated PP2/PP4 points -- i.e. it is acceptable for DP32 to fail without blocking the rest of the sweep" + - "Recipes: benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/{1k1k/agg-h100-tep8pp4.yaml, 8k1k/agg-h100-tep8pp4.yaml, 1k1k/disagg-h100-dep32.yaml}. Same vllm/vllm-openai:v0.21.0 image, dynamo 1.2.0.dev20260426, h100-disagg runner pool as the PP2 baseline" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2196 diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh index 1334c95542..29aeab9330 100644 --- a/runners/launch_h100-dgxc-slurm.sh +++ b/runners/launch_h100-dgxc-slurm.sh @@ -16,7 +16,30 @@ if [[ "$IS_MULTINODE" == "true" ]]; then # MODEL_PATH: Override with pre-downloaded paths on H100 runner # The yaml files specify HuggingFace model IDs for portability, but we use # local paths to avoid repeated downloading on the shared H100 cluster. - if [[ $FRAMEWORK == "dynamo-sglang" ]]; then + if [[ $FRAMEWORK == "dynamo-vllm" ]]; then + if [[ $MODEL_PREFIX == "dsv4" && $PRECISION == "fp8" ]]; then + SELECTED_MODEL_PATH="" + if [[ -n "${MODEL_PATH:-}" && -d "${MODEL_PATH}" ]]; then + SELECTED_MODEL_PATH="$MODEL_PATH" + else + for candidate in \ + /mnt/nfs/lustre/models/dsv4-fp8 \ + /mnt/nfs/lustre/models/deepseek-v4-pro \ + /mnt/nfs/lustre/models/dsv4-pro \ + /mnt/nfs/lustre/models/DeepSeek-V4-Pro; do + if [[ -d "$candidate" ]]; then + SELECTED_MODEL_PATH="$candidate" + break + fi + done + fi + export MODEL_PATH="${SELECTED_MODEL_PATH:-/mnt/nfs/lustre/models/dsv4-fp8}" + export SRT_SLURM_MODEL_PREFIX="deepseek-v4-pro" + else + echo "Unsupported model prefix/precision for dynamo-vllm: $MODEL_PREFIX/$PRECISION" + exit 1 + fi + elif [[ $FRAMEWORK == "dynamo-sglang" ]]; then if [[ $MODEL_PREFIX == "dsr1" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/mnt/nfs/lustre/models/dsr1-fp8" export SRT_SLURM_MODEL_PREFIX="dsr1-fp8" @@ -34,7 +57,7 @@ if [[ "$IS_MULTINODE" == "true" ]]; then exit 1 fi else - echo "Unsupported framework: $FRAMEWORK. Supported frameworks are: dynamo-trt, dynamo-sglang" + echo "Unsupported framework: $FRAMEWORK. Supported frameworks are: dynamo-trt, dynamo-sglang, dynamo-vllm" exit 1 fi @@ -49,6 +72,15 @@ if [[ "$IS_MULTINODE" == "true" ]]; then if [[ "$IS_AGENTIC" == "1" ]]; then git clone --branch cam/sa-submission-q2-2026 --single-branch https://github.com/cquil11/srt-slurm-nv.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" + elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "dsv4" ]]; then + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + git checkout aflowers/vllm-gb200-v0.20.0 + mkdir -p recipes/vllm/deepseek-v4 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4" recipes/vllm/deepseek-v4 + cp \ + "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/configs/vllm-container-deps.sh" \ + configs/patches/vllm-container-deps.sh else git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" @@ -84,6 +116,23 @@ if [[ "$IS_MULTINODE" == "true" ]]; then # SGLang container mapping SQUASH_FILE="/mnt/nfs/lustre/containers/lmsysorg_sglang_v0.5.8.post1-cu130.sqsh" CONTAINER_KEY="lmsysorg/sglang:v0.5.8-cu130" + elif [[ $FRAMEWORK == "dynamo-vllm" ]]; then + # vLLM container mapping. Import the recipe image once into the shared + # squash directory so every Slurm node uses the exact configured tag. + CONTAINER_KEY="$IMAGE" + SQUASH_FILE="/mnt/nfs/sa-shared/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/+/g').sqsh" + mkdir -p "$(dirname "$SQUASH_FILE")" + ( + exec 9>"${SQUASH_FILE}.lock" + flock -w 1800 9 || { echo "Failed to acquire lock for $SQUASH_FILE" >&2; exit 1; } + if unsquashfs -l "$SQUASH_FILE" >/dev/null 2>&1; then + echo "Squash file already exists and is valid: $SQUASH_FILE" + else + rm -f "$SQUASH_FILE" "${SQUASH_FILE}.tmp."* + enroot import -o "${SQUASH_FILE}.tmp.$$" "docker://$IMAGE" + mv -f "${SQUASH_FILE}.tmp.$$" "$SQUASH_FILE" + fi + ) elif [[ $FRAMEWORK == "dynamo-trt" ]]; then # TRT-LLM container mapping - convert IMAGE to srt-slurm format (nvcr.io/ -> nvcr.io#) CONTAINER_KEY=$(echo "$IMAGE" | sed 's|nvcr.io/|nvcr.io#|') @@ -115,6 +164,7 @@ model_paths: containers: dynamo-trtllm: "${SQUASH_FILE}" dynamo-sglang: "${SQUASH_FILE}" + dynamo-vllm: "${SQUASH_FILE}" nginx-sqsh: "${NGINX_SQUASH_FILE}" latest: "${SQUASH_FILE}" "${CONTAINER_KEY}": "${SQUASH_FILE}" @@ -143,8 +193,10 @@ EOF # Override the job name in the config file with the runner name sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_FILE" - # Raise sglang's torch-distributed TCPStore timeout from the 600s gloo default - sed -i '/^ watchdog-timeout:/a\ dist-timeout: 1800' "${CONFIG_FILE%%:*}" + # Raise sglang's torch-distributed TCPStore timeout from the 600s gloo default. + if [[ $FRAMEWORK == "dynamo-sglang" ]]; then + sed -i '/^ watchdog-timeout:/a\ dist-timeout: 1800' "${CONFIG_FILE%%:*}" + fi SRTCTL_OUTPUT=$(srtctl apply -f "$CONFIG_FILE" --tags "h100,${MODEL_PREFIX},${PRECISION},${ISL}x${OSL},infmax-$(date +%Y%m%d)" 2>&1) echo "$SRTCTL_OUTPUT"