diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/1p1d-dep2-tp4-c1-c16-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/1p1d-dep2-tp4-c1-c16-8k1k.yaml new file mode 100644 index 000000000..dfbe048f1 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/1p1d-dep2-tp4-c1-c16-8k1k.yaml @@ -0,0 +1,62 @@ +name: "minimax-m3-vllm-disagg-b200-1p1d-fp4-dep2-tp4-8k1k" +model: + path: "nvidia/MiniMax-M3-NVFP4" + container: "vllm/vllm-openai:cu129-nightly-8e981630c9336233ca9de91452f68918bddbc4e2" + precision: "fp4" +resources: + gpu_type: "b200" + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 2 + gpus_per_decode: 4 +dynamo: {install: true, version: 1.2.1} +frontend: {type: dynamo, enable_multiple_frontends: false} +backend: + type: vllm + connector: null + allow_prefill_decode_colocation: true + prefill_environment: {VLLM_FLOAT32_MATMUL_PRECISION: high, UCX_TLS: "cuda_ipc,cuda_copy,rc"} + decode_environment: {VLLM_FLOAT32_MATMUL_PRECISION: high, UCX_TLS: "cuda_ipc,cuda_copy,rc"} + vllm_config: + prefill: + served-model-name: nvidia/MiniMax-M3-NVFP4 + no-enable-flashinfer-autotune: true + tensor-parallel-size: 1 + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + enable-expert-parallel: true + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + max-num-batched-tokens: 16384 + decode: + served-model-name: nvidia/MiniMax-M3-NVFP4 + no-enable-flashinfer-autotune: true + tensor-parallel-size: 4 + enable-expert-parallel: false + trust-remote-code: true + no-enable-prefix-caching: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + kv-cache-dtype: fp8 + block-size: 128 + gpu-memory-utilization: 0.90 + max-model-len: 9472 + language-model-only: true + stream-interval: 32 + max-num-seqs: 1024 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 +health_check: {max_attempts: 360, interval_seconds: 10} +benchmark: {type: "sa-bench", isl: 8192, osl: 1024, random_range_ratio: 0.8, concurrencies: "1x4x8x16", req_rate: "inf"} diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/1p2d-dep2-tp4-c64-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/1p2d-dep2-tp4-c64-8k1k.yaml new file mode 100644 index 000000000..8b887b3bf --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/1p2d-dep2-tp4-c64-8k1k.yaml @@ -0,0 +1,127 @@ +name: minimax-m3-vllm-disagg-b200-1p2d-fp4-dep2-tp4-c64-8k1k +model: + path: nvidia/MiniMax-M3-NVFP4 + container: vllm/vllm-openai:cu129-nightly-8e981630c9336233ca9de91452f68918bddbc4e2 + precision: fp4 +resources: + gpu_type: b200 + prefill_nodes: 1 + prefill_workers: 1 + gpus_per_prefill: 2 + decode_nodes: 2 + decode_workers: 2 + spread_workers: true + gpus_per_decode: 4 + gpus_per_node: 8 +backend: + type: vllm + connector: null + prefill_environment: + VLLM_USE_DEEP_GEMM: '1' + VLLM_SKIP_P2P_CHECK: '1' + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' + NVIDIA_GDRCOPY: '1' + PYTHONUNBUFFERED: '1' + VLLM_LOG_STATS_INTERVAL: '1' + NVSHMEM_IB_ENABLE_IBGDA: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NCCL_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL: '0' + VLLM_USE_NCCL_SYMM_MEM: '1' + UCX_IB_ROCE_REACHABILITY_MODE: local_subnet + VLLM_NIXL_SIDE_CHANNEL_PORT: '5600' + VLLM_NIXL_ABORT_REQUEST_TIMEOUT: '300' + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: cuda_ipc,cuda_copy,rc + decode_environment: + VLLM_USE_DEEP_GEMM: '1' + VLLM_SKIP_P2P_CHECK: '1' + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' + NVIDIA_GDRCOPY: '1' + PYTHONUNBUFFERED: '1' + VLLM_LOG_STATS_INTERVAL: '1' + NVSHMEM_IB_ENABLE_IBGDA: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NCCL_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL: '0' + VLLM_USE_NCCL_SYMM_MEM: '1' + UCX_IB_ROCE_REACHABILITY_MODE: local_subnet + VLLM_NIXL_SIDE_CHANNEL_PORT: '5600' + VLLM_NIXL_ABORT_REQUEST_TIMEOUT: '300' + VLLM_MOE_DP_CHUNK_SIZE: '384' + VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD: '8192' + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: cuda_ipc,cuda_copy,rc + vllm_config: + prefill: + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + enable-expert-parallel: true + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + max-model-len: 9472 + max-num-seqs: 16 + enforce-eager: true + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 16384 + no-enable-chunked-prefill: true + kv-cache-dtype: fp8 + async-scheduling: true + no-enable-prefix-caching: true + trust-remote-code: true + served-model-name: nvidia/MiniMax-M3-NVFP4 + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + decode: + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + enable-expert-parallel: false + max-model-len: 9472 + max-num-seqs: 32 + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+rms_norm"],"pass_config":{}}' + kv-cache-dtype: fp8 + all2all-backend: deepep_low_latency + async-scheduling: true + stream-interval: 32 + enable-dbo: true + dbo-decode-token-threshold: 32 + no-enable-prefix-caching: true + trust-remote-code: true + served-model-name: nvidia/MiniMax-M3-NVFP4 + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + language-model-only: true +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '64' + req_rate: inf +frontend: + type: dynamo + enable_multiple_frontends: false + args: + dyn-chat-processor: vllm + reasoning-parser: minimax_m3 +dynamo: + install: false + version: 1.2.1 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/2p2d-dep2-tp4-c128-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/2p2d-dep2-tp4-c128-8k1k.yaml new file mode 100644 index 000000000..f1014157d --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/2p2d-dep2-tp4-c128-8k1k.yaml @@ -0,0 +1,127 @@ +name: minimax-m3-vllm-disagg-b200-2p2d-fp4-dep2-tp4-c128-8k1k +model: + path: nvidia/MiniMax-M3-NVFP4 + container: vllm/vllm-openai:cu129-nightly-8e981630c9336233ca9de91452f68918bddbc4e2 + precision: fp4 +resources: + gpu_type: b200 + prefill_nodes: 1 + prefill_workers: 2 + gpus_per_prefill: 2 + decode_nodes: 2 + decode_workers: 2 + spread_workers: true + gpus_per_decode: 4 + gpus_per_node: 8 +backend: + type: vllm + connector: null + prefill_environment: + VLLM_USE_DEEP_GEMM: '1' + VLLM_SKIP_P2P_CHECK: '1' + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' + NVIDIA_GDRCOPY: '1' + PYTHONUNBUFFERED: '1' + VLLM_LOG_STATS_INTERVAL: '1' + NVSHMEM_IB_ENABLE_IBGDA: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NCCL_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL: '0' + VLLM_USE_NCCL_SYMM_MEM: '1' + UCX_IB_ROCE_REACHABILITY_MODE: local_subnet + VLLM_NIXL_SIDE_CHANNEL_PORT: '5600' + VLLM_NIXL_ABORT_REQUEST_TIMEOUT: '300' + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: cuda_ipc,cuda_copy,rc + decode_environment: + VLLM_USE_DEEP_GEMM: '1' + VLLM_SKIP_P2P_CHECK: '1' + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' + NVIDIA_GDRCOPY: '1' + PYTHONUNBUFFERED: '1' + VLLM_LOG_STATS_INTERVAL: '1' + NVSHMEM_IB_ENABLE_IBGDA: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NCCL_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL: '0' + VLLM_USE_NCCL_SYMM_MEM: '1' + UCX_IB_ROCE_REACHABILITY_MODE: local_subnet + VLLM_NIXL_SIDE_CHANNEL_PORT: '5600' + VLLM_NIXL_ABORT_REQUEST_TIMEOUT: '300' + VLLM_MOE_DP_CHUNK_SIZE: '384' + VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD: '8192' + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: cuda_ipc,cuda_copy,rc + vllm_config: + prefill: + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + enable-expert-parallel: true + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + max-model-len: 9472 + max-num-seqs: 16 + enforce-eager: true + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 16384 + no-enable-chunked-prefill: true + kv-cache-dtype: fp8 + async-scheduling: true + no-enable-prefix-caching: true + trust-remote-code: true + served-model-name: nvidia/MiniMax-M3-NVFP4 + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + decode: + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + enable-expert-parallel: false + max-model-len: 9472 + max-num-seqs: 64 + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+rms_norm"],"pass_config":{}}' + kv-cache-dtype: fp8 + all2all-backend: deepep_low_latency + async-scheduling: true + stream-interval: 32 + enable-dbo: true + dbo-decode-token-threshold: 32 + no-enable-prefix-caching: true + trust-remote-code: true + served-model-name: nvidia/MiniMax-M3-NVFP4 + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + language-model-only: true +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '128' + req_rate: inf +frontend: + type: dynamo + enable_multiple_frontends: false + args: + dyn-chat-processor: vllm + reasoning-parser: minimax_m3 +dynamo: + install: false + version: 1.2.1 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/3p2d-dep2-tp4-c256-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/3p2d-dep2-tp4-c256-8k1k.yaml new file mode 100644 index 000000000..7becb72ab --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/3p2d-dep2-tp4-c256-8k1k.yaml @@ -0,0 +1,127 @@ +name: minimax-m3-vllm-disagg-b200-3p2d-fp4-dep2-tp4-c256-8k1k +model: + path: nvidia/MiniMax-M3-NVFP4 + container: vllm/vllm-openai:cu129-nightly-8e981630c9336233ca9de91452f68918bddbc4e2 + precision: fp4 +resources: + gpu_type: b200 + prefill_nodes: 1 + prefill_workers: 3 + gpus_per_prefill: 2 + decode_nodes: 2 + decode_workers: 2 + spread_workers: true + gpus_per_decode: 4 + gpus_per_node: 8 +backend: + type: vllm + connector: null + prefill_environment: + VLLM_USE_DEEP_GEMM: '1' + VLLM_SKIP_P2P_CHECK: '1' + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' + NVIDIA_GDRCOPY: '1' + PYTHONUNBUFFERED: '1' + VLLM_LOG_STATS_INTERVAL: '1' + NVSHMEM_IB_ENABLE_IBGDA: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NCCL_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL: '0' + VLLM_USE_NCCL_SYMM_MEM: '1' + UCX_IB_ROCE_REACHABILITY_MODE: local_subnet + VLLM_NIXL_SIDE_CHANNEL_PORT: '5600' + VLLM_NIXL_ABORT_REQUEST_TIMEOUT: '300' + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: cuda_ipc,cuda_copy,rc + decode_environment: + VLLM_USE_DEEP_GEMM: '1' + VLLM_SKIP_P2P_CHECK: '1' + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' + NVIDIA_GDRCOPY: '1' + PYTHONUNBUFFERED: '1' + VLLM_LOG_STATS_INTERVAL: '1' + NVSHMEM_IB_ENABLE_IBGDA: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NCCL_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL: '0' + VLLM_USE_NCCL_SYMM_MEM: '1' + UCX_IB_ROCE_REACHABILITY_MODE: local_subnet + VLLM_NIXL_SIDE_CHANNEL_PORT: '5600' + VLLM_NIXL_ABORT_REQUEST_TIMEOUT: '300' + VLLM_MOE_DP_CHUNK_SIZE: '384' + VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD: '8192' + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: cuda_ipc,cuda_copy,rc + vllm_config: + prefill: + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + enable-expert-parallel: true + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + max-model-len: 9472 + max-num-seqs: 16 + enforce-eager: true + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 16384 + no-enable-chunked-prefill: true + kv-cache-dtype: fp8 + async-scheduling: true + no-enable-prefix-caching: true + trust-remote-code: true + served-model-name: nvidia/MiniMax-M3-NVFP4 + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + decode: + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + enable-expert-parallel: false + max-model-len: 9472 + max-num-seqs: 128 + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+rms_norm"],"pass_config":{}}' + kv-cache-dtype: fp8 + all2all-backend: deepep_low_latency + async-scheduling: true + stream-interval: 32 + enable-dbo: true + dbo-decode-token-threshold: 32 + no-enable-prefix-caching: true + trust-remote-code: true + served-model-name: nvidia/MiniMax-M3-NVFP4 + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + language-model-only: true +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '256' + req_rate: inf +frontend: + type: dynamo + enable_multiple_frontends: false + args: + dyn-chat-processor: vllm + reasoning-parser: minimax_m3 +dynamo: + install: false + version: 1.2.1 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/3p2d-dep2-tp4-c512-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/3p2d-dep2-tp4-c512-8k1k.yaml new file mode 100644 index 000000000..25d4e5dba --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/3p2d-dep2-tp4-c512-8k1k.yaml @@ -0,0 +1,127 @@ +name: minimax-m3-vllm-disagg-b200-3p2d-fp4-dep2-tp4-c512-8k1k +model: + path: nvidia/MiniMax-M3-NVFP4 + container: vllm/vllm-openai:cu129-nightly-8e981630c9336233ca9de91452f68918bddbc4e2 + precision: fp4 +resources: + gpu_type: b200 + prefill_nodes: 1 + prefill_workers: 3 + gpus_per_prefill: 2 + decode_nodes: 2 + decode_workers: 2 + spread_workers: true + gpus_per_decode: 4 + gpus_per_node: 8 +backend: + type: vllm + connector: null + prefill_environment: + VLLM_USE_DEEP_GEMM: '1' + VLLM_SKIP_P2P_CHECK: '1' + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' + NVIDIA_GDRCOPY: '1' + PYTHONUNBUFFERED: '1' + VLLM_LOG_STATS_INTERVAL: '1' + NVSHMEM_IB_ENABLE_IBGDA: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NCCL_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL: '0' + VLLM_USE_NCCL_SYMM_MEM: '1' + UCX_IB_ROCE_REACHABILITY_MODE: local_subnet + VLLM_NIXL_SIDE_CHANNEL_PORT: '5600' + VLLM_NIXL_ABORT_REQUEST_TIMEOUT: '300' + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: cuda_ipc,cuda_copy,rc + decode_environment: + VLLM_USE_DEEP_GEMM: '1' + VLLM_SKIP_P2P_CHECK: '1' + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' + NVIDIA_GDRCOPY: '1' + PYTHONUNBUFFERED: '1' + VLLM_LOG_STATS_INTERVAL: '1' + NVSHMEM_IB_ENABLE_IBGDA: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NCCL_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL: '0' + VLLM_USE_NCCL_SYMM_MEM: '1' + UCX_IB_ROCE_REACHABILITY_MODE: local_subnet + VLLM_NIXL_SIDE_CHANNEL_PORT: '5600' + VLLM_NIXL_ABORT_REQUEST_TIMEOUT: '300' + VLLM_MOE_DP_CHUNK_SIZE: '384' + VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD: '8192' + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: cuda_ipc,cuda_copy,rc + vllm_config: + prefill: + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + enable-expert-parallel: true + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + max-model-len: 9472 + max-num-seqs: 16 + enforce-eager: true + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 16384 + no-enable-chunked-prefill: true + kv-cache-dtype: fp8 + async-scheduling: true + no-enable-prefix-caching: true + trust-remote-code: true + served-model-name: nvidia/MiniMax-M3-NVFP4 + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + decode: + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + enable-expert-parallel: false + max-model-len: 9472 + max-num-seqs: 256 + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+rms_norm"],"pass_config":{}}' + kv-cache-dtype: fp8 + all2all-backend: deepep_low_latency + async-scheduling: true + stream-interval: 32 + enable-dbo: true + dbo-decode-token-threshold: 32 + no-enable-prefix-caching: true + trust-remote-code: true + served-model-name: nvidia/MiniMax-M3-NVFP4 + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + language-model-only: true +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '512' + req_rate: inf +frontend: + type: dynamo + enable_multiple_frontends: false + args: + dyn-chat-processor: vllm + reasoning-parser: minimax_m3 +dynamo: + install: false + version: 1.2.1 diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/4p2d-dep2-tp4-c1024-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/4p2d-dep2-tp4-c1024-8k1k.yaml new file mode 100644 index 000000000..8b2708447 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4/8k1k/4p2d-dep2-tp4-c1024-8k1k.yaml @@ -0,0 +1,127 @@ +name: minimax-m3-vllm-disagg-b200-4p2d-fp4-dep2-tp4-c1024-8k1k +model: + path: nvidia/MiniMax-M3-NVFP4 + container: vllm/vllm-openai:cu129-nightly-8e981630c9336233ca9de91452f68918bddbc4e2 + precision: fp4 +resources: + gpu_type: b200 + prefill_nodes: 2 + prefill_workers: 4 + gpus_per_prefill: 2 + decode_nodes: 2 + decode_workers: 2 + spread_workers: true + gpus_per_decode: 4 + gpus_per_node: 8 +backend: + type: vllm + connector: null + prefill_environment: + VLLM_USE_DEEP_GEMM: '1' + VLLM_SKIP_P2P_CHECK: '1' + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' + NVIDIA_GDRCOPY: '1' + PYTHONUNBUFFERED: '1' + VLLM_LOG_STATS_INTERVAL: '1' + NVSHMEM_IB_ENABLE_IBGDA: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NCCL_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL: '0' + VLLM_USE_NCCL_SYMM_MEM: '1' + UCX_IB_ROCE_REACHABILITY_MODE: local_subnet + VLLM_NIXL_SIDE_CHANNEL_PORT: '5600' + VLLM_NIXL_ABORT_REQUEST_TIMEOUT: '300' + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: cuda_ipc,cuda_copy,rc + decode_environment: + VLLM_USE_DEEP_GEMM: '1' + VLLM_SKIP_P2P_CHECK: '1' + VLLM_RANDOMIZE_DP_DUMMY_INPUTS: '1' + NVIDIA_GDRCOPY: '1' + PYTHONUNBUFFERED: '1' + VLLM_LOG_STATS_INTERVAL: '1' + NVSHMEM_IB_ENABLE_IBGDA: '1' + NCCL_CUMEM_ENABLE: '1' + NCCL_MNNVL_ENABLE: '1' + NCCL_NVLS_ENABLE: '1' + NCCL_TIMEOUT: '1800' + TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC: '1800' + VLLM_USE_FLASHINFER_MOE_FP4: '1' + VLLM_USE_TRTLLM_RAGGED_DEEPSEEK_PREFILL: '0' + VLLM_USE_NCCL_SYMM_MEM: '1' + UCX_IB_ROCE_REACHABILITY_MODE: local_subnet + VLLM_NIXL_SIDE_CHANNEL_PORT: '5600' + VLLM_NIXL_ABORT_REQUEST_TIMEOUT: '300' + VLLM_MOE_DP_CHUNK_SIZE: '384' + VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD: '8192' + VLLM_FLOAT32_MATMUL_PRECISION: high + UCX_TLS: cuda_ipc,cuda_copy,rc + vllm_config: + prefill: + tensor-parallel-size: 1 + pipeline-parallel-size: 1 + enable-expert-parallel: true + data-parallel-size: 2 + data-parallel-rpc-port: 13345 + data-parallel-hybrid-lb: true + max-model-len: 9472 + max-num-seqs: 16 + enforce-eager: true + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 16384 + no-enable-chunked-prefill: true + kv-cache-dtype: fp8 + async-scheduling: true + no-enable-prefix-caching: true + trust-remote-code: true + served-model-name: nvidia/MiniMax-M3-NVFP4 + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + language-model-only: true + stream-interval: 32 + max-cudagraph-capture-size: 2048 + decode: + tensor-parallel-size: 4 + pipeline-parallel-size: 1 + enable-expert-parallel: false + max-model-len: 9472 + max-num-seqs: 512 + gpu-memory-utilization: 0.9 + max-num-batched-tokens: 16384 + max-cudagraph-capture-size: 2048 + compilation-config: '{"cudagraph_mode":"FULL_DECODE_ONLY","custom_ops":["+rms_norm"],"pass_config":{}}' + kv-cache-dtype: fp8 + all2all-backend: deepep_low_latency + async-scheduling: true + stream-interval: 32 + enable-dbo: true + dbo-decode-token-threshold: 32 + no-enable-prefix-caching: true + trust-remote-code: true + served-model-name: nvidia/MiniMax-M3-NVFP4 + no-enable-flashinfer-autotune: true + kv-transfer-config: '{"kv_connector": "NixlConnector", "kv_role": "kv_both"}' + attention-config: '{"backend": "FLASHINFER", "use_trtllm_attention": true, "indexer_kv_dtype": "fp8"}' + block-size: 128 + language-model-only: true +benchmark: + type: sa-bench + isl: 8192 + osl: 1024 + concurrencies: '1024' + req_rate: inf +frontend: + type: dynamo + enable_multiple_frontends: false + args: + dyn-chat-processor: vllm + reasoning-parser: minimax_m3 +dynamo: + install: false + version: 1.2.1 diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 33e9118b7..5ed47cfb8 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -13211,6 +13211,101 @@ qwen3.5-fp8-h100-sglang-agentic: - { tp: 8, ep: 8, kv-offloading: none, conc-list: [1, 2, 4, 8, 12, 14, 16] } - { tp: 8, ep: 8, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [12, 14, 16, 20, 24, 28, 32, 42] } +minimaxm3-fp4-b200-dynamo-vllm: + image: vllm/vllm-openai:cu129-nightly-8e981630c9336233ca9de91452f68918bddbc4e2 + model: nvidia/MiniMax-M3-NVFP4 + model-prefix: minimaxm3 + runner: b200-multinode + precision: fp4 + framework: dynamo-vllm + router: { name: dynamo-router, version: "1.2.1" } + kv-p2p-transfer: nixl + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - conc-list: [1, 4, 8, 16] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/1p1d-dep2-tp4-c1-c16-8k1k.yaml" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + - conc-list: [64] + prefill: + num-worker: 1 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/1p2d-dep2-tp4-c64-8k1k.yaml" + decode: + num-worker: 2 + tp: 4 + ep: 1 + dp-attn: false + - conc-list: [128] + prefill: + num-worker: 2 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/2p2d-dep2-tp4-c128-8k1k.yaml" + decode: + num-worker: 2 + tp: 4 + ep: 1 + dp-attn: false + - conc-list: [256] + prefill: + num-worker: 3 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/3p2d-dep2-tp4-c256-8k1k.yaml" + decode: + num-worker: 2 + tp: 4 + ep: 1 + dp-attn: false + - conc-list: [512] + prefill: + num-worker: 3 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/3p2d-dep2-tp4-c512-8k1k.yaml" + decode: + num-worker: 2 + tp: 4 + ep: 1 + dp-attn: false + - conc-list: [1024] + prefill: + num-worker: 4 + tp: 2 + ep: 2 + dp-attn: true + additional-settings: + - "CONFIG_FILE=recipes/vllm/minimax-m3/b200-fp4/8k1k/4p2d-dep2-tp4-c1024-8k1k.yaml" + decode: + num-worker: 2 + tp: 4 + ep: 1 + dp-attn: false + # MiniMax-M3 NVFP4 disagg sweep on the same B300 topology matrix as the MXFP8 # baseline above. The image includes vLLM PR #46380, so no runtime patch is # needed. diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 04991f247..2184310f1 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4875,3 +4875,12 @@ - "Add SGLANG_MAMBA_SSM_DTYPE=bfloat16 in both non-MTP and MTP benchmark scripts" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2201 +- config-keys: + - minimaxm3-fp4-b200-dynamo-vllm + description: + - "Add MiniMax-M3 NVFP4 B200 Dynamo-vLLM disaggregated benchmarks at 8k/1k." + - "Use validated configurations from concurrency 1 through 1024, including the corrected spread-decode c64 point." + - "Image: vllm/vllm-openai:cu129-nightly-8e981630c9336233ca9de91452f68918bddbc4e2." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2230 + + diff --git a/runners/launch_b200-dgxc.sh b/runners/launch_b200-dgxc.sh index 58051e640..23d7a1aa3 100644 --- a/runners/launch_b200-dgxc.sh +++ b/runners/launch_b200-dgxc.sh @@ -69,9 +69,8 @@ elif [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp8" ]]; then export MODEL_PATH="/lustre/fsw/gharunners/models/MiniMax-M3-MXFP8" export SRT_SLURM_MODEL_PREFIX="minimax-m3-mxfp8" elif [[ $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp4" ]]; then - # NVFP4 checkpoint, pre-staged on the b200-dgxc scratch tree. export MODEL_PATH="/scratch/fsw/models/MiniMax-M3-NVFP4" - export SRT_SLURM_MODEL_PREFIX="minimax-m3-nvfp4" + export SRT_SLURM_MODEL_PREFIX="nvidia/MiniMax-M3-NVFP4" else echo "Unsupported model prefix/precision: $MODEL_PREFIX/$PRECISION" echo "Available models under /lustre/fsw/models:" @@ -113,6 +112,13 @@ if [[ "$IS_MULTINODE" == "true" ]]; then git checkout aflowers/vllm-gb200-v0.20.0 mkdir -p recipes/vllm/deepseek-v4 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4" recipes/vllm/deepseek-v4 + elif [[ $FRAMEWORK == "dynamo-vllm" && $MODEL_PREFIX == "minimaxm3" && $PRECISION == "fp4" ]]; then + git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" || exit 1 + mkdir -p recipes/vllm/minimax-m3/b200-fp4 + cp -rT \ + "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/minimax-m3/b200-fp4" \ + recipes/vllm/minimax-m3/b200-fp4 elif [[ $FRAMEWORK == "dynamo-sglang" && $MODEL_PREFIX == "glm5" && $PRECISION == "fp8" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" || exit 1