[Klaud Cold] Try TEP4 and DEP4 for DeepSeek V4.1 Flash AgentX on B300 / 在 B300 上尝试 DeepSeek V4.1 Flash AgentX 的 TEP4 与 DEP4

benchmarks/single_node/agentic/dsv41flash_fp4_vllm_mtp.sh

@@ -8,6 +8,13 @@ check_env_vars MODEL TP CONC KV_OFFLOADING TOTAL_CPU_DRAM_GB RESULT_DIR DURATION
 require_agentic_kv_offload_none
 export GPU_COUNT="$TP"
 
+# Parallelism arms. The matrix always sets EP_SIZE and DP_ATTENTION; stand-alone
+# runs default to the pure-TP recipe. TEP keeps TP attention and shards the
+# routed experts (--enable-expert-parallel). DEP runs one attention rank per
+# GPU (--data-parallel-size TP) behind vllm-router with session affinity.
+EP_SIZE="${EP_SIZE:-1}"
+DP_ATTENTION="${DP_ATTENTION:-false}"
+
 # Complete/resume partial downloads instead of trusting nonempty directories.
 if [[ -n "${MODEL_PATH:-}" && "$MODEL_PATH" != "$MODEL" ]]; then
     hf download "$MODEL" --local-dir "$MODEL_PATH"
@@ -21,21 +28,42 @@ resolve_trace_source
 install_agentic_deps
 mkdir -p "$RESULT_DIR"
 SERVER_LOG="$RESULT_DIR/server.log"
+ROUTER_LOG="$RESULT_DIR/router.log"
 export VLLM_ENGINE_READY_TIMEOUT_S="${VLLM_ENGINE_READY_TIMEOUT_S:-3600}"
 export VLLM_USE_RUST_FRONTEND=1
 export PYTHONUNBUFFERED=1
 
 # Preserve the upstream scheduler defaults; size graph capture for the sweep.
+# Each DEP rank sees about CONC/TP sequences; capture for twice that so
+# consistent-hash imbalance across ranks still lands on captured graphs.
 NUM_SPEC_TOKENS=5
+if [[ "$DP_ATTENTION" == "true" ]]; then
+    CAPTURE_TARGET=$(( 2 * ((CONC + TP - 1) / TP) * (1 + NUM_SPEC_TOKENS) ))
+else
+    CAPTURE_TARGET=$(( CONC * (1 + NUM_SPEC_TOKENS) ))
+fi
 CAPTURE_SIZE=1
-while (( CAPTURE_SIZE < CONC * (1 + NUM_SPEC_TOKENS) && CAPTURE_SIZE < 2048 )); do
+while (( CAPTURE_SIZE < CAPTURE_TARGET && CAPTURE_SIZE < 2048 )); do
     CAPTURE_SIZE=$((CAPTURE_SIZE * 2))
 done
 
 # Pyxis shares the host network; port 8888 can already belong to a host service.
 select_available_server_port
+VLLM_BACKEND_PORT="$PORT"
+if [[ "$DP_ATTENTION" == "true" ]]; then
+    # vllm-router fronts the DP ranks on PORT and expands the one HTTP backend
+    # into one logical worker per rank. Bind every turn of a conversation to
+    # the same rank by mapping AIPerf's correlation ID to X-Session-ID.
+    VLLM_BACKEND_PORT=$((PORT + 1))
+    VLLM_ROUTER_VERSION=0.1.14
+    VLLM_ROUTER_METRICS_PORT=$((PORT + 10000))
+    export AIPERF_HTTP_X_SESSION_ID_FROM_CORRELATION_ID=1
+    agentic_pip_install --quiet "vllm-router==$VLLM_ROUTER_VERSION"
+fi
 export AIPERF_SERVER_URL="http://localhost:${PORT}"
-export AIPERF_SERVER_METRICS_URLS="${AIPERF_SERVER_URL}/metrics"
+# AIPerf scrapes the public endpoint's /metrics on its own; under DEP that is the
+# router, so name the engine endpoint explicitly (deduplicated for pure TP).
+export AIPERF_SERVER_METRICS_URLS="http://localhost:${VLLM_BACKEND_PORT}/metrics"
 export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:"
 echo "Using vLLM endpoint ${AIPERF_SERVER_URL}"
 
@@ -46,9 +74,23 @@ if [[ "${EVAL_ONLY:-false}" == true ]]; then
 else
     SPEC_CONFIG='{"method":"dspark","num_speculative_tokens":5,"draft_sample_method":"probabilistic","rejection_sample_method":"synthetic","synthetic_acceptance_length":3.51,"enable_adaptive_verification":false}'
 fi
+PARALLEL_ARGS=(--tensor-parallel-size "$TP")
+if [[ "$DP_ATTENTION" == "true" ]]; then
+    PARALLEL_ARGS=(--tensor-parallel-size 1 --data-parallel-size "$TP")
+    # Under DEP every rank's expert layer sees the tokens dispatched from all
+    # DP ranks, so the TRT-LLM FP4 MoE workspace is larger than under TP. With
+    # the image default of 0.92 the autotuner warmup died 4.45 GiB short after
+    # the KV cache was sized (run 34655656558, DEP4 c64); reserve headroom and
+    # let the allocator grow segments instead of fragmenting.
+    PARALLEL_ARGS+=(--gpu-memory-utilization 0.85)
+    export PYTORCH_ALLOC_CONF=expandable_segments:True
+fi
+if [[ "$EP_SIZE" -gt 1 ]]; then
+    PARALLEL_ARGS+=(--enable-expert-parallel)
+fi
 VLLM_CMD=(
     vllm serve "$MODEL_PATH" --served-model-name "$MODEL"
-    --host 0.0.0.0 --port "$PORT" --tensor-parallel-size "$TP"
+    --host 0.0.0.0 --port "$VLLM_BACKEND_PORT" "${PARALLEL_ARGS[@]}"
     --language-model-only
     --tokenizer-mode deepseek_v41
     --tool-call-parser deepseek_v41 --enable-auto-tool-choice
@@ -63,7 +105,21 @@ printf '%q ' "${VLLM_CMD[@]}" | tee "$RESULT_DIR/vllm_command.txt"
 printf '\n' | tee -a "$RESULT_DIR/vllm_command.txt"
 "${VLLM_CMD[@]}" > "$SERVER_LOG" 2>&1 &
 SERVER_PID=$!
-wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
+wait_for_server_ready --port "$VLLM_BACKEND_PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID"
+
+if [[ "$DP_ATTENTION" == "true" ]]; then
+    echo "Starting vllm-router on port $PORT for $TP DP ranks..."
+    vllm-router \
+        --worker-urls "http://localhost:$VLLM_BACKEND_PORT" \
+        --policy consistent_hash \
+        --intra-node-data-parallel-size "$TP" \
+        --host 0.0.0.0 --port "$PORT" \
+        --prometheus-host 127.0.0.1 --prometheus-port "$VLLM_ROUTER_METRICS_PORT" \
+        --request-timeout-secs 14400 \
+        --disable-retries > "$ROUTER_LOG" 2>&1 &
+    ROUTER_PID=$!
+    wait_for_server_ready --port "$PORT" --server-log "$ROUTER_LOG" --server-pid "$ROUTER_PID"
+fi
 
 if [[ "${EVAL_ONLY:-false}" == true ]]; then
     run_eval --port "$PORT"

configs/nvidia-master.yaml

@@ -10379,6 +10379,11 @@ dsv41flash-fp4-b300-vllm-agentic-dspark:
       search-space:
       # Engram weights use UVA DRAM; the KV cache stays GPU-resident.
       - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128] }
+      # TEP4: TP attention, routed experts sharded across the 4 GPUs (--enable-expert-parallel).
+      - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [32, 64, 128] }
+      # DEP4: one attention rank per GPU (--data-parallel-size 4) behind vllm-router
+      # consistent-hash session affinity; experts sharded as in TEP4.
+      - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [32, 64, 128], router: { name: vllm-router, version: "0.1.14" } }
 
 dsv41flash-fp4-b200-vllm-agentic-dspark:
   image: vllm/vllm-openai:deepseekv41-flash-0909

docs/configuration-procedures.md

@@ -178,7 +178,7 @@ weights determine the recipe's `precision: fp4` label.
 
 The GPU-specific entry points share the text-only serving script, `deepseek_v41` tokenizer and
 parsers, 1M context, and the shared AgentX trace replay, power, metrics, and eval
-helpers. Concurrency is 1–128. Model-runner selection and scheduler batching follow the
+helpers. Concurrency is 1–128 at TP4. On B300 the TP4 grid is joined by TEP4 (`ep: 4`, `--enable-expert-parallel`) and DEP4 (`dp-attn: true`, `--data-parallel-size 4` behind vllm-router 0.1.14 consistent-hash session affinity, engine on `PORT+1`, `--gpu-memory-utilization 0.85` because the TRT-LLM FP4 MoE autotuner warmup OOMs at the image default under DP) arms at concurrency 32–128; the shared script gates both on `EP_SIZE` and `DP_ATTENTION`, so every other row still serves the pure TP4 command. Model-runner selection and scheduler batching follow the
 official single-node TP recipe defaults; graph capture covers concurrency times
 the six-token DSpark verification block. The launchers mount the repository at `/ix` for this recipe so
 AgentX runtime directories are not created under `/workspace`. Launcher-specific model paths and persistent caches are reused.

docs/configuration-procedures_zh.md

@@ -177,7 +177,7 @@ llm-d 不是 srt-slurm 路径：InferenceX 自己持有 Slurm allocation，并
 专家权重为 MXFP4，因此配方标记为 `precision: fp4`。
 
 各 GPU 入口共用纯文本服务脚本，使用 `deepseek_v41` tokenizer 和解析器、1M 上下文，
-以及共享的 AgentX 轨迹回放、功耗、指标和 eval helper。并发范围为 1–128。模型 runner 选择和调度批处理沿用官方单节点 TP 配方的默认值；
+以及共享的 AgentX 轨迹回放、功耗、指标和 eval helper。TP4 的并发范围为 1–128。B300 在 TP4 网格之外新增 TEP4（`ep: 4`、`--enable-expert-parallel`）和 DEP4（`dp-attn: true`、`--data-parallel-size 4`，由 vllm-router 0.1.14 一致性哈希保证会话亲和，引擎监听 `PORT+1`，并设置 `--gpu-memory-utilization 0.85`，因为 DP 下 TRT-LLM FP4 MoE 自动调优预热在镜像默认值下会 OOM）分支，并发 32–128；共享脚本按 `EP_SIZE` 与 `DP_ATTENTION` 门控，其余各行仍执行纯 TP4 命令。模型 runner 选择和调度批处理沿用官方单节点 TP 配方的默认值；
 CUDA graph capture 覆盖并发数乘以六 token DSpark 验证块。launcher 都为该配方将仓库挂载到 `/ix`，避免在 `/workspace`
 下创建 AgentX 运行目录。沿用各 launcher 的模型路径和持久化缓存。配方在计算节点探测服务端口，首选端口被占用时选择可用端口，
 服务、回放、指标和 eval 共用同一端点。所有配方都必须获得 GPU sweep 和 eval

perf-changelog.yaml

@@ -7338,3 +7338,10 @@
     - "Resolve the requested H100 SGLang container instead of a hardcoded older image, and stop benchmark/eval clients when their ready server or required worker exits."
     - "H100 SGLang 使用所请求的容器而非硬编码旧镜像；已就绪的服务或必需工作进程退出后，停止其 benchmark/评测客户端。"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3020
+
+- config-keys:
+    - dsv41flash-fp4-b300-vllm-agentic-dspark
+  description:
+    - "Add B300 TEP4 and DEP4 AgentX arms at concurrency 32, 64 and 128 alongside the unchanged TP4 grid: TEP4 shards the routed experts with --enable-expert-parallel; DEP4 runs data-parallel attention (--data-parallel-size 4) behind vllm-router 0.1.14 consistent-hash session affinity with --gpu-memory-utilization 0.85 and expandable allocator segments, because the TRT-LLM FP4 MoE autotuner warmup ran out of memory at the image default (run 34655656558). The shared DSpark script gates both on EP_SIZE and DP_ATTENTION, so the existing TP4 rows on every SKU serve the same command as before."
+    - "为 B300 新增 TEP4 与 DEP4 AgentX 分支（并发 32、64、128），TP4 网格保持不变：TEP4 通过 --enable-expert-parallel 切分路由专家；DEP4 以 --data-parallel-size 4 运行数据并行注意力，并由 vllm-router 0.1.14 一致性哈希保证会话亲和，并设置 --gpu-memory-utilization 0.85 与可扩展分配器段，因为在镜像默认值下 TRT-LLM FP4 MoE 自动调优预热会内存不足（运行 34655656558）。共享 DSpark 脚本按 EP_SIZE 与 DP_ATTENTION 门控，各 SKU 现有 TP4 行的服务命令与以前完全一致。"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3028