[PowerX] collect native llm-d 8k1k power / 采集 llm-d 8k1k 原生功耗

.github/workflows/benchmark-multinode-tmpl.yml

@@ -538,6 +538,7 @@ jobs:
             ${{ env.RESULT_FILENAME }}_*.json
             agg_${{ env.RESULT_FILENAME }}_*.json
             power_validation_${{ env.RESULT_FILENAME }}_*.json
+            slurm_job_*_outcome.txt
             LOGS/power/**
             LOGS/native_power/**
             result_processing_${{ env.RESULT_FILENAME }}.json

.github/workflows/benchmark-tmpl.yml

@@ -474,6 +474,7 @@ jobs:
             gpu_metrics_identity.json
             gpu_metrics_identity.csv
             power_validation_${{ env.RESULT_FILENAME }}.json
+            slurm_job_*_outcome.txt
             results/gpu_metrics*.csv
             results/gpu_metrics*_context.json
             results/gpu_metrics_identity.json

.github/workflows/test-changelog-gate.yml

@@ -31,6 +31,8 @@ on:
       - "runners/launch_*.sh"
       - "runners/slurm_utils.sh"
       - "runners/test_slurm_utils.py"
+      - "runners/test_llmd_lifecycle.py"
+      - "benchmarks/multi_node/llm-d/**"
       - "utils/ci_priority.py"
       - "utils/test_ci_priority.py"
       - ".github/workflows/reuse-sweep-comment.yml"
@@ -123,4 +125,5 @@ jobs:
             utils/evals/test_batched_eval.py \
             utils/evals/test_run_eval_dispatch.py \
             runners/test_slurm_utils.py \
+            runners/test_llmd_lifecycle.py \
             -v -n 4

.github/workflows/test-process-result.yml

@@ -47,6 +47,8 @@ on:
       - 'benchmarks/multi_node/tilert_utils/**'
       - 'benchmarks/multi_node/glm5.1_fp8_b200_tilert-disagg.sh'
       - 'runners/test_tilert_power_lifecycle.py'
+      - 'benchmarks/multi_node/llm-d/**'
+      - 'runners/test_llmd_power_lifecycle.py'
 
 permissions:
   contents: read
@@ -68,7 +70,7 @@ jobs:
         run: |
           cd utils
           uv run --no-project --exclude-newer PT12H --python 3.12 --with pytest --with pyyaml \
-            python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py agentic/aggregation/ test_gb300_power_official_contract.py test_inject_srt_power_concurrencies.py test_process_result.py test_native_multinode_power.py ../runners/test_native_collector_barriers.py ../runners/test_native_collector_receipts.py ../runners/test_kimik3_bh_power.py ../runners/test_tilert_power_lifecycle.py -v
+            python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py agentic/aggregation/ test_gb300_power_official_contract.py test_inject_srt_power_concurrencies.py test_process_result.py test_native_multinode_power.py ../runners/test_native_collector_barriers.py ../runners/test_native_collector_receipts.py ../runners/test_kimik3_bh_power.py ../runners/test_tilert_power_lifecycle.py ../runners/test_llmd_power_lifecycle.py -v
 
       - name: Test serving client result persistence
         run: |

benchmarks/multi_node/llm-d/job.slurm

@@ -63,25 +63,46 @@ export DOCKER_CONT_NAME
 : "${BENCHMARK_LOGS_DIR:?BENCHMARK_LOGS_DIR not set}"
 DOCKER_MOUNT_PATH="/workspace"
 
-cleanup() {
-    echo "[${SLURM_JOB_ID}] cleanup on $(hostname)"
-    [[ -n "${WATCHER_PID:-}" ]] && kill "$WATCHER_PID" 2>/dev/null || true
-}
-trap cleanup INT TERM HUP EXIT
-
-# Coordinator-done watcher. server.sh on the decode coordinator writes
-# this marker after the bench finishes; we then scancel the allocation
-# from outside the container (the image has no SLURM client tools).
-# Without this, workers `wait` on local vLLM forever and the job runs
-# to TIME_LIMIT.
+# Normal completion is coordinated inside server.sh. Cancelling a successful
+# allocation would discard its terminal success state and interrupt collectors.
 BENCH_DONE_MARKER="$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID"
 rm -f "$BENCH_DONE_MARKER"
-(
-    while [[ ! -f "$BENCH_DONE_MARKER" ]]; do sleep 5; done
-    echo "[${SLURM_JOB_ID}] coordinator finished; scancel'ing job"
-    scancel "$SLURM_JOB_ID" 2>/dev/null || true
-) &
-WATCHER_PID=$!
+POWERX_NATIVE_ENABLED="${POWERX_NATIVE_ENABLED:-0}"
+export POWERX_HOST_UID="" POWERX_HOST_GID="" POWERX_COLLECTOR_REVISION=""
+POWERX_DOCKER_MOUNT=""
+if [[ "$POWERX_NATIVE_ENABLED" == 1 ]]; then
+    export POWERX_HOST_UID="$(id -u)" POWERX_HOST_GID="$(id -g)"
+    export POWERX_COLLECTOR_REVISION="$(git -C "$DI_REPO_DIR" rev-parse HEAD)"
+    export POWERX_RAW_ROOT="${POWERX_RAW_ROOT:-/tmp/inferencex-native-$SLURM_JOB_ID}"
+    POWERX_DOCKER_MOUNT="-v ${POWERX_RAW_ROOT}:/powerx_native"
+    srun --nodes="$NUM_NODES" --ntasks-per-node=1 mkdir -p "$POWERX_RAW_ROOT"
+    mkdir -p "$BENCHMARK_LOGS_DIR/power_control-$SLURM_JOB_ID" "$DI_REPO_DIR/LOGS/native_power"
+    chmod 777 "$BENCHMARK_LOGS_DIR/power_control-$SLURM_JOB_ID"
+    srun --nodes="$NUM_NODES" --ntasks-per-node=1 chmod 777 "$POWERX_RAW_ROOT"
+    srun --nodes="$NUM_NODES" --ntasks-per-node=1 bash -c '
+        { hostname; timedatectl show -p NTPSynchronized --value 2>/dev/null || echo false; } \
+            > "$BENCHMARK_LOGS_DIR/power_control-$SLURM_JOB_ID/host-$SLURM_PROCID"
+    '
+fi
+main_rc=0
+finish_llmd_job() {
+    local rc=$?
+    trap - EXIT
+    [[ "$rc" == 0 ]] || main_rc=$rc
+
+    if [[ "$POWERX_NATIVE_ENABLED" == 1 ]]; then
+        # Copy as the host user only after collectors and serving containers drain.
+        srun --nodes="$NUM_NODES" --ntasks-per-node=1 bash -c '
+            source_dir="$POWERX_RAW_ROOT/node-$SLURM_PROCID"
+            [[ -d "$source_dir" ]] || exit 1
+            cp -R "$source_dir" "$DI_REPO_DIR/LOGS/native_power/"
+        ' || main_rc=$?
+    fi
+    exit "$main_rc"
+}
+trap finish_llmd_job EXIT
+trap 'exit 143' TERM HUP
+trap 'exit 130' INT
 
 # Container engine: 'docker' (default) for clusters where the SLURM
 # user can talk to /var/run/docker.sock (e.g. h200-dgxc-slurm); 'pyxis'
@@ -119,10 +140,15 @@ exec docker run --rm \
     --privileged \
     -v ${MODEL_DIR}:/models:ro \
     -v ${BENCHMARK_LOGS_DIR}:/benchmark_logs \
+    $POWERX_DOCKER_MOUNT \
     -v ${DI_REPO_DIR}:${DOCKER_MOUNT_PATH} \
     -v ${DI_REPO_DIR}/benchmarks/multi_node/llm-d-recipes:/etc/llmd-recipes:ro \
     -v ${DI_REPO_DIR}/benchmarks/llm-d/epp-config.yaml:/etc/epp/config.yaml:ro \
     -v ${DI_REPO_DIR}/benchmarks/llm-d/envoy.yaml:/etc/envoy/envoy.yaml:ro \
+    -e POWERX_NATIVE_ENABLED=$POWERX_NATIVE_ENABLED \
+    -e POWERX_HOST_UID=$POWERX_HOST_UID \
+    -e POWERX_HOST_GID=$POWERX_HOST_GID \
+    -e POWERX_COLLECTOR_REVISION=$POWERX_COLLECTOR_REVISION \
     -e SLURM_JOB_ID=\$SLURM_JOB_ID \
     -e NODE_RANK=\$SLURM_PROCID \
     -e NUM_NODES=$NUM_NODES \
@@ -173,7 +199,7 @@ exec docker run --rm \
         ${DOCKER_MOUNT_PATH}/benchmarks/multi_node/llm-d/server.sh \
             2>&1 | tee /benchmark_logs/slurm_job-'\"\$SLURM_JOB_ID\"'_rank_'\"\$SLURM_PROCID\"'.log
     '
-"
+" || main_rc=$?
 
     srun bash -c "docker ps -aq --filter name=\"^${DOCKER_CONT_NAME}_\" | xargs -r docker rm -f" || true
 
@@ -200,7 +226,7 @@ elif [[ "$LLMD_CONTAINER_ENGINE" == "pyxis" ]]; then
     export IS_AGENTIC SCENARIO_TYPE FRAMEWORK PRECISION MODEL_PREFIX
     export RUNNER_TYPE RESULT_FILENAME SPEC_DECODING IS_MULTINODE CONFIG_FILE
 
-    PYXIS_ENV_LIST="NUM_NODES,PREFILL_NODES,DECODE_NODES,ALL_IPS,PREFILL_LEADER_IP,DECODE_LEADER_IP"
+    PYXIS_ENV_LIST="POWERX_NATIVE_ENABLED,POWERX_HOST_UID,POWERX_HOST_GID,POWERX_COLLECTOR_REVISION,NUM_NODES,PREFILL_NODES,DECODE_NODES,ALL_IPS,PREFILL_LEADER_IP,DECODE_LEADER_IP"
     PYXIS_ENV_LIST+=",PREFILL_WORKERS,DECODE_WORKERS"
     PYXIS_ENV_LIST+=",PREFILL_DP_ADDR,DECODE_DP_ADDR,MODEL_NAME,GPUS_PER_NODE"
     PYXIS_ENV_LIST+=",PREFILL_DP_SIZE,DECODE_DP_SIZE"
@@ -213,6 +239,9 @@ elif [[ "$LLMD_CONTAINER_ENGINE" == "pyxis" ]]; then
 
     PYXIS_MOUNTS="${MODEL_DIR}:/models:ro"
     PYXIS_MOUNTS+=",${BENCHMARK_LOGS_DIR}:/benchmark_logs"
+    if [[ "$POWERX_NATIVE_ENABLED" == 1 ]]; then
+        PYXIS_MOUNTS+=",${POWERX_RAW_ROOT}:/powerx_native"
+    fi
     PYXIS_MOUNTS+=",${DI_REPO_DIR}:${DOCKER_MOUNT_PATH}"
     PYXIS_MOUNTS+=",${DI_REPO_DIR}/benchmarks/multi_node/llm-d-recipes:/etc/llmd-recipes:ro"
     PYXIS_MOUNTS+=",${DI_REPO_DIR}/benchmarks/llm-d/epp-config.yaml:/etc/epp/config.yaml:ro"
@@ -257,9 +286,11 @@ export MODEL_DIR=/models
 export BENCHMARK_LOGS_DIR=/benchmark_logs
 '"$DOCKER_MOUNT_PATH"'/benchmarks/multi_node/llm-d/server.sh \
     2>&1 | tee /benchmark_logs/slurm_job-${SLURM_JOB_ID}_rank_${SLURM_PROCID}.log
-'
+' || main_rc=$?
 
 else
     echo "Unsupported LLMD_CONTAINER_ENGINE: $LLMD_CONTAINER_ENGINE (expected docker|pyxis)" >&2
     exit 1
 fi
+
+exit "$main_rc"

benchmarks/multi_node/llm-d/server.sh

@@ -114,6 +114,45 @@ else
     DP_ADDR="$DECODE_DP_ADDR"
 fi
 
+# Exactly one coordinator owns the benchmark and global shutdown, even when
+# decode has several independent engine leaders.
+BENCH_DONE_MARKER="$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID"
+BENCH_RC=0
+source /workspace/benchmarks/native_power_lifecycle.sh
+
+finish_llmd_node() {
+    local rc=$? pid
+    trap - EXIT
+    if [[ "${POWERX_NATIVE_ENABLED:-0}" == 1 && -n "${POWERX_COLLECTOR_PID:-}" ]]; then
+        if [[ "$NODE_RANK" == "$PREFILL_NODES" || "$rc" != 0 ]]; then
+            powerx_stop_collectors || rc=$?
+        else
+            powerx_reap_collector || rc=$?
+        fi
+    fi
+    if [[ "$NODE_RANK" == "$PREFILL_NODES" ]]; then
+        printf '%s\n' "$rc" > "$BENCH_DONE_MARKER.tmp" || rc=1
+        if [[ -n "${POWERX_HOST_UID:-}" ]]; then
+            chown "$POWERX_HOST_UID:$POWERX_HOST_GID" "$BENCH_DONE_MARKER.tmp" || rc=1
+        fi
+        mv -f "$BENCH_DONE_MARKER.tmp" "$BENCH_DONE_MARKER" || rc=1
+    fi
+    for pid in "${ENVOY_PID:-}" "${EPP_PID:-}" "${SIDECAR_PID:-}" "${VLLM_PID:-}"; do
+        [[ -z "$pid" ]] || kill -TERM "$pid" 2>/dev/null || true
+    done
+    exit "$rc"
+}
+trap finish_llmd_node EXIT
+trap 'exit 143' TERM HUP
+trap 'exit 130' INT
+if [[ "${POWERX_NATIVE_ENABLED:-0}" == 1 ]]; then
+    _power_control="$BENCHMARK_LOGS_DIR/power_control-$SLURM_JOB_ID"
+    export POWERX_NODE_NAME="$(sed -n '1p' "$_power_control/host-$NODE_RANK")"
+    export POWERX_CLOCK_SYNCHRONIZED="$(sed -n '2p' "$_power_control/host-$NODE_RANK")"
+    powerx_start_collector "/powerx_native/node-$NODE_RANK" "$_power_control" \
+        nvidia "$NODE_RANK" "$ROLE" "$GPUS_PER_NODE" "$NUM_NODES"
+fi
+
 DP_SIZE_LOCAL="$GPUS_PER_NODE"
 START_RANK=$((LWS_WORKER_INDEX * DP_SIZE_LOCAL))
 
@@ -327,11 +366,7 @@ fi
 # ================================================================
 # Coordinator (decode leader): endpoints, EPP, Envoy, bench, eval
 # ================================================================
-if [[ "$ROLE" == "decode" && "$LWS_WORKER_INDEX" -eq 0 ]]; then
-
-    # Release the allocation whenever the coordinator exits.
-    BENCH_DONE_MARKER="$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID"
-    trap 'touch "$BENCH_DONE_MARKER" 2>/dev/null || true' EXIT
+if [[ "$NODE_RANK" == "$PREFILL_NODES" ]]; then
 
     # ---- Write endpoints.yaml (file-discovery) ----
     # namespace must match EPP's --pool-namespace (file-discovery filters by it;
@@ -551,6 +586,10 @@ PY
     done
     echo "All ${#_prefill_ips[@]} prefill vLLM endpoint(s) ready"
 
+    if [[ "${POWERX_NATIVE_ENABLED:-0}" == 1 ]]; then
+        powerx_wait_collectors ready
+    fi
+
     # ---- Benchmark sweep (one run per concurrency level) ----
     # BENCH_MAX_CONCURRENCY is an 'x'-delimited list from submit.sh (e.g. "1024x512").
     IFS='x' read -r -a CONCURRENCIES <<< "$BENCH_MAX_CONCURRENCY"
@@ -582,10 +621,8 @@ PY
             )
         fi
 
-        # Non-fatal: a failed or timed-out conc point must not abort the sweep
-        # or (under set -e) skip the allocation release below. The EXIT trap
-        # releases the allocation regardless, but continuing here lets a
-        # multi-conc sweep record every point it can.
+        # Continue collecting available points after a failure, retaining the
+        # nonzero verdict for the coordinator's final status and worker shutdown.
         run_benchmark_serving \
             --bench-serving-dir /workspace \
             --tokenizer /models \
@@ -600,7 +637,7 @@ PY
             --result-filename "${RESULT_FILENAME}_c${max_concurrency}_gpus_${_bench_total_gpus}_ctx_${_bench_prefill_gpus}_gen_${_bench_decode_gpus}" \
             --result-dir "$BENCHMARK_LOGS_DIR/" \
             "${bench_extra_args[@]}" \
-            || echo "WARNING: benchmark conc=$max_concurrency failed/timed out (rc=$?)"
+            || { BENCH_RC=$?; echo "WARNING: benchmark conc=$max_concurrency failed/timed out (rc=$BENCH_RC)"; }
     done
     fi
 
@@ -631,10 +668,12 @@ PY
         )
     fi
 
-    # Signal job.slurm (outside the container, where scancel exists) to release
-    # the allocation; without it workers wait until TIME_LIMIT.
-    touch "$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID"
+    # EXIT drains every collector before signaling workers to stop serving.
+    exit "$BENCH_RC"
 else
-    # Workers (prefill leader, prefill/decode workers): keep vLLM alive.
-    wait
+    while [[ ! -f "$BENCH_DONE_MARKER" ]]; do
+        kill -0 "$VLLM_PID" 2>/dev/null || exit 1
+        sleep 2
+    done
+    exit "$(cat "$BENCH_DONE_MARKER")"
 fi

benchmarks/multi_node/llm-d/submit.sh

@@ -70,6 +70,10 @@ export PREFILL_DP_SIZE=$(( PREFILL_NODES / PREFILL_WORKERS * GPUS_PER_NODE ))
 export DECODE_DP_SIZE=$((  DECODE_NODES  / DECODE_WORKERS  * GPUS_PER_NODE ))
 export BENCH_INPUT_LEN=$ISL
 export BENCH_OUTPUT_LEN=$OSL
+export POWERX_NATIVE_ENABLED=0
+if [[ "${REQUIRE_POWER:-0}" =~ ^(1|true|TRUE|yes|YES)$ && "$ISL" == 8192 && "$OSL" == 1024 && "${IS_AGENTIC:-0}" != 1 && "${SCENARIO_TYPE:-}" != agentic-coding && "${EVAL_ONLY:-false}" != true ]]; then
+    export POWERX_NATIVE_ENABLED=1
+fi
 export BENCH_MAX_CONCURRENCY=$CONCURRENCIES
 export BENCH_REQUEST_RATE=$REQUEST_RATE
 export BENCH_RANDOM_RANGE_RATIO=$RANDOM_RANGE_RATIO

configs/nvidia-master.yaml

@@ -4466,6 +4466,7 @@ dsv4-fp4-gb200-llmd-vllm:
     fixed-seq-len:
     - isl: 8192
       osl: 1024
+      require-power: true
       search-space:
       # Low latency: 1 prefill DEP8 + 1 decode TP8.
       - spec-decoding: "none"

docs/configuration-procedures.md

@@ -120,6 +120,12 @@ For GLM-5.1 on B200 Nscale, `MODEL_PATH` can select an existing shared checkpoin
 
 Only fixed 8192/1024 `glm5.1-fp8-b200-tilert` requires native power. TileRT runs inside its returned `salloc` allocation, retains both role exit codes and drains collectors before staging audits. Exactly one physical node per role is supported. Other sequence lengths, AgentX and eval-only do not enable this collector. Hardware qualification and publication remain pending.
 
+## Native llm-d power
+
+This integration depends on the Slurm completion and normal worker shutdown changes proposed in [#3052](https://github.com/SemiAnalysisAI/InferenceX/pull/3052); those changes remain an unmerged prerequisite.
+
+Only fixed 8192/1024 `dsv4-fp4-gb200-llmd-vllm` requires native power. Every serving node collects its selected devices; the coordinator waits for readiness and drain before normal shutdown. Results and audits are retained together. Other sequence lengths, AgentX and eval-only do not enable this collector. Hardware qualification and publication remain pending.
+
 ## Register an srt-slurm recipe
 
 Mapping source: [`benchmarks/multi_node/srt-slurm-recipes/RECIPES.md`](../benchmarks/multi_node/srt-slurm-recipes/RECIPES.md). Checked-in recipes: [`benchmarks/multi_node/srt-slurm-recipes/`](../benchmarks/multi_node/srt-slurm-recipes/).

docs/configuration-procedures_zh.md

@@ -120,6 +120,12 @@ B200 Nscale 的 GLM-5.1 可用 `MODEL_PATH` 指定已有共享权重，覆盖默
 
 仅固定 8192/1024 的 `glm5.1-fp8-b200-tilert` 要求原生功耗。TileRT 在 `salloc` 返回的分配内运行，保留两个角色的退出码，并在保存审计数据前等待采集器排空。每个角色仅支持一个物理节点。其他序列长度、AgentX 和 eval-only 不启用此采集器。硬件资格验证与发布仍待完成。
 
+## llm-d 原生功耗
+
+此集成依赖 [#3052](https://github.com/SemiAnalysisAI/InferenceX/pull/3052) 中的 Slurm 完成状态检查和工作进程正常退出变更；这些前置变更尚未合并。
+
+仅固定 8192/1024 的 `dsv4-fp4-gb200-llmd-vllm` 要求原生功耗。每个服务节点采集所选设备；协调进程等待 ready 与 drain 后正常退出。结果与审计数据共同保留。其他序列长度、AgentX 和 eval-only 不启用此采集器。硬件资格验证与发布仍待完成。
+
 ## 注册 srt-slurm 配方
 
 映射来源：[`benchmarks/multi_node/srt-slurm-recipes/RECIPES.md`](../benchmarks/multi_node/srt-slurm-recipes/RECIPES.md)。检入的配方：[`benchmarks/multi_node/srt-slurm-recipes/`](../benchmarks/multi_node/srt-slurm-recipes/)。

docs/results-and-ingestion.md

@@ -114,6 +114,10 @@ Processing and diagnostic power-audit uploads run after launcher or validation f
 
 The native collector sets UTC and records context beside its CSV for portable replay; existing benchmark monitors keep their current behavior. Its launcher integration requires separate hardware qualification. The offline adapter accepts this context without changing producers. Unusable samples outside the formal window do not establish coverage; `boundary_degenerate_rows` retains their per-GPU counts.
 
+### Slurm completion receipts
+
+Shared Slurm waiting verifies the terminal allocation state and exit code, consulting `scontrol` when `sacct` is missing or non-terminal and retaining `slurm_job_*_outcome.txt`. Launchers stage available evidence before returning failure. llm-d workers exit using the coordinator’s atomically published status-bearing completion marker; normal completion no longer cancels the allocation.
+
 ## Eval artifacts
 
 ### Per-config identity and collection

docs/results-and-ingestion_zh.md

@@ -114,6 +114,10 @@ PR changelog 选择具有代表性的 NVIDIA 和 AMD 覆盖，并非所有受影
 
 原生采集器单独设置 UTC，并在 CSV 旁记录上下文以支持跨环境回放；现有基准监控行为保持不变。启动器接入需要另行完成硬件验证。离线适配器接受该上下文，不改变现有生产端。正式窗口外的无效样本不能构成覆盖；`boundary_degenerate_rows` 保留其逐 GPU 计数。
 
+### Slurm 完成状态文件
+
+共享 Slurm 等待逻辑检查分配的最终状态和退出码；当 `sacct` 记录缺失或尚未进入最终状态时查询 `scontrol`，并保留 `slurm_job_*_outcome.txt`。启动器先保存已有证据再返回失败。llm-d 工作进程根据协调进程原子发布的完成状态退出；正常结束不再取消 Slurm 分配。
+
 ## 评测工件
 
 ### 单配置身份和收集

perf-changelog.yaml

@@ -7503,3 +7503,13 @@
     - "Enable required power for the H200 Kimi-K3 latency recipe and route their launch, concurrency injection, failure handling and result validation through the pinned producer and shared collector."
     - "仅为 H200 Kimi-K3 latency 配方启用必需实测功耗。"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3044
+
+- config-keys:
+  - dsv4-fp4-gb200-llmd-vllm
+  scenario-type:
+  - fixed-seq-len
+  description:
+  - Require native power only for llm-d 8k1k and preserve per-node collector readiness,
+    drain and completion evidence.
+  - 仅对 llm-d 8k1k 要求原生功耗，并保留每节点采集器 ready、drain 和完成证据。
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3056

runners/launch_b200-nscale-compat.sh

@@ -449,10 +449,12 @@ EOF
     tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null
 
     wait $POLL_PID
+    SRT_JOB_RC=0
+    verify_slurm_job_completion "$JOB_ID" || SRT_JOB_RC=$?
 
     set -x
 
-    echo "Job $JOB_ID completed!"
+    echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence"
     echo "Collecting results..."
 
     if [ ! -d "$LOGS_DIR" ]; then
@@ -504,6 +506,8 @@ EOF
     done
     find . -name '.nfs*' -delete 2>/dev/null || true
 
+    if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi
+
 else
 
     SQUASH_FILE="/data/home/sa-shared/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh"