diff --git a/.github/workflows/benchmark-multinode-tmpl.yml b/.github/workflows/benchmark-multinode-tmpl.yml index 81f4cdb806..1dea154aa2 100644 --- a/.github/workflows/benchmark-multinode-tmpl.yml +++ b/.github/workflows/benchmark-multinode-tmpl.yml @@ -538,6 +538,7 @@ jobs: ${{ env.RESULT_FILENAME }}_*.json agg_${{ env.RESULT_FILENAME }}_*.json power_validation_${{ env.RESULT_FILENAME }}_*.json + slurm_job_*_outcome.txt LOGS/power/** LOGS/native_power/** result_processing_${{ env.RESULT_FILENAME }}.json diff --git a/.github/workflows/benchmark-tmpl.yml b/.github/workflows/benchmark-tmpl.yml index 529b04b665..7c20674e5f 100644 --- a/.github/workflows/benchmark-tmpl.yml +++ b/.github/workflows/benchmark-tmpl.yml @@ -474,6 +474,7 @@ jobs: gpu_metrics_identity.json gpu_metrics_identity.csv power_validation_${{ env.RESULT_FILENAME }}.json + slurm_job_*_outcome.txt results/gpu_metrics*.csv results/gpu_metrics*_context.json results/gpu_metrics_identity.json diff --git a/.github/workflows/test-changelog-gate.yml b/.github/workflows/test-changelog-gate.yml index 7d49c2ff11..d88effabec 100644 --- a/.github/workflows/test-changelog-gate.yml +++ b/.github/workflows/test-changelog-gate.yml @@ -31,6 +31,8 @@ on: - "runners/launch_*.sh" - "runners/slurm_utils.sh" - "runners/test_slurm_utils.py" + - "runners/test_llmd_lifecycle.py" + - "benchmarks/multi_node/llm-d/**" - "utils/ci_priority.py" - "utils/test_ci_priority.py" - ".github/workflows/reuse-sweep-comment.yml" @@ -123,4 +125,5 @@ jobs: utils/evals/test_batched_eval.py \ utils/evals/test_run_eval_dispatch.py \ runners/test_slurm_utils.py \ + runners/test_llmd_lifecycle.py \ -v -n 4 diff --git a/.github/workflows/test-process-result.yml b/.github/workflows/test-process-result.yml index 5dc71b1eef..d1df4bfdf9 100644 --- a/.github/workflows/test-process-result.yml +++ b/.github/workflows/test-process-result.yml @@ -47,6 +47,8 @@ on: - 'benchmarks/multi_node/tilert_utils/**' - 'benchmarks/multi_node/glm5.1_fp8_b200_tilert-disagg.sh' - 'runners/test_tilert_power_lifecycle.py' + - 'benchmarks/multi_node/llm-d/**' + - 'runners/test_llmd_power_lifecycle.py' permissions: contents: read @@ -68,7 +70,7 @@ jobs: run: | cd utils uv run --no-project --exclude-newer PT12H --python 3.12 --with pytest --with pyyaml \ - python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py agentic/aggregation/ test_gb300_power_official_contract.py test_inject_srt_power_concurrencies.py test_process_result.py test_native_multinode_power.py ../runners/test_native_collector_barriers.py ../runners/test_native_collector_receipts.py ../runners/test_kimik3_bh_power.py ../runners/test_tilert_power_lifecycle.py -v + python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py agentic/aggregation/ test_gb300_power_official_contract.py test_inject_srt_power_concurrencies.py test_process_result.py test_native_multinode_power.py ../runners/test_native_collector_barriers.py ../runners/test_native_collector_receipts.py ../runners/test_kimik3_bh_power.py ../runners/test_tilert_power_lifecycle.py ../runners/test_llmd_power_lifecycle.py -v - name: Test serving client result persistence run: | diff --git a/benchmarks/multi_node/llm-d/job.slurm b/benchmarks/multi_node/llm-d/job.slurm index 536507bee5..eb3a42b846 100644 --- a/benchmarks/multi_node/llm-d/job.slurm +++ b/benchmarks/multi_node/llm-d/job.slurm @@ -63,25 +63,46 @@ export DOCKER_CONT_NAME : "${BENCHMARK_LOGS_DIR:?BENCHMARK_LOGS_DIR not set}" DOCKER_MOUNT_PATH="/workspace" -cleanup() { - echo "[${SLURM_JOB_ID}] cleanup on $(hostname)" - [[ -n "${WATCHER_PID:-}" ]] && kill "$WATCHER_PID" 2>/dev/null || true -} -trap cleanup INT TERM HUP EXIT - -# Coordinator-done watcher. server.sh on the decode coordinator writes -# this marker after the bench finishes; we then scancel the allocation -# from outside the container (the image has no SLURM client tools). -# Without this, workers `wait` on local vLLM forever and the job runs -# to TIME_LIMIT. +# Normal completion is coordinated inside server.sh. Cancelling a successful +# allocation would discard its terminal success state and interrupt collectors. BENCH_DONE_MARKER="$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID" rm -f "$BENCH_DONE_MARKER" -( - while [[ ! -f "$BENCH_DONE_MARKER" ]]; do sleep 5; done - echo "[${SLURM_JOB_ID}] coordinator finished; scancel'ing job" - scancel "$SLURM_JOB_ID" 2>/dev/null || true -) & -WATCHER_PID=$! +POWERX_NATIVE_ENABLED="${POWERX_NATIVE_ENABLED:-0}" +export POWERX_HOST_UID="" POWERX_HOST_GID="" POWERX_COLLECTOR_REVISION="" +POWERX_DOCKER_MOUNT="" +if [[ "$POWERX_NATIVE_ENABLED" == 1 ]]; then + export POWERX_HOST_UID="$(id -u)" POWERX_HOST_GID="$(id -g)" + export POWERX_COLLECTOR_REVISION="$(git -C "$DI_REPO_DIR" rev-parse HEAD)" + export POWERX_RAW_ROOT="${POWERX_RAW_ROOT:-/tmp/inferencex-native-$SLURM_JOB_ID}" + POWERX_DOCKER_MOUNT="-v ${POWERX_RAW_ROOT}:/powerx_native" + srun --nodes="$NUM_NODES" --ntasks-per-node=1 mkdir -p "$POWERX_RAW_ROOT" + mkdir -p "$BENCHMARK_LOGS_DIR/power_control-$SLURM_JOB_ID" "$DI_REPO_DIR/LOGS/native_power" + chmod 777 "$BENCHMARK_LOGS_DIR/power_control-$SLURM_JOB_ID" + srun --nodes="$NUM_NODES" --ntasks-per-node=1 chmod 777 "$POWERX_RAW_ROOT" + srun --nodes="$NUM_NODES" --ntasks-per-node=1 bash -c ' + { hostname; timedatectl show -p NTPSynchronized --value 2>/dev/null || echo false; } \ + > "$BENCHMARK_LOGS_DIR/power_control-$SLURM_JOB_ID/host-$SLURM_PROCID" + ' +fi +main_rc=0 +finish_llmd_job() { + local rc=$? + trap - EXIT + [[ "$rc" == 0 ]] || main_rc=$rc + + if [[ "$POWERX_NATIVE_ENABLED" == 1 ]]; then + # Copy as the host user only after collectors and serving containers drain. + srun --nodes="$NUM_NODES" --ntasks-per-node=1 bash -c ' + source_dir="$POWERX_RAW_ROOT/node-$SLURM_PROCID" + [[ -d "$source_dir" ]] || exit 1 + cp -R "$source_dir" "$DI_REPO_DIR/LOGS/native_power/" + ' || main_rc=$? + fi + exit "$main_rc" +} +trap finish_llmd_job EXIT +trap 'exit 143' TERM HUP +trap 'exit 130' INT # Container engine: 'docker' (default) for clusters where the SLURM # user can talk to /var/run/docker.sock (e.g. h200-dgxc-slurm); 'pyxis' @@ -119,10 +140,15 @@ exec docker run --rm \ --privileged \ -v ${MODEL_DIR}:/models:ro \ -v ${BENCHMARK_LOGS_DIR}:/benchmark_logs \ + $POWERX_DOCKER_MOUNT \ -v ${DI_REPO_DIR}:${DOCKER_MOUNT_PATH} \ -v ${DI_REPO_DIR}/benchmarks/multi_node/llm-d-recipes:/etc/llmd-recipes:ro \ -v ${DI_REPO_DIR}/benchmarks/llm-d/epp-config.yaml:/etc/epp/config.yaml:ro \ -v ${DI_REPO_DIR}/benchmarks/llm-d/envoy.yaml:/etc/envoy/envoy.yaml:ro \ + -e POWERX_NATIVE_ENABLED=$POWERX_NATIVE_ENABLED \ + -e POWERX_HOST_UID=$POWERX_HOST_UID \ + -e POWERX_HOST_GID=$POWERX_HOST_GID \ + -e POWERX_COLLECTOR_REVISION=$POWERX_COLLECTOR_REVISION \ -e SLURM_JOB_ID=\$SLURM_JOB_ID \ -e NODE_RANK=\$SLURM_PROCID \ -e NUM_NODES=$NUM_NODES \ @@ -173,7 +199,7 @@ exec docker run --rm \ ${DOCKER_MOUNT_PATH}/benchmarks/multi_node/llm-d/server.sh \ 2>&1 | tee /benchmark_logs/slurm_job-'\"\$SLURM_JOB_ID\"'_rank_'\"\$SLURM_PROCID\"'.log ' -" +" || main_rc=$? srun bash -c "docker ps -aq --filter name=\"^${DOCKER_CONT_NAME}_\" | xargs -r docker rm -f" || true @@ -200,7 +226,7 @@ elif [[ "$LLMD_CONTAINER_ENGINE" == "pyxis" ]]; then export IS_AGENTIC SCENARIO_TYPE FRAMEWORK PRECISION MODEL_PREFIX export RUNNER_TYPE RESULT_FILENAME SPEC_DECODING IS_MULTINODE CONFIG_FILE - PYXIS_ENV_LIST="NUM_NODES,PREFILL_NODES,DECODE_NODES,ALL_IPS,PREFILL_LEADER_IP,DECODE_LEADER_IP" + PYXIS_ENV_LIST="POWERX_NATIVE_ENABLED,POWERX_HOST_UID,POWERX_HOST_GID,POWERX_COLLECTOR_REVISION,NUM_NODES,PREFILL_NODES,DECODE_NODES,ALL_IPS,PREFILL_LEADER_IP,DECODE_LEADER_IP" PYXIS_ENV_LIST+=",PREFILL_WORKERS,DECODE_WORKERS" PYXIS_ENV_LIST+=",PREFILL_DP_ADDR,DECODE_DP_ADDR,MODEL_NAME,GPUS_PER_NODE" PYXIS_ENV_LIST+=",PREFILL_DP_SIZE,DECODE_DP_SIZE" @@ -213,6 +239,9 @@ elif [[ "$LLMD_CONTAINER_ENGINE" == "pyxis" ]]; then PYXIS_MOUNTS="${MODEL_DIR}:/models:ro" PYXIS_MOUNTS+=",${BENCHMARK_LOGS_DIR}:/benchmark_logs" + if [[ "$POWERX_NATIVE_ENABLED" == 1 ]]; then + PYXIS_MOUNTS+=",${POWERX_RAW_ROOT}:/powerx_native" + fi PYXIS_MOUNTS+=",${DI_REPO_DIR}:${DOCKER_MOUNT_PATH}" PYXIS_MOUNTS+=",${DI_REPO_DIR}/benchmarks/multi_node/llm-d-recipes:/etc/llmd-recipes:ro" PYXIS_MOUNTS+=",${DI_REPO_DIR}/benchmarks/llm-d/epp-config.yaml:/etc/epp/config.yaml:ro" @@ -257,9 +286,11 @@ export MODEL_DIR=/models export BENCHMARK_LOGS_DIR=/benchmark_logs '"$DOCKER_MOUNT_PATH"'/benchmarks/multi_node/llm-d/server.sh \ 2>&1 | tee /benchmark_logs/slurm_job-${SLURM_JOB_ID}_rank_${SLURM_PROCID}.log -' +' || main_rc=$? else echo "Unsupported LLMD_CONTAINER_ENGINE: $LLMD_CONTAINER_ENGINE (expected docker|pyxis)" >&2 exit 1 fi + +exit "$main_rc" diff --git a/benchmarks/multi_node/llm-d/server.sh b/benchmarks/multi_node/llm-d/server.sh index 89894b210c..d1fd8ee9e6 100755 --- a/benchmarks/multi_node/llm-d/server.sh +++ b/benchmarks/multi_node/llm-d/server.sh @@ -114,6 +114,45 @@ else DP_ADDR="$DECODE_DP_ADDR" fi +# Exactly one coordinator owns the benchmark and global shutdown, even when +# decode has several independent engine leaders. +BENCH_DONE_MARKER="$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID" +BENCH_RC=0 +source /workspace/benchmarks/native_power_lifecycle.sh + +finish_llmd_node() { + local rc=$? pid + trap - EXIT + if [[ "${POWERX_NATIVE_ENABLED:-0}" == 1 && -n "${POWERX_COLLECTOR_PID:-}" ]]; then + if [[ "$NODE_RANK" == "$PREFILL_NODES" || "$rc" != 0 ]]; then + powerx_stop_collectors || rc=$? + else + powerx_reap_collector || rc=$? + fi + fi + if [[ "$NODE_RANK" == "$PREFILL_NODES" ]]; then + printf '%s\n' "$rc" > "$BENCH_DONE_MARKER.tmp" || rc=1 + if [[ -n "${POWERX_HOST_UID:-}" ]]; then + chown "$POWERX_HOST_UID:$POWERX_HOST_GID" "$BENCH_DONE_MARKER.tmp" || rc=1 + fi + mv -f "$BENCH_DONE_MARKER.tmp" "$BENCH_DONE_MARKER" || rc=1 + fi + for pid in "${ENVOY_PID:-}" "${EPP_PID:-}" "${SIDECAR_PID:-}" "${VLLM_PID:-}"; do + [[ -z "$pid" ]] || kill -TERM "$pid" 2>/dev/null || true + done + exit "$rc" +} +trap finish_llmd_node EXIT +trap 'exit 143' TERM HUP +trap 'exit 130' INT +if [[ "${POWERX_NATIVE_ENABLED:-0}" == 1 ]]; then + _power_control="$BENCHMARK_LOGS_DIR/power_control-$SLURM_JOB_ID" + export POWERX_NODE_NAME="$(sed -n '1p' "$_power_control/host-$NODE_RANK")" + export POWERX_CLOCK_SYNCHRONIZED="$(sed -n '2p' "$_power_control/host-$NODE_RANK")" + powerx_start_collector "/powerx_native/node-$NODE_RANK" "$_power_control" \ + nvidia "$NODE_RANK" "$ROLE" "$GPUS_PER_NODE" "$NUM_NODES" +fi + DP_SIZE_LOCAL="$GPUS_PER_NODE" START_RANK=$((LWS_WORKER_INDEX * DP_SIZE_LOCAL)) @@ -327,11 +366,7 @@ fi # ================================================================ # Coordinator (decode leader): endpoints, EPP, Envoy, bench, eval # ================================================================ -if [[ "$ROLE" == "decode" && "$LWS_WORKER_INDEX" -eq 0 ]]; then - - # Release the allocation whenever the coordinator exits. - BENCH_DONE_MARKER="$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID" - trap 'touch "$BENCH_DONE_MARKER" 2>/dev/null || true' EXIT +if [[ "$NODE_RANK" == "$PREFILL_NODES" ]]; then # ---- Write endpoints.yaml (file-discovery) ---- # namespace must match EPP's --pool-namespace (file-discovery filters by it; @@ -551,6 +586,10 @@ PY done echo "All ${#_prefill_ips[@]} prefill vLLM endpoint(s) ready" + if [[ "${POWERX_NATIVE_ENABLED:-0}" == 1 ]]; then + powerx_wait_collectors ready + fi + # ---- Benchmark sweep (one run per concurrency level) ---- # BENCH_MAX_CONCURRENCY is an 'x'-delimited list from submit.sh (e.g. "1024x512"). IFS='x' read -r -a CONCURRENCIES <<< "$BENCH_MAX_CONCURRENCY" @@ -582,10 +621,8 @@ PY ) fi - # Non-fatal: a failed or timed-out conc point must not abort the sweep - # or (under set -e) skip the allocation release below. The EXIT trap - # releases the allocation regardless, but continuing here lets a - # multi-conc sweep record every point it can. + # Continue collecting available points after a failure, retaining the + # nonzero verdict for the coordinator's final status and worker shutdown. run_benchmark_serving \ --bench-serving-dir /workspace \ --tokenizer /models \ @@ -600,7 +637,7 @@ PY --result-filename "${RESULT_FILENAME}_c${max_concurrency}_gpus_${_bench_total_gpus}_ctx_${_bench_prefill_gpus}_gen_${_bench_decode_gpus}" \ --result-dir "$BENCHMARK_LOGS_DIR/" \ "${bench_extra_args[@]}" \ - || echo "WARNING: benchmark conc=$max_concurrency failed/timed out (rc=$?)" + || { BENCH_RC=$?; echo "WARNING: benchmark conc=$max_concurrency failed/timed out (rc=$BENCH_RC)"; } done fi @@ -631,10 +668,12 @@ PY ) fi - # Signal job.slurm (outside the container, where scancel exists) to release - # the allocation; without it workers wait until TIME_LIMIT. - touch "$BENCHMARK_LOGS_DIR/.bench_done.$SLURM_JOB_ID" + # EXIT drains every collector before signaling workers to stop serving. + exit "$BENCH_RC" else - # Workers (prefill leader, prefill/decode workers): keep vLLM alive. - wait + while [[ ! -f "$BENCH_DONE_MARKER" ]]; do + kill -0 "$VLLM_PID" 2>/dev/null || exit 1 + sleep 2 + done + exit "$(cat "$BENCH_DONE_MARKER")" fi diff --git a/benchmarks/multi_node/llm-d/submit.sh b/benchmarks/multi_node/llm-d/submit.sh index 9d00125c1c..54c11b563e 100755 --- a/benchmarks/multi_node/llm-d/submit.sh +++ b/benchmarks/multi_node/llm-d/submit.sh @@ -70,6 +70,10 @@ export PREFILL_DP_SIZE=$(( PREFILL_NODES / PREFILL_WORKERS * GPUS_PER_NODE )) export DECODE_DP_SIZE=$(( DECODE_NODES / DECODE_WORKERS * GPUS_PER_NODE )) export BENCH_INPUT_LEN=$ISL export BENCH_OUTPUT_LEN=$OSL +export POWERX_NATIVE_ENABLED=0 +if [[ "${REQUIRE_POWER:-0}" =~ ^(1|true|TRUE|yes|YES)$ && "$ISL" == 8192 && "$OSL" == 1024 && "${IS_AGENTIC:-0}" != 1 && "${SCENARIO_TYPE:-}" != agentic-coding && "${EVAL_ONLY:-false}" != true ]]; then + export POWERX_NATIVE_ENABLED=1 +fi export BENCH_MAX_CONCURRENCY=$CONCURRENCIES export BENCH_REQUEST_RATE=$REQUEST_RATE export BENCH_RANDOM_RANGE_RATIO=$RANDOM_RANGE_RATIO diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 302ff0a277..cfb18be39d 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -4466,6 +4466,7 @@ dsv4-fp4-gb200-llmd-vllm: fixed-seq-len: - isl: 8192 osl: 1024 + require-power: true search-space: # Low latency: 1 prefill DEP8 + 1 decode TP8. - spec-decoding: "none" diff --git a/docs/configuration-procedures.md b/docs/configuration-procedures.md index 157b45b311..3db88beed4 100644 --- a/docs/configuration-procedures.md +++ b/docs/configuration-procedures.md @@ -120,6 +120,12 @@ For GLM-5.1 on B200 Nscale, `MODEL_PATH` can select an existing shared checkpoin Only fixed 8192/1024 `glm5.1-fp8-b200-tilert` requires native power. TileRT runs inside its returned `salloc` allocation, retains both role exit codes and drains collectors before staging audits. Exactly one physical node per role is supported. Other sequence lengths, AgentX and eval-only do not enable this collector. Hardware qualification and publication remain pending. +## Native llm-d power + +This integration depends on the Slurm completion and normal worker shutdown changes proposed in [#3052](https://github.com/SemiAnalysisAI/InferenceX/pull/3052); those changes remain an unmerged prerequisite. + +Only fixed 8192/1024 `dsv4-fp4-gb200-llmd-vllm` requires native power. Every serving node collects its selected devices; the coordinator waits for readiness and drain before normal shutdown. Results and audits are retained together. Other sequence lengths, AgentX and eval-only do not enable this collector. Hardware qualification and publication remain pending. + ## Register an srt-slurm recipe Mapping source: [`benchmarks/multi_node/srt-slurm-recipes/RECIPES.md`](../benchmarks/multi_node/srt-slurm-recipes/RECIPES.md). Checked-in recipes: [`benchmarks/multi_node/srt-slurm-recipes/`](../benchmarks/multi_node/srt-slurm-recipes/). diff --git a/docs/configuration-procedures_zh.md b/docs/configuration-procedures_zh.md index 0e64e87a24..d7698e70e1 100644 --- a/docs/configuration-procedures_zh.md +++ b/docs/configuration-procedures_zh.md @@ -120,6 +120,12 @@ B200 Nscale 的 GLM-5.1 可用 `MODEL_PATH` 指定已有共享权重,覆盖默 仅固定 8192/1024 的 `glm5.1-fp8-b200-tilert` 要求原生功耗。TileRT 在 `salloc` 返回的分配内运行,保留两个角色的退出码,并在保存审计数据前等待采集器排空。每个角色仅支持一个物理节点。其他序列长度、AgentX 和 eval-only 不启用此采集器。硬件资格验证与发布仍待完成。 +## llm-d 原生功耗 + +此集成依赖 [#3052](https://github.com/SemiAnalysisAI/InferenceX/pull/3052) 中的 Slurm 完成状态检查和工作进程正常退出变更;这些前置变更尚未合并。 + +仅固定 8192/1024 的 `dsv4-fp4-gb200-llmd-vllm` 要求原生功耗。每个服务节点采集所选设备;协调进程等待 ready 与 drain 后正常退出。结果与审计数据共同保留。其他序列长度、AgentX 和 eval-only 不启用此采集器。硬件资格验证与发布仍待完成。 + ## 注册 srt-slurm 配方 映射来源:[`benchmarks/multi_node/srt-slurm-recipes/RECIPES.md`](../benchmarks/multi_node/srt-slurm-recipes/RECIPES.md)。检入的配方:[`benchmarks/multi_node/srt-slurm-recipes/`](../benchmarks/multi_node/srt-slurm-recipes/)。 diff --git a/docs/results-and-ingestion.md b/docs/results-and-ingestion.md index b24c0f625a..2984b18d6f 100644 --- a/docs/results-and-ingestion.md +++ b/docs/results-and-ingestion.md @@ -114,6 +114,10 @@ Processing and diagnostic power-audit uploads run after launcher or validation f The native collector sets UTC and records context beside its CSV for portable replay; existing benchmark monitors keep their current behavior. Its launcher integration requires separate hardware qualification. The offline adapter accepts this context without changing producers. Unusable samples outside the formal window do not establish coverage; `boundary_degenerate_rows` retains their per-GPU counts. +### Slurm completion receipts + +Shared Slurm waiting verifies the terminal allocation state and exit code, consulting `scontrol` when `sacct` is missing or non-terminal and retaining `slurm_job_*_outcome.txt`. Launchers stage available evidence before returning failure. llm-d workers exit using the coordinator’s atomically published status-bearing completion marker; normal completion no longer cancels the allocation. + ## Eval artifacts ### Per-config identity and collection diff --git a/docs/results-and-ingestion_zh.md b/docs/results-and-ingestion_zh.md index a10b82e22b..eece846db2 100644 --- a/docs/results-and-ingestion_zh.md +++ b/docs/results-and-ingestion_zh.md @@ -114,6 +114,10 @@ PR changelog 选择具有代表性的 NVIDIA 和 AMD 覆盖,并非所有受影 原生采集器单独设置 UTC,并在 CSV 旁记录上下文以支持跨环境回放;现有基准监控行为保持不变。启动器接入需要另行完成硬件验证。离线适配器接受该上下文,不改变现有生产端。正式窗口外的无效样本不能构成覆盖;`boundary_degenerate_rows` 保留其逐 GPU 计数。 +### Slurm 完成状态文件 + +共享 Slurm 等待逻辑检查分配的最终状态和退出码;当 `sacct` 记录缺失或尚未进入最终状态时查询 `scontrol`,并保留 `slurm_job_*_outcome.txt`。启动器先保存已有证据再返回失败。llm-d 工作进程根据协调进程原子发布的完成状态退出;正常结束不再取消 Slurm 分配。 + ## 评测工件 ### 单配置身份和收集 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index ef367b14a0..a8185bde5b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7503,3 +7503,13 @@ - "Enable required power for the H200 Kimi-K3 latency recipe and route their launch, concurrency injection, failure handling and result validation through the pinned producer and shared collector." - "仅为 H200 Kimi-K3 latency 配方启用必需实测功耗。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3044 + +- config-keys: + - dsv4-fp4-gb200-llmd-vllm + scenario-type: + - fixed-seq-len + description: + - Require native power only for llm-d 8k1k and preserve per-node collector readiness, + drain and completion evidence. + - 仅对 llm-d 8k1k 要求原生功耗,并保留每节点采集器 ready、drain 和完成证据。 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3056 diff --git a/runners/launch_b200-nscale-compat.sh b/runners/launch_b200-nscale-compat.sh index 5f822ec530..f70cbab3f8 100644 --- a/runners/launch_b200-nscale-compat.sh +++ b/runners/launch_b200-nscale-compat.sh @@ -449,10 +449,12 @@ EOF tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null wait $POLL_PID + SRT_JOB_RC=0 + verify_slurm_job_completion "$JOB_ID" || SRT_JOB_RC=$? set -x - echo "Job $JOB_ID completed!" + echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence" echo "Collecting results..." if [ ! -d "$LOGS_DIR" ]; then @@ -504,6 +506,8 @@ EOF done find . -name '.nfs*' -delete 2>/dev/null || true + if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi + else SQUASH_FILE="/data/home/sa-shared/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" diff --git a/runners/launch_b200-nscale-slurm.sh b/runners/launch_b200-nscale-slurm.sh index fb95bd7ae4..b3e74e9e2e 100755 --- a/runners/launch_b200-nscale-slurm.sh +++ b/runners/launch_b200-nscale-slurm.sh @@ -397,13 +397,10 @@ LOG_FILE="$LOGS_DIR/sweep_${JOB_ID}.log" # streams until the job leaves the queue. SRT_JOB_RC=0 stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || SRT_JOB_RC=$? -if [[ "$SRT_JOB_RC" != "0" && "$USES_AGENTX_POWER" != "1" ]]; then - exit "$SRT_JOB_RC" -fi set -x -echo "Job $JOB_ID completed!" +echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence" echo "Collecting results..." if [ ! -d "$LOGS_DIR" ]; then @@ -428,7 +425,7 @@ fi cp -r "$LOGS_DIR" "$GITHUB_WORKSPACE/LOGS" bundle_server_logs "$LOGS_DIR" "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" -if [[ "$AGENTX_POWER_RC" != "0" ]]; then +if [[ "$AGENTX_POWER_RC" != "0" && "$SRT_JOB_RC" == "0" ]]; then echo "ERROR: AgentX power validation failed; available audit and server artifacts were staged" >&2 exit "$AGENTX_POWER_RC" fi @@ -487,3 +484,5 @@ for i in 1 2 3 4 5; do sleep 10 done find . -name '.nfs*' -delete 2>/dev/null || true + +if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi diff --git a/runners/launch_b300-dsxe.sh b/runners/launch_b300-dsxe.sh index c0c959d252..e68951f730 100755 --- a/runners/launch_b300-dsxe.sh +++ b/runners/launch_b300-dsxe.sh @@ -338,10 +338,12 @@ echo "Tailing LOG_FILE: $LOG_FILE" tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null wait $POLL_PID +SRT_JOB_RC=0 +verify_slurm_job_completion "$JOB_ID" || SRT_JOB_RC=$? set -x -echo "Job $JOB_ID completed!" +echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence" echo "Collecting results..." if [ ! -d "$LOGS_DIR" ]; then @@ -393,6 +395,8 @@ for i in 1 2 3 4 5; do done find . -name '.nfs*' -delete 2>/dev/null || true +if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi + else # HF_HUB_CACHE is set to help with dataset download inside the container # for eval jobs. diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index d122051bec..f4bf125361 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -159,7 +159,8 @@ if [[ "$FRAMEWORK" == "llmd-vllm" ]]; then trap 'bundle_server_logs "$BENCHMARK_LOGS_DIR" "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz"; scancel "$JOB_ID" 2>/dev/null || true' EXIT INT TERM HUP LOG_FILE="${BENCHMARK_LOGS_DIR}/slurm_job-${JOB_ID}.out" - stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || exit 1 + SRT_JOB_RC=0 + stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || SRT_JOB_RC=$? while IFS= read -r -d '' result_file; do copy_to_workspace "$result_file" "$GITHUB_WORKSPACE/$(basename "$result_file")" || exit 1 @@ -173,8 +174,10 @@ if [[ "$FRAMEWORK" == "llmd-vllm" ]]; then copy_eval_artifacts "$EVAL_DIR" "$GITHUB_WORKSPACE" || exit 1 fi + mkdir -p "$GITHUB_WORKSPACE/LOGS" + cp -a "$BENCHMARK_LOGS_DIR/." "$GITHUB_WORKSPACE/LOGS/" || exit 1 scancel "$JOB_ID" 2>/dev/null || true - exit 0 + exit "$SRT_JOB_RC" fi # MODEL_PATH: Override with pre-downloaded paths on GB200 runner @@ -852,14 +855,12 @@ LOGS_DIR="outputs/$JOB_ID/logs" LOG_FILE="$LOGS_DIR/sweep_${JOB_ID}.log" AGENTX_POWER_RC=0 -stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || AGENTX_POWER_RC=$? -if [[ "$AGENTX_POWER_RC" != "0" && "$USES_AGENTX_POWER" != "1" ]]; then - exit 1 -fi +SRT_JOB_RC=0 +stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || SRT_JOB_RC=$? set -x -echo "Job $JOB_ID finished!" +echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence" echo "Collecting results..." if [[ "$USES_AGENTX_POWER" == "1" && "${EVAL_ONLY:-false}" != "true" ]]; then @@ -916,7 +917,7 @@ else echo "Warning: Logs directory not found at $LOGS_DIR" fi -if [[ "$AGENTX_POWER_RC" != "0" ]]; then +if [[ "$AGENTX_POWER_RC" != "0" && "$SRT_JOB_RC" == "0" ]]; then echo "ERROR: AgentX job or power validation failed; available audit and server artifacts were staged" >&2 exit "$AGENTX_POWER_RC" fi @@ -987,3 +988,5 @@ fi if [[ "${RUN_EVAL:-false}" == "true" || "${EVAL_ONLY:-false}" == "true" ]]; then copy_eval_artifacts "$LOGS_DIR/eval_results" "$GITHUB_WORKSPACE" || exit 1 fi + +exit "$SRT_JOB_RC" diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index ff3a70035c..58fd1df60f 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -664,39 +664,12 @@ _snapshot_server_logs() { trap _snapshot_server_logs EXIT AGENTX_POWER_RC=0 -if [[ "$USES_AGENTX_POWER" == "1" ]]; then - stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || AGENTX_POWER_RC=$? -else - # Wait for log file to appear (also check job is still alive) - while ! ls "$LOG_FILE" &>/dev/null; do - if ! squeue -j "$JOB_ID" --noheader 2>/dev/null | grep -q "$JOB_ID"; then - echo "ERROR: Job $JOB_ID failed before creating log file" - scontrol show job "$JOB_ID" - exit 1 - fi - echo "Waiting for JOB_ID $JOB_ID to begin and $LOG_FILE to appear..." - sleep 5 - done - - # Poll for job completion in background - ( - while squeue -j "$JOB_ID" --noheader 2>/dev/null | grep -q "$JOB_ID"; do - sleep 10 - done - ) & - POLL_PID=$! - - echo "Tailing LOG_FILE: $LOG_FILE" - - # Stream the log file until job completes (-F follows by name, polls instead of inotify for NFS) - tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null - - wait $POLL_PID -fi +SRT_JOB_RC=0 +stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || SRT_JOB_RC=$? set -x -echo "Job $JOB_ID completed!" +echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence" echo "Collecting results..." if [[ "$USES_AGENTX_POWER" == "1" && "${EVAL_ONLY:-false}" != "true" ]]; then @@ -716,7 +689,7 @@ else echo "Warning: Logs directory not found at $LOGS_DIR" fi -if [[ "$AGENTX_POWER_RC" != "0" ]]; then +if [[ "$AGENTX_POWER_RC" != "0" && "$SRT_JOB_RC" == "0" ]]; then echo "ERROR: AgentX job or power validation failed; EXIT will stage audit artifacts" >&2 exit "$AGENTX_POWER_RC" fi @@ -781,3 +754,5 @@ for i in 1 2 3 4 5; do sleep 10 done find . -name '.nfs*' -delete 2>/dev/null || true + +if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh index af43c740c0..de50907f6a 100644 --- a/runners/launch_h100-dgxc-slurm.sh +++ b/runners/launch_h100-dgxc-slurm.sh @@ -195,10 +195,12 @@ EOF tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null wait $POLL_PID + SRT_JOB_RC=0 + verify_slurm_job_completion "$JOB_ID" || SRT_JOB_RC=$? set -x - echo "Job $JOB_ID completed!" + echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence" echo "Collecting results..." if [ ! -d "$LOGS_DIR" ]; then @@ -244,6 +246,8 @@ EOF done find . -name '.nfs*' -delete 2>/dev/null || true + if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi + else HF_HUB_CACHE_MOUNT="/mnt/nfs/sa-shared/gharunners/hf-hub-cache/" diff --git a/runners/launch_h200-dgxc-slurm.sh b/runners/launch_h200-dgxc-slurm.sh index 0711e2e250..0c091da5a7 100755 --- a/runners/launch_h200-dgxc-slurm.sh +++ b/runners/launch_h200-dgxc-slurm.sh @@ -362,13 +362,10 @@ EOF SRT_JOB_RC=0 stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || SRT_JOB_RC=$? - if [[ "$SRT_JOB_RC" != "0" && "$USES_KIMIK3_POWER" != "1" ]]; then - exit "$SRT_JOB_RC" - fi set -x - echo "Job $JOB_ID completed!" + echo "Job $JOB_ID finished with status $SRT_JOB_RC; collecting evidence" echo "Collecting results..." if [ ! -d "$LOGS_DIR" ]; then @@ -378,7 +375,7 @@ EOF echo "Found logs directory: $LOGS_DIR" - AGENTX_POWER_RC="$SRT_JOB_RC" + AGENTX_POWER_RC=0 if [[ "$USES_KIMIK3_POWER" == "1" && "${EVAL_ONLY:-false}" != "true" ]]; then read -r -a POWER_CONCURRENCIES <<< "$CONC_LIST" collect_agentic_power_results "$JOB_ID" "$LOGS_DIR" \ @@ -413,7 +410,7 @@ EOF cp -r "$LOGS_DIR" "$GITHUB_WORKSPACE/LOGS" bundle_server_logs "$LOGS_DIR" "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" - if [[ "$AGENTX_POWER_RC" != "0" ]]; then + if [[ "$AGENTX_POWER_RC" != "0" && "$SRT_JOB_RC" == "0" ]]; then echo "ERROR: AgentX power validation failed; available audit and server artifacts were staged" >&2 exit "$AGENTX_POWER_RC" fi @@ -451,6 +448,8 @@ EOF done find . -name '.nfs*' -delete 2>/dev/null || true + if [[ "$SRT_JOB_RC" != "0" ]]; then exit "$SRT_JOB_RC"; fi + else SQUASH_FILE="/data/containers/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index accd850219..6e07dad917 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -123,6 +123,10 @@ if [[ "$IS_MULTINODE" == "true" ]]; then wait $POLL_PID + source "$GITHUB_WORKSPACE/runners/slurm_utils.sh" + slurm_outcome_rc=0 + verify_slurm_job_completion "$JOB_ID" || slurm_outcome_rc=$? + set -x # FIXME: The below is bad and is a result of the indirection of the ways in which @@ -249,6 +253,7 @@ PY sudo rm -rf "$BENCHMARK_LOGS_DIR/logs" 2>/dev/null || true # Log preservation and cleanup handled by EXIT trap (cleanup_and_save_logs) + exit "$slurm_outcome_rc" else diff --git a/runners/slurm_utils.sh b/runners/slurm_utils.sh index 49f33f6b22..8360fe0f83 100644 --- a/runners/slurm_utils.sh +++ b/runners/slurm_utils.sh @@ -45,6 +45,32 @@ slurm_job_is_active() { squeue -j "$job_id" --noheader 2>/dev/null | grep -q "$job_id" } +verify_slurm_job_completion() { + local job_id="$1" records record state="" exit_code="" + # Disappearance from squeue only means the job is no longer active. Read + # the allocation's terminal record, not a successful batch/extern step. + records=$(sacct -j "$job_id" --noheader --parsable2 --format=JobIDRaw,State,ExitCode 2>/dev/null) || records="" + record=$(printf '%s\n' "$records" | awk -F'|' -v job="$job_id" '$1 == job {print; exit}') + if [[ -n "$record" ]]; then + IFS='|' read -r _ state exit_code <<< "$record" + fi + case "$state" in + COMPLETED|FAILED|CANCELLED*|TIMEOUT|NODE_FAIL|OUT_OF_MEMORY|PREEMPTED|DEADLINE|BOOT_FAIL) ;; + *) + # Accounting may be missing or still report RUNNING after squeue + # empties. Ask the controller before judging the final outcome. + record=$(scontrol show job "$job_id" --oneliner 2>/dev/null) || record="" + state=$(printf '%s\n' "$record" | sed -n 's/.*JobState=\([^ ]*\).*/\1/p') + exit_code=$(printf '%s\n' "$record" | tr ' ' '\n' | sed -n 's/^ExitCode=//p') + ;; + esac + printf '%s\n' "$record" > "${GITHUB_WORKSPACE:-.}/slurm_job_${job_id}_outcome.txt" + if [[ "$state" != "COMPLETED" || "$exit_code" != "0:0" ]]; then + echo "ERROR: Slurm job $job_id ended with state=${state:-unknown} exit=${exit_code:-unknown}" >&2 + return 1 + fi +} + stream_slurm_job_log() { local job_id="$1" local log_file="$2" @@ -52,7 +78,7 @@ stream_slurm_job_log() { while [[ ! -f "$log_file" ]]; do if ! slurm_job_is_active "$job_id"; then echo "ERROR: job $job_id failed before creating $log_file" >&2 - scontrol show job "$job_id" || true + verify_slurm_job_completion "$job_id" || true return 1 fi sleep 5 @@ -68,6 +94,7 @@ stream_slurm_job_log() { echo "Tailing $log_file" tail -F -s 2 -n+1 "$log_file" --pid="$poll_pid" 2>/dev/null wait "$poll_pid" + verify_slurm_job_completion "$job_id" } copy_to_workspace() { diff --git a/runners/test_kimik3_bh_power.py b/runners/test_kimik3_bh_power.py index e4342788e0..907c33295b 100644 --- a/runners/test_kimik3_bh_power.py +++ b/runners/test_kimik3_bh_power.py @@ -86,34 +86,61 @@ def test_kimi_power_selects_verified_runtime( assert "edwingao28/srt-slurm.git" not in (tmp_path / "route.log").read_text() -@pytest.mark.parametrize("hardware", ["b200", "h200"]) -def test_kimi_failed_power_stages_evidence_before_exit(tmp_path: Path, hardware: str) -> None: +@pytest.mark.parametrize( + "hardware,native_rc,adapter_rc,kimi,expected_rc", + [ + ("b200", 0, 42, True, 42), + ("h200", 0, 0, True, 0), + ("h200", 0, 42, True, 42), + ("h200", 1, 0, True, 1), + ("h200", 1, 42, True, 1), + ("h200", 7, 0, True, 7), + ("h200", 7, 42, True, 7), + ("h200", 7, 0, False, 7), + ], +) +def test_kimi_failed_power_stages_evidence_before_exit( + tmp_path: Path, hardware: str, native_rc: int, adapter_rc: int, + kimi: bool, expected_rc: int, +) -> None: filename = {"b200": "launch_b200-nscale-slurm.sh", "h200": "launch_h200-dgxc-slurm.sh"}[hardware] source = (REPO / "runners" / filename).read_text() - start = source.index('AGENTX_POWER_RC="$SRT_JOB_RC"') - end = source.index('exit "$AGENTX_POWER_RC"', start) - end = source.index("\n", end) + 1 - source = source[start:end] + "fi\n" + if hardware == "h200": + start = source.index(" SRT_JOB_RC=0") + end = source.index('\nelse\n SQUASH_FILE=', start) + source = source[start:end] + else: + start = source.index('AGENTX_POWER_RC="$SRT_JOB_RC"') + end = source.index('exit "$AGENTX_POWER_RC"', start) + end = source.index("\n", end) + 1 + source = source[start:end] + "fi\n" logs = tmp_path / "source-logs" logs.mkdir() + (logs / "server.log").write_text("retained server output\n") for name in ("exporter-image.sha256", "power-producer-sha.txt"): (tmp_path / name).write_text("retained\n") harness = ''' set -e +stream_slurm_job_log() { return "$TEST_NATIVE_RC"; } collect_agentic_power_results() { mkdir -p "$2/power" - printf 'invalid telemetry\\n' > "$2/power/validation.json" - return 42 + printf 'telemetry audit\\n' > "$2/power/validation.json" + return "$TEST_ADAPTER_RC" } bundle_server_logs() { printf 'server evidence\\n' > "$2"; } +copy_fixed_sequence_results() { :; } ''' - env = dict(os.environ, SRT_JOB_RC="0", USES_AGENTX_POWER="1", USES_KIMIK3_POWER="1", - USES_DCGM_POWER="1", EVAL_ONLY="false", JOB_ID="123", CONC_LIST="1", + env = dict(os.environ, SRT_JOB_RC=str(native_rc), TEST_NATIVE_RC=str(native_rc), + TEST_ADAPTER_RC=str(adapter_rc), USES_AGENTX_POWER="1", USES_KIMIK3_POWER=str(int(kimi)), + USES_DCGM_POWER=str(int(kimi)), EVAL_ONLY="false", JOB_ID="123", CONC_LIST="1", + LOG_FILE=str(logs / "server.log"), RUN_EVAL="false", GITHUB_WORKSPACE=str(tmp_path), LOGS_DIR=str(logs), RESULT_FILENAME="kimi-test", SELECTED_POWER_SRT_SLURM_PIN="a" * 40) result = subprocess.run(["bash"], input=harness + source, text=True, capture_output=True, cwd=tmp_path, env=env) - assert result.returncode == 42, result.stderr - assert (tmp_path / "LOGS/power/validation.json").read_text() == "invalid telemetry\n" + assert result.returncode == expected_rc, result.stderr + assert (tmp_path / "LOGS/server.log").read_text() == "retained server output\n" + if kimi: + assert (tmp_path / "LOGS/power/validation.json").read_text() == "telemetry audit\n" assert (tmp_path / "multinode_server_logs.tar.gz").read_text() == "server evidence\n" diff --git a/runners/test_llmd_lifecycle.py b/runners/test_llmd_lifecycle.py new file mode 100644 index 0000000000..0b21452490 --- /dev/null +++ b/runners/test_llmd_lifecycle.py @@ -0,0 +1,90 @@ +"""Exercise llm-d shutdown with local stand-ins for the Slurm/container boundary.""" +import os +from pathlib import Path +import re +import subprocess +import sys + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +JOB = ROOT / 'benchmarks/multi_node/llm-d/job.slurm' +SERVER = ROOT / 'benchmarks/multi_node/llm-d/server.sh' + + +@pytest.mark.parametrize('main_rc', [0, 7]) +def test_llmd_completed_coordinator_does_not_cancel_allocation(tmp_path, main_rc): + cwd = tmp_path / 'repo/benchmarks/multi_node/llm-d' + cwd.mkdir(parents=True) + model, logs, bindir = tmp_path / 'model', tmp_path / 'logs', tmp_path / 'bin' + for path in (model, logs, bindir): + path.mkdir() + squash = tmp_path / 'image.sqsh' + squash.write_text('fixture image') + commands = { + 'scontrol': '#!/bin/sh\nprintf "node-a\\nnode-b\\n"\n', + 'scancel': '#!/bin/sh\necho unexpected-cancel >> "$CANCEL_RECEIPT"\n', + 'sleep': '#!/bin/sh\n/bin/sleep 0.01\n', + 'srun': '#!' + sys.executable + '\n' + r''' +import os, pathlib, subprocess, sys, time +args = sys.argv[1:] +if any(a.startswith('--container-image=') for a in args): + marker = pathlib.Path(os.environ['BENCHMARK_LOGS_DIR']) / ('.bench_done.' + os.environ['SLURM_JOB_ID']) + marker.write_text(os.environ['MAIN_RC'] + '\n') + time.sleep(0.2) + sys.exit(int(os.environ['MAIN_RC'])) +if 'ip route' in ' '.join(args): + print('127.0.0.1') + sys.exit(0) +args = [arg for arg in args if not arg.startswith('--')] +sys.exit(subprocess.run(args).returncode) +''', + } + for name, script in commands.items(): + path = bindir / name + path.write_text(script) + path.chmod(0o755) + defaults = dict.fromkeys('PREFILL_WORKERS DECODE_WORKERS PREFILL_DP_SIZE DECODE_DP_SIZE ' + 'BENCH_INPUT_LEN BENCH_OUTPUT_LEN BENCH_MAX_CONCURRENCY ' + 'BENCH_REQUEST_RATE BENCH_RANDOM_RANGE_RATIO BENCH_NUM_PROMPTS_MULTIPLIER ' + 'RUN_EVAL EVAL_ONLY EVAL_CONC EVAL_FRAMEWORK EVAL_LIMIT EVAL_SUITE ' + 'SWEBENCH_GEN_MODE SWEBENCH_USE_MODAL MODAL_TOKEN_ID MODAL_TOKEN_SECRET ' + 'IS_AGENTIC SCENARIO_TYPE FRAMEWORK PRECISION MODEL_PREFIX ' + 'RUNNER_TYPE RESULT_FILENAME SPEC_DECODING IS_MULTINODE CONFIG_FILE'.split(), '1') + env = {**os.environ, **defaults, 'PATH': str(bindir) + os.pathsep + os.environ['PATH'], + 'SLURM_JOB_ID': 'local-fixture', 'SLURM_JOB_NODELIST': 'node-[a-b]', + 'NUM_NODES': '2', 'PREFILL_NODES': '1', 'DECODE_NODES': '1', 'GPUS_PER_NODE': '2', + 'MODEL_DIR': str(model), 'MODEL_NAME': 'fixture', 'BENCHMARK_LOGS_DIR': str(logs), + 'LLMD_CONTAINER_ENGINE': 'pyxis', 'LLMD_SQUASH_FILE': str(squash), + 'MAIN_RC': str(main_rc), 'CANCEL_RECEIPT': str(tmp_path / 'cancelled')} + result = subprocess.run(['bash', str(JOB)], cwd=cwd, env=env, + capture_output=True, text=True, timeout=10) + assert result.returncode == main_rc, result.stderr + result.stdout + assert not (tmp_path / 'cancelled').exists() + + +@pytest.mark.parametrize('node_rc', [0, 7]) +def test_llmd_node_records_status_and_stops_owned_server(tmp_path, node_rc): + function = re.search(r'^finish_llmd_node\(\) \{\n.*?^\}', + SERVER.read_text(), flags=re.MULTILINE | re.DOTALL).group() + command = function + r''' +NODE_RANK=1 PREFILL_NODES=1 +BENCH_DONE_MARKER="$1/done" +# Observe the publication boundary after shell redirection opens its target, +# but before printf can write the status seen by workers. +printf() { + if [[ -e "$BENCH_DONE_MARKER" ]]; then + echo 'worker can observe an incomplete status' >&2 + return 1 + fi + builtin printf "$@" +} +sleep 60 & +VLLM_PID=$! +trap finish_llmd_node EXIT +exit "$2" +''' + result = subprocess.run(['bash', '-c', command, 'bash', str(tmp_path), str(node_rc)], + capture_output=True, text=True, timeout=5) + assert result.returncode == node_rc, result.stderr + assert (tmp_path / 'done').read_text().strip() == str(node_rc) diff --git a/runners/test_llmd_power_lifecycle.py b/runners/test_llmd_power_lifecycle.py new file mode 100644 index 0000000000..50c01ee940 --- /dev/null +++ b/runners/test_llmd_power_lifecycle.py @@ -0,0 +1,73 @@ +"""CPU-only lifecycle checks with controlled external collector/Slurm processes.""" +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +JOB = ROOT / 'benchmarks/multi_node/llm-d/job.slurm' + + +@pytest.mark.parametrize('main_rc', [0, 7]) +@pytest.mark.parametrize('enabled', [False, True]) +def test_llmd_job_stages_native_evidence_and_keeps_main_step_status(tmp_path, main_rc, enabled): + repo = tmp_path / 'repo' + cwd = repo / 'benchmarks/multi_node/llm-d' + cwd.mkdir(parents=True) + model, logs, bindir = tmp_path / 'model', tmp_path / 'logs', tmp_path / 'bin' + for path in (model, logs, bindir): + path.mkdir() + squash = tmp_path / 'image.sqsh' + squash.write_text('fixture image') + (bindir / 'scontrol').write_text('#!/bin/sh\nprintf "node-a\\nnode-b\\n"\n') + (bindir / 'git').write_text('#!/bin/sh\necho 0123456789012345678901234567890123456789\n') + (bindir / 'srun').write_text('#!' + sys.executable + '\n' + r''' +import json, os, pathlib, subprocess, sys +args = sys.argv[1:] +with open(os.environ['CALLS'], 'a') as f: + f.write(json.dumps(args) + '\n') +if any(a.startswith('--container-image=') for a in args): + if os.environ['POWERX_NATIVE_ENABLED'] != '1': + assert not any('/powerx_native' in a for a in args) + sys.exit(int(os.environ['MAIN_RC'])) + for rank in range(2): + out = pathlib.Path(os.environ['POWERX_RAW_ROOT']) / f'node-{rank}' + out.mkdir(parents=True, exist_ok=True) + (out / 'manifest.json').write_text(json.dumps({'rank':rank,'synthetic':True})) + sys.exit(int(os.environ['MAIN_RC'])) +args = [a for a in args if not a.startswith('--')] +if 'ip route' in ' '.join(args): + print('127.0.0.1') + sys.exit(0) +for rank in range(2): + subprocess.run(args, env={**os.environ,'SLURM_PROCID':str(rank)}, check=True) +''') + for path in bindir.iterdir(): + path.chmod(0o755) + defaults = dict.fromkeys('PREFILL_WORKERS DECODE_WORKERS PREFILL_DP_SIZE DECODE_DP_SIZE ' + 'BENCH_INPUT_LEN BENCH_OUTPUT_LEN BENCH_MAX_CONCURRENCY ' + 'BENCH_REQUEST_RATE BENCH_RANDOM_RANGE_RATIO BENCH_NUM_PROMPTS_MULTIPLIER ' + 'RUN_EVAL EVAL_ONLY EVAL_CONC EVAL_FRAMEWORK EVAL_LIMIT EVAL_SUITE ' + 'SWEBENCH_GEN_MODE SWEBENCH_USE_MODAL MODAL_TOKEN_ID MODAL_TOKEN_SECRET ' + 'IS_AGENTIC SCENARIO_TYPE FRAMEWORK PRECISION MODEL_PREFIX ' + 'RUNNER_TYPE RESULT_FILENAME SPEC_DECODING IS_MULTINODE CONFIG_FILE'.split(), '1') + env = {**os.environ, **defaults, 'PATH':str(bindir) + os.pathsep + os.environ['PATH'], + 'SLURM_JOB_ID':f'local-{os.getpid()}-{main_rc}', 'SLURM_JOB_NODELIST':'node-[a-b]', + 'NUM_NODES':'2','PREFILL_NODES':'1','DECODE_NODES':'1','GPUS_PER_NODE':'2', + 'MODEL_DIR':str(model),'MODEL_NAME':'fixture','BENCHMARK_LOGS_DIR':str(logs), + 'LLMD_CONTAINER_ENGINE':'pyxis','LLMD_SQUASH_FILE':str(squash), + 'POWERX_NATIVE_ENABLED':'1' if enabled else '0','POWERX_RAW_ROOT':str(tmp_path/'raw'), + 'MAIN_RC':str(main_rc),'CALLS':str(tmp_path/'calls.jsonl')} + result = subprocess.run(['bash', str(JOB)], cwd=cwd, env=env, + capture_output=True, text=True, timeout=20) + assert result.returncode == main_rc, result.stderr + result.stdout + if not enabled: + assert not (repo / 'LOGS/native_power').exists() + assert not (tmp_path / 'raw').exists() + return + for rank in range(2): + saved = repo / f'LOGS/native_power/node-{rank}/manifest.json' + assert json.loads(saved.read_text()) == {'rank':rank,'synthetic':True} diff --git a/runners/test_slurm_utils.py b/runners/test_slurm_utils.py index 31fa6f2183..7d52e1f661 100644 --- a/runners/test_slurm_utils.py +++ b/runners/test_slurm_utils.py @@ -631,3 +631,70 @@ def test_mi355x_agentic_model_mount_and_routing( script = f"benchmarks/single_node/agentic/{prefix}_fp4_mi355x_vllm_mtp.sh" assert args[-2] == script assert (REPO_ROOT / script).is_file() + + +@pytest.mark.parametrize('state,exit_code,expected', [ + ('COMPLETED', '0:0', 0), ('FAILED', '1:0', 1), ('TIMEOUT', '0:15', 1), + ('COMPLETED', '1:0', 1), ('CANCELLED', '0:15', 1), +]) +def test_slurm_terminal_allocation_status_is_required(tmp_path, state, exit_code, expected): + result = run_bash( + f'source "$1"; export GITHUB_WORKSPACE="$2"; ' + f'sacct() {{ printf "42|{state}|{exit_code}\\n42.batch|COMPLETED|0:0\\n"; }}; ' + 'scontrol() { echo "JobId=42 JobState=COMPLETED ExitCode=0:0"; }; ' + 'verify_slurm_job_completion 42', SLURM_UTILS, tmp_path, + ) + assert result.returncode == expected, result.stderr + assert (tmp_path / 'slurm_job_42_outcome.txt').read_text().strip() == f'42|{state}|{exit_code}' + + +@pytest.mark.parametrize('accounting_state,controller_state,exit_code,expected', [ + ('', 'COMPLETED', '0:0', 0), + ('RUNNING', 'COMPLETED', '0:0', 0), + ('COMPLETING', 'COMPLETED', '0:0', 0), + ('RUNNING', 'FAILED', '1:0', 1), + ('RUNNING', 'RUNNING', '0:0', 1), +]) +def test_slurm_recent_completion_falls_back_to_controller( + tmp_path, accounting_state, controller_state, exit_code, expected, +): + record = f'42|{accounting_state}|0:0' if accounting_state else '' + controller = f'JobId=42 JobState={controller_state} ExitCode={exit_code}' + result = run_bash( + 'source "$1"; export GITHUB_WORKSPACE="$2"; ' + f'sacct() {{ echo "{record}"; }}; ' + f'scontrol() {{ echo "{controller}"; }}; ' + 'verify_slurm_job_completion 42', SLURM_UTILS, tmp_path, + ) + assert result.returncode == expected, result.stderr + assert (tmp_path / 'slurm_job_42_outcome.txt').read_text().strip() == controller + + +def test_slurm_unknown_terminal_state_is_not_success(tmp_path): + result = run_bash( + 'source "$1"; export GITHUB_WORKSPACE="$2"; sacct() { return 1; }; ' + 'scontrol() { return 1; }; verify_slurm_job_completion 42', SLURM_UTILS, tmp_path, + ) + assert result.returncode == 1 + assert 'state=unknown' in result.stderr + + +def test_slurm_exit_before_log_retains_terminal_receipt(tmp_path): + result = run_bash( + 'source "$1"; export GITHUB_WORKSPACE="$2"; ' + 'slurm_job_is_active() { return 1; }; ' + 'sacct() { printf "42|FAILED|1:0\\n"; }; ' + 'stream_slurm_job_log 42 "$2/missing.log"', SLURM_UTILS, tmp_path, + ) + assert result.returncode == 1 + assert (tmp_path / 'slurm_job_42_outcome.txt').read_text().strip() == '42|FAILED|1:0' + + +@pytest.mark.parametrize('exit_code,derived,expected', [('0:0', '7:0', 0), ('1:0', '0:0', 1)]) +def test_slurm_controller_checks_allocation_not_derived_exit(tmp_path, exit_code, derived, expected): + result = run_bash( + 'source "$1"; export GITHUB_WORKSPACE="$2"; sacct() { return 1; }; ' + f'scontrol() {{ echo "JobId=42 JobState=COMPLETED ExitCode={exit_code} DerivedExitCode={derived}"; }}; ' + 'verify_slurm_job_completion 42', SLURM_UTILS, tmp_path, + ) + assert result.returncode == expected, result.stderr diff --git a/utils/test_gb300_power_official_contract.py b/utils/test_gb300_power_official_contract.py index f270b6fb16..3619aaa99a 100644 --- a/utils/test_gb300_power_official_contract.py +++ b/utils/test_gb300_power_official_contract.py @@ -318,12 +318,19 @@ def test_gb200_native_status_waits_for_terminal_and_fails_closed( assert (tmp_path / "logs/power/native-job-status-attempts.txt").read_text() == f"{attempts}\n" -def test_gb200_agentx_window_injection_and_failure_artifacts(tmp_path: Path) -> None: - launcher = (REPO_ROOT / "runners/launch_gb200-nv.sh").read_text() - injection_end = launcher.index("# Don't leak the login-node venv") +@pytest.mark.parametrize("launcher_name", ["launch_gb200-nv.sh", "launch_gb300-nv.sh"]) +@pytest.mark.parametrize("stream_rc,expected_rc", [(0, 1), (1, 1), (7, 7)]) +def test_agentx_window_injection_and_failure_artifacts( + tmp_path: Path, launcher_name: str, stream_rc: int, expected_rc: int +) -> None: + launcher = (REPO_ROOT / "runners" / launcher_name).read_text() + injection_end = launcher.index("# --no-preflight skips" if launcher_name == "launch_gb300-nv.sh" else "# Don't leak the login-node venv") injection_start = launcher.rfind('if [[ "$USES_AGENTX_POWER" == "1" ]]; then', 0, injection_end) collection_start = launcher.index("AGENTX_POWER_RC=0") - collection_end = launcher.index('\nif [[ "${EVAL_ONLY:-false}" != "true" ]]; then', collection_start) + collection_end = len(launcher) + snapshot_source = "" + if launcher_name == "launch_gb300-nv.sh": + snapshot_source = launcher[launcher.index("_snapshot_server_logs() {"):collection_start] workspace, compute, producer = (tmp_path / name for name in ("workspace", "compute", "producer")) for path in (workspace, compute, producer): path.mkdir() @@ -342,19 +349,20 @@ def test_gb200_agentx_window_injection_and_failure_artifacts(tmp_path: Path) -> env = os.environ.copy() env.update(GITHUB_WORKSPACE=str(workspace), INFMAX_WORKSPACE=str(compute), PYTHONPATH=str(REPO_ROOT), USES_AGENTX_POWER="1", USES_DCGM_POWER="1", - CONFIG_PATH=str(recipe), CONC_LIST="1", RESULT_FILENAME="result", EVAL_ONLY="false", + CONFIG_PATH=str(recipe), CONC_LIST="1", RESULT_FILENAME="result", EVAL_ONLY="false", IS_AGENTIC="1", JOB_ID="42", LOGS_DIR="outputs/42/logs", LOG_FILE="outputs/42/logs/sweep_42.log", AGENTX_POWER_SRT_SLURM_PIN=AGENTX_PRODUCER_PIN) result = subprocess.run(["/bin/bash"], input=( "set -euo pipefail\n" f"source {shlex.quote(str(REPO_ROOT / 'runners/slurm_utils.sh'))}\n" f"python3() {{ {shlex.quote(sys.executable)} \"$@\"; }}\n" - "stream_slurm_job_log() { return 1; }\n" + f"stream_slurm_job_log() {{ return {stream_rc}; }}\n" "sacct() { printf '42|FAILED|1:0\\n'; }\n" + launcher[injection_start:injection_end] + + snapshot_source + launcher[collection_start:collection_end] ), text=True, capture_output=True, cwd=producer, env=env) - assert result.returncode == 1, result.stderr + assert result.returncode == expected_rc, result.stderr assert yaml.safe_load(recipe.read_text())["benchmark"]["concurrencies"] == [1] aggregate = json.loads((workspace / "result_conc1.json").read_text()) assert aggregate["power_valid"] == 0