From 09f2cf9a35668ca90e9ff21f0adf03cffea823d2 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Fri, 11 Sep 2026 16:37:23 -0700 Subject: [PATCH 1/2] feat: enable required Kimi-K3 AgentX power backfills MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为 Kimi-K3 AgentX 补测启用必需功耗采集，保留失败诊断并修正 AMD 收尾与硬件元数据。 Signed-off-by: Wenyao Gao --- benchmarks/benchmark_lib.sh | 166 +++++++- .../agg-b200-tp8pp2-mooncake-c1-agentic.yaml | 15 +- .../agg-b200-tp8pp2-mooncake-c14-agentic.yaml | 15 +- .../agg-b200-tp8pp2-mooncake-c24-agentic.yaml | 15 +- .../agg-b200-tp8pp2-mooncake-c4-agentic.yaml | 15 +- .../agg-b200-tp8pp2-mooncake-c48-agentic.yaml | 15 +- .../agg-b200-tp8pp2-mooncake-c8-agentic.yaml | 15 +- .../agg-b200-tp8pp2-mooncake-c96-agentic.yaml | 15 +- ...cp16-dspark4-maxseq2-mooncake-agentic.yaml | 17 + ...g-gb200-dcp16-nospec-mooncake-agentic.yaml | 17 + ...agg-gb200-tp8pp2-mooncake-c16-agentic.yaml | 17 + ...agg-gb200-tp8pp2-mooncake-c32-agentic.yaml | 17 + ...agg-gb200-tp8pp2-mooncake-c48-agentic.yaml | 17 + ...agg-gb200-tp8pp2-mooncake-c72-agentic.yaml | 17 + ...agg-gb200-tp8pp2-mooncake-c96-agentic.yaml | 17 + ...dcp8-dspark7-maxseq2-mooncake-agentic.yaml | 17 + ...gg-gb300-dcp8-nospec-mooncake-agentic.yaml | 17 + .../agg-h200-tp16dp2ep32-latency-agentic.yaml | 14 + .../agg-h200-tp8dp4ep32-balanced-agentic.yaml | 14 + ...g-h200-tp8dp4ep32-vllm-simple-agentic.yaml | 14 + ...1d-dcp8-dcp8-dspark4-mooncake-agentic.yaml | 17 + ...2d-dcp8-dcp8-dspark4-mooncake-agentic.yaml | 17 + ...3d-dcp8-dcp8-dspark4-mooncake-agentic.yaml | 17 + ...3d-dcp8-dcp8-dspark7-mooncake-agentic.yaml | 17 + configs/nvidia-master.yaml | 34 +- docs/ci-procedures.md | 8 + docs/ci-procedures_zh.md | 8 + infx/results/power/single_node.py | 28 +- perf-changelog.yaml | 20 + runners/launch_b200-nscale-slurm.sh | 53 ++- runners/launch_gb200-nv.sh | 70 +++- runners/launch_gb300-nv.sh | 92 +++-- runners/launch_h200-dgxc-slurm.sh | 53 ++- runners/slurm_utils.sh | 46 +++ runners/test_kimik3_bh_power.py | 141 +++++++ .../aggregation/test_power_lifecycle.py | 355 +++++++++++++++++- utils/test_aggregate_power.py | 203 +++++++++- utils/test_gb300_power_official_contract.py | 108 +++++- utils/test_process_result.py | 19 +- 39 files changed, 1637 insertions(+), 135 deletions(-) create mode 100644 runners/test_kimik3_bh_power.py diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index 305a118ef3..c640c53e08 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -153,8 +153,12 @@ unset _benchmark_caller # -------------------------------- GPU_MONITOR_PID="" +GPU_MONITOR_SOURCE_PID="" +GPU_MONITOR_PIPE="" GPU_MONITOR_VENDOR="" GPU_MONITOR_INTERVAL=1 +# Bounded wait for AMD telemetry to cover a stop request; 0 skips the wait. +AMD_MONITOR_STOP_TIMEOUT_S="${AMD_MONITOR_STOP_TIMEOUT_S:-30}" GPU_METRICS_CSV="${GPU_METRICS_CSV:-gpu_metrics.csv}" NVIDIA_GPU_MONITOR_QUERY="timestamp,index,power.draw,temperature.gpu,clocks.current.sm,clocks.current.memory,utilization.gpu,utilization.memory" export GPU_METRICS_CSV @@ -196,8 +200,15 @@ start_gpu_monitor() { # Python; measured on MI355X: trailing ticks were lost at kill without it). # Pipe through awk to: skip preamble lines, keep first CSV header, skip repeated # headers, and flush every row so killing the pipe cannot discard buffered samples. - PYTHONUNBUFFERED=1 amd-smi metric -p -c -t -u -w "$interval" --csv 2>/dev/null \ - | awk '/^timestamp,/{if(!h){print;h=1};next} h{print;fflush()}' > "$output" & + # Track both processes: killing only awk can leave amd-smi alive until + # its next write. Keep the FIFO beside this run's raw CSV, never shared. + GPU_MONITOR_PIPE="${output}.pipe.$$" + mkfifo "$GPU_MONITOR_PIPE" || return 1 + PYTHONUNBUFFERED=1 amd-smi metric -p -c -t -u -w "$interval" --csv \ + > "$GPU_MONITOR_PIPE" 2>/dev/null & + GPU_MONITOR_SOURCE_PID=$! + awk '/^timestamp,/{if(!h){print;h=1};next} h{print;fflush()}' \ + < "$GPU_MONITOR_PIPE" > "$output" & GPU_MONITOR_PID=$! # Hardware energy-accumulator + identity snapshots; the end-side twin in # stop_gpu_monitor lets auditors cross-check the integrated energy @@ -215,19 +226,21 @@ start_gpu_monitor() { # Stop the background GPU monitor and report file size. stop_gpu_monitor() { if [[ -n "$GPU_MONITOR_PID" ]] && kill -0 "$GPU_MONITOR_PID" 2>/dev/null; then - # benchmark_end_time_unix is recorded shortly before the benchmark - # process exits, so the stream must cover one more sample past it for - # deterministic boundary interpolation. NVIDIA appends a one-shot - # post-exit sample below; amd-smi one-shot CSV has no timestamp column, - # so the AMD path instead lets the watch stream emit final ticks before - # the kill. Two extra intervals: amd-smi stamps integer seconds, so a - # tick in the same second as the window end still fails bracketing — - # the stream needs a tick at the NEXT whole second (measured on MI355X: - # end=...153.325 vs last sample ...153.0). + # The aggregator requires, for every GPU, a usable sample stamped at or + # after the (fractional) benchmark window end, which is always <= the + # wall clock when this stop runs. NVIDIA appends a one-shot post-exit + # sample below; amd-smi one-shot CSV has no timestamp column, so the + # AMD path polls the output file until every GPU's watch stream shows + # a usable tick at the next whole second — amd-smi stamps integer + # seconds, so that tick strictly covers any fractional window end + # (measured on MI355X: end=...609.157 vs last sample ...605). Observing + # the file rather than sleeping also defeats pipe-buffer loss when the + # awk consumer is killed: covered rows are already on disk. if [[ "$GPU_MONITOR_VENDOR" == "amd" ]]; then - sleep $(( ${GPU_MONITOR_INTERVAL:-1} + 2 )) + _wait_for_amd_stop_coverage fi - kill "$GPU_MONITOR_PID" 2>/dev/null + # The monitor may exit during the coverage wait; still finish cleanup. + kill "$GPU_MONITOR_PID" 2>/dev/null || true wait "$GPU_MONITOR_PID" 2>/dev/null || true case "$GPU_MONITOR_VENDOR" in nvidia) @@ -249,6 +262,13 @@ stop_gpu_monitor() { echo "[GPU Monitor] Collected $lines rows -> $GPU_METRICS_CSV" fi fi + if [[ -n "$GPU_MONITOR_SOURCE_PID" ]]; then + kill "$GPU_MONITOR_SOURCE_PID" 2>/dev/null || true + wait "$GPU_MONITOR_SOURCE_PID" 2>/dev/null || true + fi + [[ -z "$GPU_MONITOR_PIPE" ]] || rm -f "$GPU_MONITOR_PIPE" + GPU_MONITOR_SOURCE_PID="" + GPU_MONITOR_PIPE="" GPU_MONITOR_PID="" GPU_MONITOR_VENDOR="" } @@ -271,6 +291,109 @@ _repair_truncated_gpu_metrics_tail() { return 0 } +# Print the newest telemetry tick (whole epoch seconds) that EVERY observed +# GPU has covered with a usable sample (numeric epoch timestamp, numeric +# power > 0), or nothing when the stream holds no usable epoch-stamped row +# (e.g. an amd-smi build emitting ISO timestamps). Column detection mirrors +# _POWER_COL_RE/_POWER_EXCLUDE_RE/_GPU_INDEX_COL_RE in utils/aggregate_power.py. +# POSIX awk only: the ROCm container images ship mawk/busybox awk. +_amd_monitor_min_covered_tick() { + [[ -f "$GPU_METRICS_CSV" ]] || return 0 + awk -F, ' + NR == 1 { + for (i = 1; i <= NF; i++) { + name = tolower($i) + gsub(/^ +| +$/, "", name) + sub(/\r$/, "", name) + if (!power_col && name ~ /power/ && name !~ /limit|cap|max|min/) + power_col = i + if (!gpu_col && name ~ /^(index|gpu|gpu_id|gpu_index|card|device)$/) + gpu_col = i + } + next + } + !power_col || !gpu_col { next } + { + # amd-smi quotes list-valued cells that embed commas; neutralize + # them so the power cell keeps its header-relative position. + line = $0 + sub(/\r$/, "", line) + if (line ~ /"/) { + n = split(line, seg, /"/) + line = "" + for (i = 1; i <= n; i++) { + if (i % 2 == 0) gsub(/,/, ";", seg[i]) + line = line seg[i] + } + } + count = split(line, cell, /,/) + if (count < power_col || count < gpu_col) next + if (cell[1] !~ /^[0-9]+(\.[0-9]+)?$/) next + if (cell[power_col] !~ /^[0-9]+(\.[0-9]+)?$/) next + if (cell[power_col] + 0 <= 0) next + if (cell[gpu_col] == "") next + ts = cell[1] + 0 + # Mirror _parse_timestamp in utils/aggregate_power.py: normalize + # millisecond epochs so a ms-stamping amd-smi build cannot + # trivially satisfy any second-scale stop target. + if (ts > 1e12) ts /= 1000 + gpu = cell[gpu_col] + if (!(gpu in newest) || ts > newest[gpu]) + newest[gpu] = ts + } + END { + have = 0 + for (gpu in newest) + if (!have || newest[gpu] < min) { min = newest[gpu]; have = 1 } + if (have) printf "%d\n", min + } + ' "$GPU_METRICS_CSV" 2>/dev/null + return 0 +} + +# Block until every observed GPU has a usable tick at/after the first whole +# second past stop entry, so any window end preceding the stop request is +# bracketed on file. Bounded by AMD_MONITOR_STOP_TIMEOUT_S; always returns 0 — +# on timeout or early monitor death it warns and lets aggregation attribute +# the missing coverage (fail-safe, never fail-silent). +_wait_for_amd_stop_coverage() { + local target deadline covered timeout_s + # A non-integer timeout (e.g. "30s") would abort the whole stop_gpu_monitor + # call under `set -e` at the arithmetic below, leaking the monitor process + # and skipping tail repair + the energy sidecar; fall back to the default. + timeout_s="${AMD_MONITOR_STOP_TIMEOUT_S:-30}" + if [[ ! "$timeout_s" =~ ^-?[0-9]+$ ]]; then + echo "[GPU Monitor] Warning: ignoring non-integer AMD_MONITOR_STOP_TIMEOUT_S='$timeout_s', using 30" >&2 + timeout_s=30 + fi + if [[ "$timeout_s" -le 0 ]]; then + return 0 + fi + target=$(( $(date +%s) + 1 )) + deadline=$(( target + timeout_s )) + while :; do + covered=$(_amd_monitor_min_covered_tick) + if [[ -z "$covered" ]]; then + # Non-epoch timestamps or an unusable stream: keep the legacy + # fixed tail so older amd-smi builds behave exactly as before. + sleep $(( ${GPU_MONITOR_INTERVAL:-1} + 2 )) + return 0 + fi + if [[ "$covered" -ge "$target" ]]; then + return 0 + fi + if ! _background_process_is_running "$GPU_MONITOR_PID"; then + echo "[GPU Monitor] Warning: AMD monitor exited before covering the stop request (covered=$covered target=$target)" >&2 + return 0 + fi + if [[ "$(date +%s)" -ge "$deadline" ]]; then + echo "[GPU Monitor] Warning: AMD telemetry never covered the stop request within ${timeout_s}s (covered=$covered target=$target)" >&2 + return 0 + fi + sleep 1 + done +} + # Write one best-effort amd-smi snapshot; remove the file rather than keep a # partial one when the invocation fails. _write_amd_smi_sidecar() { @@ -3284,9 +3407,15 @@ run_agentic_replay_and_write_outputs() ( esac _stop_agentx_power_monitor() { + local mode="${1:-}" if [ "$agentx_monitor_stopped" = "0" ]; then - agentx_monitor_stopped=1 + if [ "$mode" = "abort" ]; then + # A cancelled run's power validity is moot; skip the AMD + # coverage wait so signal teardown stays fast. + AMD_MONITOR_STOP_TIMEOUT_S=0 + fi stop_gpu_monitor + agentx_monitor_stopped=1 fi } @@ -3330,10 +3459,11 @@ run_agentic_replay_and_write_outputs() ( agentx_monitor_stopped=0 # This function runs in a subshell, so these handlers cannot replace # launcher-owned traps. The stopped flag keeps explicit and signal/EXIT - # cleanup idempotent. - trap '_stop_agentx_power_monitor' EXIT - trap '_stop_agentx_power_monitor; exit 130' INT - trap '_stop_agentx_power_monitor; exit 143' TERM + # cleanup idempotent after stopping completes. If a signal interrupts + # the normal coverage wait, abort cleanup must still kill the monitor. + trap '_stop_agentx_power_monitor abort' EXIT + trap '_stop_agentx_power_monitor abort; exit 130' INT + trap '_stop_agentx_power_monitor abort; exit 143' TERM fi echo "$REPLAY_CMD" > "$result_dir/benchmark_command.txt" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c1-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c1-agentic.yaml index 5a48fa41f4..5510b41124 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c1-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c1-agentic.yaml @@ -111,14 +111,27 @@ sbatch_directives: srun_options: container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + benchmark: type: custom + concurrencies: [1] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c14-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c14-agentic.yaml index 2c5358f81a..a4f0f4eb45 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c14-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c14-agentic.yaml @@ -111,14 +111,27 @@ sbatch_directives: srun_options: container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + benchmark: type: custom + concurrencies: [14] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c24-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c24-agentic.yaml index f3071b339c..6dab857310 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c24-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c24-agentic.yaml @@ -111,14 +111,27 @@ sbatch_directives: srun_options: container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + benchmark: type: custom + concurrencies: [24] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c4-agentic.yaml index dd72387250..1769cd164a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c4-agentic.yaml @@ -111,14 +111,27 @@ sbatch_directives: srun_options: container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + benchmark: type: custom + concurrencies: [4] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c48-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c48-agentic.yaml index fd1f09784a..3c64e5a12d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c48-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c48-agentic.yaml @@ -111,14 +111,27 @@ sbatch_directives: srun_options: container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + benchmark: type: custom + concurrencies: [48] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c8-agentic.yaml index f8e5e0bdae..390395736a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c8-agentic.yaml @@ -111,14 +111,27 @@ sbatch_directives: srun_options: container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + benchmark: type: custom + concurrencies: [8] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c96-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c96-agentic.yaml index 99479d3263..8b4b2d6491 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c96-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c96-agentic.yaml @@ -110,14 +110,27 @@ sbatch_directives: srun_options: container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + benchmark: type: custom + concurrencies: [96] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dcp16-dspark4-maxseq2-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dcp16-dspark4-maxseq2-mooncake-agentic.yaml index 6878c47b00..db51c92d65 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dcp16-dspark4-maxseq2-mooncake-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dcp16-dspark4-maxseq2-mooncake-agentic.yaml @@ -130,10 +130,27 @@ srun_options: mem: "0" container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + benchmark: type: custom + client_placement: head + concurrencies: [1, 2, 4, 8, 16] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: + ENABLE_AGENTX_POWER: "1" + REQUIRE_POWER: "1" INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dcp16-nospec-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dcp16-nospec-mooncake-agentic.yaml index e10ec63b24..081b9ebc1d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dcp16-nospec-mooncake-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-dcp16-nospec-mooncake-agentic.yaml @@ -126,10 +126,27 @@ srun_options: mem: "0" container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + benchmark: type: custom + client_placement: head + concurrencies: [8, 40, 48] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: + ENABLE_AGENTX_POWER: "1" + REQUIRE_POWER: "1" INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c16-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c16-agentic.yaml index 706d461202..cca69c2880 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c16-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c16-agentic.yaml @@ -111,10 +111,27 @@ sbatch_directives: srun_options: container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + benchmark: type: custom + client_placement: head + concurrencies: [16] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: + ENABLE_AGENTX_POWER: "1" + REQUIRE_POWER: "1" INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c32-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c32-agentic.yaml index c380657c35..947a66ed0f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c32-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c32-agentic.yaml @@ -111,10 +111,27 @@ sbatch_directives: srun_options: container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + benchmark: type: custom + client_placement: head + concurrencies: [32] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: + ENABLE_AGENTX_POWER: "1" + REQUIRE_POWER: "1" INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c48-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c48-agentic.yaml index c5be318803..4b48d7a86a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c48-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c48-agentic.yaml @@ -110,10 +110,27 @@ sbatch_directives: srun_options: container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + benchmark: type: custom + client_placement: head + concurrencies: [48] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: + ENABLE_AGENTX_POWER: "1" + REQUIRE_POWER: "1" INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c72-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c72-agentic.yaml index bac97e2781..8150ada43d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c72-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c72-agentic.yaml @@ -110,10 +110,27 @@ sbatch_directives: srun_options: container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + benchmark: type: custom + client_placement: head + concurrencies: [72] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: + ENABLE_AGENTX_POWER: "1" + REQUIRE_POWER: "1" INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c96-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c96-agentic.yaml index a76cc01a96..14ba652cf2 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c96-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb200-tp8pp2-mooncake-c96-agentic.yaml @@ -110,10 +110,27 @@ sbatch_directives: srun_options: container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + benchmark: type: custom + client_placement: head + concurrencies: [96] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: + ENABLE_AGENTX_POWER: "1" + REQUIRE_POWER: "1" INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml index dafc046abe..25963ec257 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml @@ -114,10 +114,27 @@ srun_options: mem: "0" container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + benchmark: type: custom + client_placement: head + concurrencies: [1, 4] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: + ENABLE_AGENTX_POWER: "1" + REQUIRE_POWER: "1" INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-nospec-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-nospec-mooncake-agentic.yaml index 9b61dd4922..7e8c4d4ea1 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-nospec-mooncake-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-nospec-mooncake-agentic.yaml @@ -115,10 +115,27 @@ srun_options: mem: "0" container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + benchmark: type: custom + client_placement: head + concurrencies: [48, 70] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: + ENABLE_AGENTX_POWER: "1" + REQUIRE_POWER: "1" INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp16dp2ep32-latency-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp16dp2ep32-latency-agentic.yaml index d871a1c498..a4c9a03372 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp16dp2ep32-latency-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp16dp2ep32-latency-agentic.yaml @@ -80,8 +80,22 @@ backend: srun_options: container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + benchmark: type: custom + concurrencies: [1, 2, 3, 4, 5, 6, 7, 8, 10, 12] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp8dp4ep32-balanced-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp8dp4ep32-balanced-agentic.yaml index ba2c7f226d..a87c19fbd3 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp8dp4ep32-balanced-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp8dp4ep32-balanced-agentic.yaml @@ -81,8 +81,22 @@ backend: srun_options: container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + benchmark: type: custom + concurrencies: [1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp8dp4ep32-vllm-simple-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp8dp4ep32-vllm-simple-agentic.yaml index 8f76304fbb..3f16c9610f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp8dp4ep32-vllm-simple-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-h200-tp8dp4ep32-vllm-simple-agentic.yaml @@ -82,8 +82,22 @@ backend: srun_options: container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + benchmark: type: custom + concurrencies: [8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p1d-dcp8-dcp8-dspark4-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p1d-dcp8-dcp8-dspark4-mooncake-agentic.yaml index afe0d4a653..4d1ff23ef5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p1d-dcp8-dcp8-dspark4-mooncake-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p1d-dcp8-dcp8-dspark4-mooncake-agentic.yaml @@ -153,10 +153,27 @@ srun_options: mem: "0" container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + benchmark: type: custom + client_placement: head + concurrencies: [48, 52, 56] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: + ENABLE_AGENTX_POWER: "1" + REQUIRE_POWER: "1" INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p2d-dcp8-dcp8-dspark4-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p2d-dcp8-dcp8-dspark4-mooncake-agentic.yaml index 29388bbb67..55d3a39dad 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p2d-dcp8-dcp8-dspark4-mooncake-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p2d-dcp8-dcp8-dspark4-mooncake-agentic.yaml @@ -153,10 +153,27 @@ srun_options: mem: "0" container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + benchmark: type: custom + client_placement: head + concurrencies: [32, 48, 64] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: + ENABLE_AGENTX_POWER: "1" + REQUIRE_POWER: "1" INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark4-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark4-mooncake-agentic.yaml index 39269d445e..3c6a311e4f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark4-mooncake-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark4-mooncake-agentic.yaml @@ -153,10 +153,27 @@ srun_options: mem: "0" container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + benchmark: type: custom + client_placement: head + concurrencies: [32, 48] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: + ENABLE_AGENTX_POWER: "1" + REQUIRE_POWER: "1" INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark7-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark7-mooncake-agentic.yaml index 2df5ea74f3..d53fc6f486 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark7-mooncake-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark7-mooncake-agentic.yaml @@ -153,10 +153,27 @@ srun_options: mem: "0" container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + benchmark: type: custom + client_placement: head + concurrencies: [1] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: + ENABLE_AGENTX_POWER: "1" + REQUIRE_POWER: "1" INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 31a571b388..478aa82497 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1490,7 +1490,7 @@ kimik3-fp4-b300-vllm-agentic-dspark: image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-5894fdf model: moonshotai/Kimi-K3 model-prefix: kimik3 - runner: cluster:b300-nv + runner: cluster:b300-dsxe precision: fp4 framework: vllm multinode: false @@ -9995,8 +9995,8 @@ kimik3-fp4-gb300-dynamo-vllm-agentic-mooncake-dcp8-agg: # section 5): the single worker serves both phases, so no P/D KV transfer. # Serving is DIRECT vllm serve (srt-slurm frontend.type: vllm), not the dynamo # worker entrypoint -- framework stays dynamo-vllm for launcher routing only. -# DCP8 over a2a with the direct symmetric-memory kernels, Mooncake DRAM KV -# offload, and DSpark speculation sized per concurrency: 7 draft tokens (golden +# DCP8 over a2a with the direct symmetric-memory kernels and DSpark +# speculation sized per concurrency: 7 draft tokens (golden # AL 3.84) through conc 14, where the decode batch is small enough that deeper # speculation pays; 4 draft tokens (AL 3.36) at conc 24/48; and no speculation # at conc 96, where the batch already saturates the two nodes. @@ -10014,14 +10014,14 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: - dram-utilization: 0.8 search-space: - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } + kv-offloading: none conc-list: [1] num-nodes: 2 worker: num-worker: 1 tp: 8 pp: 2 + dcp-size: 8 ep: 1 dp-attn: false additional-settings: @@ -10029,14 +10029,14 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } + kv-offloading: none conc-list: [4] num-nodes: 2 worker: num-worker: 1 tp: 8 pp: 2 + dcp-size: 8 ep: 1 dp-attn: false additional-settings: @@ -10044,14 +10044,14 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } + kv-offloading: none conc-list: [8] num-nodes: 2 worker: num-worker: 1 tp: 8 pp: 2 + dcp-size: 8 ep: 1 dp-attn: false additional-settings: @@ -10059,14 +10059,14 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } + kv-offloading: none conc-list: [14] num-nodes: 2 worker: num-worker: 1 tp: 8 pp: 2 + dcp-size: 8 ep: 1 dp-attn: false additional-settings: @@ -10074,14 +10074,14 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } + kv-offloading: none conc-list: [24] num-nodes: 2 worker: num-worker: 1 tp: 8 pp: 2 + dcp-size: 8 ep: 1 dp-attn: false additional-settings: @@ -10089,28 +10089,28 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.36" - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } + kv-offloading: none conc-list: [48] num-nodes: 2 worker: num-worker: 1 tp: 8 pp: 2 + dcp-size: 8 ep: 1 dp-attn: false additional-settings: - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c48-agentic.yaml" - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.36" - - kv-offloading: dram - kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } + - kv-offloading: none conc-list: [96] num-nodes: 2 worker: num-worker: 1 tp: 8 pp: 2 + dcp-size: 8 ep: 1 dp-attn: false additional-settings: diff --git a/docs/ci-procedures.md b/docs/ci-procedures.md index f82c205fe4..76271f5c3f 100644 --- a/docs/ci-procedures.md +++ b/docs/ci-procedures.md @@ -222,6 +222,14 @@ RUN_ID=$(gh run list \ Do not continue if `RUN_ID` is empty. Run metadata describes the dispatch workflow ref, which may not equal input `ref`. Verify the unique title, generator command, and checkout ref in `get-jobs` before interpreting GPU results. +## Kimi-K3 AgentX power backfills + +The Kimi-K3 B200, H200, GB200 and GB300 multi-node recipes enable required DCGM telemetry and run the custom AgentX client on the serving head. Their launchers select the immutable AgentX power runtime, stamp its commit, wait for Slurm completion and telemetry drain, then validate each requested concurrency before staging results. The shared result collector preserves failed native jobs and stages available power diagnostics before returning a failure. Single-node B300 and MI355X use the shared AgentX collector; it waits for a real sample beyond the measurement end before stopping the monitor. + +For a missing-power backfill, generate only the missing recipe/concurrency combinations, set `require-power: true`, and leave `agentx-fast: false` and the duration override empty. The normal AgentX profile is one hour. Verify a first missing point on each newly enabled runtime/cluster before scheduling its remaining points. A rendered recipe or an online GitHub runner does not establish live collector readiness or Slurm capacity. Preserve the existing validated points, and keep new performance and power values paired with their own run; never attach a new run's energy to an older performance row. Manual `e2e-tests.yml` artifacts still require the normal reviewed ingestion path before they appear in the dashboard. + +B200 Kimi recipes use DCP8 with Mooncake offload disabled. The master config records `dcp-size: 8` and `kv-offloading: none` to match those commands; this metadata correction does not enable offload. + ## PR primary and modifier labels Sweep labels authorize GPU work for same-repository PRs whether draft or ready. Draft status controls review readiness, not sweep eligibility; fork PRs retain their trusted-dispatch path. Adding a sweep label or pushing with one present can start a sweep. Marking ready does not dispatch or repeat one. To start an already-labeled draft that has no run, remove and reapply its sweep label. diff --git a/docs/ci-procedures_zh.md b/docs/ci-procedures_zh.md index 1c8917da69..e21778702e 100644 --- a/docs/ci-procedures_zh.md +++ b/docs/ci-procedures_zh.md @@ -214,6 +214,14 @@ RUN_ID=$(gh run list \ 如果 `RUN_ID` 为空，不得继续。Run Metadata 描述派发 Workflow 的 Ref，可能不等于输入 `ref`。解释 GPU 结果前，必须在 `get-jobs` 中确认唯一标题、生成器命令与 Checkout Ref。 +## Kimi-K3 AgentX 功耗补测 + +Kimi-K3 的 B200、H200、GB200 和 GB300 多节点配方启用必需的 DCGM 遥测，并在服务头节点运行自定义 AgentX 客户端。Launcher 选择固定提交的 AgentX 功耗运行时、记录提交 SHA，等待 Slurm 结束和遥测收尾，然后逐一验证所请求的并发点并保存结果。共享结果收集器会保留原生任务的失败状态，先保存可用的功耗诊断文件，再返回失败。单节点 B300 和 MI355X 使用共享 AgentX 收集器；它会等到真实采样覆盖测量结束时刻之后，再停止监控。 + +补测缺失功耗时，只生成缺失的配方与并发组合，设置 `require-power: true`，保持 `agentx-fast: false`，并留空时长覆盖。标准 AgentX Profile 为一小时。对新启用的运行时或集群，先验证一个缺失点，再调度其余点。配方渲染通过或 GitHub Runner 在线并不能证明实时采集已就绪，也不能证明 Slurm 有空闲资源。保留现有有效点，新性能与功耗必须来自同一次运行；不得把新运行的能耗附加到旧性能点上。手动 `e2e-tests.yml` 产物仍须经过正常审查和入库流程，才会显示在 Dashboard 中。 + +B200 Kimi 配方采用 DCP8，且关闭 Mooncake Offload。Master Config 记录 `dcp-size: 8` 和 `kv-offloading: none` 以匹配实际命令；这项元数据修正不会启用 Offload。 + ## PR 主标签与修饰标签 同仓库 PR 无论处于草稿还是 ready 状态，都由 sweep 标签授权 GPU 运行。草稿状态控制是否开始审阅，不决定 sweep 资格；fork PR 仍使用受信任调度路径。添加 sweep 标签或在保留标签时推送提交可以启动 sweep。标记为 ready 不会调度或重复运行。已带标签但尚无运行的草稿，可先移除再重新添加对应 sweep 标签来启动。 diff --git a/infx/results/power/single_node.py b/infx/results/power/single_node.py index 54669a1a80..ee4daf85d9 100644 --- a/infx/results/power/single_node.py +++ b/infx/results/power/single_node.py @@ -9,7 +9,11 @@ aggregate and a validation sidecar, but does not fail the benchmark. Power studies can set ``REQUIRE_POWER=1`` to fail after those audit artifacts exist. The aggregate carries numeric ``power_valid`` (1/0) for metric ingestion; the -sidecar is the canonical source for boolean validity and reason codes. +sidecar is the canonical source for boolean validity and reason codes. Rows in +the ingest band but outside the formal window whose power is missing, +non-finite, or <= 0 are teardown noise: they are skipped and counted in the +sidecar's ``boundary_degenerate_rows`` instead of poisoning validity or faking +window bracketing. """ from __future__ import annotations @@ -22,7 +26,7 @@ import os import re import sys -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import datetime, timezone from pathlib import Path from statistics import mean @@ -65,6 +69,10 @@ class PowerIntegration: per_gpu_max_sample_gap_s: dict[str, float] per_gpu_energy_j: dict[str, float] device_issues: dict[str, list[str]] + # Rows in the ingest band but outside the formal window whose power was + # missing/N-A/non-finite/<=0, skipped and counted per GPU; "unknown" + # buckets rows without a GPU identity. + boundary_degenerate_rows: dict[str, int] = field(default_factory=dict) avg_power_w: float | None = None p75_power_w: float | None = None p75_total_gpu_power_w: float | None = None @@ -237,6 +245,7 @@ def _empty_integration( *, expected_num_gpus: int | None, reasons: list[str], + boundary_degenerate_rows: dict[str, int] | None = None, ) -> PowerIntegration: """Build an invalid integration result when no device data is available.""" return PowerIntegration( @@ -248,6 +257,7 @@ def _empty_integration( per_gpu_max_sample_gap_s={}, per_gpu_energy_j={}, device_issues={}, + boundary_degenerate_rows=boundary_degenerate_rows or {}, ) @@ -308,6 +318,7 @@ def integrate_power( # expose timestamps at lower resolution than their sampling cadence, so # duplicate-timestamp readings are averaged rather than treated as corrupt. raw_samples: dict[str, dict[float, list[float]]] = {} + boundary_degenerate: dict[str, int] = {} saw_missing_gpu_identity = False try: with csv_path.open("r", newline="", encoding="utf-8", errors="replace") as f: @@ -348,6 +359,15 @@ def integrate_power( power = _parse_power((row.get(power_col) or "").strip()) gpu_id = (row.get(gpu_col) or "").strip() + if (power is None or not math.isfinite(power) or power <= 0.0) and ( + timestamp < start_unix or timestamp > end_unix + ): + # SMI teardown rows can carry N/A or 0 W cells: outside the + # formal window they are counted, never used to satisfy + # bracketing or to poison in-window validity. + key = gpu_id or "unknown" + boundary_degenerate[key] = boundary_degenerate.get(key, 0) + 1 + continue if power is None: _append_reason(reasons, "invalid_power_sample") continue @@ -364,6 +384,7 @@ def integrate_power( return _empty_integration( expected_num_gpus=expected_num_gpus, reasons=reasons, + boundary_degenerate_rows=boundary_degenerate, ) if saw_missing_gpu_identity: @@ -373,6 +394,7 @@ def integrate_power( return _empty_integration( expected_num_gpus=expected_num_gpus, reasons=reasons, + boundary_degenerate_rows=boundary_degenerate, ) observed_gpu_ids = tuple(sorted(raw_samples, key=_gpu_sort_key)) @@ -450,6 +472,7 @@ def integrate_power( per_gpu_max_sample_gap_s=per_gpu_max_sample_gap_s, per_gpu_energy_j=per_gpu_energy_j, device_issues=device_issues, + boundary_degenerate_rows=boundary_degenerate, avg_power_w=avg_power_w, p75_power_w=p75_total / len(observed_gpu_ids) if p75_total is not None else None, p75_total_gpu_power_w=p75_total, @@ -679,6 +702,7 @@ def _validation_payload( "per_gpu_max_sample_gap_s": integration.per_gpu_max_sample_gap_s, "per_gpu_energy_j": integration.per_gpu_energy_j, "device_issues": integration.device_issues, + "boundary_degenerate_rows": integration.boundary_degenerate_rows, "accumulator_check": accumulator_check, "metrics": audit_metrics(metrics), } diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f1e48d3766..8cce2b53f7 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7338,3 +7338,23 @@ - "Resolve the requested H100 SGLang container instead of a hardcoded older image, and stop benchmark/eval clients when their ready server or required worker exits." - "H100 SGLang 使用所请求的容器而非硬编码旧镜像；已就绪的服务或必需工作进程退出后，停止其 benchmark/评测客户端。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3020 + +- config-keys: + - kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-dcp16-agg + - kimik3-fp4-gb200-dynamo-vllm-agentic-mooncake-dcp16-agg + - kimik3-fp4-gb200-dynamo-vllm-agentic-dspark-mooncake-tp8pp2 + - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg + - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg + - kimik3-fp4-gb300-dynamo-vllm-agentic-mooncake-dcp8-agg + - kimik3-fp4-b200-dynamo-vllm-agentic-dspark + - kimik3-fp4-h200-vllm-agentic + - kimik3-fp4-b300-vllm-agentic-dspark + - kimik3-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Enable required Kimi-K3 AgentX power on B200, H200, GB200 and GB300 with exact-window validation and failed-job audit artifacts." + - "Wait for complete AMD telemetry before teardown, use the active B300 cluster label, and correct B200 DCP/offload metadata to match its serving commands." + - "为 B200、H200、GB200 和 GB300 启用 Kimi-K3 AgentX 必需功耗采集、精确测量窗口验证及失败任务审计产物。" + - "AMD 遥测覆盖完整窗口后再收尾；使用现有 B300 集群标签，并修正 B200 的 DCP/Offload 元数据以匹配实际服务命令。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3030 diff --git a/runners/launch_b200-nscale-slurm.sh b/runners/launch_b200-nscale-slurm.sh index b5e0ef627c..f0d6b8ae96 100755 --- a/runners/launch_b200-nscale-slurm.sh +++ b/runners/launch_b200-nscale-slurm.sh @@ -5,7 +5,7 @@ # Self-contained because Nscale has its own Slurm and storage layout. # # Scope: multi-node Dynamo-vLLM DeepSeek-V4-Pro and Kimi K2.6 FP4 runs, plus -# DeepSeek-V4-Pro FP4 Dynamo-SGLang STP and MTP runs, on the +# DeepSeek-V4-Pro FP4 Dynamo-SGLang STP and MTP runs and Kimi-K3 AgentX, on the # b200-nscale runner label. # Anything else exits non-zero. @@ -13,6 +13,7 @@ SLURM_PARTITION="batch_1" SLURM_ACCOUNT="benchmark" POWER_SRT_SLURM_URL="https://github.com/edwingao28/srt-slurm.git" POWER_SRT_SLURM_PIN="e5c837f06a362dc888dfea2ee588e9f19c298270" +AGENTX_POWER_SRT_SLURM_PIN="80d7203e424f903c9017de4608ee2044afce9574" TILERT_SRT_SLURM_URL="https://github.com/SemiAnalysisAI/srt-slurm.git" TILERT_SRT_SLURM_PIN="d1e6c97b3baf3e87103b6d83189544c3c7d61c38" @@ -63,6 +64,7 @@ if [[ $FRAMEWORK != "dynamo-vllm" ]] && fi USES_DCGM_POWER=0 +USES_AGENTX_POWER=0 _POWER_CONFIG_FILE="${CONFIG_FILE:-}" if [[ "${EVAL_ONLY:-false}" == "true" && -n "${EVAL_CONFIG_FILE:-}" ]]; then _POWER_CONFIG_FILE="$EVAL_CONFIG_FILE" @@ -78,14 +80,17 @@ if [[ -n "$_POWER_CONFIG_FILE" && -f "$_RECIPE_SRC" ]] && awk ' ' "$_RECIPE_SRC"; then USES_DCGM_POWER=1 fi -if [[ "$USES_DCGM_POWER" == "1" && ( +if [[ "$USES_DCGM_POWER" == "1" && "$IS_AGENTIC" == "1" && + "$MODEL_PREFIX" == "kimik3" && "$PRECISION" == "fp4" && "$FRAMEWORK" == "dynamo-vllm" ]]; then + USES_AGENTX_POWER=1 +elif [[ "$USES_DCGM_POWER" == "1" && ( "${IS_AGENTIC:-0}" == "1" || "$PRECISION" != "fp4" || ( "$MODEL_PREFIX" == "dsv4" && "$FRAMEWORK" != "dynamo-sglang" && "$FRAMEWORK" != "dynamo-vllm" ) || ( "$MODEL_PREFIX" == "kimik2.6" && "$FRAMEWORK" != "dynamo-vllm" ) || ( "$MODEL_PREFIX" != "dsv4" && "$MODEL_PREFIX" != "kimik2.6" ) ) ]]; then - echo "Error: B200 nscale dcgm-power is limited to fixed-sequence DSV4/Kimi-K2.6 FP4 lanes" >&2 + echo "Error: B200 nscale dcgm-power requires a supported fixed-sequence lane or Kimi-K3 AgentX vLLM" >&2 exit 1 fi @@ -95,12 +100,20 @@ echo "Cloning srt-slurm repository..." SRT_REPO_DIR="srt-slurm" rm -rf "$SRT_REPO_DIR" if [[ "$USES_DCGM_POWER" == "1" ]]; then + SELECTED_POWER_SRT_SLURM_PIN="$POWER_SRT_SLURM_PIN" + if [[ "$USES_AGENTX_POWER" == "1" ]]; then + SELECTED_POWER_SRT_SLURM_PIN="$AGENTX_POWER_SRT_SLURM_PIN" + fi git clone "$POWER_SRT_SLURM_URL" "$SRT_REPO_DIR" || exit 1 cd "$SRT_REPO_DIR" || exit 1 - git checkout "$POWER_SRT_SLURM_PIN" || exit 1 - test "$(git rev-parse HEAD)" = "$POWER_SRT_SLURM_PIN" || { echo "Error: srt-slurm HEAD does not match POWER_SRT_SLURM_PIN=$POWER_SRT_SLURM_PIN" >&2; exit 1; } + git checkout "$SELECTED_POWER_SRT_SLURM_PIN" || exit 1 + test "$(git rev-parse HEAD)" = "$SELECTED_POWER_SRT_SLURM_PIN" || { echo "Error: srt-slurm HEAD does not match selected power producer $SELECTED_POWER_SRT_SLURM_PIN" >&2; exit 1; } git rev-parse HEAD > "$GITHUB_WORKSPACE/power-producer-sha.txt" - if [[ "$MODEL_PREFIX" == "dsv4" && "$FRAMEWORK" == "dynamo-sglang" ]]; then + if [[ "$USES_AGENTX_POWER" == "1" ]]; then + mkdir -p recipes/vllm/kimi-k3/agentic || exit 1 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ + recipes/vllm/kimi-k3/agentic || exit 1 + elif [[ "$MODEL_PREFIX" == "dsv4" && "$FRAMEWORK" == "dynamo-sglang" ]]; then mkdir -p recipes/sglang/deepseek-v4 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4" recipes/sglang/deepseek-v4 elif [[ "$MODEL_PREFIX" == "dsv4" ]]; then @@ -243,9 +256,8 @@ fi if [[ "$USES_DCGM_POWER" == "1" ]]; then DCGM_EXPORTER_IMAGE="nvcr.io/nvidia/k8s/dcgm-exporter:4.6.0-4.8.3-distroless" - DCGM_EXPORTER_ENROOT_REF="${DCGM_EXPORTER_IMAGE/nvcr.io\//nvcr.io#}" DCGM_EXPORTER_SQSH="$SQUASH_DIR/$(echo "$DCGM_EXPORTER_IMAGE" | sed 's/[\/:@#]/_/g').sqsh" - import_squash "$DCGM_EXPORTER_SQSH" "$DCGM_EXPORTER_ENROOT_REF" || exit 1 + import_squash "$DCGM_EXPORTER_SQSH" "$DCGM_EXPORTER_IMAGE" || exit 1 test -r "$DCGM_EXPORTER_SQSH" || { echo "Error: DCGM exporter squash not readable: $DCGM_EXPORTER_SQSH" >&2; exit 1; } unsquashfs -l "$DCGM_EXPORTER_SQSH" > /dev/null || { echo "Error: DCGM exporter squash invalid: $DCGM_EXPORTER_SQSH" >&2; exit 1; } sha256sum "$DCGM_EXPORTER_SQSH" > "$GITHUB_WORKSPACE/exporter-image.sha256" @@ -346,6 +358,12 @@ sed -i 's/^ max_attempts: [0-9]*/ max_attempts: 720/' "$CONFIG_PATH" inject_synthetic_acceptance "$CONFIG_PATH" "$FRAMEWORK" || exit 1 +if [[ "$USES_AGENTX_POWER" == "1" ]]; then + read -r -a POWER_CONCURRENCIES <<< "$CONC_LIST" + python "$GITHUB_WORKSPACE/runners/inject_srt_power_concurrencies.py" \ + "$CONFIG_PATH" "${POWER_CONCURRENCIES[@]}" || exit 1 +fi + SRTCTL_PREFLIGHT_ARGS=() # These weights are staged on the Slurm compute nodes, not the login node. if [[ $MODEL_PREFIX == "kimik2.6" ]] || @@ -373,7 +391,11 @@ LOG_FILE="$LOGS_DIR/sweep_${JOB_ID}.log" # Waits for the log file to appear, fails fast if the job dies first, then # streams until the job leaves the queue. -stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || exit 1 +SRT_JOB_RC=0 +stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || SRT_JOB_RC=$? +if [[ "$SRT_JOB_RC" != "0" && "$USES_AGENTX_POWER" != "1" ]]; then + exit "$SRT_JOB_RC" +fi set -x @@ -385,6 +407,14 @@ if [ ! -d "$LOGS_DIR" ]; then exit 1 fi +AGENTX_POWER_RC="$SRT_JOB_RC" +if [[ "$USES_AGENTX_POWER" == "1" && "${EVAL_ONLY:-false}" != "true" ]]; then + read -r -a POWER_CONCURRENCIES <<< "$CONC_LIST" + collect_agentic_power_results "$JOB_ID" "$LOGS_DIR" \ + "$GITHUB_WORKSPACE" "$GITHUB_WORKSPACE" "$RESULT_FILENAME" \ + "$SELECTED_POWER_SRT_SLURM_PIN" "${POWER_CONCURRENCIES[@]}" || AGENTX_POWER_RC=$? +fi + if [[ "$USES_DCGM_POWER" == "1" ]]; then mkdir -p "$LOGS_DIR/power" cp "$GITHUB_WORKSPACE/exporter-image.sha256" "$LOGS_DIR/power/exporter-image.sha256" @@ -394,6 +424,11 @@ fi cp -r "$LOGS_DIR" "$GITHUB_WORKSPACE/LOGS" bundle_server_logs "$LOGS_DIR" "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" +if [[ "$AGENTX_POWER_RC" != "0" ]]; then + echo "ERROR: AgentX power validation failed; available audit and server artifacts were staged" >&2 + exit "$AGENTX_POWER_RC" +fi + if [[ "${EVAL_ONLY:-false}" != "true" ]]; then RESULT_SUBDIRS=$(find "$LOGS_DIR" -maxdepth 1 -type d -name "*isl*osl*" 2>/dev/null) diff --git a/runners/launch_gb200-nv.sh b/runners/launch_gb200-nv.sh index b6090491bd..98a59d1ba2 100755 --- a/runners/launch_gb200-nv.sh +++ b/runners/launch_gb200-nv.sh @@ -15,6 +15,7 @@ SQUASH_DIR="/mnt/lustre01/users-public/sa-shared" # srt-slurm merge lands. POWER_SRT_SLURM_URL="https://github.com/edwingao28/srt-slurm.git" POWER_SRT_SLURM_PIN="6fc1bed01a0b82dae0088a105c03ce0cfb353443" +AGENTX_POWER_SRT_SLURM_PIN="80d7203e424f903c9017de4608ee2044afce9574" # Enroot 3.x does not parse Docker's tag@digest syntax. For digest-pinned # images, use its explicit registry syntax and pass the digest as the @@ -332,12 +333,15 @@ if [[ -n "$CONFIG_FILE" && -f "$_RECIPE_SRC" ]] && awk ' USES_DCGM_POWER=1 fi -# Note (wenyao): the producer pin follows the srt-slurm main lineage that the -# dynamo-sglang lanes run on (fp8 validated end-to-end, fp4 recipes -# parse-verified against the pin); other frameworks clone diverging refs -# (aflowers branch, sa-submission), so fail fast for them instead. -if [[ "$USES_DCGM_POWER" == "1" && "$FRAMEWORK" != "dynamo-sglang" ]]; then - echo "Error: dcgm-power lanes are only validated for FRAMEWORK=dynamo-sglang, got: $FRAMEWORK" >&2 +USES_AGENTX_POWER=0 +if [[ "$USES_DCGM_POWER" == "1" && "$IS_AGENTIC" == "1" && + "$MODEL_PREFIX" == "kimik3" && "$PRECISION" == "fp4" && + "$FRAMEWORK" == "dynamo-vllm" && + "$_RECIPE_REL" == recipes/vllm/kimi-k3/agentic/* ]]; then + USES_AGENTX_POWER=1 +fi +if [[ "$USES_DCGM_POWER" == "1" && "$FRAMEWORK" != "dynamo-sglang" && "$USES_AGENTX_POWER" != "1" ]]; then + echo "Error: dcgm-power requires dynamo-sglang or the supported Kimi-K3 AgentX route" >&2 exit 1 fi @@ -472,12 +476,20 @@ elif [[ "$IS_AGENTIC" == "1" && (( "$MODEL_PREFIX" == "qwen3.5" && "$PRECISION" fi # Kimi-K3 requires direct multi-node vLLM frontend support from srt-slurm. elif [[ "$IS_AGENTIC" == "1" && "$MODEL_PREFIX" == "kimik3" ]]; then - git clone --branch v1.0.53 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 - cd "$SRT_REPO_DIR" || exit 1 - test "$(git rev-parse HEAD)" = "217f94387abeddfed7149a71955dc523e07cd765" || { - echo "Error: NVIDIA/srt-slurm v1.0.53 resolved to an unexpected commit" >&2 - exit 1 - } + if [[ "$USES_AGENTX_POWER" == "1" ]]; then + git clone "$POWER_SRT_SLURM_URL" "$SRT_REPO_DIR" || exit 1 + cd "$SRT_REPO_DIR" || exit 1 + git checkout "$AGENTX_POWER_SRT_SLURM_PIN" || exit 1 + test "$(git rev-parse HEAD)" = "$AGENTX_POWER_SRT_SLURM_PIN" || exit 1 + git rev-parse HEAD > "$GITHUB_WORKSPACE/power-producer-sha.txt" + else + git clone --branch v1.0.53 --single-branch https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" || exit 1 + cd "$SRT_REPO_DIR" || exit 1 + test "$(git rev-parse HEAD)" = "217f94387abeddfed7149a71955dc523e07cd765" || { + echo "Error: NVIDIA/srt-slurm v1.0.53 resolved to an unexpected commit" >&2 + exit 1 + } + fi python3 "$GITHUB_WORKSPACE/runners/patch_srt_vllm_dp_ranks.py" "$(pwd)" || exit 1 mkdir -p recipes/vllm/kimi-k3/agentic || exit 1 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ @@ -759,6 +771,12 @@ sed -i "s/^name:.*/name: \"${SRT_SLURM_JOB_NAME}\"/" "$CONFIG_PATH" python3 "$GITHUB_WORKSPACE/runners/inject_synthetic_acceptance.py" \ "$CONFIG_PATH" "$FRAMEWORK" || exit 1 +if [[ "$USES_AGENTX_POWER" == "1" ]]; then + read -r -a POWER_CONCURRENCIES <<< "$CONC_LIST" + python3 "$GITHUB_WORKSPACE/runners/inject_srt_power_concurrencies.py" \ + "$CONFIG_PATH" "${POWER_CONCURRENCIES[@]}" || exit 1 +fi + # Don't leak the login-node venv to the compute-node orchestrator. sbatch's # default --export=ALL propagates VIRTUAL_ENV (set by `source # .venv/bin/activate` above) into job_script_minimal.j2, whose @@ -822,13 +840,24 @@ trap 'exit 143' TERM HUP LOGS_DIR="outputs/$JOB_ID/logs" LOG_FILE="$LOGS_DIR/sweep_${JOB_ID}.log" -stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || exit 1 +AGENTX_POWER_RC=0 +stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || AGENTX_POWER_RC=$? +if [[ "$AGENTX_POWER_RC" != "0" && "$USES_AGENTX_POWER" != "1" ]]; then + exit "$AGENTX_POWER_RC" +fi set -x echo "Job $JOB_ID completed!" echo "Collecting results..." +if [[ "$USES_AGENTX_POWER" == "1" && "${EVAL_ONLY:-false}" != "true" ]]; then + read -r -a POWER_CONCURRENCIES <<< "$CONC_LIST" + collect_agentic_power_results "$JOB_ID" "$LOGS_DIR" "$INFMAX_WORKSPACE" \ + "$GITHUB_WORKSPACE" "$RESULT_FILENAME" "$AGENTX_POWER_SRT_SLURM_PIN" \ + "${POWER_CONCURRENCIES[@]}" || AGENTX_POWER_RC=$? +fi + if [ -d "$LOGS_DIR" ]; then echo "Found logs directory: $LOGS_DIR" # Provenance markers travel inside the server-logs bundle so the offline @@ -844,6 +873,11 @@ else echo "Warning: Logs directory not found at $LOGS_DIR" fi +if [[ "$AGENTX_POWER_RC" != "0" ]]; then + echo "ERROR: AgentX job or power validation failed after staging audit artifacts" >&2 + exit "$AGENTX_POWER_RC" +fi + if [[ "${EVAL_ONLY:-false}" != "true" ]]; then if [ ! -d "$LOGS_DIR" ]; then exit 1 @@ -854,10 +888,12 @@ if [[ "${EVAL_ONLY:-false}" != "true" ]]; then # INFMAX_WORKSPACE mount. Its aggregation step writes one # ${RESULT_FILENAME}_conc.json there per point; stage those files # back to GITHUB_WORKSPACE for the workflow guard and artifact upload. - copy_agentic_results \ - "$INFMAX_WORKSPACE" \ - "$GITHUB_WORKSPACE" \ - "$RESULT_FILENAME" || exit 1 + if [[ "$USES_AGENTX_POWER" != "1" ]]; then + copy_agentic_results \ + "$INFMAX_WORKSPACE" \ + "$GITHUB_WORKSPACE" \ + "$RESULT_FILENAME" || exit 1 + fi else # Find all fixed-sequence result subdirectories. RESULT_SUBDIRS=$(find "$LOGS_DIR" -maxdepth 1 -type d -name "*isl*osl*" 2>/dev/null) diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 9249e91f27..ff3a70035c 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -193,12 +193,15 @@ if [[ -n "$CONFIG_FILE" && -f "$_RECIPE_SRC" ]] && awk ' USES_DCGM_POWER=1 fi -# Note (wenyao): the producer pin follows the srt-slurm main lineage that the -# dynamo-sglang lanes run on (fp8 validated end-to-end, fp4 recipes -# parse-verified against the pin); other frameworks clone diverging refs -# (aflowers branch, sa-submission), so fail fast for them instead. -if [[ "$USES_DCGM_POWER" == "1" && "$FRAMEWORK" != "dynamo-sglang" ]]; then - echo "Error: dcgm-power lanes are only validated for FRAMEWORK=dynamo-sglang, got: $FRAMEWORK" >&2 +USES_AGENTX_POWER=0 +if [[ "$USES_DCGM_POWER" == "1" && "$IS_AGENTIC" == "1" && + "$MODEL_PREFIX" == "kimik3" && "$PRECISION" == "fp4" && + "$FRAMEWORK" == "dynamo-vllm" && + "$_RECIPE_REL" == recipes/vllm/kimi-k3/agentic/* ]]; then + USES_AGENTX_POWER=1 +fi +if [[ "$USES_DCGM_POWER" == "1" && "$FRAMEWORK" != "dynamo-sglang" && "$USES_AGENTX_POWER" != "1" ]]; then + echo "Error: dcgm-power requires dynamo-sglang or the supported Kimi-K3 AgentX route" >&2 exit 1 fi @@ -207,6 +210,7 @@ fi # srt-slurm merge lands. POWER_SRT_SLURM_URL="https://github.com/edwingao28/srt-slurm.git" POWER_SRT_SLURM_PIN="6fc1bed01a0b82dae0088a105c03ce0cfb353443" +AGENTX_POWER_SRT_SLURM_PIN="80d7203e424f903c9017de4608ee2044afce9574" if [[ "$USES_DCGM_POWER" == "1" ]]; then DCGM_EXPORTER_IMAGE="nvcr.io/nvidia/k8s/dcgm-exporter:4.6.0-4.8.3-distroless" @@ -238,7 +242,16 @@ SRT_REPO_DIR="${GITHUB_WORKSPACE}/srt-slurm-${GITHUB_RUN_ID:-manual}-${GITHUB_RU SRTCTL_SETUP_SCRIPT="" rm -rf "$SRT_REPO_DIR" -if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "qwen3.5" ]]; then +if [[ "$USES_AGENTX_POWER" == "1" ]]; then + git clone "$POWER_SRT_SLURM_URL" "$SRT_REPO_DIR" || exit 1 + cd "$SRT_REPO_DIR" || exit 1 + git checkout "$AGENTX_POWER_SRT_SLURM_PIN" || exit 1 + test "$(git rev-parse HEAD)" = "$AGENTX_POWER_SRT_SLURM_PIN" || exit 1 + git rev-parse HEAD > "$GITHUB_WORKSPACE/power-producer-sha.txt" + mkdir -p recipes/vllm/kimi-k3/agentic + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ + recipes/vllm/kimi-k3/agentic +elif [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "qwen3.5" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" git checkout v1.0.50 @@ -570,6 +583,12 @@ sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_PATH" # verification. inject_synthetic_acceptance "$CONFIG_PATH" "$FRAMEWORK" || exit 1 +if [[ "$USES_AGENTX_POWER" == "1" ]]; then + read -r -a POWER_CONCURRENCIES <<< "$CONC_LIST" + python3 "$GITHUB_WORKSPACE/runners/inject_srt_power_concurrencies.py" \ + "$CONFIG_PATH" "${POWER_CONCURRENCIES[@]}" || exit 1 +fi + # --no-preflight skips srtctl's pre-submit model-path stat, which runs on # the GHA runner host (im-gb300-login-02, an x86 login node). It's required # whenever model.path resolves to the node-local /scratch NVMe that the login @@ -644,37 +663,49 @@ _snapshot_server_logs() { } trap _snapshot_server_logs EXIT -# Wait for log file to appear (also check job is still alive) -while ! ls "$LOG_FILE" &>/dev/null; do - if ! squeue -j "$JOB_ID" --noheader 2>/dev/null | grep -q "$JOB_ID"; then - echo "ERROR: Job $JOB_ID failed before creating log file" - scontrol show job "$JOB_ID" - exit 1 - fi - echo "Waiting for JOB_ID $JOB_ID to begin and $LOG_FILE to appear..." - sleep 5 -done - -# Poll for job completion in background -( - while squeue -j "$JOB_ID" --noheader 2>/dev/null | grep -q "$JOB_ID"; do - sleep 10 +AGENTX_POWER_RC=0 +if [[ "$USES_AGENTX_POWER" == "1" ]]; then + stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || AGENTX_POWER_RC=$? +else + # Wait for log file to appear (also check job is still alive) + while ! ls "$LOG_FILE" &>/dev/null; do + if ! squeue -j "$JOB_ID" --noheader 2>/dev/null | grep -q "$JOB_ID"; then + echo "ERROR: Job $JOB_ID failed before creating log file" + scontrol show job "$JOB_ID" + exit 1 + fi + echo "Waiting for JOB_ID $JOB_ID to begin and $LOG_FILE to appear..." + sleep 5 done -) & -POLL_PID=$! -echo "Tailing LOG_FILE: $LOG_FILE" + # Poll for job completion in background + ( + while squeue -j "$JOB_ID" --noheader 2>/dev/null | grep -q "$JOB_ID"; do + sleep 10 + done + ) & + POLL_PID=$! -# Stream the log file until job completes (-F follows by name, polls instead of inotify for NFS) -tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null + echo "Tailing LOG_FILE: $LOG_FILE" -wait $POLL_PID + # Stream the log file until job completes (-F follows by name, polls instead of inotify for NFS) + tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null + + wait $POLL_PID +fi set -x echo "Job $JOB_ID completed!" echo "Collecting results..." +if [[ "$USES_AGENTX_POWER" == "1" && "${EVAL_ONLY:-false}" != "true" ]]; then + read -r -a POWER_CONCURRENCIES <<< "$CONC_LIST" + collect_agentic_power_results "$JOB_ID" "$LOGS_DIR" "$INFMAX_WORKSPACE" \ + "$GITHUB_WORKSPACE" "$RESULT_FILENAME" "$AGENTX_POWER_SRT_SLURM_PIN" \ + "${POWER_CONCURRENCIES[@]}" || AGENTX_POWER_RC=$? +fi + if [ -d "$LOGS_DIR" ]; then echo "Found logs directory: $LOGS_DIR" # Tarball + LOGS copy + power provenance markers are produced by the EXIT @@ -685,6 +716,11 @@ else echo "Warning: Logs directory not found at $LOGS_DIR" fi +if [[ "$AGENTX_POWER_RC" != "0" ]]; then + echo "ERROR: AgentX job or power validation failed; EXIT will stage audit artifacts" >&2 + exit "$AGENTX_POWER_RC" +fi + if [[ "${EVAL_ONLY:-false}" != "true" ]]; then if [ ! -d "$LOGS_DIR" ]; then exit 1 diff --git a/runners/launch_h200-dgxc-slurm.sh b/runners/launch_h200-dgxc-slurm.sh index 95f519fd3f..0711e2e250 100755 --- a/runners/launch_h200-dgxc-slurm.sh +++ b/runners/launch_h200-dgxc-slurm.sh @@ -12,6 +12,8 @@ AIPERF_MMAP_CACHE_HOST_PATH="${AIPERF_MMAP_CACHE_HOST_PATH:-/home/sa-shared/ghar # commit and re-running the H200 hardware gate. POWER_SRT_SLURM_URL="https://github.com/edwingao28/srt-slurm.git" POWER_SRT_SLURM_PIN="e5c837f06a362dc888dfea2ee588e9f19c298270" +AGENTX_POWER_SRT_SLURM_PIN="80d7203e424f903c9017de4608ee2044afce9574" +SELECTED_POWER_SRT_SLURM_PIN="$POWER_SRT_SLURM_PIN" set -x @@ -41,16 +43,18 @@ if [[ "$IS_MULTINODE" == "true" ]]; then USES_DCGM_POWER=1 fi - # Only explicitly reviewed H200 FP8 AgentX recipes may use dcgm-power. - # Future recipes must earn a separate cluster smoke instead of inheriting - # this lane. - if [[ "$USES_DCGM_POWER" == "1" && ( + USES_KIMIK3_POWER=0 + if [[ "$USES_DCGM_POWER" == "1" && "$IS_AGENTIC" == "1" && + "$MODEL_PREFIX" == "kimik3" && "$PRECISION" == "fp4" && "$FRAMEWORK" == "vllm" ]]; then + USES_KIMIK3_POWER=1 + SELECTED_POWER_SRT_SLURM_PIN="$AGENTX_POWER_SRT_SLURM_PIN" + elif [[ "$USES_DCGM_POWER" == "1" && ( "$IS_AGENTIC" != "1" || "$FRAMEWORK" != "dynamo-sglang" || ( "$MODEL_PREFIX" != "glm5.2" && "$MODEL_PREFIX" != "dsv4" ) || "$PRECISION" != "fp8" ) ]]; then - echo "Error: H200 dcgm-power is validated only for AgentX dynamo-sglang glm5.2/fp8 or dsv4/fp8" >&2 + echo "Error: H200 dcgm-power requires AgentX dynamo-sglang glm5.2/dsv4 FP8 or Kimi-K3 vLLM FP4" >&2 exit 1 fi @@ -135,9 +139,17 @@ if [[ "$IS_MULTINODE" == "true" ]]; then cd "$SRT_REPO_DIR" fi elif [[ $IS_AGENTIC == "1" && $FRAMEWORK == "vllm" && $MODEL_PREFIX == "kimik3" ]]; then - git clone https://github.com/functionstackx/srt-slurm-nv.git "$SRT_REPO_DIR" - cd "$SRT_REPO_DIR" - git checkout df5baa93f4caf5169dea2a4236ad2cc742fe40e7 + if [[ "$USES_KIMIK3_POWER" == "1" ]]; then + git clone "$POWER_SRT_SLURM_URL" "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + git checkout "$SELECTED_POWER_SRT_SLURM_PIN" + test "$(git rev-parse HEAD)" = "$SELECTED_POWER_SRT_SLURM_PIN" || exit 1 + git rev-parse HEAD > "$GITHUB_WORKSPACE/power-producer-sha.txt" + else + git clone https://github.com/functionstackx/srt-slurm-nv.git "$SRT_REPO_DIR" + cd "$SRT_REPO_DIR" + git checkout df5baa93f4caf5169dea2a4236ad2cc742fe40e7 + fi mkdir -p recipes/vllm/kimi-k3/agentic cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ recipes/vllm/kimi-k3/agentic @@ -348,7 +360,11 @@ EOF LOG_FILE="$LOGS_DIR/sweep_${JOB_ID}.log" trap 'rc=$?; bundle_server_logs "$LOGS_DIR" "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz"; scancel "$JOB_ID" 2>/dev/null || true; exit "$rc"' EXIT INT TERM HUP - stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || exit 1 + SRT_JOB_RC=0 + stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || SRT_JOB_RC=$? + if [[ "$SRT_JOB_RC" != "0" && "$USES_KIMIK3_POWER" != "1" ]]; then + exit "$SRT_JOB_RC" + fi set -x @@ -362,7 +378,13 @@ EOF echo "Found logs directory: $LOGS_DIR" - if [[ "$USES_DCGM_POWER" == "1" ]]; then + AGENTX_POWER_RC="$SRT_JOB_RC" + if [[ "$USES_KIMIK3_POWER" == "1" && "${EVAL_ONLY:-false}" != "true" ]]; then + read -r -a POWER_CONCURRENCIES <<< "$CONC_LIST" + collect_agentic_power_results "$JOB_ID" "$LOGS_DIR" \ + "$GITHUB_WORKSPACE" "$GITHUB_WORKSPACE" "$RESULT_FILENAME" \ + "$SELECTED_POWER_SRT_SLURM_PIN" "${POWER_CONCURRENCIES[@]}" || AGENTX_POWER_RC=$? + elif [[ "$USES_DCGM_POWER" == "1" && "${EVAL_ONLY:-false}" != "true" ]]; then POWER_LOGS_ROOT=$(cd "$LOGS_DIR" && pwd -P) read -r -a POWER_CONCURRENCIES <<< "$CONC_LIST" for concurrency in "${POWER_CONCURRENCIES[@]}"; do @@ -371,7 +393,7 @@ EOF --agg-result "$GITHUB_WORKSPACE/${RESULT_FILENAME}_conc${concurrency}.json" --power-dir "$POWER_LOGS_ROOT/power" --logs-root "$POWER_LOGS_ROOT" - --expected-producer-sha "$POWER_SRT_SLURM_PIN" + --expected-producer-sha "$SELECTED_POWER_SRT_SLURM_PIN" ) case "${REQUIRE_POWER:-0}" in 1|true|TRUE|yes|YES) power_args+=(--require-power) ;; @@ -379,8 +401,10 @@ EOF ( cd "$GITHUB_WORKSPACE" python -m infx.results.agentic.power_adapter "${power_args[@]}" - ) || exit 1 + ) || AGENTX_POWER_RC=$? done + fi + if [[ "$USES_DCGM_POWER" == "1" ]]; then mkdir -p "$LOGS_DIR/power" cp "$GITHUB_WORKSPACE/exporter-image.sha256" "$LOGS_DIR/power/exporter-image.sha256" cp "$GITHUB_WORKSPACE/power-producer-sha.txt" "$LOGS_DIR/power/power-producer-sha.txt" @@ -389,6 +413,11 @@ EOF cp -r "$LOGS_DIR" "$GITHUB_WORKSPACE/LOGS" bundle_server_logs "$LOGS_DIR" "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" + if [[ "$AGENTX_POWER_RC" != "0" ]]; then + echo "ERROR: AgentX power validation failed; available audit and server artifacts were staged" >&2 + exit "$AGENTX_POWER_RC" + fi + if [[ "${EVAL_ONLY:-false}" != "true" ]]; then copy_fixed_sequence_results "$LOGS_DIR" "$GITHUB_WORKSPACE" "$RESULT_FILENAME" else diff --git a/runners/slurm_utils.sh b/runners/slurm_utils.sh index 8a6462f121..16a1b320a8 100644 --- a/runners/slurm_utils.sh +++ b/runners/slurm_utils.sh @@ -161,6 +161,52 @@ copy_agentic_results() { echo "Copied $copied agentic result file(s)" } +collect_agentic_power_results() { + local job_id="$1" logs_dir="$2" source_dir="$3" workspace="$4" + local result_filename="$5" producer_sha="$6" + shift 6 + local rc=0 concurrency attempt + [[ "$#" -gt 0 ]] || return 1 + mkdir -p "$logs_dir/power" || return 1 + logs_dir="$(cd "$logs_dir" && pwd -P)" || return 1 + workspace="$(cd "$workspace" && pwd -P)" || return 1 + + # Accounting can lag squeue removal; retry only missing or nonterminal rows. + for attempt in 1 2 3; do + echo "$attempt" > "$logs_dir/power/native-job-status-attempts.txt" + sacct -X -n -P -j "$job_id" --format=JobIDRaw,State,ExitCode \ + > "$logs_dir/power/native-job-status.txt" \ + 2>> "$logs_dir/power/native-job-status.stderr" || true + if awk -F'|' -v job="$job_id" ' + $1 == job && $2 !~ /^(PENDING|RUNNING|COMPLETING)$/ { found = 1 } + END { exit !found } + ' "$logs_dir/power/native-job-status.txt"; then + break + fi + if [[ "$attempt" != "3" ]]; then sleep 5; fi + done + if ! awk -F'|' -v job="$job_id" ' + $1 == job { found = 1; if ($2 != "COMPLETED" || $3 != "0:0") failed = 1 } + END { exit (!found || failed) } + ' "$logs_dir/power/native-job-status.txt"; then + rc=1 + fi + copy_agentic_results "$source_dir" "$workspace" "$result_filename" || rc=$? + for concurrency in "$@"; do + ( + cd "$workspace" || exit 1 + python3 -m infx.results.agentic.power_adapter \ + --result-dir "$logs_dir/agentic/conc_${concurrency}" \ + --agg-result "$workspace/${result_filename}_conc${concurrency}.json" \ + --power-dir "$logs_dir/power" \ + --logs-root "$logs_dir" \ + --expected-producer-sha "$producer_sha" \ + --require-power + ) || rc=$? + done + return "$rc" +} + copy_eval_artifacts() { local eval_dir="$1" local workspace="$2" diff --git a/runners/test_kimik3_bh_power.py b/runners/test_kimik3_bh_power.py new file mode 100644 index 0000000000..e5b92fc021 --- /dev/null +++ b/runners/test_kimik3_bh_power.py @@ -0,0 +1,141 @@ +import os +from pathlib import Path +import re +import subprocess + +import pytest +import yaml + +REPO = Path(__file__).resolve().parents[1] +RECIPE_DIR = "benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" + + +@pytest.mark.parametrize("hardware", ["b200", "h200"]) +@pytest.mark.parametrize("power,wrong_head", [(True, False), (False, False), (True, True)]) +def test_kimi_power_selects_verified_runtime( + tmp_path: Path, hardware: str, power: bool, wrong_head: bool, +) -> None: + launcher_name = { + "b200": "launch_b200-nscale-slurm.sh", + "h200": "launch_h200-dgxc-slurm.sh", + }[hardware] + source = (REPO / "runners" / launcher_name).read_text() + source = source[:source.index('echo "Installing srtctl..."')] + if hardware == "h200": + source += "\nfi\n" + source = source.replace( + 'source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh"', + 'source "$TEST_SLURM_UTILS"', + ) + source = re.sub( + r'^AGENTX_POWER_SRT_SLURM_PIN="[0-9a-f]+"$', + 'AGENTX_POWER_SRT_SLURM_PIN="' + "a" * 40 + '"', + source, flags=re.MULTILINE, + ) + recipe = next((REPO / RECIPE_DIR).glob(f"agg-{hardware}*")) + data = yaml.safe_load(recipe.read_text()) + if not power: + data.pop("telemetry") + destination = tmp_path / RECIPE_DIR / recipe.name + destination.parent.mkdir(parents=True) + destination.write_text(yaml.safe_dump(data)) + recipe_ref = f"recipes/vllm/kimi-k3/agentic/{recipe.name}" + harness = ''' +set -e +function git() { + printf '%s\\n' "$*" >> "$TEST_ROUTE_LOG" + case "$1" in + clone) for target in "$@"; do :; done; mkdir -p "$target" ;; + checkout) printf '%s\\n' "$2" > .test-head ;; + rev-parse) if [[ "$TEST_WRONG_HEAD" == 1 ]]; then printf 'bad-head\\n'; else cat .test-head; fi ;; + *) return 99 ;; + esac +} +function cp() { + if [[ "$1" == -rT ]]; then + mkdir -p "$3" + command cp -R "$2"/. "$3" + else + command cp "$@" + fi +} +''' + env = dict(os.environ, GITHUB_WORKSPACE=str(tmp_path), TEST_ROUTE_LOG=str(tmp_path / "route.log"), + TEST_SLURM_UTILS=str(REPO / "runners/slurm_utils.sh"), TEST_WRONG_HEAD=str(int(wrong_head)), + IS_MULTINODE="true", IS_AGENTIC="1", MODEL_PREFIX="kimik3", MODEL="moonshotai/Kimi-K3", + PRECISION="fp4", FRAMEWORK="dynamo-vllm" if hardware == "b200" else "vllm", + CONFIG_FILE=recipe_ref, SPEC_DECODING="mtp", EVAL_ONLY="false", EVAL_FRAMEWORK="lm-eval") + result = subprocess.run(["bash"], input=harness + source, text=True, capture_output=True, cwd=tmp_path, env=env) + stamp = tmp_path / "power-producer-sha.txt" + if wrong_head: + assert result.returncode != 0 + assert not stamp.exists() + return + assert result.returncode == 0, result.stderr + routed_recipe = tmp_path / "srt-slurm" / recipe_ref + assert yaml.safe_load(routed_recipe.read_text()) == data + if power: + assert stamp.read_text().strip() == "a" * 40 + assert "edwingao28/srt-slurm.git" in (tmp_path / "route.log").read_text() + else: + assert not stamp.exists() + assert "edwingao28/srt-slurm.git" not in (tmp_path / "route.log").read_text() + + +@pytest.mark.parametrize("hardware", ["b200", "h200"]) +def test_kimi_failed_power_stages_evidence_before_exit(tmp_path: Path, hardware: str) -> None: + filename = {"b200": "launch_b200-nscale-slurm.sh", "h200": "launch_h200-dgxc-slurm.sh"}[hardware] + source = (REPO / "runners" / filename).read_text() + start = source.index('AGENTX_POWER_RC="$SRT_JOB_RC"') + end = source.index('exit "$AGENTX_POWER_RC"', start) + end = source.index("\n", end) + 1 + source = source[start:end] + "fi\n" + logs = tmp_path / "source-logs" + logs.mkdir() + for name in ("exporter-image.sha256", "power-producer-sha.txt"): + (tmp_path / name).write_text("retained\n") + harness = ''' +set -e +collect_agentic_power_results() { + mkdir -p "$2/power" + printf 'invalid telemetry\\n' > "$2/power/validation.json" + return 42 +} +bundle_server_logs() { printf 'server evidence\\n' > "$2"; } +''' + env = dict(os.environ, SRT_JOB_RC="0", USES_AGENTX_POWER="1", USES_KIMIK3_POWER="1", + USES_DCGM_POWER="1", EVAL_ONLY="false", JOB_ID="123", CONC_LIST="1", + GITHUB_WORKSPACE=str(tmp_path), LOGS_DIR=str(logs), RESULT_FILENAME="kimi-test", + SELECTED_POWER_SRT_SLURM_PIN="a" * 40) + result = subprocess.run(["bash"], input=harness + source, text=True, capture_output=True, cwd=tmp_path, env=env) + assert result.returncode == 42, result.stderr + assert (tmp_path / "LOGS/power/validation.json").read_text() == "invalid telemetry\n" + assert (tmp_path / "multinode_server_logs.tar.gz").read_text() == "server evidence\n" + + +def test_b200_exporter_cold_import_uses_one_registry_separator(tmp_path: Path) -> None: + source = (REPO / "runners/launch_b200-nscale-slurm.sh").read_text() + start = source.index("enroot_uri_for_image() {") + end = source.index('\nimport_squash "$SQUASH_FILE"', start) + helpers = source[start:end] + start = source.index('if [[ "$USES_DCGM_POWER" == "1" ]]; then\n DCGM_EXPORTER_IMAGE=') + end = source.index("\nfi", start) + len("\nfi") + harness = ''' +set -euo pipefail +flock() { :; } +unsquashfs() { test -s "$2"; } +enroot() { + printf '%s\\n' "$4" > "$GITHUB_WORKSPACE/import-uri" + printf 'exporter image\\n' > "$3" +} +sha256sum() { printf 'fixture-hash %s\\n' "$1"; } +''' + env = dict(os.environ, GITHUB_WORKSPACE=str(tmp_path), SQUASH_DIR=str(tmp_path), + SQUASH_LOCK_TIMEOUT="1", USES_DCGM_POWER="1") + result = subprocess.run(["bash"], input=harness + helpers + source[start:end], text=True, + capture_output=True, cwd=tmp_path, env=env) + assert result.returncode == 0, result.stderr + uri = (tmp_path / "import-uri").read_text().strip() + assert uri.startswith("docker://nvcr.io#nvidia/k8s/dcgm-exporter:") + assert uri.count("#") == 1 + assert (tmp_path / "exporter-image.sha256").is_file() diff --git a/utils/agentic/aggregation/test_power_lifecycle.py b/utils/agentic/aggregation/test_power_lifecycle.py index 20710e68eb..74811d2f18 100644 --- a/utils/agentic/aggregation/test_power_lifecycle.py +++ b/utils/agentic/aggregation/test_power_lifecycle.py @@ -2,6 +2,7 @@ from __future__ import annotations +import csv import json import os import re @@ -278,7 +279,9 @@ def test_signal_stops_monitor_once_without_replacing_parent_trap( start_gpu_monitor() {{ printf 'monitor-pid:%s\n' "${{BASHPID:-$$}}" >> {str(event_log)!r} }} -stop_gpu_monitor() {{ printf 'monitor-stop\n' >> {str(event_log)!r}; }} +stop_gpu_monitor() {{ + printf 'monitor-stop:%s\n' "${{AMD_MONITOR_STOP_TIMEOUT_S:-unset}}" >> {str(event_log)!r} +}} fake_replay() {{ printf 'replay-ready\n' >> {str(event_log)!r} exec sleep 30 @@ -321,7 +324,355 @@ def test_signal_stops_monitor_once_without_replacing_parent_trap( assert proc.returncode == expected_rc, stderr events = _events(tmp_path) - assert events.count("monitor-stop") == 1 + stop_events = [event for event in events if event.startswith("monitor-stop")] + # Signal teardown must stop exactly once, in abort mode: the coverage wait + # is skipped by setting AMD_MONITOR_STOP_TIMEOUT_S=0 before stopping. + assert stop_events == ["monitor-stop:0"] expected_parent_event = "parent-int" if sent_signal == signal.SIGINT else "parent-term" assert expected_parent_event in events assert events[-1] == "parent-exit" + + + +@pytest.mark.parametrize( + ("sent_signal", "expected_rc"), + [(signal.SIGINT, 130), (signal.SIGTERM, 143)], +) +def test_signal_during_amd_coverage_wait_stops_monitor( + tmp_path: Path, sent_signal: signal.Signals, expected_rc: int +): + result_dir = tmp_path / "results" + result_dir.mkdir() + event_log = tmp_path / "events.log" + script = f""" +source {str(BENCHMARK_LIB)!r} +start_gpu_monitor() {{ + GPU_METRICS_CSV="$2" + printf 'timestamp,gpu,socket_power\n1,0,500\n' > "$GPU_METRICS_CSV" + command sleep 60 >/dev/null 2>&1 & + GPU_MONITOR_PID=$! + GPU_MONITOR_VENDOR=amd + printf 'monitor:%s\nlifecycle:%s\n' "$GPU_MONITOR_PID" "${{BASHPID:-$(exec sh -c 'echo "$PPID"')}}" >> {str(event_log)!r} +}} +sleep() {{ + printf 'coverage-wait\n' >> {str(event_log)!r} + command sleep "$@" +}} +_write_amd_smi_sidecar() {{ :; }} +fake_replay() {{ :; }} +trap 'printf "parent-exit\\n" >> {str(event_log)!r}' EXIT +REPLAY_CMD=fake_replay +ENABLE_AGENTX_POWER=1 +IS_MULTINODE=false +AMD_MONITOR_STOP_TIMEOUT_S=30 +run_agentic_replay_and_write_outputs {str(result_dir)!r} +exit $? +""" + proc = subprocess.Popen( + ["bash", "-c", script], + env={**os.environ, "PATH": "/usr/bin:/bin"}, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + start_new_session=True, + ) + try: + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + events = _events(tmp_path) if event_log.exists() else [] + if "coverage-wait" in events: + break + time.sleep(0.01) + else: + pytest.fail("AMD coverage wait did not start") + + pids = dict(event.split(":") for event in events if ":" in event) + # Signal only the lifecycle shell: signalling the whole group would + # kill the monitor directly and hide a broken cleanup handler. + os.kill(int(pids["lifecycle"]), sent_signal) + _, stderr = proc.communicate(timeout=5) + assert proc.returncode == expected_rc, stderr + assert _events(tmp_path)[-1] == "parent-exit" + with pytest.raises(ProcessLookupError): + os.kill(int(pids["monitor"]), 0) + finally: + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + proc.communicate() + + +# AMDSMI 26.2.0 `metric -p -c -t -u -w 1 --csv` header (order-faithful subset, +# measured on MI355X; mirrors test_detect_columns_amd_watch_mode_real_header). +_MI355X_WATCH_HEADER = ( + "timestamp,gpu,gfx_activity,umc_activity,mm_activity,vcn_activity," + "jpeg_activity,gfx_busy_inst_xcp_0,jpeg_busy_xcp_0,vcn_busy_xcp_0," + "socket_power,gfx_voltage,soc_voltage,mem_voltage,throttle_status," + "power_management,gfx_0_clk,mem_0_clk,edge,hotspot,mem" +) + +# amd-smi quotes list-valued cells with embedded commas; the coverage helper +# must keep the power cell at its header-relative position through them. +_WATCH_ROW_FORMAT = ( + "%s,%s,0,0,N/A,\"['N/A', 'N/A']\",\"['N/A', 'N/A']\",\"[0, 0]\"," + "\"[0, 0]\",\"[0, 0]\",%s,N/A,N/A,N/A,N/A,ENABLED,1404,2000,N/A,40,25\\n" +) + + +def _bash_single_quote(text: str) -> str: + return "'" + text.replace("'", "'\\''") + "'" + + +def _run_amd_stop( + tmp_path: Path, + *, + producer_script: str, + timeout_s: int | str, + interval: int = 1, + setup_script: str = "", +) -> subprocess.CompletedProcess[str]: + """Run the real stop_gpu_monitor against a scripted AMD telemetry producer.""" + csv_path = tmp_path / "gpu_metrics.csv" + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + amd_smi = bin_dir / "amd-smi" + amd_smi.write_text("#!/bin/bash\nprintf 'gpu,total_energy_consumption\\n0,100.0\\n'\n") + amd_smi.chmod(0o755) + script = f""" +set -e +source {str(BENCHMARK_LIB)!r} +GPU_METRICS_CSV={str(csv_path)!r} +printf '%s\\n' {_bash_single_quote(_MI355X_WATCH_HEADER)} > "$GPU_METRICS_CSV" +emit_row() {{ + printf {_bash_single_quote(_WATCH_ROW_FORMAT)} "$1" "$2" "$3" >> "$GPU_METRICS_CSV" +}} +{setup_script} +( {producer_script} ) & +GPU_MONITOR_PID=$! +printf '%s\\n' "$GPU_MONITOR_PID" > {str(tmp_path / "producer.pid")!r} +GPU_MONITOR_VENDOR=amd +GPU_MONITOR_INTERVAL={interval} +AMD_MONITOR_STOP_TIMEOUT_S={timeout_s} +date +%s > {str(tmp_path / "pre.txt")!r} +stop_gpu_monitor +date +%s > {str(tmp_path / "post.txt")!r} +""" + return subprocess.run( + ["bash", "-c", script], + env={ + **os.environ, + "PATH": f"{bin_dir}:/usr/bin:/bin", + "PYTHONDONTWRITEBYTECODE": "1", + }, + capture_output=True, + text=True, + check=False, + timeout=60, + ) + + +def _min_covered_tick(csv_path: Path) -> int: + """Newest usable tick (numeric ts, power > 0) covered by every GPU.""" + newest: dict[str, float] = {} + with csv_path.open(newline="", encoding="utf-8") as f: + for row in csv.DictReader(f): + try: + timestamp = float((row.get("timestamp") or "").strip()) + power = float((row.get("socket_power") or "").strip()) + except ValueError: + continue + if timestamp > 1e12: # millisecond epoch, mirror _parse_timestamp + timestamp /= 1000.0 + gpu = (row.get("gpu") or "").strip() + if not gpu or power <= 0: + continue + newest[gpu] = max(newest.get(gpu, 0.0), timestamp) + assert newest, "no usable telemetry rows" + return int(min(newest.values())) + + +def _stop_epochs(tmp_path: Path) -> tuple[int, int]: + pre = int((tmp_path / "pre.txt").read_text().strip()) + post = int((tmp_path / "post.txt").read_text().strip()) + return pre, post + + +def _assert_producer_dead(tmp_path: Path) -> None: + producer_pid = int((tmp_path / "producer.pid").read_text().strip()) + with pytest.raises(ProcessLookupError): + os.kill(producer_pid, 0) + + +def test_amd_stop_waits_until_every_gpu_covers_stop_request(tmp_path: Path): + producer = """ +while :; do + now=$(date +%s) + emit_row "$now" 0 500 + emit_row "$now" 1 505 + sleep 0.2 +done +""" + started = time.monotonic() + result = _run_amd_stop(tmp_path, producer_script=producer, timeout_s=30) + duration = time.monotonic() - started + + assert result.returncode == 0, result.stderr + assert "never covered the stop request" not in result.stdout + result.stderr + pre, _ = _stop_epochs(tmp_path) + # Every GPU has a usable tick at/after the first whole second past stop + # entry, so any fractional window end before the stop is bracketed. + assert _min_covered_tick(tmp_path / "gpu_metrics.csv") >= pre + 1 + _assert_producer_dead(tmp_path) + assert duration < 10 + + +def test_amd_stop_ignores_degenerate_rows_for_coverage(tmp_path: Path): + setup = """ +stale=$(( $(date +%s) - 30 )) +emit_row "$stale" 0 500 +emit_row "$stale" 1 505 +""" + producer = """ +while :; do + now=$(date +%s) + emit_row "$now" 0 N/A + emit_row "$now" 1 N/A + sleep 0.2 +done +""" + result = _run_amd_stop( + tmp_path, + producer_script=producer, + timeout_s=2, + setup_script=setup, + ) + + assert result.returncode == 0, result.stderr + assert "never covered the stop request" in result.stderr + pre, post = _stop_epochs(tmp_path) + assert post - pre >= 2 + _assert_producer_dead(tmp_path) + + +def test_amd_stop_requires_coverage_per_gpu(tmp_path: Path): + setup = """ +stale=$(( $(date +%s) - 30 )) +emit_row "$stale" 1 505 +""" + producer = """ +while :; do + emit_row "$(date +%s)" 0 500 + sleep 0.2 +done +""" + result = _run_amd_stop( + tmp_path, + producer_script=producer, + timeout_s=2, + setup_script=setup, + ) + + assert result.returncode == 0, result.stderr + # GPU 1 never covers the stop request, so min-over-GPUs coverage times out + # even though GPU 0 keeps producing fresh usable ticks. + assert "never covered the stop request" in result.stderr + pre, post = _stop_epochs(tmp_path) + assert post - pre >= 2 + _assert_producer_dead(tmp_path) + + +def test_amd_stop_preserves_outputs_when_monitor_exits_during_wait(tmp_path: Path): + setup = """ +emit_row 1 0 500 +sleep() { + # End the real producer on the first coverage poll without a timing race. + kill "$GPU_MONITOR_PID" + wait "$GPU_MONITOR_PID" 2>/dev/null || true +} +""" + result = _run_amd_stop( + tmp_path, + producer_script="exec /bin/sleep 60", + timeout_s=3, + setup_script=setup, + ) + + assert result.returncode == 0, result.stderr + assert "AMD monitor exited before covering the stop request" in result.stderr + assert (tmp_path / "gpu_metrics_energy_end.csv").read_text() == ( + "gpu,total_energy_consumption\n0,100.0\n" + ) + _stop_epochs(tmp_path) # The caller continued after stop under set -e. + _assert_producer_dead(tmp_path) + + +def test_amd_stop_survives_non_integer_timeout(tmp_path: Path): + producer = """ +while :; do + now=$(date +%s) + emit_row "$now" 0 500 + emit_row "$now" 1 505 + sleep 0.2 +done +""" + started = time.monotonic() + result = _run_amd_stop(tmp_path, producer_script=producer, timeout_s="30s") + duration = time.monotonic() - started + + assert result.returncode == 0, result.stderr + # A non-integer timeout must not unwind stop_gpu_monitor via a bash + # arithmetic error (which would leak the monitor and skip tail repair + # and the energy sidecar): it warns, falls back to 30, and still waits. + assert "ignoring non-integer AMD_MONITOR_STOP_TIMEOUT_S='30s'" in result.stderr + assert "never covered the stop request" not in result.stdout + result.stderr + pre, _ = _stop_epochs(tmp_path) + assert _min_covered_tick(tmp_path / "gpu_metrics.csv") >= pre + 1 + _assert_producer_dead(tmp_path) + assert duration < 10 + + +def test_amd_stop_normalizes_millisecond_epoch_timestamps(tmp_path: Path): + producer = """ +while :; do + now=$(( $(date +%s) * 1000 + 123 )) + emit_row "$now" 0 500 + emit_row "$now" 1 505 + sleep 0.2 +done +""" + started = time.monotonic() + result = _run_amd_stop(tmp_path, producer_script=producer, timeout_s=30) + duration = time.monotonic() - started + + assert result.returncode == 0, result.stderr + # Raw millisecond epochs (~1.8e12) dwarf any second-scale target, so + # without normalization the poll would return + # instantly with zero tail coverage; mirrored _parse_timestamp + # normalization makes the poll wait for real coverage instead. + assert "never covered the stop request" not in result.stdout + result.stderr + pre, _ = _stop_epochs(tmp_path) + assert _min_covered_tick(tmp_path / "gpu_metrics.csv") >= pre + 1 + _assert_producer_dead(tmp_path) + assert duration < 10 + + +def test_amd_stop_falls_back_to_fixed_tail_for_iso_timestamps(tmp_path: Path): + producer = """ +while :; do + emit_row "$(date +%Y-%m-%dT%H:%M:%S)" 0 500 + sleep 0.2 +done +""" + started = time.monotonic() + result = _run_amd_stop(tmp_path, producer_script=producer, timeout_s=30, interval=0) + duration = time.monotonic() - started + + assert result.returncode == 0, result.stderr + assert "never covered the stop request" not in result.stdout + result.stderr + assert "exited before covering" not in result.stdout + result.stderr + pre, post = _stop_epochs(tmp_path) + # Non-epoch timestamps keep the legacy interval+2 fixed tail (one shot). + assert post - pre >= 2 + assert duration < 10 + _assert_producer_dead(tmp_path) diff --git a/utils/test_aggregate_power.py b/utils/test_aggregate_power.py index aa3bf23f97..660f6d49ae 100644 --- a/utils/test_aggregate_power.py +++ b/utils/test_aggregate_power.py @@ -574,6 +574,201 @@ def test_run_rejects_malformed_telemetry_inside_window( assert audit["reasons"] == [expected_reason] + +# AMDSMI 26.2.0 `metric -p -c -t -u -w 1 --csv` header (order-faithful subset, +# measured on MI355X; see test_detect_columns_amd_watch_mode_real_header). +_MI355X_WATCH_HEADER = ( + "timestamp,gpu,gfx_activity,umc_activity,mm_activity,vcn_activity," + "jpeg_activity,gfx_busy_inst_xcp_0,jpeg_busy_xcp_0,vcn_busy_xcp_0," + "socket_power,gfx_voltage,soc_voltage,mem_voltage,throttle_status," + "power_management,gfx_0_clk,mem_0_clk,edge,hotspot,mem" +) + + +def _mi355x_watch_row(timestamp: int, gpu: int, socket_power: str) -> str: + """One data row in the shape captured from run 32433563482 (conc1): + integer-second epoch, quoted list cells with embedded commas, N/A cells, + and a trailing carriage return.""" + return ( + f"{timestamp},{gpu},0,0,N/A,\"['N/A', 'N/A', 'N/A', 'N/A']\"," + "\"['N/A', 'N/A']\",\"[0, 0, 0, 0, 0, 0, 0, 0]\",\"[0, 0]\",\"[0, 0]\"," + f"{socket_power},N/A,N/A,N/A,N/A,ENABLED,1404,2000,N/A,40,25\r" + ) + + +def test_integrate_power_skips_na_power_rows_outside_window(tmp_path: Path): + """N/A-power teardown rows past the window end are counted, not poisonous.""" + csv = tmp_path / "gpu_metrics.csv" + base = 1_700_000_000.0 + lines = ["timestamp,gpu,socket_power,temperature"] + for offset in range(-2, 13): + for gpu in range(2): + lines.append(f"{base + offset},{gpu},500.0,65") + for gpu in range(2): + lines.append(f"{base + 11},{gpu},N/A,65") + csv.write_text("\n".join(lines) + "\n", encoding="utf-8") + + result = integrate_power( + csv, + start_unix=base, + end_unix=base + 10, + expected_num_gpus=2, + ) + + assert result.power_valid is True + assert result.invalid_reasons == () + assert result.boundary_degenerate_rows == {"0": 1, "1": 1} + assert result.total_gpu_energy_j == pytest.approx(10_000.0) + + +def test_integrate_power_does_not_bracket_with_zero_power_tail(tmp_path: Path): + """A 0 W teardown row past the window end must not fake end bracketing. + + Legacy behavior accepted the 0 W row as a valid boundary sample, corrupting + the end interpolation with a bogus value; this intentionally flips that + case to an explicit benchmark_window_not_bracketed failure.""" + csv = tmp_path / "gpu_metrics.csv" + base = 1_700_000_000.0 + lines = ["timestamp,gpu,socket_power,temperature"] + # Good rows stop at end-4; the only post-end sample per GPU has power 0. + for offset in range(-1, 7): + for gpu in range(2): + lines.append(f"{base + offset},{gpu},500.0,65") + for gpu in range(2): + lines.append(f"{base + 11},{gpu},0,65") + csv.write_text("\n".join(lines) + "\n", encoding="utf-8") + + result = integrate_power( + csv, + start_unix=base, + end_unix=base + 10, + expected_num_gpus=2, + ) + + assert result.power_valid is False + assert "benchmark_window_not_bracketed" in result.invalid_reasons + assert result.boundary_degenerate_rows == {"0": 1, "1": 1} + + +def test_integrate_power_preserves_boundary_counts_without_usable_samples(tmp_path: Path): + csv = tmp_path / "gpu_metrics.csv" + csv.write_text( + "timestamp,gpu,socket_power\n" + "1699999999,0,N/A\n" + "1700000011,0,0\n" + "1700000011,1,N/A\n", + encoding="utf-8", + ) + + result = integrate_power( + csv, + start_unix=1_700_000_000, + end_unix=1_700_000_010, + expected_num_gpus=2, + ) + + assert result.power_valid is False + assert "no_usable_power_samples" in result.invalid_reasons + assert result.boundary_degenerate_rows == {"0": 2, "1": 1} + + +def test_integrate_power_keeps_zero_power_semantics_inside_window(tmp_path: Path): + """Frozen legacy behavior: an in-window 0 W sample still integrates.""" + csv = tmp_path / "gpu_metrics.csv" + base = 1_700_000_000.0 + lines = ["timestamp,gpu,socket_power,temperature"] + for offset in range(-1, 12): + watts = "0.0" if offset == 5 else "500.0" + lines.append(f"{base + offset},0,{watts},65") + csv.write_text("\n".join(lines) + "\n", encoding="utf-8") + + result = integrate_power( + csv, + start_unix=base, + end_unix=base + 10, + expected_num_gpus=1, + ) + + assert result.power_valid is True + assert result.invalid_reasons == () + assert result.boundary_degenerate_rows == {} + # Trapezoids dip to 0 at t=5: 8 x 500 + 2 x 250 = 4500 J. + assert result.total_gpu_energy_j == pytest.approx(4_500.0) + + +def test_run_writes_boundary_degenerate_rows_to_sidecar(tmp_path: Path): + base = 1_700_000_000.0 + csv = tmp_path / "gpu_metrics.csv" + _write_constant_window_samples( + csv, + start=base, + end=base + 10, + watts_per_gpu=500.0, + num_gpus=2, + ) + bench = tmp_path / "bench.json" + agg = tmp_path / "agg.json" + validation = tmp_path / "power_validation.json" + _write_bench_result( + bench, + start=base, + end=base + 10, + duration=10.0, + total_output=2_000, + total_input=10_000, + ) + agg.write_text(json.dumps({"hw": "mi355x"}), encoding="utf-8") + + exit_code = run(csv, bench, agg, expected_num_gpus=2, validation_result=validation) + + assert exit_code == 0 + audit = json.loads(validation.read_text()) + # Present-and-empty for clean streams: readers can rely on the key. + assert audit["boundary_degenerate_rows"] == {} + + +def test_integrate_power_regression_mi355x_integer_ticks_end_gap(tmp_path: Path): + """Run-32433563482 conc1 regression: amd-smi integer-second ticks stop 4 s + before the fractional aiperf window end (last tick 1787277605 vs end + ...609.157497), so bracketing fails and the producer-side telemetry loss is + attributed as benchmark_window_not_bracketed on every GPU. + + The retrieved artifact's trailing rows all carry valid socket_power + (254-264 W) with N/A activity/voltage cells; the N/A- and 0-power teardown + rows appended past the window end are the documented synthetic degenerate + shapes, asserting they are counted rather than used for bracketing.""" + csv = tmp_path / "gpu_metrics.csv" + start = 1_787_277_560.155891 + end = 1_787_277_609.157497 + last_tick = 1_787_277_605 + powers = [259, 255, 263, 264, 256, 254, 259, 259] + lines = [_MI355X_WATCH_HEADER] + for tick in range(1_787_277_555, last_tick + 1): + for gpu in range(8): + lines.append(_mi355x_watch_row(tick, gpu, str(powers[gpu]))) + # amd-smi watch mode emits a blank line between tick groups. + lines.append("") + for gpu in range(8): + lines.append(_mi355x_watch_row(1_787_277_610, gpu, "N/A")) + for gpu in range(8): + lines.append(_mi355x_watch_row(1_787_277_611, gpu, "0")) + csv.write_text("\n".join(lines) + "\n", encoding="utf-8") + + result = integrate_power( + csv, + start_unix=start, + end_unix=end, + expected_num_gpus=8, + ) + + assert result.power_valid is False + assert result.invalid_reasons == ("benchmark_window_not_bracketed",) + assert result.device_issues == { + str(gpu): ["benchmark_window_not_bracketed"] for gpu in range(8) + } + assert result.boundary_degenerate_rows == {str(gpu): 2 for gpu in range(8)} + + def test_run_patches_agg_with_power_and_joules(tmp_path: Path): base = 1_700_000_000.0 csv = tmp_path / "gpu_metrics.csv" @@ -1433,12 +1628,12 @@ def test_power_percentiles_uses_synchronized_total_not_device_percentiles(tmp_pa def test_power_percentiles_weights_time_and_clips_the_validated_window(tmp_path): csv_path = tmp_path / "power.csv" - # Dense readings near the high end must not bias a uniform linear ramp. - _write_amd_csv(csv_path, [(0, 0, 0), (1, 0, 100), (1.9, 0, 190), (2, 0, 200)]) + # Dense readings must not bias the uniform 150-250 W ramp inside the window. + _write_amd_csv(csv_path, [(0, 0, 100), (1, 0, 200), (1.9, 0, 290), (2, 0, 300)]) result = integrate_power(csv_path, start_unix=0.5, end_unix=1.5, expected_num_gpus=1) assert result.power_valid - assert result.p75_power_w == pytest.approx(125) - assert result.p90_power_w == pytest.approx(140) + assert result.p75_power_w == pytest.approx(225) + assert result.p90_power_w == pytest.approx(240) def test_power_percentiles_is_withheld_for_invalid_telemetry(tmp_path): diff --git a/utils/test_gb300_power_official_contract.py b/utils/test_gb300_power_official_contract.py index 32f5e284d7..a12cfd36d6 100644 --- a/utils/test_gb300_power_official_contract.py +++ b/utils/test_gb300_power_official_contract.py @@ -1,6 +1,8 @@ """Exercise launcher routing and exporter imports without Slurm or network access.""" import os +import json +import sys import subprocess from collections.abc import Iterator from pathlib import Path @@ -16,13 +18,15 @@ PRODUCER_PIN = "a" * 40 -def _launcher_routing_source() -> str: +def _launcher_routing_source(launcher_name: str = "launch_gb300-nv.sh") -> str: """Extract the real clone-routing chain, not a copy of its implementation.""" - launcher = LAUNCHER_PATH.read_text() - route_start = launcher.index( - 'if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" ' - '&& $MODEL_PREFIX == "qwen3.5" ]]; then' + launcher = (REPO_ROOT / "runners" / launcher_name).read_text() + start_marker = ( + 'if [[ "$USES_AGENTX_POWER" == "1" ]]; then' + if launcher_name == "launch_gb300-nv.sh" + else 'if [[ "$IS_AGENTIC" == "1" && "$MODEL_PREFIX" == "glm5.2"' ) + route_start = launcher.index(start_marker, launcher.index('echo "Cloning srt-slurm repository..."')) route_end_marker = '\nfi\n\necho "Installing srtctl..."' route_end = launcher.index(route_end_marker, route_start) + len("\nfi") return launcher[route_start:route_end] @@ -34,15 +38,16 @@ def _write_executable(path: Path, text: str) -> None: def _run_dsv4_route( - tmp_path: Path, uses_dcgm_power: bool, *, reported_head: str = "" + tmp_path: Path, uses_dcgm_power: bool, *, reported_head: str = "", + launcher_name: str = "launch_gb300-nv.sh", model: str = "dsv4" ) -> tuple[list[str], Path, Path, Path]: """Execute only the real launcher routing region in a temporary checkout.""" workspace = tmp_path / "workspace" stub_bin = tmp_path / "bin" - source = ( - workspace - / "benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k" + recipe_directory = ( + "vllm/kimi-k3/agentic" if model == "kimik3" else "sglang/deepseek-v4/8k1k" ) + source = workspace / "benchmarks/multi_node/srt-slurm-recipes" / recipe_directory source.mkdir(parents=True) (source / "overlay-marker.txt").write_text("from-workspace\n") stub_bin.mkdir() @@ -96,7 +101,7 @@ def _run_dsv4_route( """, ) - routing = _launcher_routing_source() + routing = _launcher_routing_source(launcher_name) repo_dir = workspace / "srt-slurm-route-test" harness = tmp_path / "route.sh" harness.write_text( @@ -104,14 +109,17 @@ def _run_dsv4_route( set -eo pipefail POWER_SRT_SLURM_URL={FORK_URL} POWER_SRT_SLURM_PIN={PRODUCER_PIN} -IS_AGENTIC=0 -FRAMEWORK=dynamo-sglang -MODEL_PREFIX=dsv4 +AGENTX_POWER_SRT_SLURM_PIN={PRODUCER_PIN} +USES_AGENTX_POWER={int(model == 'kimik3' and uses_dcgm_power)} +IS_AGENTIC={int(model == 'kimik3')} +FRAMEWORK={'dynamo-vllm' if model == 'kimik3' else 'dynamo-sglang'} +MODEL_PREFIX={model} PRECISION=fp4 SPEC_DECODING= USES_DCGM_POWER={int(uses_dcgm_power)} GITHUB_WORKSPACE={workspace!s} SRT_REPO_DIR={repo_dir!s} +python3() {{ :; }} {routing} """ ) @@ -121,7 +129,7 @@ def _run_dsv4_route( env["STUB_HEAD"] = reported_head subprocess.run(["/bin/bash", str(harness)], env=env, check=True) - marker = repo_dir / "recipes/sglang/deepseek-v4/8k1k/overlay-marker.txt" + marker = repo_dir / "recipes" / recipe_directory / "overlay-marker.txt" return route_log.read_text().splitlines(), workspace, repo_dir, marker @@ -255,3 +263,75 @@ def test_gb300_dsv4_recipe_images_match_their_master_configs(): assert recipe_path.is_file(), (key, config_file) recipe_image = yaml.safe_load(recipe_path.read_text())["model"]["container"] assert recipe_image == config["image"], (key, config_file) + + +@pytest.mark.parametrize("launcher_name", ["launch_gb200-nv.sh", "launch_gb300-nv.sh"]) +def test_kimi_agentx_route_uses_custom_power_producer(tmp_path, launcher_name): + log, workspace, repo_dir, marker = _run_dsv4_route( + tmp_path, True, launcher_name=launcher_name, model="kimik3" + ) + assert f"git clone {FORK_URL} {repo_dir}" in log + assert f"git checkout {PRODUCER_PIN}" in log + assert (workspace / "power-producer-sha.txt").read_text() == f"{PRODUCER_PIN}\n" + assert marker.read_text() == "from-workspace\n" + + +@pytest.mark.parametrize("launcher_name", ["launch_gb200-nv.sh", "launch_gb300-nv.sh"]) +def test_kimi_power_route_rejects_wrong_commit(tmp_path, launcher_name): + with pytest.raises(subprocess.CalledProcessError): + _run_dsv4_route( + tmp_path, True, reported_head="b" * 40, + launcher_name=launcher_name, model="kimik3", + ) + assert not (tmp_path / "workspace/power-producer-sha.txt").exists() + + +@pytest.mark.parametrize( + ("job_state", "concurrencies", "expected_rc"), + [("COMPLETED|0:0", [4], 0), ("FAILED|1:0", [4], 1), ("COMPLETED|0:0", [4, 8], 1)], +) +def test_agentx_collection_preserves_failed_jobs_and_incomplete_sweeps( + tmp_path, job_state, concurrencies, expected_rc, +): + from utils.test_aggregate_power_multinode import build_package, RESULT_STEM + + package = build_package(tmp_path) + result_dir = package.logs_root / "agentic/conc_4" + result_dir.mkdir(parents=True) + stem = "agentic_power_concurrency_4" + package.original_result.replace(result_dir / f"{stem}.json") + old_window = package.windows_dir / f"{RESULT_STEM}.json" + window = json.loads(old_window.read_text()) + window.update(benchmark_type="custom", result_path=f"agentic/conc_4/{stem}.json") + old_window.unlink() + (package.windows_dir / f"{stem}.json").write_text(json.dumps(window)) + manifest_path = package.power_dir / "manifest.json" + manifest = json.loads(manifest_path.read_text()) + manifest["expected_windows"] = [{"benchmark_type": "custom", "concurrency": 4}] + manifest["window_validations"][0].update( + benchmark_type="custom", window_file=f"windows/{stem}.json", + ) + manifest_path.write_text(json.dumps(manifest)) + source = tmp_path / "compute-workspace" + source.mkdir() + (source / "run_conc4.json").write_text(json.dumps({ + "conc": 4, "disagg": True, "num_prefill_gpu": 2, "num_decode_gpu": 2, + })) + env = os.environ.copy() + env["PYTHONPATH"] = str(REPO_ROOT) + env["PATH"] = f"{Path(sys.executable).parent}:{env['PATH']}" + result = subprocess.run( + ["bash", "-c", + 'source "$1"; export TEST_JOB_STATE="$2"; sacct() { printf "12345|%s\\n" "$TEST_JOB_STATE"; }; ' + 'shift 2; collect_agentic_power_results "$@"', + "bash", str(REPO_ROOT / "runners/slurm_utils.sh"), job_state, + "12345", str(package.logs_root), str(source), str(tmp_path), + "run", PRODUCER_PIN, *map(str, concurrencies)], + env=env, text=True, capture_output=True, + ) + assert result.returncode == expected_rc, result.stdout + result.stderr + aggregate = json.loads((tmp_path / "run_conc4.json").read_text()) + assert aggregate["power_valid"] == 1 + assert aggregate["total_gpu_energy_j"] == 84000.0 + assert (result_dir / "power_validation.json").is_file() + assert (package.power_dir / "native-job-status.txt").read_text() == f"12345|{job_state}\n" diff --git a/utils/test_process_result.py b/utils/test_process_result.py index 485021b369..f931101edb 100644 --- a/utils/test_process_result.py +++ b/utils/test_process_result.py @@ -5,6 +5,7 @@ import signal import subprocess import sys +import time from pathlib import Path import pytest @@ -1148,8 +1149,13 @@ def test_stop_gpu_monitor_drops_truncated_row_before_final_sample(self, tmp_path final_sample, ] - def test_stop_gpu_monitor_amd_waits_one_tick_and_snapshots_energy(self, tmp_path): - """AMD stop lets the watch stream bracket the window, then snapshots energy.""" + def test_stop_gpu_monitor_amd_covers_stop_request_and_snapshots_energy(self, tmp_path): + """AMD stop returns once telemetry covers the stop entry, then snapshots energy. + + A usable tick stamped past the stop request satisfies the coverage + poll on its first pass: the legacy fixed tail sleep never runs, the + stream is not mutated, and the end-side accumulator snapshot is + written.""" fake_bin = tmp_path / "bin" fake_bin.mkdir() args_log = tmp_path / "amd_args.txt" @@ -1161,7 +1167,8 @@ def test_stop_gpu_monitor_amd_waits_one_tick_and_snapshots_energy(self, tmp_path ) fake_amd_smi.chmod(0o755) sleep_log = tmp_path / "sleep_args.txt" - contents = "timestamp,gpu,socket_power\n1785881113,0,238\n" + covered_tick = int(time.time()) + 30 + contents = f"timestamp,gpu,socket_power\n{covered_tick},0,238\n" metrics = tmp_path / "gpu_metrics.csv" metrics.write_text(contents) benchmark_lib = Path(__file__).parents[1] / "benchmarks/benchmark_lib.sh" @@ -1169,7 +1176,7 @@ def test_stop_gpu_monitor_amd_waits_one_tick_and_snapshots_energy(self, tmp_path source {str(benchmark_lib)!r} kill() {{ return 0; }} wait() {{ return 0; }} -sleep() {{ printf '%s\\n' "$1" > {str(sleep_log)!r}; }} +sleep() {{ printf '%s\\n' "$1" >> {str(sleep_log)!r}; }} GPU_MONITOR_PID=999 GPU_MONITOR_VENDOR=amd GPU_MONITOR_INTERVAL=3 @@ -1190,7 +1197,8 @@ def test_stop_gpu_monitor_amd_waits_one_tick_and_snapshots_energy(self, tmp_path ) assert result.returncode == 0, result.stderr - assert sleep_log.read_text().strip() == "5" + assert not sleep_log.exists() + assert "never covered the stop request" not in result.stderr assert metrics.read_text() == contents assert "metric -E --csv" in args_log.read_text() energy_end = tmp_path / "gpu_metrics_energy_end.csv" @@ -1219,6 +1227,7 @@ def test_stop_gpu_monitor_amd_drops_truncated_row_without_append(self, tmp_path) GPU_MONITOR_PID=999 GPU_MONITOR_VENDOR=amd GPU_METRICS_CSV={str(metrics)!r} +AMD_MONITOR_STOP_TIMEOUT_S=0 stop_gpu_monitor """ env = { From 2f3d637f62d85bccc9c9b41fc5c05eca5039796c Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Fri, 11 Sep 2026 19:05:32 -0700 Subject: [PATCH 2/2] fix: replace unavailable Kimi-K3 B300 image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将已下架的 5894fdf 镜像替换为可获取的 3696c77 构建，并追加性能变更记录。此提交仅修复镜像引用；DSXE RDMA 兼容性及端到端功耗验证仍待完成。 --- configs/nvidia-master.yaml | 6 +++--- perf-changelog.yaml | 9 +++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 478aa82497..ce65fcbe4f 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1485,9 +1485,9 @@ kimik3-fp4-b300-vllm-agentic-dspark: # kimik3_dspark_probabilistic_sample_method_block_rejection_sample_method.yaml), # and EVAL_ONLY switches to real block verification. # - # The image carries the Kimi-K3 DCP, DSpark-under-DCP and Mooncake-hybrid - # changes (vllm-project/vllm agentx-k3 @ 5894fdf). - image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-5894fdf + # Kimi-K3 runtime from vllm-project/vllm @ 3696c772aae308f2420a8f307b0971e6986c4818. + # This replaces the unavailable 5894fdf build; new results use a new runtime baseline. + image: vllm/vllm-openai:nightly-dev-x86_64-cu13-3696c77 model: moonshotai/Kimi-K3 model-prefix: kimik3 runner: cluster:b300-dsxe diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 8cce2b53f7..aeb86ce946 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7358,3 +7358,12 @@ - "为 B200、H200、GB200 和 GB300 启用 Kimi-K3 AgentX 必需功耗采集、精确测量窗口验证及失败任务审计产物。" - "AMD 遥测覆盖完整窗口后再收尾；使用现有 B300 集群标签，并修正 B200 的 DCP/Offload 元数据以匹配实际服务命令。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3030 + +- config-keys: + - kimik3-fp4-b300-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Replace the unavailable B300 Kimi-K3 vLLM 5894fdf image with build 3696c77. This changes the runtime baseline; DSXE RDMA compatibility and end-to-end power qualification remain pending." + - "将 B300 Kimi-K3 不可用的 vLLM 5894fdf 镜像替换为 3696c77 构建。运行时基线随之改变；DSXE RDMA 兼容性及端到端功耗验证尚未完成。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3030