[PowerX] enable Kimi-K3 AgentX power backfills / 启用功耗补测

benchmarks/benchmark_lib.sh

@@ -153,8 +153,12 @@ unset _benchmark_caller
 # --------------------------------
 
 GPU_MONITOR_PID=""
+GPU_MONITOR_SOURCE_PID=""
+GPU_MONITOR_PIPE=""
 GPU_MONITOR_VENDOR=""
 GPU_MONITOR_INTERVAL=1
+# Bounded wait for AMD telemetry to cover a stop request; 0 skips the wait.
+AMD_MONITOR_STOP_TIMEOUT_S="${AMD_MONITOR_STOP_TIMEOUT_S:-30}"
 GPU_METRICS_CSV="${GPU_METRICS_CSV:-gpu_metrics.csv}"
 NVIDIA_GPU_MONITOR_QUERY="timestamp,index,power.draw,temperature.gpu,clocks.current.sm,clocks.current.memory,utilization.gpu,utilization.memory"
 export GPU_METRICS_CSV
@@ -196,8 +200,15 @@ start_gpu_monitor() {
         # Python; measured on MI355X: trailing ticks were lost at kill without it).
         # Pipe through awk to: skip preamble lines, keep first CSV header, skip repeated
         # headers, and flush every row so killing the pipe cannot discard buffered samples.
-        PYTHONUNBUFFERED=1 amd-smi metric -p -c -t -u -w "$interval" --csv 2>/dev/null \
-            | awk '/^timestamp,/{if(!h){print;h=1};next} h{print;fflush()}' > "$output" &
+        # Track both processes: killing only awk can leave amd-smi alive until
+        # its next write. Keep the FIFO beside this run's raw CSV, never shared.
+        GPU_MONITOR_PIPE="${output}.pipe.$$"
+        mkfifo "$GPU_MONITOR_PIPE" || return 1
+        PYTHONUNBUFFERED=1 amd-smi metric -p -c -t -u -w "$interval" --csv \
+            > "$GPU_MONITOR_PIPE" 2>/dev/null &
+        GPU_MONITOR_SOURCE_PID=$!
+        awk '/^timestamp,/{if(!h){print;h=1};next} h{print;fflush()}' \
+            < "$GPU_MONITOR_PIPE" > "$output" &
         GPU_MONITOR_PID=$!
         # Hardware energy-accumulator + identity snapshots; the end-side twin in
         # stop_gpu_monitor lets auditors cross-check the integrated energy
@@ -215,19 +226,21 @@ start_gpu_monitor() {
 # Stop the background GPU monitor and report file size.
 stop_gpu_monitor() {
     if [[ -n "$GPU_MONITOR_PID" ]] && kill -0 "$GPU_MONITOR_PID" 2>/dev/null; then
-        # benchmark_end_time_unix is recorded shortly before the benchmark
-        # process exits, so the stream must cover one more sample past it for
-        # deterministic boundary interpolation. NVIDIA appends a one-shot
-        # post-exit sample below; amd-smi one-shot CSV has no timestamp column,
-        # so the AMD path instead lets the watch stream emit final ticks before
-        # the kill. Two extra intervals: amd-smi stamps integer seconds, so a
-        # tick in the same second as the window end still fails bracketing —
-        # the stream needs a tick at the NEXT whole second (measured on MI355X:
-        # end=...153.325 vs last sample ...153.0).
+        # The aggregator requires, for every GPU, a usable sample stamped at or
+        # after the (fractional) benchmark window end, which is always <= the
+        # wall clock when this stop runs. NVIDIA appends a one-shot post-exit
+        # sample below; amd-smi one-shot CSV has no timestamp column, so the
+        # AMD path polls the output file until every GPU's watch stream shows
+        # a usable tick at the next whole second — amd-smi stamps integer
+        # seconds, so that tick strictly covers any fractional window end
+        # (measured on MI355X: end=...609.157 vs last sample ...605). Observing
+        # the file rather than sleeping also defeats pipe-buffer loss when the
+        # awk consumer is killed: covered rows are already on disk.
         if [[ "$GPU_MONITOR_VENDOR" == "amd" ]]; then
-            sleep $(( ${GPU_MONITOR_INTERVAL:-1} + 2 ))
+            _wait_for_amd_stop_coverage
         fi
-        kill "$GPU_MONITOR_PID" 2>/dev/null
+        # The monitor may exit during the coverage wait; still finish cleanup.
+        kill "$GPU_MONITOR_PID" 2>/dev/null || true
         wait "$GPU_MONITOR_PID" 2>/dev/null || true
         case "$GPU_MONITOR_VENDOR" in
             nvidia)
@@ -249,6 +262,13 @@ stop_gpu_monitor() {
             echo "[GPU Monitor] Collected $lines rows -> $GPU_METRICS_CSV"
         fi
     fi
+    if [[ -n "$GPU_MONITOR_SOURCE_PID" ]]; then
+        kill "$GPU_MONITOR_SOURCE_PID" 2>/dev/null || true
+        wait "$GPU_MONITOR_SOURCE_PID" 2>/dev/null || true
+    fi
+    [[ -z "$GPU_MONITOR_PIPE" ]] || rm -f "$GPU_MONITOR_PIPE"
+    GPU_MONITOR_SOURCE_PID=""
+    GPU_MONITOR_PIPE=""
     GPU_MONITOR_PID=""
     GPU_MONITOR_VENDOR=""
 }
@@ -271,6 +291,109 @@ _repair_truncated_gpu_metrics_tail() {
     return 0
 }
 
+# Print the newest telemetry tick (whole epoch seconds) that EVERY observed
+# GPU has covered with a usable sample (numeric epoch timestamp, numeric
+# power > 0), or nothing when the stream holds no usable epoch-stamped row
+# (e.g. an amd-smi build emitting ISO timestamps). Column detection mirrors
+# _POWER_COL_RE/_POWER_EXCLUDE_RE/_GPU_INDEX_COL_RE in utils/aggregate_power.py.
+# POSIX awk only: the ROCm container images ship mawk/busybox awk.
+_amd_monitor_min_covered_tick() {
+    [[ -f "$GPU_METRICS_CSV" ]] || return 0
+    awk -F, '
+        NR == 1 {
+            for (i = 1; i <= NF; i++) {
+                name = tolower($i)
+                gsub(/^ +| +$/, "", name)
+                sub(/\r$/, "", name)
+                if (!power_col && name ~ /power/ && name !~ /limit|cap|max|min/)
+                    power_col = i
+                if (!gpu_col && name ~ /^(index|gpu|gpu_id|gpu_index|card|device)$/)
+                    gpu_col = i
+            }
+            next
+        }
+        !power_col || !gpu_col { next }
+        {
+            # amd-smi quotes list-valued cells that embed commas; neutralize
+            # them so the power cell keeps its header-relative position.
+            line = $0
+            sub(/\r$/, "", line)
+            if (line ~ /"/) {
+                n = split(line, seg, /"/)
+                line = ""
+                for (i = 1; i <= n; i++) {
+                    if (i % 2 == 0) gsub(/,/, ";", seg[i])
+                    line = line seg[i]
+                }
+            }
+            count = split(line, cell, /,/)
+            if (count < power_col || count < gpu_col) next
+            if (cell[1] !~ /^[0-9]+(\.[0-9]+)?$/) next
+            if (cell[power_col] !~ /^[0-9]+(\.[0-9]+)?$/) next
+            if (cell[power_col] + 0 <= 0) next
+            if (cell[gpu_col] == "") next
+            ts = cell[1] + 0
+            # Mirror _parse_timestamp in utils/aggregate_power.py: normalize
+            # millisecond epochs so a ms-stamping amd-smi build cannot
+            # trivially satisfy any second-scale stop target.
+            if (ts > 1e12) ts /= 1000
+            gpu = cell[gpu_col]
+            if (!(gpu in newest) || ts > newest[gpu])
+                newest[gpu] = ts
+        }
+        END {
+            have = 0
+            for (gpu in newest)
+                if (!have || newest[gpu] < min) { min = newest[gpu]; have = 1 }
+            if (have) printf "%d\n", min
+        }
+    ' "$GPU_METRICS_CSV" 2>/dev/null
+    return 0
+}
+
+# Block until every observed GPU has a usable tick at/after the first whole
+# second past stop entry, so any window end preceding the stop request is
+# bracketed on file. Bounded by AMD_MONITOR_STOP_TIMEOUT_S; always returns 0 —
+# on timeout or early monitor death it warns and lets aggregation attribute
+# the missing coverage (fail-safe, never fail-silent).
+_wait_for_amd_stop_coverage() {
+    local target deadline covered timeout_s
+    # A non-integer timeout (e.g. "30s") would abort the whole stop_gpu_monitor
+    # call under `set -e` at the arithmetic below, leaking the monitor process
+    # and skipping tail repair + the energy sidecar; fall back to the default.
+    timeout_s="${AMD_MONITOR_STOP_TIMEOUT_S:-30}"
+    if [[ ! "$timeout_s" =~ ^-?[0-9]+$ ]]; then
+        echo "[GPU Monitor] Warning: ignoring non-integer AMD_MONITOR_STOP_TIMEOUT_S='$timeout_s', using 30" >&2
+        timeout_s=30
+    fi
+    if [[ "$timeout_s" -le 0 ]]; then
+        return 0
+    fi
+    target=$(( $(date +%s) + 1 ))
+    deadline=$(( target + timeout_s ))
+    while :; do
+        covered=$(_amd_monitor_min_covered_tick)
+        if [[ -z "$covered" ]]; then
+            # Non-epoch timestamps or an unusable stream: keep the legacy
+            # fixed tail so older amd-smi builds behave exactly as before.
+            sleep $(( ${GPU_MONITOR_INTERVAL:-1} + 2 ))
+            return 0
+        fi
+        if [[ "$covered" -ge "$target" ]]; then
+            return 0
+        fi
+        if ! _background_process_is_running "$GPU_MONITOR_PID"; then
+            echo "[GPU Monitor] Warning: AMD monitor exited before covering the stop request (covered=$covered target=$target)" >&2
+            return 0
+        fi
+        if [[ "$(date +%s)" -ge "$deadline" ]]; then
+            echo "[GPU Monitor] Warning: AMD telemetry never covered the stop request within ${timeout_s}s (covered=$covered target=$target)" >&2
+            return 0
+        fi
+        sleep 1
+    done
+}
+
 # Write one best-effort amd-smi snapshot; remove the file rather than keep a
 # partial one when the invocation fails.
 _write_amd_smi_sidecar() {
@@ -3284,9 +3407,15 @@ run_agentic_replay_and_write_outputs() (
     esac
 
     _stop_agentx_power_monitor() {
+        local mode="${1:-}"
         if [ "$agentx_monitor_stopped" = "0" ]; then
-            agentx_monitor_stopped=1
+            if [ "$mode" = "abort" ]; then
+                # A cancelled run's power validity is moot; skip the AMD
+                # coverage wait so signal teardown stays fast.
+                AMD_MONITOR_STOP_TIMEOUT_S=0
+            fi
             stop_gpu_monitor
+            agentx_monitor_stopped=1
         fi
     }
 
@@ -3330,10 +3459,11 @@ run_agentic_replay_and_write_outputs() (
         agentx_monitor_stopped=0
         # This function runs in a subshell, so these handlers cannot replace
         # launcher-owned traps. The stopped flag keeps explicit and signal/EXIT
-        # cleanup idempotent.
-        trap '_stop_agentx_power_monitor' EXIT
-        trap '_stop_agentx_power_monitor; exit 130' INT
-        trap '_stop_agentx_power_monitor; exit 143' TERM
+        # cleanup idempotent after stopping completes. If a signal interrupts
+        # the normal coverage wait, abort cleanup must still kill the monitor.
+        trap '_stop_agentx_power_monitor abort' EXIT
+        trap '_stop_agentx_power_monitor abort; exit 130' INT
+        trap '_stop_agentx_power_monitor abort; exit 143' TERM
     fi
 
     echo "$REPLAY_CMD" > "$result_dir/benchmark_command.txt"

benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c1-agentic.yaml

@@ -111,14 +111,27 @@ sbatch_directives:
 srun_options:
   container-remap-root: ""
 
+telemetry:
+  enabled: true
+  provider: dcgm-power
+  default_frequency: 1.0
+  storage_subdir: power
+  required: true
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 12
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    port: 9401
+
 benchmark:
   type: custom
+  concurrencies: [1]
   command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
   env:
     INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
     AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300"
     AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25"
-    AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics"
     AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:"
     RESULT_DIR: "/logs/agentic"
     PORT: "8000"

benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c14-agentic.yaml

@@ -111,14 +111,27 @@ sbatch_directives:
 srun_options:
   container-remap-root: ""
 
+telemetry:
+  enabled: true
+  provider: dcgm-power
+  default_frequency: 1.0
+  storage_subdir: power
+  required: true
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 12
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    port: 9401
+
 benchmark:
   type: custom
+  concurrencies: [14]
   command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
   env:
     INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
     AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300"
     AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25"
-    AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics"
     AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:"
     RESULT_DIR: "/logs/agentic"
     PORT: "8000"

benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c24-agentic.yaml

@@ -111,14 +111,27 @@ sbatch_directives:
 srun_options:
   container-remap-root: ""
 
+telemetry:
+  enabled: true
+  provider: dcgm-power
+  default_frequency: 1.0
+  storage_subdir: power
+  required: true
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 12
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    port: 9401
+
 benchmark:
   type: custom
+  concurrencies: [24]
   command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
   env:
     INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
     AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300"
     AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25"
-    AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics"
     AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:"
     RESULT_DIR: "/logs/agentic"
     PORT: "8000"

benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c4-agentic.yaml

@@ -111,14 +111,27 @@ sbatch_directives:
 srun_options:
   container-remap-root: ""
 
+telemetry:
+  enabled: true
+  provider: dcgm-power
+  default_frequency: 1.0
+  storage_subdir: power
+  required: true
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 12
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    port: 9401
+
 benchmark:
   type: custom
+  concurrencies: [4]
   command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
   env:
     INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
     AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300"
     AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25"
-    AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics"
     AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:"
     RESULT_DIR: "/logs/agentic"
     PORT: "8000"

benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c48-agentic.yaml

@@ -111,14 +111,27 @@ sbatch_directives:
 srun_options:
   container-remap-root: ""
 
+telemetry:
+  enabled: true
+  provider: dcgm-power
+  default_frequency: 1.0
+  storage_subdir: power
+  required: true
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 12
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    port: 9401
+
 benchmark:
   type: custom
+  concurrencies: [48]
   command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
   env:
     INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
     AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300"
     AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25"
-    AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics"
     AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:"
     RESULT_DIR: "/logs/agentic"
     PORT: "8000"

benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c8-agentic.yaml

@@ -111,14 +111,27 @@ sbatch_directives:
 srun_options:
   container-remap-root: ""
 
+telemetry:
+  enabled: true
+  provider: dcgm-power
+  default_frequency: 1.0
+  storage_subdir: power
+  required: true
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 12
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    port: 9401
+
 benchmark:
   type: custom
+  concurrencies: [8]
   command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
   env:
     INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
     AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300"
     AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25"
-    AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics"
     AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:"
     RESULT_DIR: "/logs/agentic"
     PORT: "8000"