[PowerX] separate native multinode collector contract / 拆分原生多节点采集契约

.github/workflows/benchmark-multinode-tmpl.yml

@@ -539,6 +539,7 @@ jobs:
             agg_${{ env.RESULT_FILENAME }}_*.json
             power_validation_${{ env.RESULT_FILENAME }}_*.json
             LOGS/power/**
+            LOGS/native_power/**
             result_processing_${{ env.RESULT_FILENAME }}.json
             LOGS/*/results_*.json
             LOGS/agentic/**/agentic_power_concurrency_*.json

.github/workflows/benchmark-tmpl.yml

@@ -449,11 +449,13 @@ jobs:
           name: ${{ inputs.eval-only && 'eval_gpu_metrics_' || 'gpu_metrics_' }}${{ env.RESULT_FILENAME }}
           path: |
             gpu_metrics.csv
+            gpu_metrics*_context.json
             gpu_metrics_energy_start.csv
             gpu_metrics_energy_end.csv
             gpu_metrics_identity.json
             gpu_metrics_identity.csv
             results/gpu_metrics*.csv
+            results/gpu_metrics*_context.json
             results/gpu_metrics_identity.json
           if-no-files-found: ignore
 
@@ -466,12 +468,14 @@ jobs:
             ${{ env.RESULT_FILENAME }}.json
             agg_${{ env.RESULT_FILENAME }}.json
             gpu_metrics.csv
+            gpu_metrics*_context.json
             gpu_metrics_energy_start.csv
             gpu_metrics_energy_end.csv
             gpu_metrics_identity.json
             gpu_metrics_identity.csv
             power_validation_${{ env.RESULT_FILENAME }}.json
             results/gpu_metrics*.csv
+            results/gpu_metrics*_context.json
             results/gpu_metrics_identity.json
             results/agentic_power_window.json
             results/agentic_power_timezone_offset.txt

.github/workflows/test-process-result.yml

@@ -10,6 +10,11 @@ on:
       - '.github/workflows/e2e-tests.yml'
       - '.github/workflows/test-process-result.yml'
       - 'benchmarks/benchmark_lib.sh'
+      - 'benchmarks/native_power_collect.sh'
+      - 'benchmarks/native_power_lifecycle.sh'
+      - 'runners/test_native_collector_barriers.py'
+      - 'runners/test_native_collector_receipts.py'
+      - 'utils/test_native_multinode_power.py'
       - 'benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/**/*.yaml'
       - 'benchmarks/multi_node/srt-slurm-recipes/vllm/deepseek-v4/**/*.yaml'
       - 'benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k2.6/b200-fp4/**/*.yaml'
@@ -60,7 +65,7 @@ jobs:
         run: |
           cd utils
           uv run --no-project --exclude-newer PT12H --python 3.12 --with pytest --with pyyaml \
-            python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py agentic/aggregation/ test_gb300_power_official_contract.py test_inject_srt_power_concurrencies.py test_process_result.py ../runners/test_kimik3_bh_power.py -v
+            python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py agentic/aggregation/ test_gb300_power_official_contract.py test_inject_srt_power_concurrencies.py test_process_result.py test_native_multinode_power.py ../runners/test_native_collector_barriers.py ../runners/test_native_collector_receipts.py ../runners/test_kimik3_bh_power.py -v
 
       - name: Test serving client result persistence
         run: |

benchmarks/native_power_collect.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+# One native SMI collector per serving node. Raw files stay on node-local scratch;
+# the launcher stages them as its host user after containers stop.
+set -uo pipefail
+power_dir=$1
+control_dir=$2
+vendor=$3
+rank=$4
+role=$5
+gpu_indices=$6
+num_nodes=$7
+repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
+export TZ=UTC
+export PYTHONPATH="$repo_root${PYTHONPATH:+:$PYTHONPATH}"
+source "$repo_root/benchmarks/benchmark_lib.sh"
+mkdir -p "$power_dir"
+collector_rc=0
+finished=0
+
+write_control() {
+    local path="$control_dir/$1"
+    local pending="$path.tmp"
+    printf '%s\n' "$2" > "$pending" || return
+    # The control directory is created by the host user before Docker starts.
+    # Containers must not strand root-owned files in the shared runner tree.
+    if [[ -n "${POWERX_HOST_UID:-}" && -n "${POWERX_HOST_GID:-}" ]]; then
+        chown "$POWERX_HOST_UID:$POWERX_HOST_GID" "$pending" || return
+    fi
+    mv -f "$pending" "$path"
+}
+
+finish() {
+    local incoming_rc=$?
+    [[ "$finished" == 0 ]] || return
+    if [[ "$incoming_rc" != 0 ]]; then collector_rc=$incoming_rc; fi
+    if ! _background_process_is_running "${GPU_MONITOR_PID:-}"; then collector_rc=1; fi
+    stop_gpu_monitor
+    if [[ "$vendor" == amd ]]; then
+        amd-smi list --json > "$power_dir/gpu_metrics_devices_end.json" || collector_rc=1
+    else
+        nvidia-smi --query-gpu=index,uuid,pci.bus_id,name,driver_version --format=csv \
+            > "$power_dir/gpu_metrics_identity_end.csv" || collector_rc=1
+    fi
+    python3 -m infx.results.power.native_multinode end --directory "$power_dir" \
+        --collector-exit-code "$collector_rc" || collector_rc=1
+    if [[ -n "${POWERX_HOST_UID:-}" && -n "${POWERX_HOST_GID:-}" ]]; then
+        chown -R "$POWERX_HOST_UID:$POWERX_HOST_GID" "$power_dir" || collector_rc=1
+    fi
+    write_control "done-$rank" "$collector_rc"
+    finished=1
+}
+trap finish EXIT
+trap 'collector_rc=130; AMD_MONITOR_STOP_TIMEOUT_S=0; exit 130' INT
+trap 'collector_rc=143; AMD_MONITOR_STOP_TIMEOUT_S=0; exit 143' TERM HUP
+
+case "${POWERX_CLOCK_SYNCHRONIZED:-false}" in
+    yes|true) clock_synchronized=true ;;
+    *) clock_synchronized=false ;;
+esac
+
+python3 -m infx.results.power.native_multinode begin --directory "$power_dir" \
+    --vendor "$vendor" --rank "$rank" --role "$role" --gpu-indices "$gpu_indices" \
+    --num-nodes "$num_nodes" --clock-synchronized "$clock_synchronized" || exit 1
+printf '{"timestamp_timezone":"UTC"}\n' > "$power_dir/gpu_metrics_context.json" || exit 1
+start_gpu_monitor --output "$power_dir/gpu_metrics.csv" || exit 1
+[[ "$GPU_MONITOR_VENDOR" == "$vendor" ]] || exit 1
+if [[ "$vendor" == amd ]]; then
+    _write_amd_smi_sidecar "$power_dir/gpu_metrics_devices.json" list --json
+fi
+_background_process_is_running "$GPU_MONITOR_PID" || exit 1
+write_control "ready-$rank" ready
+while [[ ! -f "$control_dir/stop" ]]; do
+    _background_process_is_running "$GPU_MONITOR_PID" || exit 1
+    sleep 1 &
+    wait $! || true
+done

benchmarks/native_power_lifecycle.sh

@@ -0,0 +1,81 @@
+#!/usr/bin/env bash
+# Shared barriers for native collectors. Launchers own host scratch and mounts.
+
+powerx_start_collector() {
+    local power_dir="$1" control_dir="$2" vendor="$3" rank="$4" role="$5" gpus="$6" nodes="$7"
+    local indices
+    indices=$(seq -s, 0 "$((gpus - 1))") || return 1
+    POWERX_CONTROL_DIR="$control_dir"
+    POWERX_NUM_NODES="$nodes"
+    bash "$(dirname "${BASH_SOURCE[0]}")/native_power_collect.sh" \
+        "$power_dir" "$control_dir" "$vendor" "$rank" "$role" "$indices" "$nodes" &
+    POWERX_COLLECTOR_PID=$!
+}
+
+powerx_write_control() {
+    local path="$POWERX_CONTROL_DIR/$1"
+    local pending="$path.tmp"
+    printf '%s\n' "$2" > "$pending" || return
+    if [[ -n "${POWERX_HOST_UID:-}" && -n "${POWERX_HOST_GID:-}" ]]; then
+        chown "$POWERX_HOST_UID:$POWERX_HOST_GID" "$pending" || return
+    fi
+    mv -f "$pending" "$path"
+}
+
+powerx_wait_collectors() {
+    local phase="$1" deadline=$((SECONDS + ${POWERX_BARRIER_TIMEOUT_S:-60})) rank pending failed
+    while :; do
+        pending=0
+        failed=0
+        for ((rank=0; rank<POWERX_NUM_NODES; rank++)); do
+            if [[ "$phase" == ready && -f "$POWERX_CONTROL_DIR/done-$rank" ]]; then
+                echo "PowerX collector $rank stopped before benchmark readiness" >&2
+                return 1
+            fi
+            if [[ ! -f "$POWERX_CONTROL_DIR/$phase-$rank" ]]; then
+                pending=1
+            elif [[ "$phase" == done && "$(cat "$POWERX_CONTROL_DIR/done-$rank")" != 0 ]]; then
+                failed=1
+            fi
+        done
+        if [[ "$pending" == 0 ]]; then
+            [[ "$failed" == 0 ]] || echo "One or more PowerX collectors failed" >&2
+            return "$failed"
+        fi
+        if (( SECONDS >= deadline )); then
+            echo "Timed out waiting for PowerX $phase receipts" >&2
+            return 1
+        fi
+        sleep 1
+    done
+}
+
+powerx_stop_collectors() {
+    local rc=0
+    powerx_write_control stop stop || rc=$?
+    powerx_wait_collectors done || rc=$?
+    powerx_reap_collector || rc=$?
+    return "$rc"
+}
+
+powerx_reap_collector() {
+    [[ -n "${POWERX_COLLECTOR_PID:-}" ]] || return 0
+    local deadline=$((SECONDS + ${POWERX_BARRIER_TIMEOUT_S:-60})) rc=0
+    while kill -0 "$POWERX_COLLECTOR_PID" 2>/dev/null; do
+        if (( SECONDS >= deadline )); then
+            kill -TERM "$POWERX_COLLECTOR_PID" 2>/dev/null || true
+            # The shared AMD monitor drains for three seconds before writing receipts.
+            local grace_deadline=$((SECONDS + 5))
+            while kill -0 "$POWERX_COLLECTOR_PID" 2>/dev/null && (( SECONDS < grace_deadline )); do
+                sleep 1
+            done
+            kill -KILL "$POWERX_COLLECTOR_PID" 2>/dev/null || true
+            rc=1
+            break
+        fi
+        sleep 1
+    done
+    wait "$POWERX_COLLECTOR_PID" || rc=$?
+    POWERX_COLLECTOR_PID=""
+    return "$rc"
+}

docs/results-and-ingestion.md

@@ -108,6 +108,12 @@ The PR changelog selects representative NVIDIA and AMD coverage, not an exhausti
 
 Processing and diagnostic power-audit uploads run after launcher or validation failure, retaining raw and aggregate JSON. Normal `bmk_*` upload requires successful benchmark and processing steps, so an incomplete batch or failed Slurm job does not publish diagnostic rows. The main-branch ingest trigger can still publish other successful configurations from a partially failed sweep; it does not establish complete fleet coverage. Downstream importers can use the retained outcome to reject explicitly failed benchmarks.
 
+### Native multinode telemetry
+
+`native_power_collect.sh` and `native_power_lifecycle.sh` provide per-node collection and bounded ready/stop receipts. Launchers opt into the native package under `LOGS/native_power`; this prerequisite enables no new recipe. The adapter validates serving GPU identity, synchronized clocks, collector completion, and complete formal-window coverage. It preserves per-node failures, sample counts, and collector revision in the audit.
+
+The native collector sets UTC and records context beside its CSV for portable replay; existing benchmark monitors keep their current behavior. Its launcher integration requires separate hardware qualification. The offline adapter accepts this context without changing producers. Unusable samples outside the formal window do not establish coverage; `boundary_degenerate_rows` retains their per-GPU counts.
+
 ## Eval artifacts
 
 ### Per-config identity and collection

docs/results-and-ingestion_zh.md

@@ -108,6 +108,12 @@ PR changelog 选择具有代表性的 NVIDIA 和 AMD 覆盖，并非所有受影
 
 启动器或验证失败后仍会运行处理和功耗诊断上传，并在审计工件中保留原始及聚合 JSON。正常 `bmk_*` 上传要求基准和处理步骤成功，因此不完整批次或 Slurm 失败不会发布诊断数据。主分支的入库触发器仍可发布部分失败 sweep 中其他成功配置的数据；这并不证明整个硬件范围已完成覆盖。下游导入器可利用保留的状态拒绝明确失败的基准结果。
 
+### 原生多节点遥测
+
+`native_power_collect.sh` 和 `native_power_lifecycle.sh` 提供每节点采集及有时限的就绪/停止状态文件。启动器可使用 `LOGS/native_power` 下的原生产物；此前置改动不会启用新 recipe。适配器验证服务 GPU 身份、时钟同步、采集完成及正式窗口完整覆盖，并在审计中保留节点故障、样本数和采集器版本。
+
+原生采集器单独设置 UTC，并在 CSV 旁记录上下文以支持跨环境回放；现有基准监控行为保持不变。启动器接入需要另行完成硬件验证。离线适配器接受该上下文，不改变现有生产端。正式窗口外的无效样本不能构成覆盖；`boundary_degenerate_rows` 保留其逐 GPU 计数。
+
 ## 评测工件
 
 ### 单配置身份和收集

infx/results/fixed_sequence.py

@@ -248,6 +248,21 @@ def aggregate_power_result(
         expected_num_gpus = int(env['TP']) * int(env.get('PP_SIZE', '1')) * int(env.get('PCP_SIZE', '1'))
     try:
         if is_multinode:
+            native_dir = Path(env.get('POWERX_NATIVE_DIR', 'LOGS/native_power'))
+            if env.get('POWERX_NATIVE_DIR') or native_dir.is_dir():
+                if source.is_dir() and source != native_dir:
+                    raise ValueError('Both native and SRT power packages are present')
+                source = native_dir
+                from .power.native_multinode import run
+
+                return run(
+                    native_dir, bench_path, agg_path,
+                    expected_prefill_gpus=prefill_gpus,
+                    expected_decode_gpus=decode_gpus,
+                    expected_aggregate_gpus=aggregate_gpus,
+                    validation_result=validation_path,
+                    require_power=require_power,
+                )
             from .power.multinode import run
 
             return run(