diff --git a/.github/workflows/test-process-result.yml b/.github/workflows/test-process-result.yml index 4d4cf0adf4..e8f6259f12 100644 --- a/.github/workflows/test-process-result.yml +++ b/.github/workflows/test-process-result.yml @@ -25,6 +25,8 @@ on: - 'runners/launch_b300-dsxe.sh' - 'runners/launch_h200-dgxc-slurm.sh' - 'runners/inject_srt_power_concurrencies.py' + - 'runners/slurm_utils.sh' + - 'runners/test_kimik3_bh_power.py' - 'utils/aggregate_power.py' - 'utils/aggregate_power_multinode.py' - 'utils/agentic/aggregation/**' @@ -57,4 +59,4 @@ jobs: run: | cd utils uv run --no-project --exclude-newer PT12H --python 3.12 --with pytest --with pyyaml \ - python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py agentic/aggregation/ test_gb300_power_official_contract.py test_inject_srt_power_concurrencies.py test_process_result.py -v + python -m pytest test_aggregate_power.py test_aggregate_power_multinode.py agentic/aggregation/ test_gb300_power_official_contract.py test_inject_srt_power_concurrencies.py test_process_result.py ../runners/test_kimik3_bh_power.py -v diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c1-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c1-agentic.yaml index 5a48fa41f4..5510b41124 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c1-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c1-agentic.yaml @@ -111,14 +111,27 @@ sbatch_directives: srun_options: container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + benchmark: type: custom + concurrencies: [1] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c14-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c14-agentic.yaml index 2c5358f81a..a4f0f4eb45 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c14-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c14-agentic.yaml @@ -111,14 +111,27 @@ sbatch_directives: srun_options: container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + benchmark: type: custom + concurrencies: [14] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c24-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c24-agentic.yaml index f3071b339c..6dab857310 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c24-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c24-agentic.yaml @@ -111,14 +111,27 @@ sbatch_directives: srun_options: container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + benchmark: type: custom + concurrencies: [24] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c4-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c4-agentic.yaml index dd72387250..1769cd164a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c4-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c4-agentic.yaml @@ -111,14 +111,27 @@ sbatch_directives: srun_options: container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + benchmark: type: custom + concurrencies: [4] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c48-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c48-agentic.yaml index fd1f09784a..3c64e5a12d 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c48-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c48-agentic.yaml @@ -111,14 +111,27 @@ sbatch_directives: srun_options: container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + benchmark: type: custom + concurrencies: [48] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c8-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c8-agentic.yaml index f8e5e0bdae..390395736a 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c8-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c8-agentic.yaml @@ -111,14 +111,27 @@ sbatch_directives: srun_options: container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + benchmark: type: custom + concurrencies: [8] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c96-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c96-agentic.yaml index 99479d3263..8b4b2d6491 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c96-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c96-agentic.yaml @@ -110,14 +110,27 @@ sbatch_directives: srun_options: container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + benchmark: type: custom + concurrencies: [96] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" - AIPERF_SERVER_METRICS_URLS: "http://localhost:8000/metrics" AIPERF_REQUIRED_SERVER_METRIC_PREFIX: "vllm:" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index db0784ef10..cbabca1613 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -10014,14 +10014,14 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: - dram-utilization: 0.8 search-space: - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } + kv-offloading: none conc-list: [1] num-nodes: 2 worker: num-worker: 1 tp: 8 pp: 2 + dcp-size: 8 ep: 1 dp-attn: false additional-settings: @@ -10029,14 +10029,14 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } + kv-offloading: none conc-list: [4] num-nodes: 2 worker: num-worker: 1 tp: 8 pp: 2 + dcp-size: 8 ep: 1 dp-attn: false additional-settings: @@ -10044,14 +10044,14 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } + kv-offloading: none conc-list: [8] num-nodes: 2 worker: num-worker: 1 tp: 8 pp: 2 + dcp-size: 8 ep: 1 dp-attn: false additional-settings: @@ -10059,14 +10059,14 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } + kv-offloading: none conc-list: [14] num-nodes: 2 worker: num-worker: 1 tp: 8 pp: 2 + dcp-size: 8 ep: 1 dp-attn: false additional-settings: @@ -10074,14 +10074,14 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.84" - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } + kv-offloading: none conc-list: [24] num-nodes: 2 worker: num-worker: 1 tp: 8 pp: 2 + dcp-size: 8 ep: 1 dp-attn: false additional-settings: @@ -10089,28 +10089,28 @@ kimik3-fp4-b200-dynamo-vllm-agentic-dspark: - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.36" - spec-decoding: mtp - kv-offloading: dram - kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } + kv-offloading: none conc-list: [48] num-nodes: 2 worker: num-worker: 1 tp: 8 pp: 2 + dcp-size: 8 ep: 1 dp-attn: false additional-settings: - "CONFIG_FILE=recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-mooncake-c48-agentic.yaml" - "SYNTHETIC_ACCEPTANCE=true" - "SYNTHETIC_ACCEPTANCE_LENGTH=3.36" - - kv-offloading: dram - kv-offload-backend: { name: mooncake, version: "0.3.11.post1" } + - kv-offloading: none conc-list: [96] num-nodes: 2 worker: num-worker: 1 tp: 8 pp: 2 + dcp-size: 8 ep: 1 dp-attn: false additional-settings: diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 88c163461a..1f90deb80e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7399,3 +7399,12 @@ - "Disable adaptive verification in the EVAL_ONLY DSpark config as well. It trims verification requests on device, which the ROCm DeepseekV4IndexerBackend does not support, so the eval-only engine refused to start (run 34651830283, c32). Evals keep real block rejection; throughput settings are unchanged." - "EVAL_ONLY 的 DSpark 配置同样关闭自适应验证:它会在设备端裁剪验证请求,而 ROCm 的 DeepseekV4IndexerBackend 不支持该操作,导致仅评测引擎拒绝启动(运行 34651830283,c32)。评测仍保留真实块拒绝采样;吞吐设置不变。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2962 + +- config-keys: + - kimik3-fp4-b200-dynamo-vllm-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Enable required power for the seven existing B200 Kimi-K3 recipes, use the pinned AgentX producer and shared collector, fix the exporter import URI, and record DCP8 with offload disabled to match the serving commands." + - "启用 Kimi-K3 B200 实测功耗并修正 DCP 与 Offload 元数据。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3043 diff --git a/runners/launch_b200-nscale-slurm.sh b/runners/launch_b200-nscale-slurm.sh index b5e0ef627c..f0d6b8ae96 100755 --- a/runners/launch_b200-nscale-slurm.sh +++ b/runners/launch_b200-nscale-slurm.sh @@ -5,7 +5,7 @@ # Self-contained because Nscale has its own Slurm and storage layout. # # Scope: multi-node Dynamo-vLLM DeepSeek-V4-Pro and Kimi K2.6 FP4 runs, plus -# DeepSeek-V4-Pro FP4 Dynamo-SGLang STP and MTP runs, on the +# DeepSeek-V4-Pro FP4 Dynamo-SGLang STP and MTP runs and Kimi-K3 AgentX, on the # b200-nscale runner label. # Anything else exits non-zero. @@ -13,6 +13,7 @@ SLURM_PARTITION="batch_1" SLURM_ACCOUNT="benchmark" POWER_SRT_SLURM_URL="https://github.com/edwingao28/srt-slurm.git" POWER_SRT_SLURM_PIN="e5c837f06a362dc888dfea2ee588e9f19c298270" +AGENTX_POWER_SRT_SLURM_PIN="80d7203e424f903c9017de4608ee2044afce9574" TILERT_SRT_SLURM_URL="https://github.com/SemiAnalysisAI/srt-slurm.git" TILERT_SRT_SLURM_PIN="d1e6c97b3baf3e87103b6d83189544c3c7d61c38" @@ -63,6 +64,7 @@ if [[ $FRAMEWORK != "dynamo-vllm" ]] && fi USES_DCGM_POWER=0 +USES_AGENTX_POWER=0 _POWER_CONFIG_FILE="${CONFIG_FILE:-}" if [[ "${EVAL_ONLY:-false}" == "true" && -n "${EVAL_CONFIG_FILE:-}" ]]; then _POWER_CONFIG_FILE="$EVAL_CONFIG_FILE" @@ -78,14 +80,17 @@ if [[ -n "$_POWER_CONFIG_FILE" && -f "$_RECIPE_SRC" ]] && awk ' ' "$_RECIPE_SRC"; then USES_DCGM_POWER=1 fi -if [[ "$USES_DCGM_POWER" == "1" && ( +if [[ "$USES_DCGM_POWER" == "1" && "$IS_AGENTIC" == "1" && + "$MODEL_PREFIX" == "kimik3" && "$PRECISION" == "fp4" && "$FRAMEWORK" == "dynamo-vllm" ]]; then + USES_AGENTX_POWER=1 +elif [[ "$USES_DCGM_POWER" == "1" && ( "${IS_AGENTIC:-0}" == "1" || "$PRECISION" != "fp4" || ( "$MODEL_PREFIX" == "dsv4" && "$FRAMEWORK" != "dynamo-sglang" && "$FRAMEWORK" != "dynamo-vllm" ) || ( "$MODEL_PREFIX" == "kimik2.6" && "$FRAMEWORK" != "dynamo-vllm" ) || ( "$MODEL_PREFIX" != "dsv4" && "$MODEL_PREFIX" != "kimik2.6" ) ) ]]; then - echo "Error: B200 nscale dcgm-power is limited to fixed-sequence DSV4/Kimi-K2.6 FP4 lanes" >&2 + echo "Error: B200 nscale dcgm-power requires a supported fixed-sequence lane or Kimi-K3 AgentX vLLM" >&2 exit 1 fi @@ -95,12 +100,20 @@ echo "Cloning srt-slurm repository..." SRT_REPO_DIR="srt-slurm" rm -rf "$SRT_REPO_DIR" if [[ "$USES_DCGM_POWER" == "1" ]]; then + SELECTED_POWER_SRT_SLURM_PIN="$POWER_SRT_SLURM_PIN" + if [[ "$USES_AGENTX_POWER" == "1" ]]; then + SELECTED_POWER_SRT_SLURM_PIN="$AGENTX_POWER_SRT_SLURM_PIN" + fi git clone "$POWER_SRT_SLURM_URL" "$SRT_REPO_DIR" || exit 1 cd "$SRT_REPO_DIR" || exit 1 - git checkout "$POWER_SRT_SLURM_PIN" || exit 1 - test "$(git rev-parse HEAD)" = "$POWER_SRT_SLURM_PIN" || { echo "Error: srt-slurm HEAD does not match POWER_SRT_SLURM_PIN=$POWER_SRT_SLURM_PIN" >&2; exit 1; } + git checkout "$SELECTED_POWER_SRT_SLURM_PIN" || exit 1 + test "$(git rev-parse HEAD)" = "$SELECTED_POWER_SRT_SLURM_PIN" || { echo "Error: srt-slurm HEAD does not match selected power producer $SELECTED_POWER_SRT_SLURM_PIN" >&2; exit 1; } git rev-parse HEAD > "$GITHUB_WORKSPACE/power-producer-sha.txt" - if [[ "$MODEL_PREFIX" == "dsv4" && "$FRAMEWORK" == "dynamo-sglang" ]]; then + if [[ "$USES_AGENTX_POWER" == "1" ]]; then + mkdir -p recipes/vllm/kimi-k3/agentic || exit 1 + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ + recipes/vllm/kimi-k3/agentic || exit 1 + elif [[ "$MODEL_PREFIX" == "dsv4" && "$FRAMEWORK" == "dynamo-sglang" ]]; then mkdir -p recipes/sglang/deepseek-v4 cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4" recipes/sglang/deepseek-v4 elif [[ "$MODEL_PREFIX" == "dsv4" ]]; then @@ -243,9 +256,8 @@ fi if [[ "$USES_DCGM_POWER" == "1" ]]; then DCGM_EXPORTER_IMAGE="nvcr.io/nvidia/k8s/dcgm-exporter:4.6.0-4.8.3-distroless" - DCGM_EXPORTER_ENROOT_REF="${DCGM_EXPORTER_IMAGE/nvcr.io\//nvcr.io#}" DCGM_EXPORTER_SQSH="$SQUASH_DIR/$(echo "$DCGM_EXPORTER_IMAGE" | sed 's/[\/:@#]/_/g').sqsh" - import_squash "$DCGM_EXPORTER_SQSH" "$DCGM_EXPORTER_ENROOT_REF" || exit 1 + import_squash "$DCGM_EXPORTER_SQSH" "$DCGM_EXPORTER_IMAGE" || exit 1 test -r "$DCGM_EXPORTER_SQSH" || { echo "Error: DCGM exporter squash not readable: $DCGM_EXPORTER_SQSH" >&2; exit 1; } unsquashfs -l "$DCGM_EXPORTER_SQSH" > /dev/null || { echo "Error: DCGM exporter squash invalid: $DCGM_EXPORTER_SQSH" >&2; exit 1; } sha256sum "$DCGM_EXPORTER_SQSH" > "$GITHUB_WORKSPACE/exporter-image.sha256" @@ -346,6 +358,12 @@ sed -i 's/^ max_attempts: [0-9]*/ max_attempts: 720/' "$CONFIG_PATH" inject_synthetic_acceptance "$CONFIG_PATH" "$FRAMEWORK" || exit 1 +if [[ "$USES_AGENTX_POWER" == "1" ]]; then + read -r -a POWER_CONCURRENCIES <<< "$CONC_LIST" + python "$GITHUB_WORKSPACE/runners/inject_srt_power_concurrencies.py" \ + "$CONFIG_PATH" "${POWER_CONCURRENCIES[@]}" || exit 1 +fi + SRTCTL_PREFLIGHT_ARGS=() # These weights are staged on the Slurm compute nodes, not the login node. if [[ $MODEL_PREFIX == "kimik2.6" ]] || @@ -373,7 +391,11 @@ LOG_FILE="$LOGS_DIR/sweep_${JOB_ID}.log" # Waits for the log file to appear, fails fast if the job dies first, then # streams until the job leaves the queue. -stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || exit 1 +SRT_JOB_RC=0 +stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || SRT_JOB_RC=$? +if [[ "$SRT_JOB_RC" != "0" && "$USES_AGENTX_POWER" != "1" ]]; then + exit "$SRT_JOB_RC" +fi set -x @@ -385,6 +407,14 @@ if [ ! -d "$LOGS_DIR" ]; then exit 1 fi +AGENTX_POWER_RC="$SRT_JOB_RC" +if [[ "$USES_AGENTX_POWER" == "1" && "${EVAL_ONLY:-false}" != "true" ]]; then + read -r -a POWER_CONCURRENCIES <<< "$CONC_LIST" + collect_agentic_power_results "$JOB_ID" "$LOGS_DIR" \ + "$GITHUB_WORKSPACE" "$GITHUB_WORKSPACE" "$RESULT_FILENAME" \ + "$SELECTED_POWER_SRT_SLURM_PIN" "${POWER_CONCURRENCIES[@]}" || AGENTX_POWER_RC=$? +fi + if [[ "$USES_DCGM_POWER" == "1" ]]; then mkdir -p "$LOGS_DIR/power" cp "$GITHUB_WORKSPACE/exporter-image.sha256" "$LOGS_DIR/power/exporter-image.sha256" @@ -394,6 +424,11 @@ fi cp -r "$LOGS_DIR" "$GITHUB_WORKSPACE/LOGS" bundle_server_logs "$LOGS_DIR" "$GITHUB_WORKSPACE/multinode_server_logs.tar.gz" +if [[ "$AGENTX_POWER_RC" != "0" ]]; then + echo "ERROR: AgentX power validation failed; available audit and server artifacts were staged" >&2 + exit "$AGENTX_POWER_RC" +fi + if [[ "${EVAL_ONLY:-false}" != "true" ]]; then RESULT_SUBDIRS=$(find "$LOGS_DIR" -maxdepth 1 -type d -name "*isl*osl*" 2>/dev/null) diff --git a/runners/slurm_utils.sh b/runners/slurm_utils.sh index 8a6462f121..49f33f6b22 100644 --- a/runners/slurm_utils.sh +++ b/runners/slurm_utils.sh @@ -161,6 +161,52 @@ copy_agentic_results() { echo "Copied $copied agentic result file(s)" } +collect_agentic_power_results() { + local job_id="$1" logs_dir="$2" source_dir="$3" workspace="$4" + local result_filename="$5" producer_sha="$6" + shift 6 + local rc=0 concurrency attempt + [[ "$#" -gt 0 ]] || return 1 + mkdir -p "$logs_dir/power" || return 1 + logs_dir="$(cd "$logs_dir" && pwd -P)" || return 1 + workspace="$(cd "$workspace" && pwd -P)" || return 1 + + # Accounting can lag squeue removal; retry only missing or nonterminal rows. + for attempt in 1 2 3; do + echo "$attempt" > "$logs_dir/power/native-job-status-attempts.txt" + sacct -X -n -P -j "$job_id" --format=JobIDRaw,State,ExitCode \ + > "$logs_dir/power/native-job-status.txt" \ + 2>> "$logs_dir/power/native-job-status.stderr" || true + if awk -F'|' -v job="$job_id" ' + $1 == job && $2 !~ /^(PENDING|RUNNING|COMPLETING)$/ { found = 1 } + END { exit !found } + ' "$logs_dir/power/native-job-status.txt"; then + break + fi + if [[ "$attempt" != "3" ]]; then sleep 5; fi + done + if ! awk -F'|' -v job="$job_id" ' + $1 == job { found = 1; if ($2 != "COMPLETED" || $3 != "0:0") failed = 1 } + END { exit (!found || failed) } + ' "$logs_dir/power/native-job-status.txt"; then + rc=1 + fi + copy_agentic_results "$source_dir" "$workspace" "$result_filename" || rc=$? + for concurrency in "$@"; do + ( + cd "$workspace" || exit 1 + PYTHONPATH="$INFERENCEX_SLURM_UTILS_DIR/..${PYTHONPATH:+:$PYTHONPATH}" python3 -m infx.results.agentic.power_adapter \ + --result-dir "$logs_dir/agentic/conc_${concurrency}" \ + --agg-result "$workspace/${result_filename}_conc${concurrency}.json" \ + --power-dir "$logs_dir/power" \ + --logs-root "$logs_dir" \ + --expected-producer-sha "$producer_sha" \ + --require-power + ) || rc=$? + done + return "$rc" +} + copy_eval_artifacts() { local eval_dir="$1" local workspace="$2" diff --git a/runners/test_kimik3_bh_power.py b/runners/test_kimik3_bh_power.py new file mode 100644 index 0000000000..f812eaa859 --- /dev/null +++ b/runners/test_kimik3_bh_power.py @@ -0,0 +1,141 @@ +import os +from pathlib import Path +import re +import subprocess + +import pytest +import yaml + +REPO = Path(__file__).resolve().parents[1] +RECIPE_DIR = "benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" + + +@pytest.mark.parametrize("hardware", ["b200"]) +@pytest.mark.parametrize("power,wrong_head", [(True, False), (False, False), (True, True)]) +def test_kimi_power_selects_verified_runtime( + tmp_path: Path, hardware: str, power: bool, wrong_head: bool, +) -> None: + launcher_name = { + "b200": "launch_b200-nscale-slurm.sh", + "h200": "launch_h200-dgxc-slurm.sh", + }[hardware] + source = (REPO / "runners" / launcher_name).read_text() + source = source[:source.index('echo "Installing srtctl..."')] + if hardware == "h200": + source += "\nfi\n" + source = source.replace( + 'source "$(dirname "${BASH_SOURCE[0]}")/slurm_utils.sh"', + 'source "$TEST_SLURM_UTILS"', + ) + source = re.sub( + r'^AGENTX_POWER_SRT_SLURM_PIN="[0-9a-f]+"$', + 'AGENTX_POWER_SRT_SLURM_PIN="' + "a" * 40 + '"', + source, flags=re.MULTILINE, + ) + recipe = next((REPO / RECIPE_DIR).glob(f"agg-{hardware}*")) + data = yaml.safe_load(recipe.read_text()) + if not power: + data.pop("telemetry") + destination = tmp_path / RECIPE_DIR / recipe.name + destination.parent.mkdir(parents=True) + destination.write_text(yaml.safe_dump(data)) + recipe_ref = f"recipes/vllm/kimi-k3/agentic/{recipe.name}" + harness = ''' +set -e +function git() { + printf '%s\\n' "$*" >> "$TEST_ROUTE_LOG" + case "$1" in + clone) for target in "$@"; do :; done; mkdir -p "$target" ;; + checkout) printf '%s\\n' "$2" > .test-head ;; + rev-parse) if [[ "$TEST_WRONG_HEAD" == 1 ]]; then printf 'bad-head\\n'; else cat .test-head; fi ;; + *) return 99 ;; + esac +} +function cp() { + if [[ "$1" == -rT ]]; then + mkdir -p "$3" + command cp -R "$2"/. "$3" + else + command cp "$@" + fi +} +''' + env = dict(os.environ, GITHUB_WORKSPACE=str(tmp_path), TEST_ROUTE_LOG=str(tmp_path / "route.log"), + TEST_SLURM_UTILS=str(REPO / "runners/slurm_utils.sh"), TEST_WRONG_HEAD=str(int(wrong_head)), + IS_MULTINODE="true", IS_AGENTIC="1", MODEL_PREFIX="kimik3", MODEL="moonshotai/Kimi-K3", + PRECISION="fp4", FRAMEWORK="dynamo-vllm" if hardware == "b200" else "vllm", + CONFIG_FILE=recipe_ref, SPEC_DECODING="mtp", EVAL_ONLY="false", EVAL_FRAMEWORK="lm-eval") + result = subprocess.run(["bash"], input=harness + source, text=True, capture_output=True, cwd=tmp_path, env=env) + stamp = tmp_path / "power-producer-sha.txt" + if wrong_head: + assert result.returncode != 0 + assert not stamp.exists() + return + assert result.returncode == 0, result.stderr + routed_recipe = tmp_path / "srt-slurm" / recipe_ref + assert yaml.safe_load(routed_recipe.read_text()) == data + if power: + assert stamp.read_text().strip() == "a" * 40 + assert "edwingao28/srt-slurm.git" in (tmp_path / "route.log").read_text() + else: + assert not stamp.exists() + assert "edwingao28/srt-slurm.git" not in (tmp_path / "route.log").read_text() + + +@pytest.mark.parametrize("hardware", ["b200"]) +def test_kimi_failed_power_stages_evidence_before_exit(tmp_path: Path, hardware: str) -> None: + filename = {"b200": "launch_b200-nscale-slurm.sh", "h200": "launch_h200-dgxc-slurm.sh"}[hardware] + source = (REPO / "runners" / filename).read_text() + start = source.index('AGENTX_POWER_RC="$SRT_JOB_RC"') + end = source.index('exit "$AGENTX_POWER_RC"', start) + end = source.index("\n", end) + 1 + source = source[start:end] + "fi\n" + logs = tmp_path / "source-logs" + logs.mkdir() + for name in ("exporter-image.sha256", "power-producer-sha.txt"): + (tmp_path / name).write_text("retained\n") + harness = ''' +set -e +collect_agentic_power_results() { + mkdir -p "$2/power" + printf 'invalid telemetry\\n' > "$2/power/validation.json" + return 42 +} +bundle_server_logs() { printf 'server evidence\\n' > "$2"; } +''' + env = dict(os.environ, SRT_JOB_RC="0", USES_AGENTX_POWER="1", USES_KIMIK3_POWER="1", + USES_DCGM_POWER="1", EVAL_ONLY="false", JOB_ID="123", CONC_LIST="1", + GITHUB_WORKSPACE=str(tmp_path), LOGS_DIR=str(logs), RESULT_FILENAME="kimi-test", + SELECTED_POWER_SRT_SLURM_PIN="a" * 40) + result = subprocess.run(["bash"], input=harness + source, text=True, capture_output=True, cwd=tmp_path, env=env) + assert result.returncode == 42, result.stderr + assert (tmp_path / "LOGS/power/validation.json").read_text() == "invalid telemetry\n" + assert (tmp_path / "multinode_server_logs.tar.gz").read_text() == "server evidence\n" + + +def test_b200_exporter_cold_import_uses_one_registry_separator(tmp_path: Path) -> None: + source = (REPO / "runners/launch_b200-nscale-slurm.sh").read_text() + start = source.index("enroot_uri_for_image() {") + end = source.index('\nimport_squash "$SQUASH_FILE"', start) + helpers = source[start:end] + start = source.index('if [[ "$USES_DCGM_POWER" == "1" ]]; then\n DCGM_EXPORTER_IMAGE=') + end = source.index("\nfi", start) + len("\nfi") + harness = ''' +set -euo pipefail +flock() { :; } +unsquashfs() { test -s "$2"; } +enroot() { + printf '%s\\n' "$4" > "$GITHUB_WORKSPACE/import-uri" + printf 'exporter image\\n' > "$3" +} +sha256sum() { printf 'fixture-hash %s\\n' "$1"; } +''' + env = dict(os.environ, GITHUB_WORKSPACE=str(tmp_path), SQUASH_DIR=str(tmp_path), + SQUASH_LOCK_TIMEOUT="1", USES_DCGM_POWER="1") + result = subprocess.run(["bash"], input=harness + helpers + source[start:end], text=True, + capture_output=True, cwd=tmp_path, env=env) + assert result.returncode == 0, result.stderr + uri = (tmp_path / "import-uri").read_text().strip() + assert uri.startswith("docker://nvcr.io#nvidia/k8s/dcgm-exporter:") + assert uri.count("#") == 1 + assert (tmp_path / "exporter-image.sha256").is_file() diff --git a/utils/test_gb300_power_official_contract.py b/utils/test_gb300_power_official_contract.py index 32f5e284d7..4d6e60106c 100644 --- a/utils/test_gb300_power_official_contract.py +++ b/utils/test_gb300_power_official_contract.py @@ -1,6 +1,8 @@ """Exercise launcher routing and exporter imports without Slurm or network access.""" import os +import json +import sys import subprocess from collections.abc import Iterator from pathlib import Path @@ -255,3 +257,54 @@ def test_gb300_dsv4_recipe_images_match_their_master_configs(): assert recipe_path.is_file(), (key, config_file) recipe_image = yaml.safe_load(recipe_path.read_text())["model"]["container"] assert recipe_image == config["image"], (key, config_file) + + +@pytest.mark.parametrize( + ("job_state", "concurrencies", "expected_rc"), + [("COMPLETED|0:0", [4], 0), ("FAILED|1:0", [4], 1), ("COMPLETED|0:0", [4, 8], 1)], +) +def test_agentx_collection_preserves_failed_jobs_and_incomplete_sweeps( + tmp_path, job_state, concurrencies, expected_rc, +): + from utils.test_aggregate_power_multinode import build_package, RESULT_STEM + + package = build_package(tmp_path) + result_dir = package.logs_root / "agentic/conc_4" + result_dir.mkdir(parents=True) + stem = "agentic_power_concurrency_4" + package.original_result.replace(result_dir / f"{stem}.json") + old_window = package.windows_dir / f"{RESULT_STEM}.json" + window = json.loads(old_window.read_text()) + window.update(benchmark_type="custom", result_path=f"agentic/conc_4/{stem}.json") + old_window.unlink() + (package.windows_dir / f"{stem}.json").write_text(json.dumps(window)) + manifest_path = package.power_dir / "manifest.json" + manifest = json.loads(manifest_path.read_text()) + manifest["expected_windows"] = [{"benchmark_type": "custom", "concurrency": 4}] + manifest["window_validations"][0].update( + benchmark_type="custom", window_file=f"windows/{stem}.json", + ) + manifest_path.write_text(json.dumps(manifest)) + source = tmp_path / "compute-workspace" + source.mkdir() + (source / "run_conc4.json").write_text(json.dumps({ + "conc": 4, "disagg": True, "num_prefill_gpu": 2, "num_decode_gpu": 2, + })) + env = os.environ.copy() + env.pop("PYTHONPATH", None) + env["PATH"] = f"{Path(sys.executable).parent}:{env['PATH']}" + result = subprocess.run( + ["bash", "-c", + 'source "$1"; export TEST_JOB_STATE="$2"; sacct() { printf "12345|%s\\n" "$TEST_JOB_STATE"; }; ' + 'shift 2; collect_agentic_power_results "$@"', + "bash", str(REPO_ROOT / "runners/slurm_utils.sh"), job_state, + "12345", str(package.logs_root), str(source), str(tmp_path), + "run", PRODUCER_PIN, *map(str, concurrencies)], + env=env, text=True, capture_output=True, + ) + assert result.returncode == expected_rc, result.stdout + result.stderr + aggregate = json.loads((tmp_path / "run_conc4.json").read_text()) + assert aggregate["power_valid"] == 1 + assert aggregate["total_gpu_energy_j"] == 84000.0 + assert (result_dir / "power_validation.json").is_file() + assert (package.power_dir / "native-job-status.txt").read_text() == f"12345|{job_state}\n"