[AgentX GB300] enable Kimi-K3 aggregate power / 启用聚合部署功耗

benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml

@@ -114,10 +114,27 @@ srun_options:
   mem: "0"
   container-remap-root: ""
 
+telemetry:
+  enabled: true
+  provider: dcgm-power
+  default_frequency: 1.0
+  storage_subdir: power
+  required: true
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 12
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    port: 19401
+
 benchmark:
   type: custom
+  client_placement: head
+  concurrencies: [1, 4]
   command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
   env:
+    ENABLE_AGENTX_POWER: "1"
+    REQUIRE_POWER: "1"
     INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
     RESULT_DIR: "/logs/agentic"
     PORT: "8000"

benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-nospec-mooncake-agentic.yaml

@@ -115,10 +115,27 @@ srun_options:
   mem: "0"
   container-remap-root: ""
 
+telemetry:
+  enabled: true
+  provider: dcgm-power
+  default_frequency: 1.0
+  storage_subdir: power
+  required: true
+  startup_timeout_seconds: 120
+  request_timeout_seconds: 2
+  collector_join_timeout_seconds: 12
+  dcgm_exporter:
+    container_image: dcgm-exporter
+    port: 19401
+
 benchmark:
   type: custom
+  client_placement: head
+  concurrencies: [48, 70]
   command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh
   env:
+    ENABLE_AGENTX_POWER: "1"
+    REQUIRE_POWER: "1"
     INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace"
     AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300"
     AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25"

perf-changelog.yaml

@@ -7467,3 +7467,23 @@
     - "Collect shared measured GPU power for the existing two-node GB200 GLM-5.2 aggregate AgentX recipe with producer 80d7203e424f903c9017de4608ee2044afce9574, exact selected-concurrency windows, and post-job adapter validation. Preserve native status and invalid diagnostics before failure. Ordinary Slurm limits, serving settings, other AgentX recipes, and fixed-sequence producer pins are unchanged."
     - "为现有双节点 GB200 GLM-5.2 聚合 AgentX 配方接入共享 GPU 实测功耗，按所选并发绑定测量窗口，并在任务结束后通过适配器校验。返回失败前保留原生状态及无效诊断；普通 Slurm 时限、服务配置、其他 AgentX 配方和固定序列 producer 版本保持不变。"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2928
+
+- config-keys:
+    - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg
+    - kimik3-fp4-gb300-dynamo-vllm-agentic-mooncake-dcp8-agg
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Enable the two GB300 aggregate recipes and their common launcher power route, with the pinned producer, strict result collection and retained diagnostics."
+    - "为 GB300 Kimi-K3 聚合部署启用必需实测功耗。"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3046
+
+- config-keys:
+    - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg
+    - kimik3-fp4-gb300-dynamo-vllm-agentic-mooncake-dcp8-agg
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Use port 19401 for the GB300 aggregate exporter; the two-node shared lifecycle check verified ownership, all eight GPU telemetry streams, and preservation of the system service on port 9401. Full Kimi-K3 qualification remains pending."
+    - "GB300 聚合部署改用已验证的 19401 功耗端口；完整 Kimi-K3 基准验证仍待完成。"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3046

runners/launch_gb300-nv.sh

@@ -193,12 +193,15 @@ if [[ -n "$CONFIG_FILE" && -f "$_RECIPE_SRC" ]] && awk '
     USES_DCGM_POWER=1
 fi
 
-# Note (wenyao): the producer pin follows the srt-slurm main lineage that the
-# dynamo-sglang lanes run on (fp8 validated end-to-end, fp4 recipes
-# parse-verified against the pin); other frameworks clone diverging refs
-# (aflowers branch, sa-submission), so fail fast for them instead.
-if [[ "$USES_DCGM_POWER" == "1" && "$FRAMEWORK" != "dynamo-sglang" ]]; then
-    echo "Error: dcgm-power lanes are only validated for FRAMEWORK=dynamo-sglang, got: $FRAMEWORK" >&2
+USES_AGENTX_POWER=0
+if [[ "$USES_DCGM_POWER" == "1" && "$IS_AGENTIC" == "1" &&
+    "$MODEL_PREFIX" == "kimik3" && "$PRECISION" == "fp4" &&
+    "$FRAMEWORK" == "dynamo-vllm" &&
+    "$_RECIPE_REL" == recipes/vllm/kimi-k3/agentic/* ]]; then
+    USES_AGENTX_POWER=1
+fi
+if [[ "$USES_DCGM_POWER" == "1" && "$FRAMEWORK" != "dynamo-sglang" && "$USES_AGENTX_POWER" != "1" ]]; then
+    echo "Error: dcgm-power requires dynamo-sglang or the supported Kimi-K3 AgentX route" >&2
     exit 1
 fi
 
@@ -207,6 +210,7 @@ fi
 # srt-slurm merge lands.
 POWER_SRT_SLURM_URL="https://github.com/edwingao28/srt-slurm.git"
 POWER_SRT_SLURM_PIN="6fc1bed01a0b82dae0088a105c03ce0cfb353443"
+AGENTX_POWER_SRT_SLURM_PIN="80d7203e424f903c9017de4608ee2044afce9574"
 
 if [[ "$USES_DCGM_POWER" == "1" ]]; then
     DCGM_EXPORTER_IMAGE="nvcr.io/nvidia/k8s/dcgm-exporter:4.6.0-4.8.3-distroless"
@@ -238,7 +242,16 @@ SRT_REPO_DIR="${GITHUB_WORKSPACE}/srt-slurm-${GITHUB_RUN_ID:-manual}-${GITHUB_RU
 SRTCTL_SETUP_SCRIPT=""
 rm -rf "$SRT_REPO_DIR"
 
-if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "qwen3.5" ]]; then
+if [[ "$USES_AGENTX_POWER" == "1" ]]; then
+    git clone "$POWER_SRT_SLURM_URL" "$SRT_REPO_DIR" || exit 1
+    cd "$SRT_REPO_DIR" || exit 1
+    git checkout "$AGENTX_POWER_SRT_SLURM_PIN" || exit 1
+    test "$(git rev-parse HEAD)" = "$AGENTX_POWER_SRT_SLURM_PIN" || exit 1
+    git rev-parse HEAD > "$GITHUB_WORKSPACE/power-producer-sha.txt"
+    mkdir -p recipes/vllm/kimi-k3/agentic
+    cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \
+        recipes/vllm/kimi-k3/agentic
+elif [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "qwen3.5" ]]; then
     git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR"
     cd "$SRT_REPO_DIR"
     git checkout v1.0.50
@@ -570,6 +583,12 @@ sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_PATH"
 # verification.
 inject_synthetic_acceptance "$CONFIG_PATH" "$FRAMEWORK" || exit 1
 
+if [[ "$USES_AGENTX_POWER" == "1" ]]; then
+    read -r -a POWER_CONCURRENCIES <<< "$CONC_LIST"
+    python3 "$GITHUB_WORKSPACE/runners/inject_srt_power_concurrencies.py" \
+        "$CONFIG_PATH" "${POWER_CONCURRENCIES[@]}" || exit 1
+fi
+
 # --no-preflight skips srtctl's pre-submit model-path stat, which runs on
 # the GHA runner host (im-gb300-login-02, an x86 login node). It's required
 # whenever model.path resolves to the node-local /scratch NVMe that the login
@@ -644,37 +663,49 @@ _snapshot_server_logs() {
 }
 trap _snapshot_server_logs EXIT
 
-# Wait for log file to appear (also check job is still alive)
-while ! ls "$LOG_FILE" &>/dev/null; do
-    if ! squeue -j "$JOB_ID" --noheader 2>/dev/null | grep -q "$JOB_ID"; then
-        echo "ERROR: Job $JOB_ID failed before creating log file"
-        scontrol show job "$JOB_ID"
-        exit 1
-    fi
-    echo "Waiting for JOB_ID $JOB_ID to begin and $LOG_FILE to appear..."
-    sleep 5
-done
-
-# Poll for job completion in background
-(
-    while squeue -j "$JOB_ID" --noheader 2>/dev/null | grep -q "$JOB_ID"; do
-        sleep 10
+AGENTX_POWER_RC=0
+if [[ "$USES_AGENTX_POWER" == "1" ]]; then
+    stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || AGENTX_POWER_RC=$?
+else
+    # Wait for log file to appear (also check job is still alive)
+    while ! ls "$LOG_FILE" &>/dev/null; do
+        if ! squeue -j "$JOB_ID" --noheader 2>/dev/null | grep -q "$JOB_ID"; then
+            echo "ERROR: Job $JOB_ID failed before creating log file"
+            scontrol show job "$JOB_ID"
+            exit 1
+        fi
+        echo "Waiting for JOB_ID $JOB_ID to begin and $LOG_FILE to appear..."
+        sleep 5
     done
-) &
-POLL_PID=$!
 
-echo "Tailing LOG_FILE: $LOG_FILE"
+    # Poll for job completion in background
+    (
+        while squeue -j "$JOB_ID" --noheader 2>/dev/null | grep -q "$JOB_ID"; do
+            sleep 10
+        done
+    ) &
+    POLL_PID=$!
 
-# Stream the log file until job completes (-F follows by name, polls instead of inotify for NFS)
-tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null
+    echo "Tailing LOG_FILE: $LOG_FILE"
 
-wait $POLL_PID
+    # Stream the log file until job completes (-F follows by name, polls instead of inotify for NFS)
+    tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null
+
+    wait $POLL_PID
+fi
 
 set -x
 
 echo "Job $JOB_ID completed!"
 echo "Collecting results..."
 
+if [[ "$USES_AGENTX_POWER" == "1" && "${EVAL_ONLY:-false}" != "true" ]]; then
+    read -r -a POWER_CONCURRENCIES <<< "$CONC_LIST"
+    collect_agentic_power_results "$JOB_ID" "$LOGS_DIR" "$INFMAX_WORKSPACE" \
+        "$GITHUB_WORKSPACE" "$RESULT_FILENAME" "$AGENTX_POWER_SRT_SLURM_PIN" \
+        "${POWER_CONCURRENCIES[@]}" || AGENTX_POWER_RC=$?
+fi
+
 if [ -d "$LOGS_DIR" ]; then
     echo "Found logs directory: $LOGS_DIR"
     # Tarball + LOGS copy + power provenance markers are produced by the EXIT
@@ -685,6 +716,11 @@ else
     echo "Warning: Logs directory not found at $LOGS_DIR"
 fi
 
+if [[ "$AGENTX_POWER_RC" != "0" ]]; then
+    echo "ERROR: AgentX job or power validation failed; EXIT will stage audit artifacts" >&2
+    exit "$AGENTX_POWER_RC"
+fi
+
 if [[ "${EVAL_ONLY:-false}" != "true" ]]; then
     if [ ! -d "$LOGS_DIR" ]; then
         exit 1

utils/test_gb300_power_official_contract.py

@@ -27,23 +27,28 @@ def _launcher_routing_source(launcher_path: Path = LAUNCHER_PATH) -> str:
     if launcher_path.name == "launch_gb200-nv.sh":
         gate_start = launcher.index("USES_DCGM_POWER=0")
         gate_end = launcher.index('if [[ "$USES_DCGM_POWER" == "1" ]]; then\n    DCGM_EXPORTER_IMAGE=', gate_start)
-        route_start = launcher.index('if [[ "$IS_AGENTIC" == "1" && "$MODEL_PREFIX" == "glm5.2"')
+        route_start = launcher.index('if [[ "$IS_AGENTIC" == "1" && "$MODEL_PREFIX" == "glm5.2"',
+                                      launcher.index('echo "Cloning srt-slurm repository..."'))
         route_end = launcher.index('\necho "Installing srtctl..."', route_start)
         return launcher[gate_start:gate_end] + launcher[route_start:route_end]
-    route_start = launcher.index(
-        'if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" '
-        '&& $MODEL_PREFIX == "qwen3.5" ]]; then'
-    )
-    route_end_marker = '\nfi\n\necho "Installing srtctl..."'
-    route_end = launcher.index(route_end_marker, route_start) + len("\nfi")
-    return launcher[route_start:route_end]
+    route_start = launcher.index('if [[ "$USES_AGENTX_POWER" == "1" ]]; then',
+                                  launcher.index('echo "Cloning srt-slurm repository..."'))
+    route_end = launcher.index('\necho "Installing srtctl..."', route_start)
+    return _launcher_power_gate_source(launcher_path.name) + "\n" + launcher[route_start:route_end]
 
 
 def _write_executable(path: Path, text: str) -> None:
     path.write_text(text)
     path.chmod(0o755)
 
 
+def _launcher_power_gate_source(launcher_name: str) -> str:
+    launcher = (REPO_ROOT / "runners" / launcher_name).read_text()
+    start = launcher.index("USES_DCGM_POWER=0\n")
+    end = launcher.index("\nfi", launcher.index("Error: dcgm-power requires", start))
+    return launcher[start:end + len("\nfi")]
+
+
 def _run_dsv4_route(
     tmp_path: Path, uses_dcgm_power: bool, *, reported_head: str = "",
     launcher_path: Path = LAUNCHER_PATH, model_prefix: str = "dsv4",
@@ -53,6 +58,7 @@ def _run_dsv4_route(
     workspace = tmp_path / "workspace"
     stub_bin = tmp_path / "bin"
     recipe_directory = ("sglang/glm5.2/gb200-fp4/agentic" if model_prefix == "glm5.2"
+                        else "vllm/kimi-k3/agentic" if model_prefix == "kimik3"
                         else "sglang/deepseek-v4/8k1k")
     recipe_name = recipe_name or ("glm5.2-agentx-agg.yaml" if model_prefix == "glm5.2" else "recipe.yaml")
     source = workspace / "benchmarks/multi_node/srt-slurm-recipes" / recipe_directory
@@ -123,14 +129,15 @@ def _run_dsv4_route(
 POWER_SRT_SLURM_PIN={PRODUCER_PIN}
 AGENTX_POWER_SRT_SLURM_PIN={AGENTX_PRODUCER_PIN}
 IS_AGENTIC={int(is_agentic)}
-FRAMEWORK=dynamo-sglang
+FRAMEWORK={'dynamo-vllm' if model_prefix == 'kimik3' else 'dynamo-sglang'}
 MODEL_PREFIX={model_prefix}
 PRECISION=fp4
 SPEC_DECODING=
 USES_DCGM_POWER={int(uses_dcgm_power)}
-CONFIG_FILE=recipes/{recipe_directory}/{recipe_name}
+CONFIG_FILE=recipes/{recipe_directory}/{recipe_name}:fixture-override
 GITHUB_WORKSPACE={workspace!s}
 SRT_REPO_DIR={repo_dir!s}
+python3() {{ :; }}
 {routing}
 """
     )
@@ -381,6 +388,51 @@ def test_gb300_dsv4_recipe_images_match_their_master_configs():
             assert recipe_image == config["image"], (key, config_file)
 
 
+@pytest.mark.parametrize("launcher_name", ["launch_gb300-nv.sh"])
+def test_kimi_agentx_route_uses_recipe_power_gate(tmp_path, launcher_name):
+    log, workspace, repo_dir, marker = _run_dsv4_route(
+        tmp_path, True, launcher_path=REPO_ROOT / "runners" / launcher_name, model_prefix="kimik3", is_agentic=True
+    )
+    assert f"git clone {FORK_URL} {repo_dir}" in log
+    assert f"git checkout {AGENTX_PRODUCER_PIN}" in log
+    assert (workspace / "power-producer-sha.txt").read_text() == f"{AGENTX_PRODUCER_PIN}\n"
+    assert marker.read_text() == "from-workspace\n"
+
+
+@pytest.mark.parametrize(
+    "override",
+    [
+        {"CONFIG_FILE": "recipes/vllm/kimi-k3/8k1k/test.yaml"},
+        {"MODEL_PREFIX": "other-model"},
+        {"PRECISION": "fp8"},
+        {"IS_AGENTIC": "0"},
+        {"FRAMEWORK": "vllm"},
+    ],
+)
+def test_kimi_power_gate_rejects_unsupported_route(tmp_path, override):
+    env = os.environ.copy()
+    env.update(GITHUB_WORKSPACE=str(tmp_path), CONFIG_FILE="recipes/vllm/kimi-k3/agentic/test.yaml",
+               MODEL_PREFIX="kimik3", PRECISION="fp4", IS_AGENTIC="1", FRAMEWORK="dynamo-vllm")
+    env.update(override)
+    recipe = tmp_path / "benchmarks/multi_node/srt-slurm-recipes" / env["CONFIG_FILE"].removeprefix("recipes/")
+    recipe.parent.mkdir(parents=True)
+    recipe.write_text("telemetry:\n  provider: dcgm-power\n  enabled: true\n")
+    result = subprocess.run(["/bin/bash"], input=_launcher_power_gate_source("launch_gb300-nv.sh"),
+                            text=True, capture_output=True, env=env)
+    assert result.returncode == 1
+    assert "dcgm-power requires" in result.stderr
+
+
+@pytest.mark.parametrize("launcher_name", ["launch_gb300-nv.sh"])
+def test_kimi_power_route_rejects_wrong_commit(tmp_path, launcher_name):
+    with pytest.raises(subprocess.CalledProcessError):
+        _run_dsv4_route(
+            tmp_path, True, reported_head="c" * 40,
+            launcher_path=REPO_ROOT / "runners" / launcher_name, model_prefix="kimik3", is_agentic=True,
+        )
+    assert not (tmp_path / "workspace/power-producer-sha.txt").exists()
+
+
 @pytest.mark.parametrize(
     ("job_state", "concurrencies", "expected_rc"),
     [("COMPLETED|0:0", [4], 0), ("FAILED|1:0", [4], 1), ("COMPLETED|0:0", [4, 8], 1)],