From 49c49b0355591ac8b7497ca0e6a04a2e925f1915 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 17:08:53 -0700 Subject: [PATCH 1/4] feat: enable Kimi-K3 GB300 aggregate power MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 基于 main 独立启用 GB300 聚合部署实测功耗；仅复用已合入的共享采集代码，保留配方和端口修复。 --- ...dcp8-dspark7-maxseq2-mooncake-agentic.yaml | 17 ++++ ...gg-gb300-dcp8-nospec-mooncake-agentic.yaml | 17 ++++ perf-changelog.yaml | 20 ++++ runners/launch_gb300-nv.sh | 92 +++++++++++++------ utils/test_gb300_power_official_contract.py | 55 ++++++++--- 5 files changed, 159 insertions(+), 42 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml index dafc046abe..34943fda52 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml @@ -114,10 +114,27 @@ srun_options: mem: "0" container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 + benchmark: type: custom + client_placement: head + concurrencies: [1, 4] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: + ENABLE_AGENTX_POWER: "1" + REQUIRE_POWER: "1" INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-nospec-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-nospec-mooncake-agentic.yaml index 9b61dd4922..5720019946 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-nospec-mooncake-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-nospec-mooncake-agentic.yaml @@ -115,10 +115,27 @@ srun_options: mem: "0" container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 + benchmark: type: custom + client_placement: head + concurrencies: [48, 70] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: + ENABLE_AGENTX_POWER: "1" + REQUIRE_POWER: "1" INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 75b77afe1d..30ec71d6f9 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7428,3 +7428,23 @@ - "Add representative AMD coverage for fixed-sequence request diagnostics and multinode concurrency filename processing; shared processor changes also affect unlisted fixed-sequence recipes." - "增加固定序列请求诊断与多节点并发文件名处理的代表性 AMD 覆盖；共享处理器变更也影响未列出的固定序列配置。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3026 + +- config-keys: + - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg + - kimik3-fp4-gb300-dynamo-vllm-agentic-mooncake-dcp8-agg + scenario-type: + - agentic-coding + description: + - "Enable the two GB300 aggregate recipes and their common launcher power route, with the pinned producer, strict result collection and retained diagnostics." + - "为 GB300 Kimi-K3 聚合部署启用必需实测功耗。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3046 + +- config-keys: + - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg + - kimik3-fp4-gb300-dynamo-vllm-agentic-mooncake-dcp8-agg + scenario-type: + - agentic-coding + description: + - "Use port 19401 for the GB300 aggregate exporter; the two-node shared lifecycle check verified ownership, all eight GPU telemetry streams, and preservation of the system service on port 9401. Full Kimi-K3 qualification remains pending." + - "GB300 聚合部署改用已验证的 19401 功耗端口；完整 Kimi-K3 基准验证仍待完成。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3046 diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 9249e91f27..ff3a70035c 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -193,12 +193,15 @@ if [[ -n "$CONFIG_FILE" && -f "$_RECIPE_SRC" ]] && awk ' USES_DCGM_POWER=1 fi -# Note (wenyao): the producer pin follows the srt-slurm main lineage that the -# dynamo-sglang lanes run on (fp8 validated end-to-end, fp4 recipes -# parse-verified against the pin); other frameworks clone diverging refs -# (aflowers branch, sa-submission), so fail fast for them instead. -if [[ "$USES_DCGM_POWER" == "1" && "$FRAMEWORK" != "dynamo-sglang" ]]; then - echo "Error: dcgm-power lanes are only validated for FRAMEWORK=dynamo-sglang, got: $FRAMEWORK" >&2 +USES_AGENTX_POWER=0 +if [[ "$USES_DCGM_POWER" == "1" && "$IS_AGENTIC" == "1" && + "$MODEL_PREFIX" == "kimik3" && "$PRECISION" == "fp4" && + "$FRAMEWORK" == "dynamo-vllm" && + "$_RECIPE_REL" == recipes/vllm/kimi-k3/agentic/* ]]; then + USES_AGENTX_POWER=1 +fi +if [[ "$USES_DCGM_POWER" == "1" && "$FRAMEWORK" != "dynamo-sglang" && "$USES_AGENTX_POWER" != "1" ]]; then + echo "Error: dcgm-power requires dynamo-sglang or the supported Kimi-K3 AgentX route" >&2 exit 1 fi @@ -207,6 +210,7 @@ fi # srt-slurm merge lands. POWER_SRT_SLURM_URL="https://github.com/edwingao28/srt-slurm.git" POWER_SRT_SLURM_PIN="6fc1bed01a0b82dae0088a105c03ce0cfb353443" +AGENTX_POWER_SRT_SLURM_PIN="80d7203e424f903c9017de4608ee2044afce9574" if [[ "$USES_DCGM_POWER" == "1" ]]; then DCGM_EXPORTER_IMAGE="nvcr.io/nvidia/k8s/dcgm-exporter:4.6.0-4.8.3-distroless" @@ -238,7 +242,16 @@ SRT_REPO_DIR="${GITHUB_WORKSPACE}/srt-slurm-${GITHUB_RUN_ID:-manual}-${GITHUB_RU SRTCTL_SETUP_SCRIPT="" rm -rf "$SRT_REPO_DIR" -if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "qwen3.5" ]]; then +if [[ "$USES_AGENTX_POWER" == "1" ]]; then + git clone "$POWER_SRT_SLURM_URL" "$SRT_REPO_DIR" || exit 1 + cd "$SRT_REPO_DIR" || exit 1 + git checkout "$AGENTX_POWER_SRT_SLURM_PIN" || exit 1 + test "$(git rev-parse HEAD)" = "$AGENTX_POWER_SRT_SLURM_PIN" || exit 1 + git rev-parse HEAD > "$GITHUB_WORKSPACE/power-producer-sha.txt" + mkdir -p recipes/vllm/kimi-k3/agentic + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ + recipes/vllm/kimi-k3/agentic +elif [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "qwen3.5" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" git checkout v1.0.50 @@ -570,6 +583,12 @@ sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_PATH" # verification. inject_synthetic_acceptance "$CONFIG_PATH" "$FRAMEWORK" || exit 1 +if [[ "$USES_AGENTX_POWER" == "1" ]]; then + read -r -a POWER_CONCURRENCIES <<< "$CONC_LIST" + python3 "$GITHUB_WORKSPACE/runners/inject_srt_power_concurrencies.py" \ + "$CONFIG_PATH" "${POWER_CONCURRENCIES[@]}" || exit 1 +fi + # --no-preflight skips srtctl's pre-submit model-path stat, which runs on # the GHA runner host (im-gb300-login-02, an x86 login node). It's required # whenever model.path resolves to the node-local /scratch NVMe that the login @@ -644,37 +663,49 @@ _snapshot_server_logs() { } trap _snapshot_server_logs EXIT -# Wait for log file to appear (also check job is still alive) -while ! ls "$LOG_FILE" &>/dev/null; do - if ! squeue -j "$JOB_ID" --noheader 2>/dev/null | grep -q "$JOB_ID"; then - echo "ERROR: Job $JOB_ID failed before creating log file" - scontrol show job "$JOB_ID" - exit 1 - fi - echo "Waiting for JOB_ID $JOB_ID to begin and $LOG_FILE to appear..." - sleep 5 -done - -# Poll for job completion in background -( - while squeue -j "$JOB_ID" --noheader 2>/dev/null | grep -q "$JOB_ID"; do - sleep 10 +AGENTX_POWER_RC=0 +if [[ "$USES_AGENTX_POWER" == "1" ]]; then + stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || AGENTX_POWER_RC=$? +else + # Wait for log file to appear (also check job is still alive) + while ! ls "$LOG_FILE" &>/dev/null; do + if ! squeue -j "$JOB_ID" --noheader 2>/dev/null | grep -q "$JOB_ID"; then + echo "ERROR: Job $JOB_ID failed before creating log file" + scontrol show job "$JOB_ID" + exit 1 + fi + echo "Waiting for JOB_ID $JOB_ID to begin and $LOG_FILE to appear..." + sleep 5 done -) & -POLL_PID=$! -echo "Tailing LOG_FILE: $LOG_FILE" + # Poll for job completion in background + ( + while squeue -j "$JOB_ID" --noheader 2>/dev/null | grep -q "$JOB_ID"; do + sleep 10 + done + ) & + POLL_PID=$! -# Stream the log file until job completes (-F follows by name, polls instead of inotify for NFS) -tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null + echo "Tailing LOG_FILE: $LOG_FILE" -wait $POLL_PID + # Stream the log file until job completes (-F follows by name, polls instead of inotify for NFS) + tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null + + wait $POLL_PID +fi set -x echo "Job $JOB_ID completed!" echo "Collecting results..." +if [[ "$USES_AGENTX_POWER" == "1" && "${EVAL_ONLY:-false}" != "true" ]]; then + read -r -a POWER_CONCURRENCIES <<< "$CONC_LIST" + collect_agentic_power_results "$JOB_ID" "$LOGS_DIR" "$INFMAX_WORKSPACE" \ + "$GITHUB_WORKSPACE" "$RESULT_FILENAME" "$AGENTX_POWER_SRT_SLURM_PIN" \ + "${POWER_CONCURRENCIES[@]}" || AGENTX_POWER_RC=$? +fi + if [ -d "$LOGS_DIR" ]; then echo "Found logs directory: $LOGS_DIR" # Tarball + LOGS copy + power provenance markers are produced by the EXIT @@ -685,6 +716,11 @@ else echo "Warning: Logs directory not found at $LOGS_DIR" fi +if [[ "$AGENTX_POWER_RC" != "0" ]]; then + echo "ERROR: AgentX job or power validation failed; EXIT will stage audit artifacts" >&2 + exit "$AGENTX_POWER_RC" +fi + if [[ "${EVAL_ONLY:-false}" != "true" ]]; then if [ ! -d "$LOGS_DIR" ]; then exit 1 diff --git a/utils/test_gb300_power_official_contract.py b/utils/test_gb300_power_official_contract.py index 4d6e60106c..338b312536 100644 --- a/utils/test_gb300_power_official_contract.py +++ b/utils/test_gb300_power_official_contract.py @@ -18,13 +18,15 @@ PRODUCER_PIN = "a" * 40 -def _launcher_routing_source() -> str: +def _launcher_routing_source(launcher_name: str = "launch_gb300-nv.sh") -> str: """Extract the real clone-routing chain, not a copy of its implementation.""" - launcher = LAUNCHER_PATH.read_text() - route_start = launcher.index( - 'if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" ' - '&& $MODEL_PREFIX == "qwen3.5" ]]; then' + launcher = (REPO_ROOT / "runners" / launcher_name).read_text() + start_marker = ( + 'if [[ "$USES_AGENTX_POWER" == "1" ]]; then' + if launcher_name == "launch_gb300-nv.sh" + else 'if [[ "$IS_AGENTIC" == "1" && "$MODEL_PREFIX" == "glm5.2"' ) + route_start = launcher.index(start_marker, launcher.index('echo "Cloning srt-slurm repository..."')) route_end_marker = '\nfi\n\necho "Installing srtctl..."' route_end = launcher.index(route_end_marker, route_start) + len("\nfi") return launcher[route_start:route_end] @@ -36,15 +38,16 @@ def _write_executable(path: Path, text: str) -> None: def _run_dsv4_route( - tmp_path: Path, uses_dcgm_power: bool, *, reported_head: str = "" + tmp_path: Path, uses_dcgm_power: bool, *, reported_head: str = "", + launcher_name: str = "launch_gb300-nv.sh", model: str = "dsv4" ) -> tuple[list[str], Path, Path, Path]: """Execute only the real launcher routing region in a temporary checkout.""" workspace = tmp_path / "workspace" stub_bin = tmp_path / "bin" - source = ( - workspace - / "benchmarks/multi_node/srt-slurm-recipes/sglang/deepseek-v4/8k1k" + recipe_directory = ( + "vllm/kimi-k3/agentic" if model == "kimik3" else "sglang/deepseek-v4/8k1k" ) + source = workspace / "benchmarks/multi_node/srt-slurm-recipes" / recipe_directory source.mkdir(parents=True) (source / "overlay-marker.txt").write_text("from-workspace\n") stub_bin.mkdir() @@ -98,7 +101,7 @@ def _run_dsv4_route( """, ) - routing = _launcher_routing_source() + routing = _launcher_routing_source(launcher_name) repo_dir = workspace / "srt-slurm-route-test" harness = tmp_path / "route.sh" harness.write_text( @@ -106,14 +109,17 @@ def _run_dsv4_route( set -eo pipefail POWER_SRT_SLURM_URL={FORK_URL} POWER_SRT_SLURM_PIN={PRODUCER_PIN} -IS_AGENTIC=0 -FRAMEWORK=dynamo-sglang -MODEL_PREFIX=dsv4 +AGENTX_POWER_SRT_SLURM_PIN={PRODUCER_PIN} +USES_AGENTX_POWER={int(model == 'kimik3' and uses_dcgm_power)} +IS_AGENTIC={int(model == 'kimik3')} +FRAMEWORK={'dynamo-vllm' if model == 'kimik3' else 'dynamo-sglang'} +MODEL_PREFIX={model} PRECISION=fp4 SPEC_DECODING= USES_DCGM_POWER={int(uses_dcgm_power)} GITHUB_WORKSPACE={workspace!s} SRT_REPO_DIR={repo_dir!s} +python3() {{ :; }} {routing} """ ) @@ -123,7 +129,7 @@ def _run_dsv4_route( env["STUB_HEAD"] = reported_head subprocess.run(["/bin/bash", str(harness)], env=env, check=True) - marker = repo_dir / "recipes/sglang/deepseek-v4/8k1k/overlay-marker.txt" + marker = repo_dir / "recipes" / recipe_directory / "overlay-marker.txt" return route_log.read_text().splitlines(), workspace, repo_dir, marker @@ -259,6 +265,27 @@ def test_gb300_dsv4_recipe_images_match_their_master_configs(): assert recipe_image == config["image"], (key, config_file) +@pytest.mark.parametrize("launcher_name", ["launch_gb300-nv.sh"]) +def test_kimi_agentx_route_uses_custom_power_producer(tmp_path, launcher_name): + log, workspace, repo_dir, marker = _run_dsv4_route( + tmp_path, True, launcher_name=launcher_name, model="kimik3" + ) + assert f"git clone {FORK_URL} {repo_dir}" in log + assert f"git checkout {PRODUCER_PIN}" in log + assert (workspace / "power-producer-sha.txt").read_text() == f"{PRODUCER_PIN}\n" + assert marker.read_text() == "from-workspace\n" + + +@pytest.mark.parametrize("launcher_name", ["launch_gb300-nv.sh"]) +def test_kimi_power_route_rejects_wrong_commit(tmp_path, launcher_name): + with pytest.raises(subprocess.CalledProcessError): + _run_dsv4_route( + tmp_path, True, reported_head="b" * 40, + launcher_name=launcher_name, model="kimik3", + ) + assert not (tmp_path / "workspace/power-producer-sha.txt").exists() + + @pytest.mark.parametrize( ("job_state", "concurrencies", "expected_rc"), [("COMPLETED|0:0", [4], 0), ("FAILED|1:0", [4], 1), ("COMPLETED|0:0", [4, 8], 1)], From 319c0476550b077207727cc130a0b646c31cfe3c Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 17:48:52 -0700 Subject: [PATCH 2/4] test: exercise the GB300 AgentX recipe power gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 测试执行实际 recipe 遥测开关与模型、框架、精度、场景限制，避免预设功耗标志绕过生产判断。 --- utils/test_gb300_power_official_contract.py | 41 ++++++++++++++++++++- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/utils/test_gb300_power_official_contract.py b/utils/test_gb300_power_official_contract.py index 338b312536..db944ef043 100644 --- a/utils/test_gb300_power_official_contract.py +++ b/utils/test_gb300_power_official_contract.py @@ -37,6 +37,13 @@ def _write_executable(path: Path, text: str) -> None: path.chmod(0o755) +def _launcher_power_gate_source(launcher_name: str) -> str: + launcher = (REPO_ROOT / "runners" / launcher_name).read_text() + start = launcher.index("USES_DCGM_POWER=0\n") + end = launcher.index("\nfi", launcher.index("Error: dcgm-power requires", start)) + return launcher[start:end + len("\nfi")] + + def _run_dsv4_route( tmp_path: Path, uses_dcgm_power: bool, *, reported_head: str = "", launcher_name: str = "launch_gb300-nv.sh", model: str = "dsv4" @@ -50,6 +57,10 @@ def _run_dsv4_route( source = workspace / "benchmarks/multi_node/srt-slurm-recipes" / recipe_directory source.mkdir(parents=True) (source / "overlay-marker.txt").write_text("from-workspace\n") + (source / "route-test.yaml").write_text( + "telemetry:\n provider: dcgm-power\n" + f" enabled: {str(uses_dcgm_power).lower()}\n" + ) stub_bin.mkdir() route_log = tmp_path / "route.log" @@ -110,16 +121,18 @@ def _run_dsv4_route( POWER_SRT_SLURM_URL={FORK_URL} POWER_SRT_SLURM_PIN={PRODUCER_PIN} AGENTX_POWER_SRT_SLURM_PIN={PRODUCER_PIN} -USES_AGENTX_POWER={int(model == 'kimik3' and uses_dcgm_power)} IS_AGENTIC={int(model == 'kimik3')} FRAMEWORK={'dynamo-vllm' if model == 'kimik3' else 'dynamo-sglang'} MODEL_PREFIX={model} PRECISION=fp4 SPEC_DECODING= USES_DCGM_POWER={int(uses_dcgm_power)} +USES_AGENTX_POWER=0 GITHUB_WORKSPACE={workspace!s} +CONFIG_FILE=recipes/{recipe_directory}/route-test.yaml:fixture-override SRT_REPO_DIR={repo_dir!s} python3() {{ :; }} +{_launcher_power_gate_source(launcher_name) if model == 'kimik3' else ''} {routing} """ ) @@ -266,7 +279,7 @@ def test_gb300_dsv4_recipe_images_match_their_master_configs(): @pytest.mark.parametrize("launcher_name", ["launch_gb300-nv.sh"]) -def test_kimi_agentx_route_uses_custom_power_producer(tmp_path, launcher_name): +def test_kimi_agentx_route_uses_recipe_power_gate(tmp_path, launcher_name): log, workspace, repo_dir, marker = _run_dsv4_route( tmp_path, True, launcher_name=launcher_name, model="kimik3" ) @@ -276,6 +289,30 @@ def test_kimi_agentx_route_uses_custom_power_producer(tmp_path, launcher_name): assert marker.read_text() == "from-workspace\n" +@pytest.mark.parametrize( + "override", + [ + {"CONFIG_FILE": "recipes/vllm/kimi-k3/8k1k/test.yaml"}, + {"MODEL_PREFIX": "other-model"}, + {"PRECISION": "fp8"}, + {"IS_AGENTIC": "0"}, + {"FRAMEWORK": "vllm"}, + ], +) +def test_kimi_power_gate_rejects_unsupported_route(tmp_path, override): + env = os.environ.copy() + env.update(GITHUB_WORKSPACE=str(tmp_path), CONFIG_FILE="recipes/vllm/kimi-k3/agentic/test.yaml", + MODEL_PREFIX="kimik3", PRECISION="fp4", IS_AGENTIC="1", FRAMEWORK="dynamo-vllm") + env.update(override) + recipe = tmp_path / "benchmarks/multi_node/srt-slurm-recipes" / env["CONFIG_FILE"].removeprefix("recipes/") + recipe.parent.mkdir(parents=True) + recipe.write_text("telemetry:\n provider: dcgm-power\n enabled: true\n") + result = subprocess.run(["/bin/bash"], input=_launcher_power_gate_source("launch_gb300-nv.sh"), + text=True, capture_output=True, env=env) + assert result.returncode == 1 + assert "dcgm-power requires" in result.stderr + + @pytest.mark.parametrize("launcher_name", ["launch_gb300-nv.sh"]) def test_kimi_power_route_rejects_wrong_commit(tmp_path, launcher_name): with pytest.raises(subprocess.CalledProcessError): From 5844ca302119edfaa28ab71cb3aeed3da071de63 Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 17:09:32 -0700 Subject: [PATCH 3/4] feat: enable Kimi-K3 GB300 disaggregated power MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 启用 GB300 分离部署实测功耗，并仅依赖 GB300 公共启动接线。 --- ...1p1d-dcp8-dcp8-dspark4-mooncake-agentic.yaml | 17 +++++++++++++++++ ...1p2d-dcp8-dcp8-dspark4-mooncake-agentic.yaml | 17 +++++++++++++++++ ...1p3d-dcp8-dcp8-dspark4-mooncake-agentic.yaml | 17 +++++++++++++++++ ...1p3d-dcp8-dcp8-dspark7-mooncake-agentic.yaml | 17 +++++++++++++++++ docs/ci-procedures.md | 8 ++++++++ docs/ci-procedures_zh.md | 8 ++++++++ perf-changelog.yaml | 9 +++++++++ 7 files changed, 93 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p1d-dcp8-dcp8-dspark4-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p1d-dcp8-dcp8-dspark4-mooncake-agentic.yaml index afe0d4a653..4d1ff23ef5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p1d-dcp8-dcp8-dspark4-mooncake-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p1d-dcp8-dcp8-dspark4-mooncake-agentic.yaml @@ -153,10 +153,27 @@ srun_options: mem: "0" container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + benchmark: type: custom + client_placement: head + concurrencies: [48, 52, 56] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: + ENABLE_AGENTX_POWER: "1" + REQUIRE_POWER: "1" INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p2d-dcp8-dcp8-dspark4-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p2d-dcp8-dcp8-dspark4-mooncake-agentic.yaml index 29388bbb67..55d3a39dad 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p2d-dcp8-dcp8-dspark4-mooncake-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p2d-dcp8-dcp8-dspark4-mooncake-agentic.yaml @@ -153,10 +153,27 @@ srun_options: mem: "0" container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + benchmark: type: custom + client_placement: head + concurrencies: [32, 48, 64] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: + ENABLE_AGENTX_POWER: "1" + REQUIRE_POWER: "1" INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark4-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark4-mooncake-agentic.yaml index 39269d445e..3c6a311e4f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark4-mooncake-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark4-mooncake-agentic.yaml @@ -153,10 +153,27 @@ srun_options: mem: "0" container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + benchmark: type: custom + client_placement: head + concurrencies: [32, 48] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: + ENABLE_AGENTX_POWER: "1" + REQUIRE_POWER: "1" INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark7-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark7-mooncake-agentic.yaml index 2df5ea74f3..d53fc6f486 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark7-mooncake-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark7-mooncake-agentic.yaml @@ -153,10 +153,27 @@ srun_options: mem: "0" container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 9401 + benchmark: type: custom + client_placement: head + concurrencies: [1] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: + ENABLE_AGENTX_POWER: "1" + REQUIRE_POWER: "1" INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/docs/ci-procedures.md b/docs/ci-procedures.md index f82c205fe4..fc22a6eda1 100644 --- a/docs/ci-procedures.md +++ b/docs/ci-procedures.md @@ -222,6 +222,14 @@ RUN_ID=$(gh run list \ Do not continue if `RUN_ID` is empty. Run metadata describes the dispatch workflow ref, which may not equal input `ref`. Verify the unique title, generator command, and checkout ref in `get-jobs` before interpreting GPU results. +## Kimi-K3 AgentX power backfills + +The Kimi-K3 B200, GB200 and GB300 multi-node recipes enable required DCGM telemetry and run the custom AgentX client on the serving head. Their launchers select the immutable AgentX power runtime, stamp its commit, wait for Slurm completion and telemetry drain, then validate each requested concurrency before staging results. The shared result collector preserves failed native jobs and stages available power diagnostics before returning a failure. H200 routing and AMD measurement-tail fixes are separate changes pending their own integration and validation. Required telemetry configuration alone does not establish hardware qualification. + +For a missing-power backfill, generate only the missing recipe/concurrency combinations, set `require-power: true`, and leave `agentx-fast: false` and the duration override empty. The normal AgentX profile is one hour. Verify a first missing point on each newly enabled runtime/cluster before scheduling its remaining points. A rendered recipe or an online GitHub runner does not establish live collector readiness or Slurm capacity. Preserve the existing validated points, and keep new performance and power values paired with their own run; never attach a new run's energy to an older performance row. Manual `e2e-tests.yml` artifacts still require the normal reviewed ingestion path before they appear in the dashboard. + +B200 Kimi recipes use DCP8 with Mooncake offload disabled. The master config records `dcp-size: 8` and `kv-offloading: none` to match those commands; this metadata correction does not enable offload. + ## PR primary and modifier labels Sweep labels authorize GPU work for same-repository PRs whether draft or ready. Draft status controls review readiness, not sweep eligibility; fork PRs retain their trusted-dispatch path. Adding a sweep label or pushing with one present can start a sweep. Marking ready does not dispatch or repeat one. To start an already-labeled draft that has no run, remove and reapply its sweep label. diff --git a/docs/ci-procedures_zh.md b/docs/ci-procedures_zh.md index 1c8917da69..5d6b310ad3 100644 --- a/docs/ci-procedures_zh.md +++ b/docs/ci-procedures_zh.md @@ -214,6 +214,14 @@ RUN_ID=$(gh run list \ 如果 `RUN_ID` 为空，不得继续。Run Metadata 描述派发 Workflow 的 Ref，可能不等于输入 `ref`。解释 GPU 结果前，必须在 `get-jobs` 中确认唯一标题、生成器命令与 Checkout Ref。 +## Kimi-K3 AgentX 功耗补测 + +Kimi-K3 的 B200、GB200 和 GB300 多节点配方启用必需的 DCGM 遥测，并在服务头节点运行自定义 AgentX 客户端。Launcher 选择固定提交的 AgentX 功耗运行时、记录提交 SHA，等待 Slurm 结束和遥测收尾，然后逐一验证所请求的并发点并保存结果。共享结果收集器会保留原生任务的失败状态，先保存可用的功耗诊断文件，再返回失败。H200 路由和 AMD 测量尾部修复属于独立变更，仍待各自集成与验证。启用必需遥测配置本身不代表硬件验证通过。 + +补测缺失功耗时，只生成缺失的配方与并发组合，设置 `require-power: true`，保持 `agentx-fast: false`，并留空时长覆盖。标准 AgentX Profile 为一小时。对新启用的运行时或集群，先验证一个缺失点，再调度其余点。配方渲染通过或 GitHub Runner 在线并不能证明实时采集已就绪，也不能证明 Slurm 有空闲资源。保留现有有效点，新性能与功耗必须来自同一次运行；不得把新运行的能耗附加到旧性能点上。手动 `e2e-tests.yml` 产物仍须经过正常审查和入库流程，才会显示在 Dashboard 中。 + +B200 Kimi 配方采用 DCP8，且关闭 Mooncake Offload。Master Config 记录 `dcp-size: 8` 和 `kv-offloading: none` 以匹配实际命令；这项元数据修正不会启用 Offload。 + ## PR 主标签与修饰标签 同仓库 PR 无论处于草稿还是 ready 状态，都由 sweep 标签授权 GPU 运行。草稿状态控制是否开始审阅，不决定 sweep 资格；fork PR 仍使用受信任调度路径。添加 sweep 标签或在保留标签时推送提交可以启动 sweep。标记为 ready 不会调度或重复运行。已带标签但尚无运行的草稿，可先移除再重新添加对应 sweep 标签来启动。 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 30ec71d6f9..75b993fd36 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7448,3 +7448,12 @@ - "Use port 19401 for the GB300 aggregate exporter; the two-node shared lifecycle check verified ownership, all eight GPU telemetry streams, and preservation of the system service on port 9401. Full Kimi-K3 qualification remains pending." - "GB300 聚合部署改用已验证的 19401 功耗端口；完整 Kimi-K3 基准验证仍待完成。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3046 + +- config-keys: + - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg + scenario-type: + - agentic-coding + description: + - "Enable required telemetry on the four GB300 disaggregated recipes using the launcher and collector introduced below this draft in the stack. Preserve the original bilingual backfill and validation guidance." + - "为 GB300 Kimi-K3 分离部署启用必需实测功耗。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3047 From f72e2cae8ca4c278a5f57aa3114b0a29f7c5a0fe Mon Sep 17 00:00:00 2001 From: Wenyao Gao Date: Sat, 12 Sep 2026 17:48:52 -0700 Subject: [PATCH 4/4] fix: avoid the GB300 system exporter port in disaggregated recipes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GB300 分离部署改用已验证的 19401 端口；验证 prefill 与 decode 分组均传递该端口，并保留完整硬件验证待办。 --- ...gg-gb300-1p1d-dcp8-dcp8-dspark4-mooncake-agentic.yaml | 2 +- ...gg-gb300-1p2d-dcp8-dcp8-dspark4-mooncake-agentic.yaml | 2 +- ...gg-gb300-1p3d-dcp8-dcp8-dspark4-mooncake-agentic.yaml | 2 +- ...gg-gb300-1p3d-dcp8-dcp8-dspark7-mooncake-agentic.yaml | 2 +- docs/ci-procedures.md | 2 ++ docs/ci-procedures_zh.md | 2 ++ perf-changelog.yaml | 9 +++++++++ 7 files changed, 17 insertions(+), 4 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p1d-dcp8-dcp8-dspark4-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p1d-dcp8-dcp8-dspark4-mooncake-agentic.yaml index 4d1ff23ef5..dd177cf29f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p1d-dcp8-dcp8-dspark4-mooncake-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p1d-dcp8-dcp8-dspark4-mooncake-agentic.yaml @@ -164,7 +164,7 @@ telemetry: collector_join_timeout_seconds: 12 dcgm_exporter: container_image: dcgm-exporter - port: 9401 + port: 19401 benchmark: type: custom diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p2d-dcp8-dcp8-dspark4-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p2d-dcp8-dcp8-dspark4-mooncake-agentic.yaml index 55d3a39dad..c8894c06ba 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p2d-dcp8-dcp8-dspark4-mooncake-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p2d-dcp8-dcp8-dspark4-mooncake-agentic.yaml @@ -164,7 +164,7 @@ telemetry: collector_join_timeout_seconds: 12 dcgm_exporter: container_image: dcgm-exporter - port: 9401 + port: 19401 benchmark: type: custom diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark4-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark4-mooncake-agentic.yaml index 3c6a311e4f..229f4091fd 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark4-mooncake-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark4-mooncake-agentic.yaml @@ -164,7 +164,7 @@ telemetry: collector_join_timeout_seconds: 12 dcgm_exporter: container_image: dcgm-exporter - port: 9401 + port: 19401 benchmark: type: custom diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark7-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark7-mooncake-agentic.yaml index d53fc6f486..5ff2433b7e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark7-mooncake-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/disagg-gb300-1p3d-dcp8-dcp8-dspark7-mooncake-agentic.yaml @@ -164,7 +164,7 @@ telemetry: collector_join_timeout_seconds: 12 dcgm_exporter: container_image: dcgm-exporter - port: 9401 + port: 19401 benchmark: type: custom diff --git a/docs/ci-procedures.md b/docs/ci-procedures.md index fc22a6eda1..2f53927528 100644 --- a/docs/ci-procedures.md +++ b/docs/ci-procedures.md @@ -226,6 +226,8 @@ Do not continue if `RUN_ID` is empty. Run metadata describes the dispatch workfl The Kimi-K3 B200, GB200 and GB300 multi-node recipes enable required DCGM telemetry and run the custom AgentX client on the serving head. Their launchers select the immutable AgentX power runtime, stamp its commit, wait for Slurm completion and telemetry drain, then validate each requested concurrency before staging results. The shared result collector preserves failed native jobs and stages available power diagnostics before returning a failure. H200 routing and AMD measurement-tail fixes are separate changes pending their own integration and validation. Required telemetry configuration alone does not establish hardware qualification. +GB300 Kimi-K3 aggregate and disaggregated recipes use exporter port `19401` because the node system service occupies `9401`. The shared telemetry stage forwards this port to both prefill and decode Slurm groups. The retained two-node exporter lifecycle evidence covers port ownership and cleanup; it does not replace disaggregated request, power-window, or eval qualification. + For a missing-power backfill, generate only the missing recipe/concurrency combinations, set `require-power: true`, and leave `agentx-fast: false` and the duration override empty. The normal AgentX profile is one hour. Verify a first missing point on each newly enabled runtime/cluster before scheduling its remaining points. A rendered recipe or an online GitHub runner does not establish live collector readiness or Slurm capacity. Preserve the existing validated points, and keep new performance and power values paired with their own run; never attach a new run's energy to an older performance row. Manual `e2e-tests.yml` artifacts still require the normal reviewed ingestion path before they appear in the dashboard. B200 Kimi recipes use DCP8 with Mooncake offload disabled. The master config records `dcp-size: 8` and `kv-offloading: none` to match those commands; this metadata correction does not enable offload. diff --git a/docs/ci-procedures_zh.md b/docs/ci-procedures_zh.md index 5d6b310ad3..0f0a6ba9d3 100644 --- a/docs/ci-procedures_zh.md +++ b/docs/ci-procedures_zh.md @@ -218,6 +218,8 @@ RUN_ID=$(gh run list \ Kimi-K3 的 B200、GB200 和 GB300 多节点配方启用必需的 DCGM 遥测，并在服务头节点运行自定义 AgentX 客户端。Launcher 选择固定提交的 AgentX 功耗运行时、记录提交 SHA，等待 Slurm 结束和遥测收尾，然后逐一验证所请求的并发点并保存结果。共享结果收集器会保留原生任务的失败状态，先保存可用的功耗诊断文件，再返回失败。H200 路由和 AMD 测量尾部修复属于独立变更，仍待各自集成与验证。启用必需遥测配置本身不代表硬件验证通过。 +GB300 Kimi-K3 聚合与分离部署 recipe 使用 exporter 端口 `19401`，因为节点系统服务占用了 `9401`。共享遥测阶段会将此端口传入 prefill 和 decode 两个 Slurm 分组。已有双节点 exporter 生命周期证据验证了端口归属与清理行为，但不能替代分离部署的请求账目、功耗窗口或 eval 验证。 + 补测缺失功耗时，只生成缺失的配方与并发组合，设置 `require-power: true`，保持 `agentx-fast: false`，并留空时长覆盖。标准 AgentX Profile 为一小时。对新启用的运行时或集群，先验证一个缺失点，再调度其余点。配方渲染通过或 GitHub Runner 在线并不能证明实时采集已就绪，也不能证明 Slurm 有空闲资源。保留现有有效点，新性能与功耗必须来自同一次运行；不得把新运行的能耗附加到旧性能点上。手动 `e2e-tests.yml` 产物仍须经过正常审查和入库流程，才会显示在 Dashboard 中。 B200 Kimi 配方采用 DCP8，且关闭 Mooncake Offload。Master Config 记录 `dcp-size: 8` 和 `kv-offloading: none` 以匹配实际命令；这项元数据修正不会启用 Offload。 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 75b993fd36..a7eda03a46 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7457,3 +7457,12 @@ - "Enable required telemetry on the four GB300 disaggregated recipes using the launcher and collector introduced below this draft in the stack. Preserve the original bilingual backfill and validation guidance." - "为 GB300 Kimi-K3 分离部署启用必需实测功耗。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3047 + +- config-keys: + - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-disagg + scenario-type: + - agentic-coding + description: + - "Move the GB300 disaggregated DCGM exporter to port 19401 to avoid the system listener on 9401. Preserve prefill/decode telemetry grouping; full disaggregated hardware qualification remains pending." + - "GB300 分离部署的 DCGM exporter 改用 19401，避免与 9401 系统服务冲突；保留 prefill/decode 分组，完整硬件验证仍待完成。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3047