diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml index dafc046abe..34943fda52 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-dspark7-maxseq2-mooncake-agentic.yaml @@ -114,10 +114,27 @@ srun_options: mem: "0" container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 + benchmark: type: custom + client_placement: head + concurrencies: [1, 4] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: + ENABLE_AGENTX_POWER: "1" + REQUIRE_POWER: "1" INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" RESULT_DIR: "/logs/agentic" PORT: "8000" diff --git a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-nospec-mooncake-agentic.yaml b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-nospec-mooncake-agentic.yaml index 9b61dd4922..5720019946 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-nospec-mooncake-agentic.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-gb300-dcp8-nospec-mooncake-agentic.yaml @@ -115,10 +115,27 @@ srun_options: mem: "0" container-remap-root: "" +telemetry: + enabled: true + provider: dcgm-power + default_frequency: 1.0 + storage_subdir: power + required: true + startup_timeout_seconds: 120 + request_timeout_seconds: 2 + collector_join_timeout_seconds: 12 + dcgm_exporter: + container_image: dcgm-exporter + port: 19401 + benchmark: type: custom + client_placement: head + concurrencies: [48, 70] command: bash /infmax-workspace/benchmarks/multi_node/agentic_srt.sh env: + ENABLE_AGENTX_POWER: "1" + REQUIRE_POWER: "1" INFMAX_CONTAINER_WORKSPACE: "/infmax-workspace" AIPERF_TRACE_IDLE_GAP_CAP_SECONDS: "300" AIPERF_LIVE_FAILED_REQUEST_THRESHOLD: "0.25" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 187e02fee7..010d015f2e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7467,3 +7467,23 @@ - "Collect shared measured GPU power for the existing two-node GB200 GLM-5.2 aggregate AgentX recipe with producer 80d7203e424f903c9017de4608ee2044afce9574, exact selected-concurrency windows, and post-job adapter validation. Preserve native status and invalid diagnostics before failure. Ordinary Slurm limits, serving settings, other AgentX recipes, and fixed-sequence producer pins are unchanged." - "为现有双节点 GB200 GLM-5.2 聚合 AgentX 配方接入共享 GPU 实测功耗,按所选并发绑定测量窗口,并在任务结束后通过适配器校验。返回失败前保留原生状态及无效诊断;普通 Slurm 时限、服务配置、其他 AgentX 配方和固定序列 producer 版本保持不变。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2928 + +- config-keys: + - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg + - kimik3-fp4-gb300-dynamo-vllm-agentic-mooncake-dcp8-agg + scenario-type: + - agentic-coding + description: + - "Enable the two GB300 aggregate recipes and their common launcher power route, with the pinned producer, strict result collection and retained diagnostics." + - "为 GB300 Kimi-K3 聚合部署启用必需实测功耗。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3046 + +- config-keys: + - kimik3-fp4-gb300-dynamo-vllm-agentic-dspark-mooncake-dcp8-agg + - kimik3-fp4-gb300-dynamo-vllm-agentic-mooncake-dcp8-agg + scenario-type: + - agentic-coding + description: + - "Use port 19401 for the GB300 aggregate exporter; the two-node shared lifecycle check verified ownership, all eight GPU telemetry streams, and preservation of the system service on port 9401. Full Kimi-K3 qualification remains pending." + - "GB300 聚合部署改用已验证的 19401 功耗端口;完整 Kimi-K3 基准验证仍待完成。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3046 diff --git a/runners/launch_gb300-nv.sh b/runners/launch_gb300-nv.sh index 9249e91f27..ff3a70035c 100644 --- a/runners/launch_gb300-nv.sh +++ b/runners/launch_gb300-nv.sh @@ -193,12 +193,15 @@ if [[ -n "$CONFIG_FILE" && -f "$_RECIPE_SRC" ]] && awk ' USES_DCGM_POWER=1 fi -# Note (wenyao): the producer pin follows the srt-slurm main lineage that the -# dynamo-sglang lanes run on (fp8 validated end-to-end, fp4 recipes -# parse-verified against the pin); other frameworks clone diverging refs -# (aflowers branch, sa-submission), so fail fast for them instead. -if [[ "$USES_DCGM_POWER" == "1" && "$FRAMEWORK" != "dynamo-sglang" ]]; then - echo "Error: dcgm-power lanes are only validated for FRAMEWORK=dynamo-sglang, got: $FRAMEWORK" >&2 +USES_AGENTX_POWER=0 +if [[ "$USES_DCGM_POWER" == "1" && "$IS_AGENTIC" == "1" && + "$MODEL_PREFIX" == "kimik3" && "$PRECISION" == "fp4" && + "$FRAMEWORK" == "dynamo-vllm" && + "$_RECIPE_REL" == recipes/vllm/kimi-k3/agentic/* ]]; then + USES_AGENTX_POWER=1 +fi +if [[ "$USES_DCGM_POWER" == "1" && "$FRAMEWORK" != "dynamo-sglang" && "$USES_AGENTX_POWER" != "1" ]]; then + echo "Error: dcgm-power requires dynamo-sglang or the supported Kimi-K3 AgentX route" >&2 exit 1 fi @@ -207,6 +210,7 @@ fi # srt-slurm merge lands. POWER_SRT_SLURM_URL="https://github.com/edwingao28/srt-slurm.git" POWER_SRT_SLURM_PIN="6fc1bed01a0b82dae0088a105c03ce0cfb353443" +AGENTX_POWER_SRT_SLURM_PIN="80d7203e424f903c9017de4608ee2044afce9574" if [[ "$USES_DCGM_POWER" == "1" ]]; then DCGM_EXPORTER_IMAGE="nvcr.io/nvidia/k8s/dcgm-exporter:4.6.0-4.8.3-distroless" @@ -238,7 +242,16 @@ SRT_REPO_DIR="${GITHUB_WORKSPACE}/srt-slurm-${GITHUB_RUN_ID:-manual}-${GITHUB_RU SRTCTL_SETUP_SCRIPT="" rm -rf "$SRT_REPO_DIR" -if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "qwen3.5" ]]; then +if [[ "$USES_AGENTX_POWER" == "1" ]]; then + git clone "$POWER_SRT_SLURM_URL" "$SRT_REPO_DIR" || exit 1 + cd "$SRT_REPO_DIR" || exit 1 + git checkout "$AGENTX_POWER_SRT_SLURM_PIN" || exit 1 + test "$(git rev-parse HEAD)" = "$AGENTX_POWER_SRT_SLURM_PIN" || exit 1 + git rev-parse HEAD > "$GITHUB_WORKSPACE/power-producer-sha.txt" + mkdir -p recipes/vllm/kimi-k3/agentic + cp -rT "$GITHUB_WORKSPACE/benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic" \ + recipes/vllm/kimi-k3/agentic +elif [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-trt" && $MODEL_PREFIX == "qwen3.5" ]]; then git clone https://github.com/NVIDIA/srt-slurm.git "$SRT_REPO_DIR" cd "$SRT_REPO_DIR" git checkout v1.0.50 @@ -570,6 +583,12 @@ sed -i "s/^name:.*/name: \"${RUNNER_NAME}\"/" "$CONFIG_PATH" # verification. inject_synthetic_acceptance "$CONFIG_PATH" "$FRAMEWORK" || exit 1 +if [[ "$USES_AGENTX_POWER" == "1" ]]; then + read -r -a POWER_CONCURRENCIES <<< "$CONC_LIST" + python3 "$GITHUB_WORKSPACE/runners/inject_srt_power_concurrencies.py" \ + "$CONFIG_PATH" "${POWER_CONCURRENCIES[@]}" || exit 1 +fi + # --no-preflight skips srtctl's pre-submit model-path stat, which runs on # the GHA runner host (im-gb300-login-02, an x86 login node). It's required # whenever model.path resolves to the node-local /scratch NVMe that the login @@ -644,37 +663,49 @@ _snapshot_server_logs() { } trap _snapshot_server_logs EXIT -# Wait for log file to appear (also check job is still alive) -while ! ls "$LOG_FILE" &>/dev/null; do - if ! squeue -j "$JOB_ID" --noheader 2>/dev/null | grep -q "$JOB_ID"; then - echo "ERROR: Job $JOB_ID failed before creating log file" - scontrol show job "$JOB_ID" - exit 1 - fi - echo "Waiting for JOB_ID $JOB_ID to begin and $LOG_FILE to appear..." - sleep 5 -done - -# Poll for job completion in background -( - while squeue -j "$JOB_ID" --noheader 2>/dev/null | grep -q "$JOB_ID"; do - sleep 10 +AGENTX_POWER_RC=0 +if [[ "$USES_AGENTX_POWER" == "1" ]]; then + stream_slurm_job_log "$JOB_ID" "$LOG_FILE" || AGENTX_POWER_RC=$? +else + # Wait for log file to appear (also check job is still alive) + while ! ls "$LOG_FILE" &>/dev/null; do + if ! squeue -j "$JOB_ID" --noheader 2>/dev/null | grep -q "$JOB_ID"; then + echo "ERROR: Job $JOB_ID failed before creating log file" + scontrol show job "$JOB_ID" + exit 1 + fi + echo "Waiting for JOB_ID $JOB_ID to begin and $LOG_FILE to appear..." + sleep 5 done -) & -POLL_PID=$! -echo "Tailing LOG_FILE: $LOG_FILE" + # Poll for job completion in background + ( + while squeue -j "$JOB_ID" --noheader 2>/dev/null | grep -q "$JOB_ID"; do + sleep 10 + done + ) & + POLL_PID=$! -# Stream the log file until job completes (-F follows by name, polls instead of inotify for NFS) -tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null + echo "Tailing LOG_FILE: $LOG_FILE" -wait $POLL_PID + # Stream the log file until job completes (-F follows by name, polls instead of inotify for NFS) + tail -F -s 2 -n+1 "$LOG_FILE" --pid=$POLL_PID 2>/dev/null + + wait $POLL_PID +fi set -x echo "Job $JOB_ID completed!" echo "Collecting results..." +if [[ "$USES_AGENTX_POWER" == "1" && "${EVAL_ONLY:-false}" != "true" ]]; then + read -r -a POWER_CONCURRENCIES <<< "$CONC_LIST" + collect_agentic_power_results "$JOB_ID" "$LOGS_DIR" "$INFMAX_WORKSPACE" \ + "$GITHUB_WORKSPACE" "$RESULT_FILENAME" "$AGENTX_POWER_SRT_SLURM_PIN" \ + "${POWER_CONCURRENCIES[@]}" || AGENTX_POWER_RC=$? +fi + if [ -d "$LOGS_DIR" ]; then echo "Found logs directory: $LOGS_DIR" # Tarball + LOGS copy + power provenance markers are produced by the EXIT @@ -685,6 +716,11 @@ else echo "Warning: Logs directory not found at $LOGS_DIR" fi +if [[ "$AGENTX_POWER_RC" != "0" ]]; then + echo "ERROR: AgentX job or power validation failed; EXIT will stage audit artifacts" >&2 + exit "$AGENTX_POWER_RC" +fi + if [[ "${EVAL_ONLY:-false}" != "true" ]]; then if [ ! -d "$LOGS_DIR" ]; then exit 1 diff --git a/utils/test_gb300_power_official_contract.py b/utils/test_gb300_power_official_contract.py index 422b885427..f270b6fb16 100644 --- a/utils/test_gb300_power_official_contract.py +++ b/utils/test_gb300_power_official_contract.py @@ -27,16 +27,14 @@ def _launcher_routing_source(launcher_path: Path = LAUNCHER_PATH) -> str: if launcher_path.name == "launch_gb200-nv.sh": gate_start = launcher.index("USES_DCGM_POWER=0") gate_end = launcher.index('if [[ "$USES_DCGM_POWER" == "1" ]]; then\n DCGM_EXPORTER_IMAGE=', gate_start) - route_start = launcher.index('if [[ "$IS_AGENTIC" == "1" && "$MODEL_PREFIX" == "glm5.2"') + route_start = launcher.index('if [[ "$IS_AGENTIC" == "1" && "$MODEL_PREFIX" == "glm5.2"', + launcher.index('echo "Cloning srt-slurm repository..."')) route_end = launcher.index('\necho "Installing srtctl..."', route_start) return launcher[gate_start:gate_end] + launcher[route_start:route_end] - route_start = launcher.index( - 'if [[ "$IS_AGENTIC" == "1" && $FRAMEWORK == "dynamo-sglang" ' - '&& $MODEL_PREFIX == "qwen3.5" ]]; then' - ) - route_end_marker = '\nfi\n\necho "Installing srtctl..."' - route_end = launcher.index(route_end_marker, route_start) + len("\nfi") - return launcher[route_start:route_end] + route_start = launcher.index('if [[ "$USES_AGENTX_POWER" == "1" ]]; then', + launcher.index('echo "Cloning srt-slurm repository..."')) + route_end = launcher.index('\necho "Installing srtctl..."', route_start) + return _launcher_power_gate_source(launcher_path.name) + "\n" + launcher[route_start:route_end] def _write_executable(path: Path, text: str) -> None: @@ -44,6 +42,13 @@ def _write_executable(path: Path, text: str) -> None: path.chmod(0o755) +def _launcher_power_gate_source(launcher_name: str) -> str: + launcher = (REPO_ROOT / "runners" / launcher_name).read_text() + start = launcher.index("USES_DCGM_POWER=0\n") + end = launcher.index("\nfi", launcher.index("Error: dcgm-power requires", start)) + return launcher[start:end + len("\nfi")] + + def _run_dsv4_route( tmp_path: Path, uses_dcgm_power: bool, *, reported_head: str = "", launcher_path: Path = LAUNCHER_PATH, model_prefix: str = "dsv4", @@ -53,6 +58,7 @@ def _run_dsv4_route( workspace = tmp_path / "workspace" stub_bin = tmp_path / "bin" recipe_directory = ("sglang/glm5.2/gb200-fp4/agentic" if model_prefix == "glm5.2" + else "vllm/kimi-k3/agentic" if model_prefix == "kimik3" else "sglang/deepseek-v4/8k1k") recipe_name = recipe_name or ("glm5.2-agentx-agg.yaml" if model_prefix == "glm5.2" else "recipe.yaml") source = workspace / "benchmarks/multi_node/srt-slurm-recipes" / recipe_directory @@ -123,14 +129,15 @@ def _run_dsv4_route( POWER_SRT_SLURM_PIN={PRODUCER_PIN} AGENTX_POWER_SRT_SLURM_PIN={AGENTX_PRODUCER_PIN} IS_AGENTIC={int(is_agentic)} -FRAMEWORK=dynamo-sglang +FRAMEWORK={'dynamo-vllm' if model_prefix == 'kimik3' else 'dynamo-sglang'} MODEL_PREFIX={model_prefix} PRECISION=fp4 SPEC_DECODING= USES_DCGM_POWER={int(uses_dcgm_power)} -CONFIG_FILE=recipes/{recipe_directory}/{recipe_name} +CONFIG_FILE=recipes/{recipe_directory}/{recipe_name}:fixture-override GITHUB_WORKSPACE={workspace!s} SRT_REPO_DIR={repo_dir!s} +python3() {{ :; }} {routing} """ ) @@ -381,6 +388,51 @@ def test_gb300_dsv4_recipe_images_match_their_master_configs(): assert recipe_image == config["image"], (key, config_file) +@pytest.mark.parametrize("launcher_name", ["launch_gb300-nv.sh"]) +def test_kimi_agentx_route_uses_recipe_power_gate(tmp_path, launcher_name): + log, workspace, repo_dir, marker = _run_dsv4_route( + tmp_path, True, launcher_path=REPO_ROOT / "runners" / launcher_name, model_prefix="kimik3", is_agentic=True + ) + assert f"git clone {FORK_URL} {repo_dir}" in log + assert f"git checkout {AGENTX_PRODUCER_PIN}" in log + assert (workspace / "power-producer-sha.txt").read_text() == f"{AGENTX_PRODUCER_PIN}\n" + assert marker.read_text() == "from-workspace\n" + + +@pytest.mark.parametrize( + "override", + [ + {"CONFIG_FILE": "recipes/vllm/kimi-k3/8k1k/test.yaml"}, + {"MODEL_PREFIX": "other-model"}, + {"PRECISION": "fp8"}, + {"IS_AGENTIC": "0"}, + {"FRAMEWORK": "vllm"}, + ], +) +def test_kimi_power_gate_rejects_unsupported_route(tmp_path, override): + env = os.environ.copy() + env.update(GITHUB_WORKSPACE=str(tmp_path), CONFIG_FILE="recipes/vllm/kimi-k3/agentic/test.yaml", + MODEL_PREFIX="kimik3", PRECISION="fp4", IS_AGENTIC="1", FRAMEWORK="dynamo-vllm") + env.update(override) + recipe = tmp_path / "benchmarks/multi_node/srt-slurm-recipes" / env["CONFIG_FILE"].removeprefix("recipes/") + recipe.parent.mkdir(parents=True) + recipe.write_text("telemetry:\n provider: dcgm-power\n enabled: true\n") + result = subprocess.run(["/bin/bash"], input=_launcher_power_gate_source("launch_gb300-nv.sh"), + text=True, capture_output=True, env=env) + assert result.returncode == 1 + assert "dcgm-power requires" in result.stderr + + +@pytest.mark.parametrize("launcher_name", ["launch_gb300-nv.sh"]) +def test_kimi_power_route_rejects_wrong_commit(tmp_path, launcher_name): + with pytest.raises(subprocess.CalledProcessError): + _run_dsv4_route( + tmp_path, True, reported_head="c" * 40, + launcher_path=REPO_ROOT / "runners" / launcher_name, model_prefix="kimik3", is_agentic=True, + ) + assert not (tmp_path / "workspace/power-producer-sha.txt").exists() + + @pytest.mark.parametrize( ("job_state", "concurrencies", "expected_rc"), [("COMPLETED|0:0", [4], 0), ("FAILED|1:0", [4], 1), ("COMPLETED|0:0", [4, 8], 1)],