From 04dbbb0723bba89dbe0110847c10f7a5b563469a Mon Sep 17 00:00:00 2001 From: St0rmz1 Date: Wed, 29 Jul 2026 15:34:33 -0700 Subject: [PATCH 1/2] feat(kiloclaw) improve openclaw test harness --- services/kiloclaw/scripts/tests/README.md | 12 ++ .../tests/openclaw-upgrade-image-checks.sh | 1 + .../tests/openclaw-upgrade-validate.sh | 4 + .../scripts/tests/smoke-live-provider.sh | 149 ++++++++++++++++-- 4 files changed, 156 insertions(+), 10 deletions(-) diff --git a/services/kiloclaw/scripts/tests/README.md b/services/kiloclaw/scripts/tests/README.md index cf918dc64f..23e7bc7ecc 100644 --- a/services/kiloclaw/scripts/tests/README.md +++ b/services/kiloclaw/scripts/tests/README.md @@ -12,9 +12,21 @@ Validate an OpenClaw version bump before merging the bump PR. ```bash export KILOCODE_API_KEY= # for the live smoke; from app.kilo.ai/profile +export KILOCODE_ORGANIZATION_ID= # REQUIRED if the key is a personal key in an org bash services/kiloclaw/scripts/tests/openclaw-upgrade-validate.sh ``` +> **Credits.** A personal Kilo token spends **personal** credits. If your account +> is in an organization and you want the org's credits, `KILOCODE_ORGANIZATION_ID` +> must be set — the token alone does not carry org scope. Note that exporting +> `KILOCODE_API_KEY` (the line above) means the smoke can no longer read the org id +> out of your Kilo CLI config the way it does for a CLI-sourced token, so set it +> explicitly. Get it wrong and the live turn fails deep in the gateway as +> `402 add credits`, a websocket `1008` close, or "provider rejected the request +> schema" — none of which name the real cause, and all of which look like a broken +> image. The smoke now preflights this and refuses to run rather than let you +> debug the wrong thing. + It runs a preflight (Docker, bump branch, clean tree, grype, credential) then: | Script | What it checks | Key? | diff --git a/services/kiloclaw/scripts/tests/openclaw-upgrade-image-checks.sh b/services/kiloclaw/scripts/tests/openclaw-upgrade-image-checks.sh index dbe16e5362..84c9b7561d 100755 --- a/services/kiloclaw/scripts/tests/openclaw-upgrade-image-checks.sh +++ b/services/kiloclaw/scripts/tests/openclaw-upgrade-image-checks.sh @@ -434,6 +434,7 @@ run the credentialed live smoke locally too (it loads a real key into the freshly released OpenClaw, which is why nothing here runs in CI): export KILOCODE_API_KEY= # not your personal key + export KILOCODE_ORGANIZATION_ID= # required to spend ORG credits bash services/kiloclaw/scripts/tests/openclaw-upgrade-smoke.sh That covers what CI cannot without a credential: diff --git a/services/kiloclaw/scripts/tests/openclaw-upgrade-validate.sh b/services/kiloclaw/scripts/tests/openclaw-upgrade-validate.sh index 0780974a62..a7adc13d12 100755 --- a/services/kiloclaw/scripts/tests/openclaw-upgrade-validate.sh +++ b/services/kiloclaw/scripts/tests/openclaw-upgrade-validate.sh @@ -152,6 +152,10 @@ elif [ "$IS_BUMP" -eq 1 ]; then echo "Phase 2 (the live smoke) is half the coverage and needs a Kilo API key." echo "For the full validation, set a dedicated free-model key and re-run:" echo " export KILOCODE_API_KEY= # from https://app.kilo.ai/profile (bottom)" + echo " export KILOCODE_ORGANIZATION_ID= # REQUIRED to spend ORG credits;" + echo " # a personal token alone spends only" + echo " # personal credits and the live turn" + echo " # then fails with a misleading error" echo " bash $0" ask_continue_keyless_or_stop diff --git a/services/kiloclaw/scripts/tests/smoke-live-provider.sh b/services/kiloclaw/scripts/tests/smoke-live-provider.sh index a50de81649..20cf082be1 100644 --- a/services/kiloclaw/scripts/tests/smoke-live-provider.sh +++ b/services/kiloclaw/scripts/tests/smoke-live-provider.sh @@ -72,8 +72,64 @@ if [ -z "${KILOCODE_API_KEY:-}" ]; then exit 1 fi -if [ -z "${KILOCODE_ORGANIZATION_ID:-}" ] && [ "$CREDENTIAL_SOURCE" = "local Kilo CLI config" ]; then - KILOCODE_ORGANIZATION_ID="$(read_active_provider_value kilocodeOrganizationId)" +# Fall back to the Kilo CLI config's org id whatever the token's source. This +# used to be gated on CREDENTIAL_SOURCE = "local Kilo CLI config", which meant the +# DOCUMENTED path (export KILOCODE_API_KEY, as the README and the validator both +# instruct) silently skipped org discovery and ran with no org scope at all. +ORG_SOURCE="" +if [ -n "${KILOCODE_ORGANIZATION_ID:-}" ]; then + ORG_SOURCE="environment" +else + KILOCODE_ORGANIZATION_ID="$(read_active_provider_value kilocodeOrganizationId 2>/dev/null || true)" + [ -n "${KILOCODE_ORGANIZATION_ID:-}" ] && ORG_SOURCE="local Kilo CLI config" +fi + +# Credential preflight. A personal token spends PERSONAL credits; org credits are +# only reachable when the org id travels with the request. Get this wrong and the +# live turn fails deep inside the gateway as a 402, a websocket 1008 policy close, +# or "provider rejected the request schema" — none of which name the real cause, +# and all of which read as a product regression. Fail here instead, by name. +KILOCODE_API_BASE="${KILOCODE_API_BASE:-https://api.kilocode.ai}" +profile_json=$(curl -sL --max-time 20 -H "Authorization: Bearer $KILOCODE_API_KEY" \ + "$KILOCODE_API_BASE/api/profile" 2>/dev/null || true) + +if [ -z "$profile_json" ] || ! python3 -c 'import json,sys; json.load(sys.stdin)' <<< "$profile_json" >/dev/null 2>&1; then + echo "WARN: could not reach $KILOCODE_API_BASE/api/profile to validate the credential." >&2 + echo " Continuing, but a credential problem will surface later as a confusing" >&2 + echo " gateway error rather than as a credential error." >&2 +elif ! python3 -c 'import json,sys; sys.exit(0 if json.load(sys.stdin).get("user") else 1)' <<< "$profile_json" >/dev/null 2>&1; then + echo "✗ The Kilo token was rejected by $KILOCODE_API_BASE/api/profile (not a valid/active key)." >&2 + echo " Get a key from https://app.kilo.ai/profile (bottom) and export KILOCODE_API_KEY." >&2 + exit 1 +else + org_list=$(python3 -c ' +import json +import sys + +orgs = json.load(sys.stdin).get("organizations") or [] +for o in orgs: + print(f'"'"' - {o.get("name", "?")} {o.get("id", "?")}'"'"') +' <<< "$profile_json" 2>/dev/null || true) + + if [ -z "${KILOCODE_ORGANIZATION_ID:-}" ] && [ -n "$org_list" ]; then + echo "✗ KILOCODE_ORGANIZATION_ID is not set, but this token belongs to organization(s):" >&2 + echo "$org_list" >&2 + echo >&2 + echo " A personal token only spends PERSONAL credits. To spend ORG credits the org" >&2 + echo " id must be set, otherwise the live agent turn fails with a misleading error" >&2 + echo " (402 add credits / websocket 1008 / 'provider rejected the request schema')" >&2 + echo " that looks like a broken image rather than a credential problem." >&2 + echo >&2 + echo " Fix (pick the org you want billed):" >&2 + echo " export KILOCODE_ORGANIZATION_ID=" >&2 + echo >&2 + echo " If this token really does have personal credits and you want to use them," >&2 + echo " re-run with ALLOW_NO_ORG_SCOPE=true to acknowledge and skip this check." >&2 + if [ "${ALLOW_NO_ORG_SCOPE:-false}" != "true" ]; then + exit 1 + fi + echo " ALLOW_NO_ORG_SCOPE=true set — continuing without org scope." >&2 + fi fi export KILOCODE_API_KEY @@ -177,6 +233,29 @@ wait_for_ready() { return 1 } +# The controller's /_kilo/health reports `ready` as soon as the controller is up, +# which is NOT the same as the gateway being able to serve a call: the gateway is +# still loading ~32 provider plugins and running model discovery. A call landing +# in that window dies with a websocket 1006 abnormal closure, which looks like a +# product failure but is only a race. Gate on the gateway actually answering a +# lightweight method (`status`) before asserting anything that needs it, and call +# this again after any gateway RESTART, not just after first boot. +wait_for_gateway_serving() { + local label="$1" + + echo "waiting for $label gateway to serve calls ..." + for i in $(seq 1 120); do + if docker exec "$CID" openclaw gateway call status --json --timeout 5000 >/dev/null 2>&1; then + echo " gateway serving after ${i}s" + return 0 + fi + sleep 1 + done + + echo "FAIL: $label gateway did not start serving calls" + return 1 +} + assert_configured_model() { local model model=$(docker exec -i "$CID" python3 - <<'PY' @@ -277,13 +356,58 @@ PY # openclaw writes the --json payload to stdout and logs to stderr; drop stderr # (it can contain provider/credential detail) so the parsed value is pure JSON. - if ! output=$(docker exec "$CID" openclaw gateway call agent \ - --params "$params" \ - --expect-final \ - --timeout 240000 \ - --json 2>/dev/null); then + # + # Retry ONLY the warm-up race: a 1006 abnormal closure, i.e. the gateway + # vanishing mid-call with no close frame while it is still coming up. Every + # other outcome is a real result and must fail on the first attempt. That + # includes other transport closes — notably 1008 (policy violation), which is a + # DELIBERATE server-side close and reproduces identically on every attempt. + # Retrying anything broader would mask exactly the incompatibilities this smoke + # exists to catch. + local attempt + local kind="" + for attempt in 1 2 3; do + if output=$(docker exec "$CID" openclaw gateway call agent \ + --params "$params" \ + --expect-final \ + --timeout 240000 \ + --json 2>/dev/null); then + break + fi + + # Surface the structured error identity only. The message body stays + # suppressed because provider errors can carry live credentials, but without + # the type/code a transport race and a provider rejection are + # indistinguishable — which has cost real debugging time. + kind=$(python3 -c ' +import json +import sys + +try: + err = json.load(sys.stdin).get("error", {}) +except Exception: + print("unparseable") + sys.exit() +print(" ".join(str(err.get(k, "")) for k in ("type", "kind", "code") if err.get(k))) +' <<< "$output" 2>/dev/null || echo "unparseable") + + if [[ "$kind" != *"gateway_transport_error"*"1006"* ]]; then + check "live Auto Free agent turn" "nonce returned" "command failed" + echo " error: $kind (message suppressed — provider errors can contain live credentials)" + return + fi + + echo " attempt $attempt hit the gateway warm-up race [$kind]; retrying" + if [ "$attempt" -lt 3 ]; then + wait_for_gateway_serving "post-transport-error" || true + sleep 5 + fi + output="" + done + + if [ -z "$output" ]; then check "live Auto Free agent turn" "nonce returned" "command failed" - echo " Gateway output suppressed because provider errors can contain live credentials." + echo " error: $kind after 3 attempts (message suppressed — provider errors can contain live credentials)" return fi @@ -454,6 +578,7 @@ run_phase() { echo "=== $label: $image ===" start_container "$image" wait_for_ready "$label" + wait_for_gateway_serving "$label" assert_openclaw_version "$expected_version" assert_openclaw_config_valid assert_gateway_status @@ -479,6 +604,10 @@ run_phase() { # crash-looping. Leaves the config valid, so the live turn below still runs # against a working gateway — which doubles as proof the repair is sound. assert_hook_config_self_heal "$CID" "$PORT" "$TOKEN" + # That assertion restarts the gateway, and its own wait only gets the + # CONTROLLER back to ready. Re-gate on the gateway actually serving before + # the live turn below, or the turn races the gateway's plugin load. + wait_for_gateway_serving "$label post-self-heal" fi assert_exec_approvals_seeded "$CID" assert_github_gh_auth @@ -491,9 +620,9 @@ run_phase() { echo "Credential source: $CREDENTIAL_SOURCE" echo "Model under test: $KILOCODE_SMOKE_MODEL" if [ -n "${KILOCODE_ORGANIZATION_ID:-}" ]; then - echo "Organization scope: configured" + echo "Organization scope: configured (from ${ORG_SOURCE:-unknown}) — org credits will be used" else - echo "Organization scope: not configured" + echo "Organization scope: NOT configured — this run spends PERSONAL credits only" fi if [ "$MODE" = "upgrade" ]; then From 9ee988eb87fc4d9b31cb7a3924ed5dc514e6a659 Mon Sep 17 00:00:00 2001 From: St0rmz1 Date: Wed, 29 Jul 2026 16:06:26 -0700 Subject: [PATCH 2/2] fix(kiloclaw): surface the gateway close reason and correct the 1008 guidance The live smoke discarded the websocket close reason on a failed agent turn, so the string that names the cause never reached the operator. Print it for transport errors: it is generated by OpenClaw's own loopback gateway inside the container, not by the model provider, so it carries no credentials. Provider error bodies stay suppressed. This surfaced the real cause of a failing turn: 'pairing required: device is asking for more scopes than currently approved'. OpenClaw's in-container gateway auto-approves the CLI device at operator.read, then refuses its upgrade to operator.write and closes with 1008. That also disproves the guidance added earlier in this branch, which told the reader a 1008 meant a missing organization id. It does not; a 1008 reproduces with the org id correctly set, and the credential symptom is a 402. Correct the preflight message and the README to describe the two separately, and stop printing the 'message suppressed' caveat when a reason was in fact shown. --- services/kiloclaw/scripts/tests/README.md | 15 ++++--- .../scripts/tests/smoke-live-provider.sh | 39 ++++++++++++++----- 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/services/kiloclaw/scripts/tests/README.md b/services/kiloclaw/scripts/tests/README.md index 23e7bc7ecc..c922f69ac0 100644 --- a/services/kiloclaw/scripts/tests/README.md +++ b/services/kiloclaw/scripts/tests/README.md @@ -21,11 +21,16 @@ bash services/kiloclaw/scripts/tests/openclaw-upgrade-validate.sh > must be set — the token alone does not carry org scope. Note that exporting > `KILOCODE_API_KEY` (the line above) means the smoke can no longer read the org id > out of your Kilo CLI config the way it does for a CLI-sourced token, so set it -> explicitly. Get it wrong and the live turn fails deep in the gateway as -> `402 add credits`, a websocket `1008` close, or "provider rejected the request -> schema" — none of which name the real cause, and all of which look like a broken -> image. The smoke now preflights this and refuses to run rather than let you -> debug the wrong thing. +> explicitly. Get it wrong and the live turn fails with `402 Add credits to +> continue`, which reads as a broken image rather than a credential problem. The +> smoke now preflights this and refuses to run rather than let you debug the wrong +> thing. +> +> Note that a websocket `1008` close on the live turn is a *different* problem and +> not a credential one: it is OpenClaw's own in-container gateway refusing the +> CLI's device scope upgrade (`operator.read` to `operator.write`) with +> `pairing required`. The smoke now prints the close reason so the two are not +> confused. It runs a preflight (Docker, bump branch, clean tree, grype, credential) then: diff --git a/services/kiloclaw/scripts/tests/smoke-live-provider.sh b/services/kiloclaw/scripts/tests/smoke-live-provider.sh index 20cf082be1..e1e1207e60 100644 --- a/services/kiloclaw/scripts/tests/smoke-live-provider.sh +++ b/services/kiloclaw/scripts/tests/smoke-live-provider.sh @@ -116,9 +116,8 @@ for o in orgs: echo "$org_list" >&2 echo >&2 echo " A personal token only spends PERSONAL credits. To spend ORG credits the org" >&2 - echo " id must be set, otherwise the live agent turn fails with a misleading error" >&2 - echo " (402 add credits / websocket 1008 / 'provider rejected the request schema')" >&2 - echo " that looks like a broken image rather than a credential problem." >&2 + echo " id must be set, otherwise the live agent turn fails with '402 Add credits to" >&2 + echo " continue', which reads as a broken image rather than a credential problem." >&2 echo >&2 echo " Fix (pick the org you want billed):" >&2 echo " export KILOCODE_ORGANIZATION_ID=" >&2 @@ -329,6 +328,18 @@ assert_control_ui_proxy() { check "proxied Control UI HTML" "ready" "$result" } +# Print the turn's error identity. Only add the "suppressed" caveat when there is +# genuinely nothing more to show: for a transport close we already printed the +# gateway's own close reason, which explains the failure outright. +report_turn_error() { + local ident="$1" + if [[ "$ident" == *" :: "* ]]; then + echo " error: $ident" + else + echo " error: $ident (provider message suppressed — it can contain live credentials)" + fi +} + assert_live_agent_turn() { local nonce local session_id @@ -375,10 +386,13 @@ PY break fi - # Surface the structured error identity only. The message body stays - # suppressed because provider errors can carry live credentials, but without - # the type/code a transport race and a provider rejection are - # indistinguishable — which has cost real debugging time. + # Surface the structured error identity, plus the websocket close `reason` + # for transport errors. That reason is generated by OpenClaw's own gateway + # (the loopback ws://127.0.0.1:3001 inside the container), not by the model + # provider, so it carries no credentials and is safe to print — and it is the + # single most useful line here. Suppressing it hid strings like "pairing + # required: device is asking for more scopes than currently approved", which + # names the cause outright. Provider error bodies stay suppressed. kind=$(python3 -c ' import json import sys @@ -388,12 +402,17 @@ try: except Exception: print("unparseable") sys.exit() -print(" ".join(str(err.get(k, "")) for k in ("type", "kind", "code") if err.get(k))) + +ident = " ".join(str(err.get(k, "")) for k in ("type", "kind", "code") if err.get(k)) +reason = err.get("reason") +if reason and str(err.get("type", "")) == "gateway_transport_error": + ident = f"{ident} :: {reason}" +print(ident) ' <<< "$output" 2>/dev/null || echo "unparseable") if [[ "$kind" != *"gateway_transport_error"*"1006"* ]]; then check "live Auto Free agent turn" "nonce returned" "command failed" - echo " error: $kind (message suppressed — provider errors can contain live credentials)" + report_turn_error "$kind" return fi @@ -407,7 +426,7 @@ print(" ".join(str(err.get(k, "")) for k in ("type", "kind", "code") if err.get( if [ -z "$output" ]; then check "live Auto Free agent turn" "nonce returned" "command failed" - echo " error: $kind after 3 attempts (message suppressed — provider errors can contain live credentials)" + report_turn_error "$kind after 3 attempts" return fi