From c8cd98cf52e2149d6faad48cce67cb35a7c64f87 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 8 Aug 2026 18:34:01 -0700 Subject: [PATCH] refactor(gateway): centralize connect failure classification (#120505) * refactor(gateway): centralize connect failure classification * fix(gateway): project connect failure into status JSON * fix(protocol): restore remote-auth context in pairing remediation * fix(gateway): classify generic probe failures by close reason * fix(gateway): keep transport closes unreachable * fix(gateway): preserve rate-limit diagnostics * fix(gateway): preserve typed rate-limit failures --- packages/gateway-client/src/client.ts | 12 +- .../src/connect-error-details.test.ts | 159 +++++++++++++++++ .../src/connect-error-details.ts | 92 ++++++++++ src/cli/daemon-cli/probe.test.ts | 160 ++++++++++++++++++ src/cli/daemon-cli/probe.ts | 35 +++- src/cli/daemon-cli/status.gather.ts | 7 + src/cli/gateway-backed-exit.process.test.ts | 77 +++++++++ src/commands/doctor-gateway-health.test.ts | 60 +++++++ src/commands/doctor-gateway-health.ts | 20 ++- .../gateway-health-auth-diagnostic.ts | 64 ++++++- src/commands/gateway-readiness.test.ts | 63 +++++++ src/commands/health.test.ts | 73 ++++++++ src/commands/health.ts | 21 ++- src/flows/doctor-core-checks.runtime.test.ts | 20 +++ src/flows/doctor-core-checks.runtime.ts | 20 ++- src/gateway/call.test.ts | 105 ++++++++++++ src/gateway/call.ts | 15 ++ src/gateway/client.test.ts | 45 +++++ src/gateway/probe.test.ts | 17 +- src/gateway/probe.ts | 21 ++- src/tui/tui.test.ts | 52 ++++-- src/tui/tui.ts | 51 +++--- 22 files changed, 1128 insertions(+), 61 deletions(-) diff --git a/packages/gateway-client/src/client.ts b/packages/gateway-client/src/client.ts index 7d482e1a96c1..06668b3c49f9 100644 --- a/packages/gateway-client/src/client.ts +++ b/packages/gateway-client/src/client.ts @@ -875,6 +875,8 @@ export class GatewayClient { assembled: AssembledConnect, ) { const role = this.opts.role ?? "operator"; + const detailCode = + error instanceof GatewayClientRequestError ? readConnectErrorDetailCode(error.details) : null; const shouldRetryWithDeviceToken = shouldRetryGatewayWithDeviceToken({ retryBudgetUsed: this.deviceTokenRetryBudgetUsed, currentDeviceToken: assembled.resolvedDeviceToken, @@ -886,9 +888,7 @@ export class GatewayClient { if ( this.opts.deviceIdentity && assembled.usingStoredDeviceToken && - error instanceof GatewayClientRequestError && - readConnectErrorDetailCode(error.details) === - ConnectErrorDetailCodes.AUTH_DEVICE_TOKEN_MISMATCH + detailCode === ConnectErrorDetailCodes.AUTH_DEVICE_TOKEN_MISMATCH ) { const deviceId = this.opts.deviceIdentity.deviceId; try { @@ -942,7 +942,11 @@ export class GatewayClient { } this.notifyConnectError(error); const message = `gateway connect failed: ${formatGatewayClientErrorForLog(error)}`; - if (this.opts.mode === GATEWAY_CLIENT_MODES.PROBE || isGatewayClientStoppedError(error)) { + if ( + this.opts.mode === GATEWAY_CLIENT_MODES.PROBE || + isGatewayClientStoppedError(error) || + detailCode === ConnectErrorDetailCodes.AUTH_RATE_LIMITED + ) { this.logDebug(message); } else { this.logError(message); diff --git a/packages/gateway-protocol/src/connect-error-details.test.ts b/packages/gateway-protocol/src/connect-error-details.test.ts index cabb53d85580..15f5d85a039c 100644 --- a/packages/gateway-protocol/src/connect-error-details.test.ts +++ b/packages/gateway-protocol/src/connect-error-details.test.ts @@ -4,6 +4,7 @@ import { buildPairingConnectCloseReason, buildPairingConnectErrorDetails, buildPairingConnectErrorMessage, + classifyGatewayConnectFailure, describePairingConnectRequirement, formatConnectErrorMessage, formatConnectPairingRequiredMessage, @@ -67,6 +68,164 @@ describe("readConnectErrorRecoveryAdvice", () => { }); }); +describe("classifyGatewayConnectFailure", () => { + it.each([ + { + name: "structured pairing upgrade", + input: { + details: { code: "PAIRING_REQUIRED", reason: "scope-upgrade", requestId: "req-123" }, + message: "connect failed", + }, + kind: "pairing-required", + message: "scope upgrade pending approval (requestId: req-123)", + remediation: "openclaw devices approve --latest", + }, + { + name: "structured device identity requirement", + input: { details: { code: "DEVICE_IDENTITY_REQUIRED" }, message: "connect failed" }, + kind: "device-identity-required", + message: "connect failed", + remediation: undefined, + }, + { + name: "structured scope mismatch", + input: { details: { code: "AUTH_SCOPE_MISMATCH" }, message: "scope rejected" }, + kind: "scope-mismatch", + message: "scope rejected", + remediation: "openclaw devices list", + }, + { + name: "structured authentication rate limit", + input: { details: { code: "AUTH_RATE_LIMITED" }, message: "connect failed" }, + kind: "rate-limited", + message: "connect failed", + remediation: "temporary authentication lockout", + }, + { + name: "shared token mismatch", + input: { details: { code: "AUTH_TOKEN_MISMATCH" }, message: "gateway token mismatch" }, + kind: "auth-rejected", + message: "gateway token mismatch", + remediation: "gateway.remote.token", + }, + { + name: "device token mismatch", + input: { + details: { code: "AUTH_DEVICE_TOKEN_MISMATCH" }, + message: "device token mismatch", + }, + kind: "auth-rejected", + message: "device token mismatch", + remediation: "openclaw devices rotate --device --role operator", + }, + { + name: "other structured auth rejection", + input: { details: { code: "AUTH_PASSWORD_MISMATCH" }, message: "password mismatch" }, + kind: "auth-rejected", + message: "password mismatch", + remediation: undefined, + }, + { + name: "legacy pairing reason", + input: { reason: "gateway closed (1008): pairing required" }, + kind: "pairing-required", + message: "gateway closed (1008): pairing required", + remediation: "openclaw devices approve --latest", + }, + { + name: "legacy pairing reason behind a generic message", + input: { + message: "connect failed", + reason: "gateway closed (1008): pairing required", + }, + kind: "pairing-required", + message: "connect failed", + remediation: "openclaw devices approve --latest", + }, + { + name: "legacy device identity reason behind a generic message", + input: { + message: "connect failed", + reason: "gateway closed (1008): device identity required", + }, + kind: "device-identity-required", + message: "connect failed", + remediation: undefined, + }, + { + name: "legacy scope mismatch reason behind a generic message", + input: { message: "connect failed", reason: "scope mismatch" }, + kind: "scope-mismatch", + message: "connect failed", + remediation: "openclaw devices list", + }, + { + name: "legacy device token reason behind a generic message", + input: { message: "connect failed", reason: "device token mismatch" }, + kind: "auth-rejected", + message: "connect failed", + remediation: "openclaw devices rotate --device --role operator", + }, + { + name: "legacy shared token reason behind a generic message", + input: { message: "connect failed", reason: "gateway token mismatch" }, + kind: "auth-rejected", + message: "connect failed", + remediation: "gateway.remote.token", + }, + { + name: "legacy gateway close reason behind a generic message", + input: { message: "connect failed", reason: "gateway closed (1008): auth failed" }, + kind: "gateway-rejected", + message: "connect failed", + remediation: undefined, + }, + { + name: "legacy gateway close", + input: { message: "gateway closed (1008): auth failed" }, + kind: "gateway-rejected", + message: "gateway closed (1008): auth failed", + remediation: undefined, + }, + { + name: "legacy authentication rate limit", + input: { + reason: "unauthorized: too many failed authentication attempts (retry later)", + }, + kind: "rate-limited", + message: "unauthorized: too many failed authentication attempts (retry later)", + remediation: "temporary authentication lockout", + }, + { + name: "generic retry hint without the authentication lockout phrase", + input: { message: "connect failed; retry later" }, + kind: "unreachable", + message: "connect failed; retry later", + remediation: undefined, + }, + { + name: "unreachable endpoint", + input: { message: "connect ECONNREFUSED 127.0.0.1:18789" }, + kind: "unreachable", + message: "connect ECONNREFUSED 127.0.0.1:18789", + remediation: undefined, + }, + ])("classifies $name", ({ input, kind, message, remediation }) => { + const result = classifyGatewayConnectFailure(input); + expect(result.kind).toBe(kind); + expect(result.userMessage).toBe(message); + if (remediation) { + expect(result.remediation).toContain(remediation); + } else { + expect(result.remediation).toBeUndefined(); + } + if (kind === "pairing-required") { + expect(result.remediation).toContain("--url"); + expect(result.remediation).toContain("--token/--password"); + } + }); +}); + describe("resolveAuthConnectErrorDetailCode", () => { it("maps device token scope mismatches to a dedicated auth detail", () => { expect(resolveAuthConnectErrorDetailCode("scope_mismatch")).toBe("AUTH_SCOPE_MISMATCH"); diff --git a/packages/gateway-protocol/src/connect-error-details.ts b/packages/gateway-protocol/src/connect-error-details.ts index aa32d6b94da4..603f7c674ca7 100644 --- a/packages/gateway-protocol/src/connect-error-details.ts +++ b/packages/gateway-protocol/src/connect-error-details.ts @@ -473,6 +473,98 @@ export function readConnectPairingRequiredMessage( }; } +const PAIRING_APPROVAL_REMEDIATION = + "Run `openclaw devices approve --latest` to preview the pending request, then rerun the printed " + + "`openclaw devices approve ` command and reconnect (pass the same --url and " + + "--token/--password flags if you connected with explicit credentials)."; +const DEVICE_TOKEN_REMEDIATION = + "Rotate the paired-device token with `openclaw devices rotate --device --role operator`, then reconnect."; +const SHARED_TOKEN_REMEDIATION = + "Verify `gateway.remote.token` matches `gateway.auth.token`. If a paired-device token is stale, " + + "rotate it with `openclaw devices rotate --device --role operator`, then reconnect."; +const SCOPE_MISMATCH_REMEDIATION = + "Review approved scopes with `openclaw devices list`; if an upgrade is pending, preview it with " + + "`openclaw devices approve --latest`, approve the printed request, then reconnect."; +const RATE_LIMITED_REMEDIATION = + "Wait for the temporary authentication lockout to expire, then retry."; +const GATEWAY_CLOSED_MESSAGE_PATTERN = /\bgateway closed \(\d+\):/i; + +/** Classifies Gateway connect failures from structured details, with one legacy text fallback. */ +export function classifyGatewayConnectFailure(input: { + details?: unknown; + reason?: string | null; + message?: string | null; +}) { + const code = readConnectErrorDetailCode(input.details); + const message = normalizeOptionalString(input.message); + const reason = normalizeOptionalString(input.reason); + const userMessage = message ?? reason; + const classificationText = [message, reason] + .filter((value): value is string => Boolean(value)) + .join("\n"); + const normalized = classificationText.toLowerCase(); + const pairing = + readPairingConnectErrorDetails(input.details) ?? + readConnectPairingRequiredMessage(classificationText); + if (code === ConnectErrorDetailCodes.PAIRING_REQUIRED || pairing) { + return { + kind: "pairing-required" as const, + userMessage: + code === ConnectErrorDetailCodes.PAIRING_REQUIRED + ? formatConnectPairingRequiredMessage(input.details) + : (userMessage ?? "device pairing required"), + remediation: PAIRING_APPROVAL_REMEDIATION, + }; + } + const deviceIdentityRequired = + code === ConnectErrorDetailCodes.DEVICE_IDENTITY_REQUIRED || + code === ConnectErrorDetailCodes.CONTROL_UI_DEVICE_IDENTITY_REQUIRED || + normalized.includes("device identity required"); + const scopeMismatch = + code === ConnectErrorDetailCodes.AUTH_SCOPE_MISMATCH || normalized.includes("scope mismatch"); + const rateLimited = + code === ConnectErrorDetailCodes.AUTH_RATE_LIMITED || + (!code && normalized.includes("too many failed authentication attempts")); + const deviceTokenMismatch = + code === ConnectErrorDetailCodes.AUTH_DEVICE_TOKEN_MISMATCH || + normalized.includes("device token mismatch"); + const sharedTokenMismatch = + code === ConnectErrorDetailCodes.AUTH_TOKEN_MISMATCH || + normalized.includes("gateway token mismatch"); + const authRejected = + deviceTokenMismatch || + sharedTokenMismatch || + code?.startsWith("AUTH_") || + code?.startsWith("DEVICE_AUTH_"); + const kind = deviceIdentityRequired + ? ("device-identity-required" as const) + : scopeMismatch + ? ("scope-mismatch" as const) + : rateLimited + ? ("rate-limited" as const) + : authRejected + ? ("auth-rejected" as const) + : code || GATEWAY_CLOSED_MESSAGE_PATTERN.test(classificationText) + ? ("gateway-rejected" as const) + : ("unreachable" as const); + const remediation = rateLimited + ? RATE_LIMITED_REMEDIATION + : scopeMismatch + ? SCOPE_MISMATCH_REMEDIATION + : deviceTokenMismatch + ? DEVICE_TOKEN_REMEDIATION + : sharedTokenMismatch + ? SHARED_TOKEN_REMEDIATION + : undefined; + return { + kind, + userMessage: + userMessage ?? + (kind === "unreachable" ? "gateway unreachable" : "gateway rejected connection"), + ...(remediation ? { remediation } : {}), + }; +} + /** Formats pairing-required details into the canonical user-facing message. */ export function formatConnectPairingRequiredMessage(details: unknown): string { const pairing = readPairingConnectErrorDetails(details); diff --git a/src/cli/daemon-cli/probe.test.ts b/src/cli/daemon-cli/probe.test.ts index 3b36a534f127..7c11f468d7fb 100644 --- a/src/cli/daemon-cli/probe.test.ts +++ b/src/cli/daemon-cli/probe.test.ts @@ -1,6 +1,8 @@ // Daemon probe tests cover gateway probe command behavior and output. import { describe, expect, it, vi } from "vitest"; +import { gatewayProbeResultSawGateway } from "../../commands/gateway-health-auth-diagnostic.js"; import { probeGatewayStatus } from "./probe.js"; +import type { DaemonStatus } from "./status.gather.js"; const callGatewayMock = vi.hoisted(() => vi.fn()); const probeGatewayMock = vi.hoisted(() => vi.fn()); @@ -17,6 +19,19 @@ vi.mock("../progress.js", () => ({ withProgress: async (_opts: unknown, fn: () => Promise) => await fn(), })); +function createDaemonStatus(rpc: NonNullable): DaemonStatus { + return { + service: { + label: "test service", + loaded: true, + loadedText: "loaded", + notLoadedText: "not loaded", + }, + rpc, + extraServices: [], + }; +} + describe("probeGatewayStatus", () => { const pairingPendingAuth = { role: null, @@ -39,6 +54,7 @@ describe("probeGatewayStatus", () => { kind: "connect", capability: "pairing_pending", auth: pairingPendingAuth, + connectFailure: { kind: "pairing-required" }, error: "gateway closed (1008): pairing required", }); } @@ -85,6 +101,148 @@ describe("probeGatewayStatus", () => { }); }); + it("projects allowlisted connect failure details without serializing raw payloads", async () => { + probeGatewayMock.mockResolvedValueOnce({ + ok: false, + error: "connect failed", + close: { code: 1008, reason: "connect failed" }, + connectErrorDetails: { + code: "PAIRING_REQUIRED", + reason: "scope-upgrade", + secret: "do-not-print", + }, + auth: pairingPendingAuth, + }); + + const result = await probeGatewayStatus({ + url: "ws://127.0.0.1:19191", + timeoutMs: 5_000, + json: true, + }); + + expect(result.ok).toBe(false); + if (result.ok) { + throw new Error("expected failed gateway probe"); + } + expect(result.connectFailure).toEqual({ + kind: "pairing-required", + detailCode: "PAIRING_REQUIRED", + }); + expect(result).not.toHaveProperty("connectErrorDetails"); + expect(gatewayProbeResultSawGateway(result)).toBe(true); + + const json = JSON.stringify(createDaemonStatus(result)); + expect(json).not.toContain("do-not-print"); + expect(json).not.toContain('"secret"'); + expect(json).not.toContain("scope-upgrade"); + }); + + it("classifies a legacy pairing close when the probe error is generic", async () => { + probeGatewayMock.mockResolvedValueOnce({ + ok: false, + error: "connect failed", + close: { code: 1008, reason: "pairing required" }, + }); + + const result = await probeGatewayStatus({ + url: "ws://127.0.0.1:19191", + timeoutMs: 5_000, + json: true, + }); + + expect(result.ok).toBe(false); + if (result.ok) { + throw new Error("expected failed gateway probe"); + } + expect(result.error).toBe("connect failed"); + expect(result.connectFailure).toEqual({ kind: "pairing-required" }); + expect(gatewayProbeResultSawGateway(result)).toBe(true); + }); + + it("does not classify an unvalidated transport close as a reachable gateway", async () => { + probeGatewayMock.mockResolvedValueOnce({ + ok: false, + error: "connect ECONNREFUSED 127.0.0.1:19191", + close: { code: 1006, reason: "" }, + }); + + const result = await probeGatewayStatus({ + url: "ws://127.0.0.1:19191", + timeoutMs: 5_000, + json: true, + }); + + expect(result.ok).toBe(false); + if (result.ok) { + throw new Error("expected failed gateway probe"); + } + expect(result.connectFailure).toEqual({ kind: "unreachable" }); + expect(gatewayProbeResultSawGateway(result)).toBe(false); + }); + + it("projects authentication rate limits as reachable temporary lockouts", async () => { + probeGatewayMock.mockResolvedValueOnce({ + ok: false, + error: "connect failed", + close: { + code: 1008, + reason: "unauthorized: too many failed authentication attempts (retry later)", + }, + connectErrorDetails: { + code: "AUTH_RATE_LIMITED", + authReason: "rate_limited", + recommendedNextStep: "wait_then_retry", + retryAfterMs: 60_000, + }, + }); + + const result = await probeGatewayStatus({ + url: "ws://127.0.0.1:19191", + timeoutMs: 5_000, + json: true, + }); + + expect(result.ok).toBe(false); + if (result.ok) { + throw new Error("expected failed gateway probe"); + } + expect(result.connectFailure).toEqual({ + kind: "rate-limited", + detailCode: "AUTH_RATE_LIMITED", + }); + expect(gatewayProbeResultSawGateway(result)).toBe(true); + expect(JSON.stringify(createDaemonStatus(result))).not.toContain("retryAfterMs"); + }); + + it("omits unknown detail codes from serialized daemon status", async () => { + probeGatewayMock.mockResolvedValueOnce({ + ok: false, + error: "connect failed", + close: { code: 1008, reason: "connect failed" }, + connectErrorDetails: { + code: "FUTURE_SENSITIVE_CODE", + secret: "do-not-print-unknown", + }, + auth: pairingPendingAuth, + }); + + const result = await probeGatewayStatus({ + url: "ws://127.0.0.1:19191", + timeoutMs: 5_000, + json: true, + }); + + expect(result.ok).toBe(false); + if (result.ok) { + throw new Error("expected failed gateway probe"); + } + expect(result.connectFailure).toEqual({ kind: "gateway-rejected" }); + + const json = JSON.stringify(createDaemonStatus(result)); + expect(json).not.toContain("FUTURE_SENSITIVE_CODE"); + expect(json).not.toContain("do-not-print-unknown"); + }); + it("preserves gateway server version from the connect probe", async () => { callGatewayMock.mockReset(); probeGatewayMock.mockReset(); @@ -285,6 +443,7 @@ describe("probeGatewayStatus", () => { expect(result).toEqual({ ok: false, kind: "read", + connectFailure: { kind: "unreachable" }, error: "gateway status RPC skipped because configured gateway credentials are disabled for this status request", }); @@ -484,6 +643,7 @@ describe("probeGatewayStatus", () => { expect(result).toEqual({ ok: false, kind: "read", + connectFailure: { kind: "unreachable" }, error: "missing scope: operator.admin", }); expect(probeGatewayMock).not.toHaveBeenCalled(); diff --git a/src/cli/daemon-cli/probe.ts b/src/cli/daemon-cli/probe.ts index e1f74b675597..b2c32b767094 100644 --- a/src/cli/daemon-cli/probe.ts +++ b/src/cli/daemon-cli/probe.ts @@ -1,5 +1,10 @@ // Gateway status probe helper used by `gateway status` service diagnostics. import { redactSensitiveUrlLikeString } from "@openclaw/net-policy/redact-sensitive-url"; +import { + classifyGatewayConnectFailure, + ConnectErrorDetailCodes, + readConnectErrorDetailCode, +} from "../../../packages/gateway-protocol/src/connect-error-details.js"; import type { OpenClawConfig } from "../../config/types.js"; import type { GatewayProbeResult } from "../../gateway/probe.js"; import { formatErrorMessage } from "../../infra/errors.js"; @@ -14,6 +19,9 @@ type GatewayStatusRequireRpcProbeResult = { type GatewayStatusProbeResult = GatewayProbeResult | GatewayStatusRequireRpcProbeResult; const probeGatewayModuleLoader = createLazyImportLoader(() => import("../../gateway/probe.js")); +const CONNECT_ERROR_DETAIL_CODE_VALUES: ReadonlySet = new Set( + Object.values(ConnectErrorDetailCodes), +); async function loadProbeGatewayModule(): Promise { return await probeGatewayModuleLoader.load(); @@ -36,6 +44,21 @@ function resolveGatewayStatusProbeDetails(result: GatewayStatusProbeResult) { return "authProbe" in result ? result.authProbe : result; } +function projectGatewayConnectFailure(params: { + details?: unknown; + message: string; + reason?: string; +}) { + // Daemon status is serialized for diagnostics, so raw gateway details must + // stop here; only closed classification facts may cross this boundary. + const failure = classifyGatewayConnectFailure(params); + const detailCode = readConnectErrorDetailCode(params.details); + return { + kind: failure.kind, + ...(detailCode && CONNECT_ERROR_DETAIL_CODE_VALUES.has(detailCode) ? { detailCode } : {}), + }; +} + function readRuntimeVersionFromStatusPayload(payload: unknown): string | null { if (!payload || typeof payload !== "object") { return null; @@ -129,6 +152,7 @@ export async function probeGatewayStatus(opts: { ...(version != null ? { version } : {}), } as const; } + const error = redactSensitiveUrlLikeString(resolveProbeFailureMessage(result)); return { ok: false, kind, @@ -136,15 +160,22 @@ export async function probeGatewayStatus(opts: { auth, ...serverSummary, ...(version != null ? { version } : {}), + connectFailure: projectGatewayConnectFailure({ + details: probeDetails?.connectErrorDetails, + message: error, + reason: probeDetails?.close?.reason, + }), // Probe failure text can echo the credential-bearing target URL (close // reasons, transport errors); status renderers print it verbatim. - error: redactSensitiveUrlLikeString(resolveProbeFailureMessage(result)), + error, } as const; } catch (err) { + const error = redactSensitiveUrlLikeString(formatErrorMessage(err)); return { ok: false, kind, - error: redactSensitiveUrlLikeString(formatErrorMessage(err)), + connectFailure: projectGatewayConnectFailure({ message: error }), + error, } as const; } } diff --git a/src/cli/daemon-cli/status.gather.ts b/src/cli/daemon-cli/status.gather.ts index 9233350b2811..6976c2a39826 100644 --- a/src/cli/daemon-cli/status.gather.ts +++ b/src/cli/daemon-cli/status.gather.ts @@ -2,6 +2,7 @@ import fs from "node:fs/promises"; import { uniqueStrings } from "@openclaw/normalization-core/string-normalization"; import JSON5 from "json5"; +import type { classifyGatewayConnectFailure } from "../../../packages/gateway-protocol/src/connect-error-details.js"; import { createConfigIO, resolveConfigPath, @@ -123,6 +124,8 @@ type CliStatusSummary = { entrypoint?: string; }; +type GatewayConnectFailureKind = ReturnType["kind"]; + const gatewayProbeAuthModuleLoader = createLazyImportLoader( () => import("../../gateway/probe-auth.js"), ); @@ -343,6 +346,10 @@ export type DaemonStatus = { }; version?: string | null; error?: string; + connectFailure?: { + kind: GatewayConnectFailureKind; + detailCode?: string; + }; url?: string; authWarning?: string; }; diff --git a/src/cli/gateway-backed-exit.process.test.ts b/src/cli/gateway-backed-exit.process.test.ts index de69fdba53c0..ed610e2a10b3 100644 --- a/src/cli/gateway-backed-exit.process.test.ts +++ b/src/cli/gateway-backed-exit.process.test.ts @@ -78,6 +78,43 @@ async function startCronListGateway(token: string): Promise<{ url: string }> { return { url: `ws://127.0.0.1:${address.port}` }; } +async function startRateLimitedGateway(): Promise<{ url: string }> { + const wss = new WebSocketServer({ host: "127.0.0.1", port: 0 }); + activeServers.add(wss); + wss.on("connection", (ws) => { + sendMinimalGatewayConnectChallenge(ws); + ws.on("message", (data) => { + const frame = parseMinimalGatewayRequestFrame(data); + if (frame.type !== "req" || !frame.id || frame.method !== "connect") { + return; + } + const message = "unauthorized: too many failed authentication attempts (retry later)"; + ws.send( + JSON.stringify({ + type: "res", + id: frame.id, + ok: false, + error: { + code: "INVALID_REQUEST", + message, + retryable: true, + retryAfterMs: 60_000, + details: { + code: "AUTH_RATE_LIMITED", + authReason: "rate_limited", + recommendedNextStep: "wait_then_retry", + }, + }, + }), + () => ws.close(1008, message), + ); + }); + }); + await once(wss, "listening"); + const address = wss.address() as AddressInfo; + return { url: `ws://127.0.0.1:${address.port}` }; +} + async function runIsolatedGatewayCli(params: { args: string[]; root: string; @@ -298,4 +335,44 @@ describe("gateway-backed CLI process exit", () => { gateway: { url: `ws://127.0.0.1:${port}` }, }); }, 30_000); + + it("preserves pre-hello rate-limit details through the real health entry point", async () => { + const root = tempDirs.make("openclaw-gateway-rate-limit-json-"); + const stateDir = path.join(root, "state"); + const configPath = path.join(stateDir, "openclaw.json"); + const gateway = await startRateLimitedGateway(); + await fs.mkdir(stateDir, { recursive: true }); + await fs.writeFile( + configPath, + JSON.stringify({ + gateway: { + mode: "remote", + remote: { url: gateway.url, token: "test-token" }, + }, + }), + ); + + const result = await runIsolatedGatewayCli({ + args: ["health", "--json", "--timeout", "2000"], + root, + stateDir, + configPath, + }); + + expect(result, result.stderr).toMatchObject({ code: 1, signal: null, stderr: "" }); + expect(JSON.parse(result.stdout)).toEqual({ + ok: false, + error: { + type: "gateway_request_error", + code: "AUTH_RATE_LIMITED", + message: + "Gateway authentication is temporarily rate-limited. Wait for the temporary lockout to expire, then retry.", + retryable: true, + retryAfterMs: 60_000, + }, + gateway: { reachable: true }, + }); + expect(result.stdout).not.toContain("gateway.remote.token"); + expect(result.stdout).not.toContain("devices rotate"); + }, 30_000); }); diff --git a/src/commands/doctor-gateway-health.test.ts b/src/commands/doctor-gateway-health.test.ts index b1b3b759b3e5..8cbe47342697 100644 --- a/src/commands/doctor-gateway-health.test.ts +++ b/src/commands/doctor-gateway-health.test.ts @@ -1,9 +1,12 @@ // Doctor gateway health tests cover gateway probe failures, auth requirements, and repair messages. import { beforeEach, describe, expect, it, vi } from "vitest"; +import { GatewayClientRequestError } from "../../packages/gateway-client/src/index.js"; import type { OpenClawConfig } from "../config/config.js"; import { GATEWAY_HEALTH_CREDENTIALS_REQUIRED_MESSAGE, GATEWAY_HEALTH_CREDENTIALS_REQUIRED_TITLE, + GATEWAY_HEALTH_RATE_LIMITED_MESSAGE, + GATEWAY_HEALTH_RATE_LIMITED_TITLE, } from "./gateway-health-auth-diagnostic.js"; const callGateway = vi.hoisted(() => vi.fn()); @@ -334,6 +337,63 @@ describe("checkGatewayHealth", () => { expect(callGateway).toHaveBeenCalledTimes(1); }); + it("reports a temporary lockout when status auth is rate-limited", async () => { + callGateway.mockRejectedValueOnce(new Error()); + isGatewayCredentialsRequiredError.mockReturnValueOnce(true); + probeGatewayStatus.mockResolvedValueOnce({ + ok: false, + kind: "connect", + error: "connect failed", + connectFailure: { kind: "rate-limited", detailCode: "AUTH_RATE_LIMITED" }, + }); + const runtime = { log: vi.fn(), error: vi.fn(), exit: vi.fn() }; + + await expect( + checkGatewayHealth({ runtime: runtime as never, cfg, timeoutMs: 3000 }), + ).resolves.toEqual({ authenticated: false, healthOk: true }); + + expect(runtime.error).not.toHaveBeenCalled(); + expect(note).toHaveBeenCalledWith( + GATEWAY_HEALTH_RATE_LIMITED_MESSAGE, + GATEWAY_HEALTH_RATE_LIMITED_TITLE, + ); + expect(note).not.toHaveBeenCalledWith( + GATEWAY_HEALTH_CREDENTIALS_REQUIRED_MESSAGE, + GATEWAY_HEALTH_CREDENTIALS_REQUIRED_TITLE, + ); + const output = note.mock.calls.flat().join("\n"); + expect(output).not.toContain("gateway.remote.token"); + expect(output).not.toContain("devices rotate"); + }); + + it("handles the real typed rate-limit error without forcing the credentials predicate", async () => { + const error = new GatewayClientRequestError({ + code: "INVALID_REQUEST", + message: "unauthorized: too many failed authentication attempts (retry later)", + details: { + code: "AUTH_RATE_LIMITED", + authReason: "rate_limited", + recommendedNextStep: "wait_then_retry", + }, + retryable: true, + retryAfterMs: 60_000, + }); + callGateway.mockRejectedValueOnce(error); + const runtime = { log: vi.fn(), error: vi.fn(), exit: vi.fn() }; + + await expect( + checkGatewayHealth({ runtime: runtime as never, cfg, timeoutMs: 3000 }), + ).resolves.toEqual({ authenticated: false, healthOk: true }); + + expect(isGatewayCredentialsRequiredError).not.toHaveBeenCalled(); + expect(probeGatewayStatus).not.toHaveBeenCalled(); + expect(runtime.error).not.toHaveBeenCalled(); + expect(note).toHaveBeenCalledWith( + GATEWAY_HEALTH_RATE_LIMITED_MESSAGE, + GATEWAY_HEALTH_RATE_LIMITED_TITLE, + ); + }); + it("reports credentials-required when status RPC auth SecretRefs are unavailable", async () => { const error = new Error("gateway.auth.password unavailable"); callGateway.mockRejectedValueOnce(error); diff --git a/src/commands/doctor-gateway-health.ts b/src/commands/doctor-gateway-health.ts index 4acce7192a1f..0aec26d11277 100644 --- a/src/commands/doctor-gateway-health.ts +++ b/src/commands/doctor-gateway-health.ts @@ -24,7 +24,11 @@ import { VERSION } from "../version.js"; import { GATEWAY_HEALTH_CREDENTIALS_REQUIRED_MESSAGE, GATEWAY_HEALTH_CREDENTIALS_REQUIRED_TITLE, + GATEWAY_HEALTH_RATE_LIMITED_MESSAGE, + GATEWAY_HEALTH_RATE_LIMITED_TITLE, + gatewayConnectErrorWasRateLimited, gatewayProbeResultSawGateway, + gatewayProbeResultWasRateLimited, } from "./gateway-health-auth-diagnostic.js"; import { formatGatewayClosedDiagnostic, formatHealthCheckFailure } from "./health-format.js"; import { formatTelemetryExporterSummary } from "./telemetry-exporter-summary.js"; @@ -159,6 +163,10 @@ export async function checkGatewayHealth(params: { } return { healthOk, authenticated: true, status }; } catch (err) { + if (gatewayConnectErrorWasRateLimited(err)) { + note(GATEWAY_HEALTH_RATE_LIMITED_MESSAGE, GATEWAY_HEALTH_RATE_LIMITED_TITLE); + return { healthOk: true, authenticated: false }; + } if (isGatewayHealthAuthUnavailableError(err)) { const probeDetails = await buildGatewayProbeConnectionDetails({ config: params.cfg }); const probe = await probeGatewayStatus({ @@ -170,10 +178,14 @@ export async function checkGatewayHealth(params: { json: true, }); if (gatewayProbeResultSawGateway(probe)) { - note( - GATEWAY_HEALTH_CREDENTIALS_REQUIRED_MESSAGE, - GATEWAY_HEALTH_CREDENTIALS_REQUIRED_TITLE, - ); + if (gatewayProbeResultWasRateLimited(probe)) { + note(GATEWAY_HEALTH_RATE_LIMITED_MESSAGE, GATEWAY_HEALTH_RATE_LIMITED_TITLE); + } else { + note( + GATEWAY_HEALTH_CREDENTIALS_REQUIRED_MESSAGE, + GATEWAY_HEALTH_CREDENTIALS_REQUIRED_TITLE, + ); + } healthOk = true; return { healthOk, authenticated: false }; } diff --git a/src/commands/gateway-health-auth-diagnostic.ts b/src/commands/gateway-health-auth-diagnostic.ts index 65fc5776bbd6..5791fa56376c 100644 --- a/src/commands/gateway-health-auth-diagnostic.ts +++ b/src/commands/gateway-health-auth-diagnostic.ts @@ -1,4 +1,8 @@ /** Gateway health auth diagnostic helpers for reachable-but-unauthenticated probes. */ +import { + classifyGatewayConnectFailure, + ConnectErrorDetailCodes, +} from "../../packages/gateway-protocol/src/connect-error-details.js"; import type { DaemonStatus } from "../cli/daemon-cli/status.gather.js"; type GatewayProbeReachabilityEvidence = NonNullable; @@ -7,6 +11,35 @@ export const GATEWAY_HEALTH_CREDENTIALS_REQUIRED_MESSAGE = "Gateway is reachable, but this CLI has no token/password or paired device token for read-scope health RPCs."; export const GATEWAY_HEALTH_CREDENTIALS_REQUIRED_TITLE = "Gateway credentials required"; export const GATEWAY_HEALTH_REACHABLE_LINE = "Gateway: reachable"; +export const GATEWAY_HEALTH_RATE_LIMITED_MESSAGE = + "Gateway authentication is temporarily rate-limited. Wait for the temporary lockout to expire, then retry."; +export const GATEWAY_HEALTH_RATE_LIMITED_TITLE = "Gateway authentication rate-limited"; + +function gatewayProbeFailureKind(status: GatewayProbeReachabilityEvidence) { + return ( + status.connectFailure?.kind ?? classifyGatewayConnectFailure({ message: status.error }).kind + ); +} + +/** Detects the temporary authentication lockout outcome from projected or legacy probe facts. */ +export function gatewayProbeResultWasRateLimited( + status: GatewayProbeReachabilityEvidence, +): boolean { + return gatewayProbeFailureKind(status) === "rate-limited"; +} + +/** Detects a structured or legacy rate-limit connect error before close projection. */ +export function gatewayConnectErrorWasRateLimited(error: unknown): boolean { + if (!(error instanceof Error)) { + return false; + } + return ( + classifyGatewayConnectFailure({ + details: (error as Error & { details?: unknown }).details, + message: error.message, + }).kind === "rate-limited" + ); +} /** * Detects when a daemon probe reached the gateway even if read-scope auth failed. @@ -26,11 +59,7 @@ export function gatewayProbeResultSawGateway(status: GatewayProbeReachabilityEvi if (server?.version || server?.connId) { return true; } - // Older probes may only expose close/error text for auth failures; treat known gateway - // close reasons as reachability evidence so health can explain missing credentials. - return /\bgateway closed \(\d+\):|\bpairing required\b|\bdevice identity required\b/i.test( - status.error ?? "", - ); + return gatewayProbeFailureKind(status) !== "unreachable"; } /** @@ -48,3 +77,28 @@ export function buildCredentialsRequiredHealthDiagnostic() { }, }; } + +/** Builds the health diagnostic emitted for a temporary Gateway authentication lockout. */ +export function buildRateLimitedHealthDiagnostic(error?: unknown) { + const retryAfterCandidate = + error instanceof Error ? (error as Error & { retryAfterMs?: unknown }).retryAfterMs : undefined; + const retryAfterMs = + typeof retryAfterCandidate === "number" && + Number.isSafeInteger(retryAfterCandidate) && + retryAfterCandidate >= 0 + ? retryAfterCandidate + : undefined; + return { + ok: false, + error: { + type: "gateway_request_error", + code: ConnectErrorDetailCodes.AUTH_RATE_LIMITED, + message: GATEWAY_HEALTH_RATE_LIMITED_MESSAGE, + retryable: true, + ...(retryAfterMs !== undefined ? { retryAfterMs } : {}), + }, + gateway: { + reachable: true, + }, + }; +} diff --git a/src/commands/gateway-readiness.test.ts b/src/commands/gateway-readiness.test.ts index 9129c447a9f6..29f1eb3a6b85 100644 --- a/src/commands/gateway-readiness.test.ts +++ b/src/commands/gateway-readiness.test.ts @@ -302,6 +302,69 @@ describe("ensureGatewayReadyForOperation", () => { expect(runtime.log).not.toHaveBeenCalled(); }); + it("uses the projected connect failure when the daemon error text is generic", async () => { + const status = createStatus({ + service: { + label: "systemd user", + loaded: true, + loadedText: "enabled", + notLoadedText: "disabled", + command: { programArguments: ["openclaw", "gateway", "run", "--port", "18789"] }, + runtime: { status: "running" }, + }, + port: { port: 18789, status: "busy", listeners: [], hints: [] }, + rpc: { + ok: false, + error: "connect failed", + connectFailure: { kind: "pairing-required", detailCode: "PAIRING_REQUIRED" }, + url: "ws://127.0.0.1:18789", + }, + }); + const confirm = vi.fn(); + + const result = await ensureGatewayReadyForOperation({ + runtime, + operation: "open the dashboard", + readyWhenReachable: true, + interactive: true, + deps: { gatherStatus: vi.fn().mockResolvedValue(status), confirm }, + }); + + expect(result).toMatchObject({ ready: true, recovered: false }); + expect(confirm).not.toHaveBeenCalled(); + }); + + it("accepts a rate-limited Gateway as reachable without starting the service", async () => { + const status = createStatus({ + service: { + label: "systemd user", + loaded: true, + loadedText: "enabled", + notLoadedText: "disabled", + command: { programArguments: ["openclaw", "gateway", "run", "--port", "18789"] }, + runtime: { status: "running" }, + }, + port: { port: 18789, status: "busy", listeners: [], hints: [] }, + rpc: { + ok: false, + error: "connect failed", + connectFailure: { kind: "rate-limited", detailCode: "AUTH_RATE_LIMITED" }, + url: "ws://127.0.0.1:18789", + }, + }); + const startGateway = vi.fn(); + + const result = await ensureGatewayReadyForOperation({ + runtime, + operation: "open the dashboard", + readyWhenReachable: true, + deps: { gatherStatus: vi.fn().mockResolvedValue(status), startGateway }, + }); + + expect(result).toMatchObject({ ready: true, recovered: false }); + expect(startGateway).not.toHaveBeenCalled(); + }); + it("still treats a timeout on the target port as not ready", async () => { const status = createStatus({ service: { diff --git a/src/commands/health.test.ts b/src/commands/health.test.ts index b14e988c53bb..0ab22405b265 100644 --- a/src/commands/health.test.ts +++ b/src/commands/health.test.ts @@ -4,7 +4,9 @@ import { GatewayClientRequestError } from "../../packages/gateway-client/src/ind import { stripAnsi } from "../../packages/terminal-core/src/ansi.js"; import { buildCredentialsRequiredHealthDiagnostic, + buildRateLimitedHealthDiagnostic, GATEWAY_HEALTH_CREDENTIALS_REQUIRED_MESSAGE, + GATEWAY_HEALTH_RATE_LIMITED_MESSAGE, GATEWAY_HEALTH_REACHABLE_LINE, } from "./gateway-health-auth-diagnostic.js"; import { formatHealthCheckFailure } from "./health-format.js"; @@ -509,6 +511,77 @@ describe("healthCommand", () => { }, ); + it.each([ + { json: true, expectedLogs: 1 }, + { json: undefined, expectedLogs: 2 }, + ])( + "preserves a typed pre-hello authentication lockout through health output", + async ({ json, expectedLogs }) => { + const error = new GatewayClientRequestError({ + code: "INVALID_REQUEST", + message: "unauthorized: too many failed authentication attempts (retry later)", + details: { + code: "AUTH_RATE_LIMITED", + authReason: "rate_limited", + recommendedNextStep: "wait_then_retry", + }, + retryable: true, + retryAfterMs: 60_000, + }); + callGatewayMock.mockRejectedValueOnce(error); + + await healthCommand({ json, timeoutMs: 5000, config: {} }, runtime as never); + + expect(isGatewayCredentialsRequiredErrorMock).not.toHaveBeenCalled(); + expect(probeGatewayStatusMock).not.toHaveBeenCalled(); + expect(runtime.exit).toHaveBeenCalledWith(1); + expect(runtime.log).toHaveBeenCalledTimes(expectedLogs); + if (json) { + expect(JSON.parse(requireFirstRuntimeLog())).toEqual( + buildRateLimitedHealthDiagnostic(error), + ); + } else { + expect(runtime.log.mock.calls).toEqual([ + [GATEWAY_HEALTH_REACHABLE_LINE], + [GATEWAY_HEALTH_RATE_LIMITED_MESSAGE], + ]); + } + }, + ); + + it.each([ + { json: true, expectedLogs: 1 }, + { json: undefined, expectedLogs: 2 }, + ])( + "reports temporary authentication lockouts without credential-change guidance", + async ({ json, expectedLogs }) => { + callGatewayMock.mockRejectedValueOnce(new Error()); + isGatewayCredentialsRequiredErrorMock.mockReturnValueOnce(true); + probeGatewayStatusMock.mockResolvedValueOnce({ + ok: false, + kind: "connect", + error: "connect failed", + connectFailure: { kind: "rate-limited", detailCode: "AUTH_RATE_LIMITED" }, + }); + + await healthCommand({ json, timeoutMs: 5000, config: {} }, runtime as never); + + expect(runtime.exit).toHaveBeenCalledWith(1); + expect(runtime.log).toHaveBeenCalledTimes(expectedLogs); + if (json) { + expect(JSON.parse(requireFirstRuntimeLog())).toEqual(buildRateLimitedHealthDiagnostic()); + } else { + expect(runtime.log.mock.calls).toEqual([ + [GATEWAY_HEALTH_REACHABLE_LINE], + [GATEWAY_HEALTH_RATE_LIMITED_MESSAGE], + ]); + } + const output = runtime.log.mock.calls.flat().join("\n"); + expect(output).not.toContain("gateway.remote.token"); + expect(output).not.toContain("devices rotate"); + }, + ); + it("keeps credential failures machine-readable when the gateway is unreachable", async () => { const error = new Error("gateway health requires credentials"); const payload = { diff --git a/src/commands/health.ts b/src/commands/health.ts index 146aa35341e7..faf969b102ab 100644 --- a/src/commands/health.ts +++ b/src/commands/health.ts @@ -37,8 +37,11 @@ import { buildChannelAccountBindings, resolvePreferredAccountId } from "../routi import { type RuntimeEnv, writeRuntimeJson } from "../runtime.js"; import { buildCredentialsRequiredHealthDiagnostic, + buildRateLimitedHealthDiagnostic, + gatewayConnectErrorWasRateLimited, GATEWAY_HEALTH_REACHABLE_LINE, gatewayProbeResultSawGateway, + gatewayProbeResultWasRateLimited, } from "./gateway-health-auth-diagnostic.js"; import { formatHealthChannelLines } from "./health-format.js"; import { logGatewayConnectionDetails } from "./status.gateway-connection.js"; @@ -72,9 +75,21 @@ export async function emitReachableGatewayAuthDiagnostic(params: { localPortOverride?: number; json?: boolean; }): Promise { - if (!isGatewayHealthAuthUnavailableError(params.error)) { + const directRateLimit = gatewayConnectErrorWasRateLimited(params.error); + if (!directRateLimit && !isGatewayHealthAuthUnavailableError(params.error)) { return false; } + if (directRateLimit) { + const diagnostic = buildRateLimitedHealthDiagnostic(params.error); + if (params.json) { + writeRuntimeJson(params.runtime, diagnostic); + } else { + params.runtime.log(GATEWAY_HEALTH_REACHABLE_LINE); + params.runtime.log(diagnostic.error.message); + } + params.runtime.exit(1); + return true; + } const details = await buildGatewayProbeConnectionDetails({ config: params.config, token: params.token, @@ -94,7 +109,9 @@ export async function emitReachableGatewayAuthDiagnostic(params: { if (!gatewayProbeResultSawGateway(probe)) { return false; } - const diagnostic = buildCredentialsRequiredHealthDiagnostic(); + const diagnostic = gatewayProbeResultWasRateLimited(probe) + ? buildRateLimitedHealthDiagnostic() + : buildCredentialsRequiredHealthDiagnostic(); if (params.json) { writeRuntimeJson(params.runtime, diagnostic); params.runtime.exit(1); diff --git a/src/flows/doctor-core-checks.runtime.test.ts b/src/flows/doctor-core-checks.runtime.test.ts index 051867598902..2cccf6230d9e 100644 --- a/src/flows/doctor-core-checks.runtime.test.ts +++ b/src/flows/doctor-core-checks.runtime.test.ts @@ -1,6 +1,7 @@ // Doctor runtime check tests cover runtime-backed doctor checks. import { beforeEach, describe, expect, it, vi } from "vitest"; import type { AnyAgentTool } from "../agents/tools/common.js"; +import { GATEWAY_HEALTH_RATE_LIMITED_MESSAGE } from "../commands/gateway-health-auth-diagnostic.js"; import { setPluginToolMeta } from "../plugins/tools.js"; const mocks = vi.hoisted(() => ({ @@ -591,6 +592,25 @@ describe("doctor gateway runtime checks", () => { }); }); + it("reports temporary Gateway authentication lockouts with wait-and-retry guidance", async () => { + mocks.probeGatewayStatus.mockResolvedValueOnce({ + ok: false, + error: "connect failed", + connectFailure: { kind: "rate-limited", detailCode: "AUTH_RATE_LIMITED" }, + }); + + await expect( + collectGatewayHealthFindings({ cfg: { gateway: { mode: "local" } } }), + ).resolves.toContainEqual({ + checkId: "core/doctor/gateway-health", + severity: "warning", + message: GATEWAY_HEALTH_RATE_LIMITED_MESSAGE, + path: "gateway.mode", + target: "http://127.0.0.1:5829", + fixHint: "Wait for the temporary authentication lockout to expire, then rerun doctor.", + }); + }); + it("redacts sensitive remote gateway URLs from health finding targets", async () => { mocks.buildGatewayProbeConnectionDetails.mockResolvedValueOnce({ url: "wss://user:pass@gateway.example.test/rpc?token=secret&safe=value", diff --git a/src/flows/doctor-core-checks.runtime.ts b/src/flows/doctor-core-checks.runtime.ts index cf844f7d4b9f..7cb3bf38d646 100644 --- a/src/flows/doctor-core-checks.runtime.ts +++ b/src/flows/doctor-core-checks.runtime.ts @@ -31,7 +31,11 @@ import { import type { AnyAgentTool } from "../agents/tools/common.js"; import { probeGatewayStatus } from "../cli/daemon-cli/probe.js"; import { collectUnavailableAgentSkills } from "../commands/doctor-skills-core.js"; -import { gatewayProbeResultSawGateway } from "../commands/gateway-health-auth-diagnostic.js"; +import { + GATEWAY_HEALTH_RATE_LIMITED_MESSAGE, + gatewayProbeResultSawGateway, + gatewayProbeResultWasRateLimited, +} from "../commands/gateway-health-auth-diagnostic.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { getSystemdCgroupHygieneSummary, @@ -132,10 +136,22 @@ export async function collectGatewayHealthFindings( config: ctx.cfg, json: true, }); + const mode = ctx.cfg.gateway?.mode === "remote" ? "remote" : "local"; + if (gatewayProbeResultWasRateLimited(probe)) { + return [ + { + checkId: "core/doctor/gateway-health", + severity: "warning", + message: GATEWAY_HEALTH_RATE_LIMITED_MESSAGE, + path: mode === "remote" ? "gateway.remote.url" : "gateway.mode", + target: formatGatewayHealthTarget(probeDetails.url), + fixHint: "Wait for the temporary authentication lockout to expire, then rerun doctor.", + }, + ]; + } if (gatewayProbeResultSawGateway(probe)) { return []; } - const mode = ctx.cfg.gateway?.mode === "remote" ? "remote" : "local"; return [ { checkId: "core/doctor/gateway-health", diff --git a/src/gateway/call.test.ts b/src/gateway/call.test.ts index 4ab747c76d9c..6ac9edd5367a 100644 --- a/src/gateway/call.test.ts +++ b/src/gateway/call.test.ts @@ -85,6 +85,7 @@ type StartMode = | "hello" | "close" | "connect-error" + | "connect-error-close" | "silent" | "startup-retry-then-hello" | "clean-prehello-close-then-hello" @@ -144,6 +145,16 @@ function startStubGatewayClient() { lastClientOptions?.onConnectError?.( connectError ?? connectAssemblyErrorState.create("device private key invalid"), ); + } else if (startMode === "connect-error-close") { + lastClientOptions?.onConnectError?.( + connectError ?? connectAssemblyErrorState.create("device private key invalid"), + ); + lastClientOptions?.onClose?.(closeCode, closeReason, { + phase: "pre-hello", + socketOpened: true, + transportValidated: true, + transientPreHelloCleanClose: false, + }); } else if (startMode === "close") { lastClientOptions?.onClose?.(closeCode, closeReason); } @@ -1587,6 +1598,100 @@ describe("callGateway error details", () => { expect(lastRequestOptions).toBeNull(); }); + it("preserves allowlisted rate-limit details before the following close", async () => { + startMode = "connect-error-close"; + closeCode = 1008; + closeReason = "unauthorized: too many failed authentication attempts (retry later)"; + connectError = Object.assign( + new Error("unauthorized: too many failed authentication attempts (retry later)"), + { + name: "GatewayClientRequestError", + gatewayCode: "INVALID_REQUEST", + details: { + code: "AUTH_RATE_LIMITED", + authReason: "rate_limited", + recommendedNextStep: "wait_then_retry", + }, + retryable: true, + retryAfterMs: 60_000, + }, + ); + setLocalLoopbackGatewayConfig(); + + let error: unknown; + await callGateway({ method: "health" }).catch((caught: unknown) => { + error = caught; + }); + + expect(error).toBe(connectError); + expect(formatGatewayClientRequestErrorJson(error)).toEqual({ + ok: false, + error: { + type: "gateway_request_error", + code: "INVALID_REQUEST", + message: "unauthorized: too many failed authentication attempts (retry later)", + details: { + code: "AUTH_RATE_LIMITED", + authReason: "rate_limited", + recommendedNextStep: "wait_then_retry", + }, + retryable: true, + retryAfterMs: 60_000, + }, + }); + }); + + it.each([ + { + name: "another structured auth rejection", + error: Object.assign(new Error("unauthorized: gateway token mismatch"), { + name: "GatewayClientRequestError", + gatewayCode: "INVALID_REQUEST", + details: { code: "AUTH_TOKEN_MISMATCH" }, + retryable: false, + }), + }, + { + name: "rate-limit-looking text without structured details", + error: Object.assign( + new Error("unauthorized: too many failed authentication attempts (retry later)"), + { + name: "GatewayClientRequestError", + gatewayCode: "INVALID_REQUEST", + retryable: true, + }, + ), + }, + { name: "ordinary connect error", error: new Error("ordinary connect failure") }, + ])("keeps $name on the existing transport-close path", async ({ error: connectFailure }) => { + startMode = "connect-error-close"; + closeCode = 1008; + closeReason = "connect failed"; + connectError = connectFailure; + setLocalLoopbackGatewayConfig(); + + let error: unknown; + await callGateway({ method: "health" }).catch((caught: unknown) => { + error = caught; + }); + + expect(formatGatewayTransportErrorJson(error)).toEqual({ + ok: false, + error: { + type: "gateway_transport_error", + kind: "closed", + message: "gateway closed (1008): connect failed", + code: 1008, + reason: "connect failed", + }, + gateway: { + url: "ws://127.0.0.1:18789", + urlSource: "local loopback", + bindDetail: "Bind: loopback", + }, + }); + }); + it("surfaces agent runtime identity connect request errors", async () => { startMode = "connect-error"; connectError = new Error( diff --git a/src/gateway/call.ts b/src/gateway/call.ts index 4c9a67f011c0..10c5d8bdd5cd 100644 --- a/src/gateway/call.ts +++ b/src/gateway/call.ts @@ -11,6 +11,10 @@ import { type GatewayClientMode, type GatewayClientName, } from "../../packages/gateway-protocol/src/client-info.js"; +import { + ConnectErrorDetailCodes, + readConnectErrorDetailCode, +} from "../../packages/gateway-protocol/src/connect-error-details.js"; import { MIN_CLIENT_PROTOCOL_VERSION, PROTOCOL_VERSION, @@ -940,6 +944,16 @@ function isRequiredAgentRuntimeIdentityConnectError(err: Error): boolean { ); } +function isAllowlistedGatewayConnectRequestError(err: Error): boolean { + if (err.name !== "GatewayClientRequestError") { + return false; + } + return ( + readConnectErrorDetailCode((err as Error & { details?: unknown }).details) === + ConnectErrorDetailCodes.AUTH_RATE_LIMITED + ); +} + async function executeGatewayRequestWithScopes(params: { opts: CallGatewayBaseOptions; scopes: OperatorScope[] | undefined; @@ -1121,6 +1135,7 @@ async function executeGatewayRequestWithScopes(params: { const shouldSurface = isGatewayConnectAssemblyError(err) || isAgentRuntimeIdentityConnectError || + isAllowlistedGatewayConnectRequestError(err) || (surfaceGatewayClientRequestErrors && isGatewayClientRequestError); if (settled || !shouldSurface) { return; diff --git a/src/gateway/client.test.ts b/src/gateway/client.test.ts index 80bb80b537bd..c1fe9d46cead 100644 --- a/src/gateway/client.test.ts +++ b/src/gateway/client.test.ts @@ -2076,6 +2076,51 @@ describe("GatewayClient connect auth payload", () => { }); }); + it("reports AUTH_RATE_LIMITED before pausing reconnect on the following close", async () => { + const onConnectError = vi.fn(); + const onReconnectPaused = vi.fn(); + const client = new GatewayClient({ + url: "ws://127.0.0.1:18789", + token: "shared-token", + onConnectError, + onReconnectPaused, + }); + + const { ws: ws1, connect: firstConnect } = startClientAndConnect({ client }); + await expectNoReconnectAfterConnectFailure({ + client, + firstWs: ws1, + connectId: firstConnect.id, + failureDetails: { + code: "AUTH_RATE_LIMITED", + authReason: "rate_limited", + recommendedNextStep: "wait_then_retry", + }, + failureMessage: "unauthorized: too many failed authentication attempts (retry later)", + }); + + expect(onConnectError).toHaveBeenCalledOnce(); + expect(onConnectError.mock.calls[0]?.[0]).toMatchObject({ + name: "GatewayClientRequestError", + details: { + code: "AUTH_RATE_LIMITED", + authReason: "rate_limited", + recommendedNextStep: "wait_then_retry", + }, + }); + expect(onReconnectPaused).toHaveBeenCalledWith({ + code: 1008, + reason: "connect failed", + detailCode: "AUTH_RATE_LIMITED", + }); + expect(logDebugMock).toHaveBeenCalledWith( + expect.stringContaining("gateway connect failed: GatewayClientRequestError"), + ); + expect(logErrorMock).not.toHaveBeenCalledWith( + expect.stringContaining("gateway connect failed: GatewayClientRequestError"), + ); + }); + it("keeps reconnect paused callback errors inside close dispatch", async () => { const onReconnectPaused = vi.fn(() => { throw new Error("paused callback failed"); diff --git a/src/gateway/probe.test.ts b/src/gateway/probe.test.ts index 6de40be70dca..ad2366402ed2 100644 --- a/src/gateway/probe.test.ts +++ b/src/gateway/probe.test.ts @@ -674,6 +674,7 @@ describe("probeGateway", () => { it("prefers the structured connect error over the generic close reason", async () => { gatewayClientState.startMode = "connect-error-close"; gatewayClientState.socketOpened = true; + gatewayClientState.close = { code: 1008, reason: "connect failed" }; const result = await runTokenLightweightProbe({ timeoutMs: 5_000, @@ -682,11 +683,25 @@ describe("probeGateway", () => { expectProbeResultFields(result, { ok: false, error: "scope upgrade pending approval (requestId: req-123)", - close: { code: 1008, reason: "pairing required" }, + close: { code: 1008, reason: "connect failed" }, }); + expectProbeAuthFields(result, { capability: "pairing_pending" }); expect(result.connectLatencyMs).not.toBeNull(); }); + it("keeps probe capability unknown for temporary authentication lockouts", async () => { + gatewayClientState.startMode = "connect-error-close"; + gatewayClientState.connectError = + "unauthorized: too many failed authentication attempts (retry later)"; + gatewayClientState.connectErrorDetails = { code: "AUTH_RATE_LIMITED" }; + gatewayClientState.close = { code: 1008, reason: "connect failed" }; + + const result = await runTokenLightweightProbe({ timeoutMs: 5_000 }); + + expectProbeResultFields(result, { ok: false }); + expectProbeAuthFields(result, { capability: "unknown" }); + }); + it("keeps latency unknown when the opened transport fails validation", async () => { gatewayClientState.startMode = "connect-error-close"; gatewayClientState.socketOpened = true; diff --git a/src/gateway/probe.ts b/src/gateway/probe.ts index cdd056144568..605edd6e2eb0 100644 --- a/src/gateway/probe.ts +++ b/src/gateway/probe.ts @@ -6,6 +6,7 @@ import { GATEWAY_CLIENT_MODES, GATEWAY_CLIENT_NAMES, } from "../../packages/gateway-protocol/src/client-info.js"; +import { classifyGatewayConnectFailure } from "../../packages/gateway-protocol/src/connect-error-details.js"; import { readMissingScopeError, type MissingScopeErrorDetails, @@ -69,7 +70,6 @@ type GatewayProbeDetailLevel = "none" | "presence" | "config" | "full"; const MIN_PROBE_TIMEOUT_MS = 250; export const MAX_TIMER_DELAY_MS = MAX_SAFE_TIMEOUT_DELAY_MS; -const PAIRING_REQUIRED_PATTERN = /\bpairing required\b/i; const OPERATOR_READ_SCOPE = "operator.read"; const OPERATOR_WRITE_SCOPE = "operator.write"; const OPERATOR_ADMIN_SCOPE = "operator.admin"; @@ -188,6 +188,7 @@ function resolveProbeAuthSummary(params: { role?: string | null; scopes?: string[]; authMetadataPresent?: boolean; + connectErrorDetails?: unknown; error?: string | null; close?: GatewayProbeClose | null; verifiedRead?: boolean; @@ -200,6 +201,7 @@ function resolveProbeAuthSummary(params: { capability: resolveGatewayProbeCapability({ auth: { scopes }, authMetadataPresent: params.authMetadataPresent, + connectErrorDetails: params.connectErrorDetails, error: params.error, close: params.close, verifiedRead: params.verifiedRead, @@ -208,22 +210,22 @@ function resolveProbeAuthSummary(params: { }; } -function isPairingPendingProbeFailure(params: { - error?: string | null; - close?: GatewayProbeClose | null; -}): boolean { - return PAIRING_REQUIRED_PATTERN.test(params.close?.reason ?? params.error ?? ""); -} - function resolveGatewayProbeCapability(params: { auth?: Pick | null; authMetadataPresent?: boolean; + connectErrorDetails?: unknown; error?: string | null; close?: GatewayProbeClose | null; verifiedRead?: boolean; connectLatencyMs?: number | null; }): GatewayProbeCapability { - if (isPairingPendingProbeFailure(params)) { + if ( + classifyGatewayConnectFailure({ + details: params.connectErrorDetails, + reason: params.close?.reason, + message: params.error, + }).kind === "pairing-required" + ) { return "pairing_pending"; } const scopes = Array.isArray(params.auth?.scopes) ? params.auth.scopes : []; @@ -383,6 +385,7 @@ export async function probeGateway(opts: { role: auth.role, scopes: auth.scopes, authMetadataPresent, + connectErrorDetails, error: params.error, close, verifiedRead: params.verifiedRead, diff --git a/src/tui/tui.test.ts b/src/tui/tui.test.ts index 86dc834f39de..53677f301013 100644 --- a/src/tui/tui.test.ts +++ b/src/tui/tui.test.ts @@ -348,30 +348,56 @@ describe("resolveInitialTuiAgentId", () => { describe("resolveGatewayDisconnectState", () => { it("returns scope-upgrade recovery guidance when disconnect reason requires pairing", () => { - const state = resolveGatewayDisconnectState("gateway closed (1008): pairing required"); + const state = resolveGatewayDisconnectState({ + reason: "gateway closed (1008): pairing required", + }); expect(state.connectionStatus).toContain("pairing required"); expect(state.activityStatus).toBe("device approval needed: preview latest request"); - expect(state.pairingHint).toContain("openclaw devices approve --latest"); - expect(state.pairingHint).toContain("openclaw devices approve "); - expect(state.pairingHint).toContain("--token"); + expect(state.remediation).toContain("openclaw devices approve --latest"); + expect(state.remediation).toContain("openclaw devices approve "); + expect(state.remediation).toContain("--url"); + expect(state.remediation).toContain("--token/--password"); // Must steer users to `devices`, not the unrelated chat-DM `pairing` command. - expect(state.pairingHint).not.toContain("openclaw pairing"); + expect(state.remediation).not.toContain("openclaw pairing"); }); - it("returns the same guidance when the gateway reports a pending scope upgrade", () => { - const state = resolveGatewayDisconnectState( - "gateway closed (1008): scope upgrade pending approval", - ); + it("uses structured pairing details before the generic close reason", () => { + const state = resolveGatewayDisconnectState({ + details: { code: "PAIRING_REQUIRED", reason: "scope-upgrade" }, + reason: "connect failed", + }); expect(state.activityStatus).toBe("device approval needed: preview latest request"); - expect(state.pairingHint).toContain("openclaw devices approve --latest"); - expect(state.pairingHint).toContain("openclaw devices approve "); + expect(state.connectionStatus).toContain("scope upgrade pending approval"); + expect(state.remediation).toContain("openclaw devices approve --latest"); + }); + + it("shows the device-token rotation command for structured token mismatch", () => { + const state = resolveGatewayDisconnectState({ + details: { code: "AUTH_DEVICE_TOKEN_MISMATCH" }, + reason: "device token mismatch", + }); + expect(state.activityStatus).toBe("gateway authentication needs attention"); + expect(state.remediation).toContain( + "openclaw devices rotate --device --role operator", + ); + }); + + it("shows wait-and-retry guidance for a temporary authentication lockout", () => { + const state = resolveGatewayDisconnectState({ + details: { code: "AUTH_RATE_LIMITED" }, + reason: "unauthorized: too many failed authentication attempts (retry later)", + }); + expect(state.activityStatus).toBe("gateway authentication temporarily rate-limited"); + expect(state.remediation).toContain("temporary authentication lockout"); + expect(state.remediation).not.toContain("gateway.remote.token"); + expect(state.remediation).not.toContain("devices rotate"); }); it("falls back to idle for generic disconnect reasons", () => { - const state = resolveGatewayDisconnectState("network timeout"); + const state = resolveGatewayDisconnectState({ reason: "network timeout" }); expect(state.connectionStatus).toBe("gateway disconnected: network timeout"); expect(state.activityStatus).toBe("idle"); - expect(state.pairingHint).toBeUndefined(); + expect(state.remediation).toBeUndefined(); }); }); diff --git a/src/tui/tui.ts b/src/tui/tui.ts index 169489d4134d..de073fadb042 100644 --- a/src/tui/tui.ts +++ b/src/tui/tui.ts @@ -10,6 +10,7 @@ import { Text, TUI, } from "@earendil-works/pi-tui"; +import { classifyGatewayConnectFailure } from "../../packages/gateway-protocol/src/connect-error-details.js"; import type { CommandEntry } from "../../packages/gateway-protocol/src/index.js"; import { resolveAgentIdByWorkspacePath, resolveDefaultAgentId } from "../agents/agent-scope.js"; import { normalizeThinkLevel } from "../auto-reply/thinking.shared.js"; @@ -204,26 +205,37 @@ export function resolveInitialTuiAgentId(params: { return normalizeAgentId(params.fallbackAgentId); } -export function resolveGatewayDisconnectState(reason?: string): { +export function resolveGatewayDisconnectState( + input: { + details?: unknown; + reason?: string | null; + } = {}, +): { connectionStatus: string; activityStatus: string; - pairingHint?: string; + remediation?: string; } { - const reasonLabel = reason?.trim() ? reason.trim() : "closed"; - // Covers both "pairing required" and a pending "scope upgrade" for a paired device. - if (/pairing required|scope upgrade/i.test(reasonLabel)) { + const failure = classifyGatewayConnectFailure(input); + const reasonLabel = + failure.userMessage === "gateway unreachable" ? "closed" : failure.userMessage; + if (failure.kind === "pairing-required") { return { connectionStatus: `gateway disconnected: ${reasonLabel}`, activityStatus: "device approval needed: preview latest request", - pairingHint: - "Device approval needed. Run `openclaw devices approve --latest` to preview the pending request, " + - "then rerun the printed `openclaw devices approve ` command " + - "(reuse `--token` or other auth flags if needed), then reconnect.", + remediation: failure.remediation, + }; + } + if (failure.kind === "rate-limited") { + return { + connectionStatus: `gateway disconnected: ${reasonLabel}`, + activityStatus: "gateway authentication temporarily rate-limited", + remediation: failure.remediation, }; } return { connectionStatus: `gateway disconnected: ${reasonLabel}`, - activityStatus: "idle", + activityStatus: failure.remediation ? "gateway authentication needs attention" : "idle", + remediation: failure.remediation, }; } @@ -593,7 +605,7 @@ export async function runTui(opts: RunTuiOptions): Promise { const sessionIds = new Map(); let connectionGeneration = 0; let wasDisconnected = false; - let pairingHintShown = false; + let remediationShown = false; const localRunIds = createTuiRunIdTracker(); const localBtwRunIds = createTuiRunIdTracker(); @@ -1567,7 +1579,7 @@ export async function runTui(opts: RunTuiOptions): Promise { const ownsConnection = () => connectedGeneration === connectionGeneration && state.isConnected && !exitRequested; state.isConnected = true; - pairingHintShown = false; + remediationShown = false; const reconnected = wasDisconnected; wasDisconnected = false; if (reconnected) { @@ -1679,7 +1691,7 @@ export async function runTui(opts: RunTuiOptions): Promise { }); }; - const handleBackendDisconnected = (reason: string) => { + const handleBackendDisconnected = (reason: string, details?: unknown) => { if (exitRequested) { return; } @@ -1699,20 +1711,21 @@ export async function runTui(opts: RunTuiOptions): Promise { ? { connectionStatus: `local runtime stopped${reason ? `: ${reason}` : ""}`, activityStatus: "idle", - pairingHint: undefined, + remediation: undefined, } - : resolveGatewayDisconnectState(reason); + : resolveGatewayDisconnectState({ reason, details }); setConnectionStatus(disconnectState.connectionStatus, 5000); setActivityStatus(disconnectState.activityStatus); - if (disconnectState.pairingHint && !pairingHintShown) { - pairingHintShown = true; - chatLog.addSystem(disconnectState.pairingHint); + if (disconnectState.remediation && !remediationShown) { + remediationShown = true; + chatLog.addSystem(disconnectState.remediation); } updateFooter(); tui.requestRender(); }; client.onConnectError = (error) => { - handleBackendDisconnected(formatTuiErrorMessage(error)); + const details = "details" in error ? (error as { details?: unknown }).details : undefined; + handleBackendDisconnected(formatTuiErrorMessage(error), details); }; client.onDisconnected = handleBackendDisconnected;