mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
refactor(gateway): centralize connect failure classification (#120505)
* refactor(gateway): centralize connect failure classification * fix(gateway): project connect failure into status JSON * fix(protocol): restore remote-auth context in pairing remediation * fix(gateway): classify generic probe failures by close reason * fix(gateway): keep transport closes unreachable * fix(gateway): preserve rate-limit diagnostics * fix(gateway): preserve typed rate-limit failures
This commit is contained in:
committed by
GitHub
parent
75dbe52e3e
commit
c8cd98cf52
@@ -875,6 +875,8 @@ export class GatewayClient {
|
||||
assembled: AssembledConnect,
|
||||
) {
|
||||
const role = this.opts.role ?? "operator";
|
||||
const detailCode =
|
||||
error instanceof GatewayClientRequestError ? readConnectErrorDetailCode(error.details) : null;
|
||||
const shouldRetryWithDeviceToken = shouldRetryGatewayWithDeviceToken({
|
||||
retryBudgetUsed: this.deviceTokenRetryBudgetUsed,
|
||||
currentDeviceToken: assembled.resolvedDeviceToken,
|
||||
@@ -886,9 +888,7 @@ export class GatewayClient {
|
||||
if (
|
||||
this.opts.deviceIdentity &&
|
||||
assembled.usingStoredDeviceToken &&
|
||||
error instanceof GatewayClientRequestError &&
|
||||
readConnectErrorDetailCode(error.details) ===
|
||||
ConnectErrorDetailCodes.AUTH_DEVICE_TOKEN_MISMATCH
|
||||
detailCode === ConnectErrorDetailCodes.AUTH_DEVICE_TOKEN_MISMATCH
|
||||
) {
|
||||
const deviceId = this.opts.deviceIdentity.deviceId;
|
||||
try {
|
||||
@@ -942,7 +942,11 @@ export class GatewayClient {
|
||||
}
|
||||
this.notifyConnectError(error);
|
||||
const message = `gateway connect failed: ${formatGatewayClientErrorForLog(error)}`;
|
||||
if (this.opts.mode === GATEWAY_CLIENT_MODES.PROBE || isGatewayClientStoppedError(error)) {
|
||||
if (
|
||||
this.opts.mode === GATEWAY_CLIENT_MODES.PROBE ||
|
||||
isGatewayClientStoppedError(error) ||
|
||||
detailCode === ConnectErrorDetailCodes.AUTH_RATE_LIMITED
|
||||
) {
|
||||
this.logDebug(message);
|
||||
} else {
|
||||
this.logError(message);
|
||||
|
||||
@@ -4,6 +4,7 @@ import {
|
||||
buildPairingConnectCloseReason,
|
||||
buildPairingConnectErrorDetails,
|
||||
buildPairingConnectErrorMessage,
|
||||
classifyGatewayConnectFailure,
|
||||
describePairingConnectRequirement,
|
||||
formatConnectErrorMessage,
|
||||
formatConnectPairingRequiredMessage,
|
||||
@@ -67,6 +68,164 @@ describe("readConnectErrorRecoveryAdvice", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("classifyGatewayConnectFailure", () => {
|
||||
it.each([
|
||||
{
|
||||
name: "structured pairing upgrade",
|
||||
input: {
|
||||
details: { code: "PAIRING_REQUIRED", reason: "scope-upgrade", requestId: "req-123" },
|
||||
message: "connect failed",
|
||||
},
|
||||
kind: "pairing-required",
|
||||
message: "scope upgrade pending approval (requestId: req-123)",
|
||||
remediation: "openclaw devices approve --latest",
|
||||
},
|
||||
{
|
||||
name: "structured device identity requirement",
|
||||
input: { details: { code: "DEVICE_IDENTITY_REQUIRED" }, message: "connect failed" },
|
||||
kind: "device-identity-required",
|
||||
message: "connect failed",
|
||||
remediation: undefined,
|
||||
},
|
||||
{
|
||||
name: "structured scope mismatch",
|
||||
input: { details: { code: "AUTH_SCOPE_MISMATCH" }, message: "scope rejected" },
|
||||
kind: "scope-mismatch",
|
||||
message: "scope rejected",
|
||||
remediation: "openclaw devices list",
|
||||
},
|
||||
{
|
||||
name: "structured authentication rate limit",
|
||||
input: { details: { code: "AUTH_RATE_LIMITED" }, message: "connect failed" },
|
||||
kind: "rate-limited",
|
||||
message: "connect failed",
|
||||
remediation: "temporary authentication lockout",
|
||||
},
|
||||
{
|
||||
name: "shared token mismatch",
|
||||
input: { details: { code: "AUTH_TOKEN_MISMATCH" }, message: "gateway token mismatch" },
|
||||
kind: "auth-rejected",
|
||||
message: "gateway token mismatch",
|
||||
remediation: "gateway.remote.token",
|
||||
},
|
||||
{
|
||||
name: "device token mismatch",
|
||||
input: {
|
||||
details: { code: "AUTH_DEVICE_TOKEN_MISMATCH" },
|
||||
message: "device token mismatch",
|
||||
},
|
||||
kind: "auth-rejected",
|
||||
message: "device token mismatch",
|
||||
remediation: "openclaw devices rotate --device <deviceId> --role operator",
|
||||
},
|
||||
{
|
||||
name: "other structured auth rejection",
|
||||
input: { details: { code: "AUTH_PASSWORD_MISMATCH" }, message: "password mismatch" },
|
||||
kind: "auth-rejected",
|
||||
message: "password mismatch",
|
||||
remediation: undefined,
|
||||
},
|
||||
{
|
||||
name: "legacy pairing reason",
|
||||
input: { reason: "gateway closed (1008): pairing required" },
|
||||
kind: "pairing-required",
|
||||
message: "gateway closed (1008): pairing required",
|
||||
remediation: "openclaw devices approve --latest",
|
||||
},
|
||||
{
|
||||
name: "legacy pairing reason behind a generic message",
|
||||
input: {
|
||||
message: "connect failed",
|
||||
reason: "gateway closed (1008): pairing required",
|
||||
},
|
||||
kind: "pairing-required",
|
||||
message: "connect failed",
|
||||
remediation: "openclaw devices approve --latest",
|
||||
},
|
||||
{
|
||||
name: "legacy device identity reason behind a generic message",
|
||||
input: {
|
||||
message: "connect failed",
|
||||
reason: "gateway closed (1008): device identity required",
|
||||
},
|
||||
kind: "device-identity-required",
|
||||
message: "connect failed",
|
||||
remediation: undefined,
|
||||
},
|
||||
{
|
||||
name: "legacy scope mismatch reason behind a generic message",
|
||||
input: { message: "connect failed", reason: "scope mismatch" },
|
||||
kind: "scope-mismatch",
|
||||
message: "connect failed",
|
||||
remediation: "openclaw devices list",
|
||||
},
|
||||
{
|
||||
name: "legacy device token reason behind a generic message",
|
||||
input: { message: "connect failed", reason: "device token mismatch" },
|
||||
kind: "auth-rejected",
|
||||
message: "connect failed",
|
||||
remediation: "openclaw devices rotate --device <deviceId> --role operator",
|
||||
},
|
||||
{
|
||||
name: "legacy shared token reason behind a generic message",
|
||||
input: { message: "connect failed", reason: "gateway token mismatch" },
|
||||
kind: "auth-rejected",
|
||||
message: "connect failed",
|
||||
remediation: "gateway.remote.token",
|
||||
},
|
||||
{
|
||||
name: "legacy gateway close reason behind a generic message",
|
||||
input: { message: "connect failed", reason: "gateway closed (1008): auth failed" },
|
||||
kind: "gateway-rejected",
|
||||
message: "connect failed",
|
||||
remediation: undefined,
|
||||
},
|
||||
{
|
||||
name: "legacy gateway close",
|
||||
input: { message: "gateway closed (1008): auth failed" },
|
||||
kind: "gateway-rejected",
|
||||
message: "gateway closed (1008): auth failed",
|
||||
remediation: undefined,
|
||||
},
|
||||
{
|
||||
name: "legacy authentication rate limit",
|
||||
input: {
|
||||
reason: "unauthorized: too many failed authentication attempts (retry later)",
|
||||
},
|
||||
kind: "rate-limited",
|
||||
message: "unauthorized: too many failed authentication attempts (retry later)",
|
||||
remediation: "temporary authentication lockout",
|
||||
},
|
||||
{
|
||||
name: "generic retry hint without the authentication lockout phrase",
|
||||
input: { message: "connect failed; retry later" },
|
||||
kind: "unreachable",
|
||||
message: "connect failed; retry later",
|
||||
remediation: undefined,
|
||||
},
|
||||
{
|
||||
name: "unreachable endpoint",
|
||||
input: { message: "connect ECONNREFUSED 127.0.0.1:18789" },
|
||||
kind: "unreachable",
|
||||
message: "connect ECONNREFUSED 127.0.0.1:18789",
|
||||
remediation: undefined,
|
||||
},
|
||||
])("classifies $name", ({ input, kind, message, remediation }) => {
|
||||
const result = classifyGatewayConnectFailure(input);
|
||||
expect(result.kind).toBe(kind);
|
||||
expect(result.userMessage).toBe(message);
|
||||
if (remediation) {
|
||||
expect(result.remediation).toContain(remediation);
|
||||
} else {
|
||||
expect(result.remediation).toBeUndefined();
|
||||
}
|
||||
if (kind === "pairing-required") {
|
||||
expect(result.remediation).toContain("--url");
|
||||
expect(result.remediation).toContain("--token/--password");
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveAuthConnectErrorDetailCode", () => {
|
||||
it("maps device token scope mismatches to a dedicated auth detail", () => {
|
||||
expect(resolveAuthConnectErrorDetailCode("scope_mismatch")).toBe("AUTH_SCOPE_MISMATCH");
|
||||
|
||||
@@ -473,6 +473,98 @@ export function readConnectPairingRequiredMessage(
|
||||
};
|
||||
}
|
||||
|
||||
const PAIRING_APPROVAL_REMEDIATION =
|
||||
"Run `openclaw devices approve --latest` to preview the pending request, then rerun the printed " +
|
||||
"`openclaw devices approve <requestId>` command and reconnect (pass the same --url and " +
|
||||
"--token/--password flags if you connected with explicit credentials).";
|
||||
const DEVICE_TOKEN_REMEDIATION =
|
||||
"Rotate the paired-device token with `openclaw devices rotate --device <deviceId> --role operator`, then reconnect.";
|
||||
const SHARED_TOKEN_REMEDIATION =
|
||||
"Verify `gateway.remote.token` matches `gateway.auth.token`. If a paired-device token is stale, " +
|
||||
"rotate it with `openclaw devices rotate --device <deviceId> --role operator`, then reconnect.";
|
||||
const SCOPE_MISMATCH_REMEDIATION =
|
||||
"Review approved scopes with `openclaw devices list`; if an upgrade is pending, preview it with " +
|
||||
"`openclaw devices approve --latest`, approve the printed request, then reconnect.";
|
||||
const RATE_LIMITED_REMEDIATION =
|
||||
"Wait for the temporary authentication lockout to expire, then retry.";
|
||||
const GATEWAY_CLOSED_MESSAGE_PATTERN = /\bgateway closed \(\d+\):/i;
|
||||
|
||||
/** Classifies Gateway connect failures from structured details, with one legacy text fallback. */
|
||||
export function classifyGatewayConnectFailure(input: {
|
||||
details?: unknown;
|
||||
reason?: string | null;
|
||||
message?: string | null;
|
||||
}) {
|
||||
const code = readConnectErrorDetailCode(input.details);
|
||||
const message = normalizeOptionalString(input.message);
|
||||
const reason = normalizeOptionalString(input.reason);
|
||||
const userMessage = message ?? reason;
|
||||
const classificationText = [message, reason]
|
||||
.filter((value): value is string => Boolean(value))
|
||||
.join("\n");
|
||||
const normalized = classificationText.toLowerCase();
|
||||
const pairing =
|
||||
readPairingConnectErrorDetails(input.details) ??
|
||||
readConnectPairingRequiredMessage(classificationText);
|
||||
if (code === ConnectErrorDetailCodes.PAIRING_REQUIRED || pairing) {
|
||||
return {
|
||||
kind: "pairing-required" as const,
|
||||
userMessage:
|
||||
code === ConnectErrorDetailCodes.PAIRING_REQUIRED
|
||||
? formatConnectPairingRequiredMessage(input.details)
|
||||
: (userMessage ?? "device pairing required"),
|
||||
remediation: PAIRING_APPROVAL_REMEDIATION,
|
||||
};
|
||||
}
|
||||
const deviceIdentityRequired =
|
||||
code === ConnectErrorDetailCodes.DEVICE_IDENTITY_REQUIRED ||
|
||||
code === ConnectErrorDetailCodes.CONTROL_UI_DEVICE_IDENTITY_REQUIRED ||
|
||||
normalized.includes("device identity required");
|
||||
const scopeMismatch =
|
||||
code === ConnectErrorDetailCodes.AUTH_SCOPE_MISMATCH || normalized.includes("scope mismatch");
|
||||
const rateLimited =
|
||||
code === ConnectErrorDetailCodes.AUTH_RATE_LIMITED ||
|
||||
(!code && normalized.includes("too many failed authentication attempts"));
|
||||
const deviceTokenMismatch =
|
||||
code === ConnectErrorDetailCodes.AUTH_DEVICE_TOKEN_MISMATCH ||
|
||||
normalized.includes("device token mismatch");
|
||||
const sharedTokenMismatch =
|
||||
code === ConnectErrorDetailCodes.AUTH_TOKEN_MISMATCH ||
|
||||
normalized.includes("gateway token mismatch");
|
||||
const authRejected =
|
||||
deviceTokenMismatch ||
|
||||
sharedTokenMismatch ||
|
||||
code?.startsWith("AUTH_") ||
|
||||
code?.startsWith("DEVICE_AUTH_");
|
||||
const kind = deviceIdentityRequired
|
||||
? ("device-identity-required" as const)
|
||||
: scopeMismatch
|
||||
? ("scope-mismatch" as const)
|
||||
: rateLimited
|
||||
? ("rate-limited" as const)
|
||||
: authRejected
|
||||
? ("auth-rejected" as const)
|
||||
: code || GATEWAY_CLOSED_MESSAGE_PATTERN.test(classificationText)
|
||||
? ("gateway-rejected" as const)
|
||||
: ("unreachable" as const);
|
||||
const remediation = rateLimited
|
||||
? RATE_LIMITED_REMEDIATION
|
||||
: scopeMismatch
|
||||
? SCOPE_MISMATCH_REMEDIATION
|
||||
: deviceTokenMismatch
|
||||
? DEVICE_TOKEN_REMEDIATION
|
||||
: sharedTokenMismatch
|
||||
? SHARED_TOKEN_REMEDIATION
|
||||
: undefined;
|
||||
return {
|
||||
kind,
|
||||
userMessage:
|
||||
userMessage ??
|
||||
(kind === "unreachable" ? "gateway unreachable" : "gateway rejected connection"),
|
||||
...(remediation ? { remediation } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/** Formats pairing-required details into the canonical user-facing message. */
|
||||
export function formatConnectPairingRequiredMessage(details: unknown): string {
|
||||
const pairing = readPairingConnectErrorDetails(details);
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
// Daemon probe tests cover gateway probe command behavior and output.
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { gatewayProbeResultSawGateway } from "../../commands/gateway-health-auth-diagnostic.js";
|
||||
import { probeGatewayStatus } from "./probe.js";
|
||||
import type { DaemonStatus } from "./status.gather.js";
|
||||
|
||||
const callGatewayMock = vi.hoisted(() => vi.fn());
|
||||
const probeGatewayMock = vi.hoisted(() => vi.fn());
|
||||
@@ -17,6 +19,19 @@ vi.mock("../progress.js", () => ({
|
||||
withProgress: async (_opts: unknown, fn: () => Promise<unknown>) => await fn(),
|
||||
}));
|
||||
|
||||
function createDaemonStatus(rpc: NonNullable<DaemonStatus["rpc"]>): DaemonStatus {
|
||||
return {
|
||||
service: {
|
||||
label: "test service",
|
||||
loaded: true,
|
||||
loadedText: "loaded",
|
||||
notLoadedText: "not loaded",
|
||||
},
|
||||
rpc,
|
||||
extraServices: [],
|
||||
};
|
||||
}
|
||||
|
||||
describe("probeGatewayStatus", () => {
|
||||
const pairingPendingAuth = {
|
||||
role: null,
|
||||
@@ -39,6 +54,7 @@ describe("probeGatewayStatus", () => {
|
||||
kind: "connect",
|
||||
capability: "pairing_pending",
|
||||
auth: pairingPendingAuth,
|
||||
connectFailure: { kind: "pairing-required" },
|
||||
error: "gateway closed (1008): pairing required",
|
||||
});
|
||||
}
|
||||
@@ -85,6 +101,148 @@ describe("probeGatewayStatus", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("projects allowlisted connect failure details without serializing raw payloads", async () => {
|
||||
probeGatewayMock.mockResolvedValueOnce({
|
||||
ok: false,
|
||||
error: "connect failed",
|
||||
close: { code: 1008, reason: "connect failed" },
|
||||
connectErrorDetails: {
|
||||
code: "PAIRING_REQUIRED",
|
||||
reason: "scope-upgrade",
|
||||
secret: "do-not-print",
|
||||
},
|
||||
auth: pairingPendingAuth,
|
||||
});
|
||||
|
||||
const result = await probeGatewayStatus({
|
||||
url: "ws://127.0.0.1:19191",
|
||||
timeoutMs: 5_000,
|
||||
json: true,
|
||||
});
|
||||
|
||||
expect(result.ok).toBe(false);
|
||||
if (result.ok) {
|
||||
throw new Error("expected failed gateway probe");
|
||||
}
|
||||
expect(result.connectFailure).toEqual({
|
||||
kind: "pairing-required",
|
||||
detailCode: "PAIRING_REQUIRED",
|
||||
});
|
||||
expect(result).not.toHaveProperty("connectErrorDetails");
|
||||
expect(gatewayProbeResultSawGateway(result)).toBe(true);
|
||||
|
||||
const json = JSON.stringify(createDaemonStatus(result));
|
||||
expect(json).not.toContain("do-not-print");
|
||||
expect(json).not.toContain('"secret"');
|
||||
expect(json).not.toContain("scope-upgrade");
|
||||
});
|
||||
|
||||
it("classifies a legacy pairing close when the probe error is generic", async () => {
|
||||
probeGatewayMock.mockResolvedValueOnce({
|
||||
ok: false,
|
||||
error: "connect failed",
|
||||
close: { code: 1008, reason: "pairing required" },
|
||||
});
|
||||
|
||||
const result = await probeGatewayStatus({
|
||||
url: "ws://127.0.0.1:19191",
|
||||
timeoutMs: 5_000,
|
||||
json: true,
|
||||
});
|
||||
|
||||
expect(result.ok).toBe(false);
|
||||
if (result.ok) {
|
||||
throw new Error("expected failed gateway probe");
|
||||
}
|
||||
expect(result.error).toBe("connect failed");
|
||||
expect(result.connectFailure).toEqual({ kind: "pairing-required" });
|
||||
expect(gatewayProbeResultSawGateway(result)).toBe(true);
|
||||
});
|
||||
|
||||
it("does not classify an unvalidated transport close as a reachable gateway", async () => {
|
||||
probeGatewayMock.mockResolvedValueOnce({
|
||||
ok: false,
|
||||
error: "connect ECONNREFUSED 127.0.0.1:19191",
|
||||
close: { code: 1006, reason: "" },
|
||||
});
|
||||
|
||||
const result = await probeGatewayStatus({
|
||||
url: "ws://127.0.0.1:19191",
|
||||
timeoutMs: 5_000,
|
||||
json: true,
|
||||
});
|
||||
|
||||
expect(result.ok).toBe(false);
|
||||
if (result.ok) {
|
||||
throw new Error("expected failed gateway probe");
|
||||
}
|
||||
expect(result.connectFailure).toEqual({ kind: "unreachable" });
|
||||
expect(gatewayProbeResultSawGateway(result)).toBe(false);
|
||||
});
|
||||
|
||||
it("projects authentication rate limits as reachable temporary lockouts", async () => {
|
||||
probeGatewayMock.mockResolvedValueOnce({
|
||||
ok: false,
|
||||
error: "connect failed",
|
||||
close: {
|
||||
code: 1008,
|
||||
reason: "unauthorized: too many failed authentication attempts (retry later)",
|
||||
},
|
||||
connectErrorDetails: {
|
||||
code: "AUTH_RATE_LIMITED",
|
||||
authReason: "rate_limited",
|
||||
recommendedNextStep: "wait_then_retry",
|
||||
retryAfterMs: 60_000,
|
||||
},
|
||||
});
|
||||
|
||||
const result = await probeGatewayStatus({
|
||||
url: "ws://127.0.0.1:19191",
|
||||
timeoutMs: 5_000,
|
||||
json: true,
|
||||
});
|
||||
|
||||
expect(result.ok).toBe(false);
|
||||
if (result.ok) {
|
||||
throw new Error("expected failed gateway probe");
|
||||
}
|
||||
expect(result.connectFailure).toEqual({
|
||||
kind: "rate-limited",
|
||||
detailCode: "AUTH_RATE_LIMITED",
|
||||
});
|
||||
expect(gatewayProbeResultSawGateway(result)).toBe(true);
|
||||
expect(JSON.stringify(createDaemonStatus(result))).not.toContain("retryAfterMs");
|
||||
});
|
||||
|
||||
it("omits unknown detail codes from serialized daemon status", async () => {
|
||||
probeGatewayMock.mockResolvedValueOnce({
|
||||
ok: false,
|
||||
error: "connect failed",
|
||||
close: { code: 1008, reason: "connect failed" },
|
||||
connectErrorDetails: {
|
||||
code: "FUTURE_SENSITIVE_CODE",
|
||||
secret: "do-not-print-unknown",
|
||||
},
|
||||
auth: pairingPendingAuth,
|
||||
});
|
||||
|
||||
const result = await probeGatewayStatus({
|
||||
url: "ws://127.0.0.1:19191",
|
||||
timeoutMs: 5_000,
|
||||
json: true,
|
||||
});
|
||||
|
||||
expect(result.ok).toBe(false);
|
||||
if (result.ok) {
|
||||
throw new Error("expected failed gateway probe");
|
||||
}
|
||||
expect(result.connectFailure).toEqual({ kind: "gateway-rejected" });
|
||||
|
||||
const json = JSON.stringify(createDaemonStatus(result));
|
||||
expect(json).not.toContain("FUTURE_SENSITIVE_CODE");
|
||||
expect(json).not.toContain("do-not-print-unknown");
|
||||
});
|
||||
|
||||
it("preserves gateway server version from the connect probe", async () => {
|
||||
callGatewayMock.mockReset();
|
||||
probeGatewayMock.mockReset();
|
||||
@@ -285,6 +443,7 @@ describe("probeGatewayStatus", () => {
|
||||
expect(result).toEqual({
|
||||
ok: false,
|
||||
kind: "read",
|
||||
connectFailure: { kind: "unreachable" },
|
||||
error:
|
||||
"gateway status RPC skipped because configured gateway credentials are disabled for this status request",
|
||||
});
|
||||
@@ -484,6 +643,7 @@ describe("probeGatewayStatus", () => {
|
||||
expect(result).toEqual({
|
||||
ok: false,
|
||||
kind: "read",
|
||||
connectFailure: { kind: "unreachable" },
|
||||
error: "missing scope: operator.admin",
|
||||
});
|
||||
expect(probeGatewayMock).not.toHaveBeenCalled();
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
// Gateway status probe helper used by `gateway status` service diagnostics.
|
||||
import { redactSensitiveUrlLikeString } from "@openclaw/net-policy/redact-sensitive-url";
|
||||
import {
|
||||
classifyGatewayConnectFailure,
|
||||
ConnectErrorDetailCodes,
|
||||
readConnectErrorDetailCode,
|
||||
} from "../../../packages/gateway-protocol/src/connect-error-details.js";
|
||||
import type { OpenClawConfig } from "../../config/types.js";
|
||||
import type { GatewayProbeResult } from "../../gateway/probe.js";
|
||||
import { formatErrorMessage } from "../../infra/errors.js";
|
||||
@@ -14,6 +19,9 @@ type GatewayStatusRequireRpcProbeResult = {
|
||||
type GatewayStatusProbeResult = GatewayProbeResult | GatewayStatusRequireRpcProbeResult;
|
||||
|
||||
const probeGatewayModuleLoader = createLazyImportLoader(() => import("../../gateway/probe.js"));
|
||||
const CONNECT_ERROR_DETAIL_CODE_VALUES: ReadonlySet<string> = new Set(
|
||||
Object.values(ConnectErrorDetailCodes),
|
||||
);
|
||||
|
||||
async function loadProbeGatewayModule(): Promise<typeof import("../../gateway/probe.js")> {
|
||||
return await probeGatewayModuleLoader.load();
|
||||
@@ -36,6 +44,21 @@ function resolveGatewayStatusProbeDetails(result: GatewayStatusProbeResult) {
|
||||
return "authProbe" in result ? result.authProbe : result;
|
||||
}
|
||||
|
||||
function projectGatewayConnectFailure(params: {
|
||||
details?: unknown;
|
||||
message: string;
|
||||
reason?: string;
|
||||
}) {
|
||||
// Daemon status is serialized for diagnostics, so raw gateway details must
|
||||
// stop here; only closed classification facts may cross this boundary.
|
||||
const failure = classifyGatewayConnectFailure(params);
|
||||
const detailCode = readConnectErrorDetailCode(params.details);
|
||||
return {
|
||||
kind: failure.kind,
|
||||
...(detailCode && CONNECT_ERROR_DETAIL_CODE_VALUES.has(detailCode) ? { detailCode } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function readRuntimeVersionFromStatusPayload(payload: unknown): string | null {
|
||||
if (!payload || typeof payload !== "object") {
|
||||
return null;
|
||||
@@ -129,6 +152,7 @@ export async function probeGatewayStatus(opts: {
|
||||
...(version != null ? { version } : {}),
|
||||
} as const;
|
||||
}
|
||||
const error = redactSensitiveUrlLikeString(resolveProbeFailureMessage(result));
|
||||
return {
|
||||
ok: false,
|
||||
kind,
|
||||
@@ -136,15 +160,22 @@ export async function probeGatewayStatus(opts: {
|
||||
auth,
|
||||
...serverSummary,
|
||||
...(version != null ? { version } : {}),
|
||||
connectFailure: projectGatewayConnectFailure({
|
||||
details: probeDetails?.connectErrorDetails,
|
||||
message: error,
|
||||
reason: probeDetails?.close?.reason,
|
||||
}),
|
||||
// Probe failure text can echo the credential-bearing target URL (close
|
||||
// reasons, transport errors); status renderers print it verbatim.
|
||||
error: redactSensitiveUrlLikeString(resolveProbeFailureMessage(result)),
|
||||
error,
|
||||
} as const;
|
||||
} catch (err) {
|
||||
const error = redactSensitiveUrlLikeString(formatErrorMessage(err));
|
||||
return {
|
||||
ok: false,
|
||||
kind,
|
||||
error: redactSensitiveUrlLikeString(formatErrorMessage(err)),
|
||||
connectFailure: projectGatewayConnectFailure({ message: error }),
|
||||
error,
|
||||
} as const;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
import fs from "node:fs/promises";
|
||||
import { uniqueStrings } from "@openclaw/normalization-core/string-normalization";
|
||||
import JSON5 from "json5";
|
||||
import type { classifyGatewayConnectFailure } from "../../../packages/gateway-protocol/src/connect-error-details.js";
|
||||
import {
|
||||
createConfigIO,
|
||||
resolveConfigPath,
|
||||
@@ -123,6 +124,8 @@ type CliStatusSummary = {
|
||||
entrypoint?: string;
|
||||
};
|
||||
|
||||
type GatewayConnectFailureKind = ReturnType<typeof classifyGatewayConnectFailure>["kind"];
|
||||
|
||||
const gatewayProbeAuthModuleLoader = createLazyImportLoader(
|
||||
() => import("../../gateway/probe-auth.js"),
|
||||
);
|
||||
@@ -343,6 +346,10 @@ export type DaemonStatus = {
|
||||
};
|
||||
version?: string | null;
|
||||
error?: string;
|
||||
connectFailure?: {
|
||||
kind: GatewayConnectFailureKind;
|
||||
detailCode?: string;
|
||||
};
|
||||
url?: string;
|
||||
authWarning?: string;
|
||||
};
|
||||
|
||||
@@ -78,6 +78,43 @@ async function startCronListGateway(token: string): Promise<{ url: string }> {
|
||||
return { url: `ws://127.0.0.1:${address.port}` };
|
||||
}
|
||||
|
||||
async function startRateLimitedGateway(): Promise<{ url: string }> {
|
||||
const wss = new WebSocketServer({ host: "127.0.0.1", port: 0 });
|
||||
activeServers.add(wss);
|
||||
wss.on("connection", (ws) => {
|
||||
sendMinimalGatewayConnectChallenge(ws);
|
||||
ws.on("message", (data) => {
|
||||
const frame = parseMinimalGatewayRequestFrame(data);
|
||||
if (frame.type !== "req" || !frame.id || frame.method !== "connect") {
|
||||
return;
|
||||
}
|
||||
const message = "unauthorized: too many failed authentication attempts (retry later)";
|
||||
ws.send(
|
||||
JSON.stringify({
|
||||
type: "res",
|
||||
id: frame.id,
|
||||
ok: false,
|
||||
error: {
|
||||
code: "INVALID_REQUEST",
|
||||
message,
|
||||
retryable: true,
|
||||
retryAfterMs: 60_000,
|
||||
details: {
|
||||
code: "AUTH_RATE_LIMITED",
|
||||
authReason: "rate_limited",
|
||||
recommendedNextStep: "wait_then_retry",
|
||||
},
|
||||
},
|
||||
}),
|
||||
() => ws.close(1008, message),
|
||||
);
|
||||
});
|
||||
});
|
||||
await once(wss, "listening");
|
||||
const address = wss.address() as AddressInfo;
|
||||
return { url: `ws://127.0.0.1:${address.port}` };
|
||||
}
|
||||
|
||||
async function runIsolatedGatewayCli(params: {
|
||||
args: string[];
|
||||
root: string;
|
||||
@@ -298,4 +335,44 @@ describe("gateway-backed CLI process exit", () => {
|
||||
gateway: { url: `ws://127.0.0.1:${port}` },
|
||||
});
|
||||
}, 30_000);
|
||||
|
||||
it("preserves pre-hello rate-limit details through the real health entry point", async () => {
|
||||
const root = tempDirs.make("openclaw-gateway-rate-limit-json-");
|
||||
const stateDir = path.join(root, "state");
|
||||
const configPath = path.join(stateDir, "openclaw.json");
|
||||
const gateway = await startRateLimitedGateway();
|
||||
await fs.mkdir(stateDir, { recursive: true });
|
||||
await fs.writeFile(
|
||||
configPath,
|
||||
JSON.stringify({
|
||||
gateway: {
|
||||
mode: "remote",
|
||||
remote: { url: gateway.url, token: "test-token" },
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await runIsolatedGatewayCli({
|
||||
args: ["health", "--json", "--timeout", "2000"],
|
||||
root,
|
||||
stateDir,
|
||||
configPath,
|
||||
});
|
||||
|
||||
expect(result, result.stderr).toMatchObject({ code: 1, signal: null, stderr: "" });
|
||||
expect(JSON.parse(result.stdout)).toEqual({
|
||||
ok: false,
|
||||
error: {
|
||||
type: "gateway_request_error",
|
||||
code: "AUTH_RATE_LIMITED",
|
||||
message:
|
||||
"Gateway authentication is temporarily rate-limited. Wait for the temporary lockout to expire, then retry.",
|
||||
retryable: true,
|
||||
retryAfterMs: 60_000,
|
||||
},
|
||||
gateway: { reachable: true },
|
||||
});
|
||||
expect(result.stdout).not.toContain("gateway.remote.token");
|
||||
expect(result.stdout).not.toContain("devices rotate");
|
||||
}, 30_000);
|
||||
});
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
// Doctor gateway health tests cover gateway probe failures, auth requirements, and repair messages.
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { GatewayClientRequestError } from "../../packages/gateway-client/src/index.js";
|
||||
import type { OpenClawConfig } from "../config/config.js";
|
||||
import {
|
||||
GATEWAY_HEALTH_CREDENTIALS_REQUIRED_MESSAGE,
|
||||
GATEWAY_HEALTH_CREDENTIALS_REQUIRED_TITLE,
|
||||
GATEWAY_HEALTH_RATE_LIMITED_MESSAGE,
|
||||
GATEWAY_HEALTH_RATE_LIMITED_TITLE,
|
||||
} from "./gateway-health-auth-diagnostic.js";
|
||||
|
||||
const callGateway = vi.hoisted(() => vi.fn());
|
||||
@@ -334,6 +337,63 @@ describe("checkGatewayHealth", () => {
|
||||
expect(callGateway).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("reports a temporary lockout when status auth is rate-limited", async () => {
|
||||
callGateway.mockRejectedValueOnce(new Error());
|
||||
isGatewayCredentialsRequiredError.mockReturnValueOnce(true);
|
||||
probeGatewayStatus.mockResolvedValueOnce({
|
||||
ok: false,
|
||||
kind: "connect",
|
||||
error: "connect failed",
|
||||
connectFailure: { kind: "rate-limited", detailCode: "AUTH_RATE_LIMITED" },
|
||||
});
|
||||
const runtime = { log: vi.fn(), error: vi.fn(), exit: vi.fn() };
|
||||
|
||||
await expect(
|
||||
checkGatewayHealth({ runtime: runtime as never, cfg, timeoutMs: 3000 }),
|
||||
).resolves.toEqual({ authenticated: false, healthOk: true });
|
||||
|
||||
expect(runtime.error).not.toHaveBeenCalled();
|
||||
expect(note).toHaveBeenCalledWith(
|
||||
GATEWAY_HEALTH_RATE_LIMITED_MESSAGE,
|
||||
GATEWAY_HEALTH_RATE_LIMITED_TITLE,
|
||||
);
|
||||
expect(note).not.toHaveBeenCalledWith(
|
||||
GATEWAY_HEALTH_CREDENTIALS_REQUIRED_MESSAGE,
|
||||
GATEWAY_HEALTH_CREDENTIALS_REQUIRED_TITLE,
|
||||
);
|
||||
const output = note.mock.calls.flat().join("\n");
|
||||
expect(output).not.toContain("gateway.remote.token");
|
||||
expect(output).not.toContain("devices rotate");
|
||||
});
|
||||
|
||||
it("handles the real typed rate-limit error without forcing the credentials predicate", async () => {
|
||||
const error = new GatewayClientRequestError({
|
||||
code: "INVALID_REQUEST",
|
||||
message: "unauthorized: too many failed authentication attempts (retry later)",
|
||||
details: {
|
||||
code: "AUTH_RATE_LIMITED",
|
||||
authReason: "rate_limited",
|
||||
recommendedNextStep: "wait_then_retry",
|
||||
},
|
||||
retryable: true,
|
||||
retryAfterMs: 60_000,
|
||||
});
|
||||
callGateway.mockRejectedValueOnce(error);
|
||||
const runtime = { log: vi.fn(), error: vi.fn(), exit: vi.fn() };
|
||||
|
||||
await expect(
|
||||
checkGatewayHealth({ runtime: runtime as never, cfg, timeoutMs: 3000 }),
|
||||
).resolves.toEqual({ authenticated: false, healthOk: true });
|
||||
|
||||
expect(isGatewayCredentialsRequiredError).not.toHaveBeenCalled();
|
||||
expect(probeGatewayStatus).not.toHaveBeenCalled();
|
||||
expect(runtime.error).not.toHaveBeenCalled();
|
||||
expect(note).toHaveBeenCalledWith(
|
||||
GATEWAY_HEALTH_RATE_LIMITED_MESSAGE,
|
||||
GATEWAY_HEALTH_RATE_LIMITED_TITLE,
|
||||
);
|
||||
});
|
||||
|
||||
it("reports credentials-required when status RPC auth SecretRefs are unavailable", async () => {
|
||||
const error = new Error("gateway.auth.password unavailable");
|
||||
callGateway.mockRejectedValueOnce(error);
|
||||
|
||||
@@ -24,7 +24,11 @@ import { VERSION } from "../version.js";
|
||||
import {
|
||||
GATEWAY_HEALTH_CREDENTIALS_REQUIRED_MESSAGE,
|
||||
GATEWAY_HEALTH_CREDENTIALS_REQUIRED_TITLE,
|
||||
GATEWAY_HEALTH_RATE_LIMITED_MESSAGE,
|
||||
GATEWAY_HEALTH_RATE_LIMITED_TITLE,
|
||||
gatewayConnectErrorWasRateLimited,
|
||||
gatewayProbeResultSawGateway,
|
||||
gatewayProbeResultWasRateLimited,
|
||||
} from "./gateway-health-auth-diagnostic.js";
|
||||
import { formatGatewayClosedDiagnostic, formatHealthCheckFailure } from "./health-format.js";
|
||||
import { formatTelemetryExporterSummary } from "./telemetry-exporter-summary.js";
|
||||
@@ -159,6 +163,10 @@ export async function checkGatewayHealth(params: {
|
||||
}
|
||||
return { healthOk, authenticated: true, status };
|
||||
} catch (err) {
|
||||
if (gatewayConnectErrorWasRateLimited(err)) {
|
||||
note(GATEWAY_HEALTH_RATE_LIMITED_MESSAGE, GATEWAY_HEALTH_RATE_LIMITED_TITLE);
|
||||
return { healthOk: true, authenticated: false };
|
||||
}
|
||||
if (isGatewayHealthAuthUnavailableError(err)) {
|
||||
const probeDetails = await buildGatewayProbeConnectionDetails({ config: params.cfg });
|
||||
const probe = await probeGatewayStatus({
|
||||
@@ -170,10 +178,14 @@ export async function checkGatewayHealth(params: {
|
||||
json: true,
|
||||
});
|
||||
if (gatewayProbeResultSawGateway(probe)) {
|
||||
note(
|
||||
GATEWAY_HEALTH_CREDENTIALS_REQUIRED_MESSAGE,
|
||||
GATEWAY_HEALTH_CREDENTIALS_REQUIRED_TITLE,
|
||||
);
|
||||
if (gatewayProbeResultWasRateLimited(probe)) {
|
||||
note(GATEWAY_HEALTH_RATE_LIMITED_MESSAGE, GATEWAY_HEALTH_RATE_LIMITED_TITLE);
|
||||
} else {
|
||||
note(
|
||||
GATEWAY_HEALTH_CREDENTIALS_REQUIRED_MESSAGE,
|
||||
GATEWAY_HEALTH_CREDENTIALS_REQUIRED_TITLE,
|
||||
);
|
||||
}
|
||||
healthOk = true;
|
||||
return { healthOk, authenticated: false };
|
||||
}
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
/** Gateway health auth diagnostic helpers for reachable-but-unauthenticated probes. */
|
||||
import {
|
||||
classifyGatewayConnectFailure,
|
||||
ConnectErrorDetailCodes,
|
||||
} from "../../packages/gateway-protocol/src/connect-error-details.js";
|
||||
import type { DaemonStatus } from "../cli/daemon-cli/status.gather.js";
|
||||
|
||||
type GatewayProbeReachabilityEvidence = NonNullable<DaemonStatus["rpc"]>;
|
||||
@@ -7,6 +11,35 @@ export const GATEWAY_HEALTH_CREDENTIALS_REQUIRED_MESSAGE =
|
||||
"Gateway is reachable, but this CLI has no token/password or paired device token for read-scope health RPCs.";
|
||||
export const GATEWAY_HEALTH_CREDENTIALS_REQUIRED_TITLE = "Gateway credentials required";
|
||||
export const GATEWAY_HEALTH_REACHABLE_LINE = "Gateway: reachable";
|
||||
export const GATEWAY_HEALTH_RATE_LIMITED_MESSAGE =
|
||||
"Gateway authentication is temporarily rate-limited. Wait for the temporary lockout to expire, then retry.";
|
||||
export const GATEWAY_HEALTH_RATE_LIMITED_TITLE = "Gateway authentication rate-limited";
|
||||
|
||||
function gatewayProbeFailureKind(status: GatewayProbeReachabilityEvidence) {
|
||||
return (
|
||||
status.connectFailure?.kind ?? classifyGatewayConnectFailure({ message: status.error }).kind
|
||||
);
|
||||
}
|
||||
|
||||
/** Detects the temporary authentication lockout outcome from projected or legacy probe facts. */
|
||||
export function gatewayProbeResultWasRateLimited(
|
||||
status: GatewayProbeReachabilityEvidence,
|
||||
): boolean {
|
||||
return gatewayProbeFailureKind(status) === "rate-limited";
|
||||
}
|
||||
|
||||
/** Detects a structured or legacy rate-limit connect error before close projection. */
|
||||
export function gatewayConnectErrorWasRateLimited(error: unknown): boolean {
|
||||
if (!(error instanceof Error)) {
|
||||
return false;
|
||||
}
|
||||
return (
|
||||
classifyGatewayConnectFailure({
|
||||
details: (error as Error & { details?: unknown }).details,
|
||||
message: error.message,
|
||||
}).kind === "rate-limited"
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Detects when a daemon probe reached the gateway even if read-scope auth failed.
|
||||
@@ -26,11 +59,7 @@ export function gatewayProbeResultSawGateway(status: GatewayProbeReachabilityEvi
|
||||
if (server?.version || server?.connId) {
|
||||
return true;
|
||||
}
|
||||
// Older probes may only expose close/error text for auth failures; treat known gateway
|
||||
// close reasons as reachability evidence so health can explain missing credentials.
|
||||
return /\bgateway closed \(\d+\):|\bpairing required\b|\bdevice identity required\b/i.test(
|
||||
status.error ?? "",
|
||||
);
|
||||
return gatewayProbeFailureKind(status) !== "unreachable";
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -48,3 +77,28 @@ export function buildCredentialsRequiredHealthDiagnostic() {
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Builds the health diagnostic emitted for a temporary Gateway authentication lockout. */
|
||||
export function buildRateLimitedHealthDiagnostic(error?: unknown) {
|
||||
const retryAfterCandidate =
|
||||
error instanceof Error ? (error as Error & { retryAfterMs?: unknown }).retryAfterMs : undefined;
|
||||
const retryAfterMs =
|
||||
typeof retryAfterCandidate === "number" &&
|
||||
Number.isSafeInteger(retryAfterCandidate) &&
|
||||
retryAfterCandidate >= 0
|
||||
? retryAfterCandidate
|
||||
: undefined;
|
||||
return {
|
||||
ok: false,
|
||||
error: {
|
||||
type: "gateway_request_error",
|
||||
code: ConnectErrorDetailCodes.AUTH_RATE_LIMITED,
|
||||
message: GATEWAY_HEALTH_RATE_LIMITED_MESSAGE,
|
||||
retryable: true,
|
||||
...(retryAfterMs !== undefined ? { retryAfterMs } : {}),
|
||||
},
|
||||
gateway: {
|
||||
reachable: true,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -302,6 +302,69 @@ describe("ensureGatewayReadyForOperation", () => {
|
||||
expect(runtime.log).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("uses the projected connect failure when the daemon error text is generic", async () => {
|
||||
const status = createStatus({
|
||||
service: {
|
||||
label: "systemd user",
|
||||
loaded: true,
|
||||
loadedText: "enabled",
|
||||
notLoadedText: "disabled",
|
||||
command: { programArguments: ["openclaw", "gateway", "run", "--port", "18789"] },
|
||||
runtime: { status: "running" },
|
||||
},
|
||||
port: { port: 18789, status: "busy", listeners: [], hints: [] },
|
||||
rpc: {
|
||||
ok: false,
|
||||
error: "connect failed",
|
||||
connectFailure: { kind: "pairing-required", detailCode: "PAIRING_REQUIRED" },
|
||||
url: "ws://127.0.0.1:18789",
|
||||
},
|
||||
});
|
||||
const confirm = vi.fn();
|
||||
|
||||
const result = await ensureGatewayReadyForOperation({
|
||||
runtime,
|
||||
operation: "open the dashboard",
|
||||
readyWhenReachable: true,
|
||||
interactive: true,
|
||||
deps: { gatherStatus: vi.fn().mockResolvedValue(status), confirm },
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ ready: true, recovered: false });
|
||||
expect(confirm).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("accepts a rate-limited Gateway as reachable without starting the service", async () => {
|
||||
const status = createStatus({
|
||||
service: {
|
||||
label: "systemd user",
|
||||
loaded: true,
|
||||
loadedText: "enabled",
|
||||
notLoadedText: "disabled",
|
||||
command: { programArguments: ["openclaw", "gateway", "run", "--port", "18789"] },
|
||||
runtime: { status: "running" },
|
||||
},
|
||||
port: { port: 18789, status: "busy", listeners: [], hints: [] },
|
||||
rpc: {
|
||||
ok: false,
|
||||
error: "connect failed",
|
||||
connectFailure: { kind: "rate-limited", detailCode: "AUTH_RATE_LIMITED" },
|
||||
url: "ws://127.0.0.1:18789",
|
||||
},
|
||||
});
|
||||
const startGateway = vi.fn();
|
||||
|
||||
const result = await ensureGatewayReadyForOperation({
|
||||
runtime,
|
||||
operation: "open the dashboard",
|
||||
readyWhenReachable: true,
|
||||
deps: { gatherStatus: vi.fn().mockResolvedValue(status), startGateway },
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ ready: true, recovered: false });
|
||||
expect(startGateway).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("still treats a timeout on the target port as not ready", async () => {
|
||||
const status = createStatus({
|
||||
service: {
|
||||
|
||||
@@ -4,7 +4,9 @@ import { GatewayClientRequestError } from "../../packages/gateway-client/src/ind
|
||||
import { stripAnsi } from "../../packages/terminal-core/src/ansi.js";
|
||||
import {
|
||||
buildCredentialsRequiredHealthDiagnostic,
|
||||
buildRateLimitedHealthDiagnostic,
|
||||
GATEWAY_HEALTH_CREDENTIALS_REQUIRED_MESSAGE,
|
||||
GATEWAY_HEALTH_RATE_LIMITED_MESSAGE,
|
||||
GATEWAY_HEALTH_REACHABLE_LINE,
|
||||
} from "./gateway-health-auth-diagnostic.js";
|
||||
import { formatHealthCheckFailure } from "./health-format.js";
|
||||
@@ -509,6 +511,77 @@ describe("healthCommand", () => {
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
{ json: true, expectedLogs: 1 },
|
||||
{ json: undefined, expectedLogs: 2 },
|
||||
])(
|
||||
"preserves a typed pre-hello authentication lockout through health output",
|
||||
async ({ json, expectedLogs }) => {
|
||||
const error = new GatewayClientRequestError({
|
||||
code: "INVALID_REQUEST",
|
||||
message: "unauthorized: too many failed authentication attempts (retry later)",
|
||||
details: {
|
||||
code: "AUTH_RATE_LIMITED",
|
||||
authReason: "rate_limited",
|
||||
recommendedNextStep: "wait_then_retry",
|
||||
},
|
||||
retryable: true,
|
||||
retryAfterMs: 60_000,
|
||||
});
|
||||
callGatewayMock.mockRejectedValueOnce(error);
|
||||
|
||||
await healthCommand({ json, timeoutMs: 5000, config: {} }, runtime as never);
|
||||
|
||||
expect(isGatewayCredentialsRequiredErrorMock).not.toHaveBeenCalled();
|
||||
expect(probeGatewayStatusMock).not.toHaveBeenCalled();
|
||||
expect(runtime.exit).toHaveBeenCalledWith(1);
|
||||
expect(runtime.log).toHaveBeenCalledTimes(expectedLogs);
|
||||
if (json) {
|
||||
expect(JSON.parse(requireFirstRuntimeLog())).toEqual(
|
||||
buildRateLimitedHealthDiagnostic(error),
|
||||
);
|
||||
} else {
|
||||
expect(runtime.log.mock.calls).toEqual([
|
||||
[GATEWAY_HEALTH_REACHABLE_LINE],
|
||||
[GATEWAY_HEALTH_RATE_LIMITED_MESSAGE],
|
||||
]);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
{ json: true, expectedLogs: 1 },
|
||||
{ json: undefined, expectedLogs: 2 },
|
||||
])(
|
||||
"reports temporary authentication lockouts without credential-change guidance",
|
||||
async ({ json, expectedLogs }) => {
|
||||
callGatewayMock.mockRejectedValueOnce(new Error());
|
||||
isGatewayCredentialsRequiredErrorMock.mockReturnValueOnce(true);
|
||||
probeGatewayStatusMock.mockResolvedValueOnce({
|
||||
ok: false,
|
||||
kind: "connect",
|
||||
error: "connect failed",
|
||||
connectFailure: { kind: "rate-limited", detailCode: "AUTH_RATE_LIMITED" },
|
||||
});
|
||||
|
||||
await healthCommand({ json, timeoutMs: 5000, config: {} }, runtime as never);
|
||||
|
||||
expect(runtime.exit).toHaveBeenCalledWith(1);
|
||||
expect(runtime.log).toHaveBeenCalledTimes(expectedLogs);
|
||||
if (json) {
|
||||
expect(JSON.parse(requireFirstRuntimeLog())).toEqual(buildRateLimitedHealthDiagnostic());
|
||||
} else {
|
||||
expect(runtime.log.mock.calls).toEqual([
|
||||
[GATEWAY_HEALTH_REACHABLE_LINE],
|
||||
[GATEWAY_HEALTH_RATE_LIMITED_MESSAGE],
|
||||
]);
|
||||
}
|
||||
const output = runtime.log.mock.calls.flat().join("\n");
|
||||
expect(output).not.toContain("gateway.remote.token");
|
||||
expect(output).not.toContain("devices rotate");
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps credential failures machine-readable when the gateway is unreachable", async () => {
|
||||
const error = new Error("gateway health requires credentials");
|
||||
const payload = {
|
||||
|
||||
+19
-2
@@ -37,8 +37,11 @@ import { buildChannelAccountBindings, resolvePreferredAccountId } from "../routi
|
||||
import { type RuntimeEnv, writeRuntimeJson } from "../runtime.js";
|
||||
import {
|
||||
buildCredentialsRequiredHealthDiagnostic,
|
||||
buildRateLimitedHealthDiagnostic,
|
||||
gatewayConnectErrorWasRateLimited,
|
||||
GATEWAY_HEALTH_REACHABLE_LINE,
|
||||
gatewayProbeResultSawGateway,
|
||||
gatewayProbeResultWasRateLimited,
|
||||
} from "./gateway-health-auth-diagnostic.js";
|
||||
import { formatHealthChannelLines } from "./health-format.js";
|
||||
import { logGatewayConnectionDetails } from "./status.gateway-connection.js";
|
||||
@@ -72,9 +75,21 @@ export async function emitReachableGatewayAuthDiagnostic(params: {
|
||||
localPortOverride?: number;
|
||||
json?: boolean;
|
||||
}): Promise<boolean> {
|
||||
if (!isGatewayHealthAuthUnavailableError(params.error)) {
|
||||
const directRateLimit = gatewayConnectErrorWasRateLimited(params.error);
|
||||
if (!directRateLimit && !isGatewayHealthAuthUnavailableError(params.error)) {
|
||||
return false;
|
||||
}
|
||||
if (directRateLimit) {
|
||||
const diagnostic = buildRateLimitedHealthDiagnostic(params.error);
|
||||
if (params.json) {
|
||||
writeRuntimeJson(params.runtime, diagnostic);
|
||||
} else {
|
||||
params.runtime.log(GATEWAY_HEALTH_REACHABLE_LINE);
|
||||
params.runtime.log(diagnostic.error.message);
|
||||
}
|
||||
params.runtime.exit(1);
|
||||
return true;
|
||||
}
|
||||
const details = await buildGatewayProbeConnectionDetails({
|
||||
config: params.config,
|
||||
token: params.token,
|
||||
@@ -94,7 +109,9 @@ export async function emitReachableGatewayAuthDiagnostic(params: {
|
||||
if (!gatewayProbeResultSawGateway(probe)) {
|
||||
return false;
|
||||
}
|
||||
const diagnostic = buildCredentialsRequiredHealthDiagnostic();
|
||||
const diagnostic = gatewayProbeResultWasRateLimited(probe)
|
||||
? buildRateLimitedHealthDiagnostic()
|
||||
: buildCredentialsRequiredHealthDiagnostic();
|
||||
if (params.json) {
|
||||
writeRuntimeJson(params.runtime, diagnostic);
|
||||
params.runtime.exit(1);
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
// Doctor runtime check tests cover runtime-backed doctor checks.
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { AnyAgentTool } from "../agents/tools/common.js";
|
||||
import { GATEWAY_HEALTH_RATE_LIMITED_MESSAGE } from "../commands/gateway-health-auth-diagnostic.js";
|
||||
import { setPluginToolMeta } from "../plugins/tools.js";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
@@ -591,6 +592,25 @@ describe("doctor gateway runtime checks", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("reports temporary Gateway authentication lockouts with wait-and-retry guidance", async () => {
|
||||
mocks.probeGatewayStatus.mockResolvedValueOnce({
|
||||
ok: false,
|
||||
error: "connect failed",
|
||||
connectFailure: { kind: "rate-limited", detailCode: "AUTH_RATE_LIMITED" },
|
||||
});
|
||||
|
||||
await expect(
|
||||
collectGatewayHealthFindings({ cfg: { gateway: { mode: "local" } } }),
|
||||
).resolves.toContainEqual({
|
||||
checkId: "core/doctor/gateway-health",
|
||||
severity: "warning",
|
||||
message: GATEWAY_HEALTH_RATE_LIMITED_MESSAGE,
|
||||
path: "gateway.mode",
|
||||
target: "http://127.0.0.1:5829",
|
||||
fixHint: "Wait for the temporary authentication lockout to expire, then rerun doctor.",
|
||||
});
|
||||
});
|
||||
|
||||
it("redacts sensitive remote gateway URLs from health finding targets", async () => {
|
||||
mocks.buildGatewayProbeConnectionDetails.mockResolvedValueOnce({
|
||||
url: "wss://user:pass@gateway.example.test/rpc?token=secret&safe=value",
|
||||
|
||||
@@ -31,7 +31,11 @@ import {
|
||||
import type { AnyAgentTool } from "../agents/tools/common.js";
|
||||
import { probeGatewayStatus } from "../cli/daemon-cli/probe.js";
|
||||
import { collectUnavailableAgentSkills } from "../commands/doctor-skills-core.js";
|
||||
import { gatewayProbeResultSawGateway } from "../commands/gateway-health-auth-diagnostic.js";
|
||||
import {
|
||||
GATEWAY_HEALTH_RATE_LIMITED_MESSAGE,
|
||||
gatewayProbeResultSawGateway,
|
||||
gatewayProbeResultWasRateLimited,
|
||||
} from "../commands/gateway-health-auth-diagnostic.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import {
|
||||
getSystemdCgroupHygieneSummary,
|
||||
@@ -132,10 +136,22 @@ export async function collectGatewayHealthFindings(
|
||||
config: ctx.cfg,
|
||||
json: true,
|
||||
});
|
||||
const mode = ctx.cfg.gateway?.mode === "remote" ? "remote" : "local";
|
||||
if (gatewayProbeResultWasRateLimited(probe)) {
|
||||
return [
|
||||
{
|
||||
checkId: "core/doctor/gateway-health",
|
||||
severity: "warning",
|
||||
message: GATEWAY_HEALTH_RATE_LIMITED_MESSAGE,
|
||||
path: mode === "remote" ? "gateway.remote.url" : "gateway.mode",
|
||||
target: formatGatewayHealthTarget(probeDetails.url),
|
||||
fixHint: "Wait for the temporary authentication lockout to expire, then rerun doctor.",
|
||||
},
|
||||
];
|
||||
}
|
||||
if (gatewayProbeResultSawGateway(probe)) {
|
||||
return [];
|
||||
}
|
||||
const mode = ctx.cfg.gateway?.mode === "remote" ? "remote" : "local";
|
||||
return [
|
||||
{
|
||||
checkId: "core/doctor/gateway-health",
|
||||
|
||||
@@ -85,6 +85,7 @@ type StartMode =
|
||||
| "hello"
|
||||
| "close"
|
||||
| "connect-error"
|
||||
| "connect-error-close"
|
||||
| "silent"
|
||||
| "startup-retry-then-hello"
|
||||
| "clean-prehello-close-then-hello"
|
||||
@@ -144,6 +145,16 @@ function startStubGatewayClient() {
|
||||
lastClientOptions?.onConnectError?.(
|
||||
connectError ?? connectAssemblyErrorState.create("device private key invalid"),
|
||||
);
|
||||
} else if (startMode === "connect-error-close") {
|
||||
lastClientOptions?.onConnectError?.(
|
||||
connectError ?? connectAssemblyErrorState.create("device private key invalid"),
|
||||
);
|
||||
lastClientOptions?.onClose?.(closeCode, closeReason, {
|
||||
phase: "pre-hello",
|
||||
socketOpened: true,
|
||||
transportValidated: true,
|
||||
transientPreHelloCleanClose: false,
|
||||
});
|
||||
} else if (startMode === "close") {
|
||||
lastClientOptions?.onClose?.(closeCode, closeReason);
|
||||
}
|
||||
@@ -1587,6 +1598,100 @@ describe("callGateway error details", () => {
|
||||
expect(lastRequestOptions).toBeNull();
|
||||
});
|
||||
|
||||
it("preserves allowlisted rate-limit details before the following close", async () => {
|
||||
startMode = "connect-error-close";
|
||||
closeCode = 1008;
|
||||
closeReason = "unauthorized: too many failed authentication attempts (retry later)";
|
||||
connectError = Object.assign(
|
||||
new Error("unauthorized: too many failed authentication attempts (retry later)"),
|
||||
{
|
||||
name: "GatewayClientRequestError",
|
||||
gatewayCode: "INVALID_REQUEST",
|
||||
details: {
|
||||
code: "AUTH_RATE_LIMITED",
|
||||
authReason: "rate_limited",
|
||||
recommendedNextStep: "wait_then_retry",
|
||||
},
|
||||
retryable: true,
|
||||
retryAfterMs: 60_000,
|
||||
},
|
||||
);
|
||||
setLocalLoopbackGatewayConfig();
|
||||
|
||||
let error: unknown;
|
||||
await callGateway({ method: "health" }).catch((caught: unknown) => {
|
||||
error = caught;
|
||||
});
|
||||
|
||||
expect(error).toBe(connectError);
|
||||
expect(formatGatewayClientRequestErrorJson(error)).toEqual({
|
||||
ok: false,
|
||||
error: {
|
||||
type: "gateway_request_error",
|
||||
code: "INVALID_REQUEST",
|
||||
message: "unauthorized: too many failed authentication attempts (retry later)",
|
||||
details: {
|
||||
code: "AUTH_RATE_LIMITED",
|
||||
authReason: "rate_limited",
|
||||
recommendedNextStep: "wait_then_retry",
|
||||
},
|
||||
retryable: true,
|
||||
retryAfterMs: 60_000,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
name: "another structured auth rejection",
|
||||
error: Object.assign(new Error("unauthorized: gateway token mismatch"), {
|
||||
name: "GatewayClientRequestError",
|
||||
gatewayCode: "INVALID_REQUEST",
|
||||
details: { code: "AUTH_TOKEN_MISMATCH" },
|
||||
retryable: false,
|
||||
}),
|
||||
},
|
||||
{
|
||||
name: "rate-limit-looking text without structured details",
|
||||
error: Object.assign(
|
||||
new Error("unauthorized: too many failed authentication attempts (retry later)"),
|
||||
{
|
||||
name: "GatewayClientRequestError",
|
||||
gatewayCode: "INVALID_REQUEST",
|
||||
retryable: true,
|
||||
},
|
||||
),
|
||||
},
|
||||
{ name: "ordinary connect error", error: new Error("ordinary connect failure") },
|
||||
])("keeps $name on the existing transport-close path", async ({ error: connectFailure }) => {
|
||||
startMode = "connect-error-close";
|
||||
closeCode = 1008;
|
||||
closeReason = "connect failed";
|
||||
connectError = connectFailure;
|
||||
setLocalLoopbackGatewayConfig();
|
||||
|
||||
let error: unknown;
|
||||
await callGateway({ method: "health" }).catch((caught: unknown) => {
|
||||
error = caught;
|
||||
});
|
||||
|
||||
expect(formatGatewayTransportErrorJson(error)).toEqual({
|
||||
ok: false,
|
||||
error: {
|
||||
type: "gateway_transport_error",
|
||||
kind: "closed",
|
||||
message: "gateway closed (1008): connect failed",
|
||||
code: 1008,
|
||||
reason: "connect failed",
|
||||
},
|
||||
gateway: {
|
||||
url: "ws://127.0.0.1:18789",
|
||||
urlSource: "local loopback",
|
||||
bindDetail: "Bind: loopback",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("surfaces agent runtime identity connect request errors", async () => {
|
||||
startMode = "connect-error";
|
||||
connectError = new Error(
|
||||
|
||||
@@ -11,6 +11,10 @@ import {
|
||||
type GatewayClientMode,
|
||||
type GatewayClientName,
|
||||
} from "../../packages/gateway-protocol/src/client-info.js";
|
||||
import {
|
||||
ConnectErrorDetailCodes,
|
||||
readConnectErrorDetailCode,
|
||||
} from "../../packages/gateway-protocol/src/connect-error-details.js";
|
||||
import {
|
||||
MIN_CLIENT_PROTOCOL_VERSION,
|
||||
PROTOCOL_VERSION,
|
||||
@@ -940,6 +944,16 @@ function isRequiredAgentRuntimeIdentityConnectError(err: Error): boolean {
|
||||
);
|
||||
}
|
||||
|
||||
function isAllowlistedGatewayConnectRequestError(err: Error): boolean {
|
||||
if (err.name !== "GatewayClientRequestError") {
|
||||
return false;
|
||||
}
|
||||
return (
|
||||
readConnectErrorDetailCode((err as Error & { details?: unknown }).details) ===
|
||||
ConnectErrorDetailCodes.AUTH_RATE_LIMITED
|
||||
);
|
||||
}
|
||||
|
||||
async function executeGatewayRequestWithScopes<T>(params: {
|
||||
opts: CallGatewayBaseOptions;
|
||||
scopes: OperatorScope[] | undefined;
|
||||
@@ -1121,6 +1135,7 @@ async function executeGatewayRequestWithScopes<T>(params: {
|
||||
const shouldSurface =
|
||||
isGatewayConnectAssemblyError(err) ||
|
||||
isAgentRuntimeIdentityConnectError ||
|
||||
isAllowlistedGatewayConnectRequestError(err) ||
|
||||
(surfaceGatewayClientRequestErrors && isGatewayClientRequestError);
|
||||
if (settled || !shouldSurface) {
|
||||
return;
|
||||
|
||||
@@ -2076,6 +2076,51 @@ describe("GatewayClient connect auth payload", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("reports AUTH_RATE_LIMITED before pausing reconnect on the following close", async () => {
|
||||
const onConnectError = vi.fn();
|
||||
const onReconnectPaused = vi.fn();
|
||||
const client = new GatewayClient({
|
||||
url: "ws://127.0.0.1:18789",
|
||||
token: "shared-token",
|
||||
onConnectError,
|
||||
onReconnectPaused,
|
||||
});
|
||||
|
||||
const { ws: ws1, connect: firstConnect } = startClientAndConnect({ client });
|
||||
await expectNoReconnectAfterConnectFailure({
|
||||
client,
|
||||
firstWs: ws1,
|
||||
connectId: firstConnect.id,
|
||||
failureDetails: {
|
||||
code: "AUTH_RATE_LIMITED",
|
||||
authReason: "rate_limited",
|
||||
recommendedNextStep: "wait_then_retry",
|
||||
},
|
||||
failureMessage: "unauthorized: too many failed authentication attempts (retry later)",
|
||||
});
|
||||
|
||||
expect(onConnectError).toHaveBeenCalledOnce();
|
||||
expect(onConnectError.mock.calls[0]?.[0]).toMatchObject({
|
||||
name: "GatewayClientRequestError",
|
||||
details: {
|
||||
code: "AUTH_RATE_LIMITED",
|
||||
authReason: "rate_limited",
|
||||
recommendedNextStep: "wait_then_retry",
|
||||
},
|
||||
});
|
||||
expect(onReconnectPaused).toHaveBeenCalledWith({
|
||||
code: 1008,
|
||||
reason: "connect failed",
|
||||
detailCode: "AUTH_RATE_LIMITED",
|
||||
});
|
||||
expect(logDebugMock).toHaveBeenCalledWith(
|
||||
expect.stringContaining("gateway connect failed: GatewayClientRequestError"),
|
||||
);
|
||||
expect(logErrorMock).not.toHaveBeenCalledWith(
|
||||
expect.stringContaining("gateway connect failed: GatewayClientRequestError"),
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps reconnect paused callback errors inside close dispatch", async () => {
|
||||
const onReconnectPaused = vi.fn(() => {
|
||||
throw new Error("paused callback failed");
|
||||
|
||||
@@ -674,6 +674,7 @@ describe("probeGateway", () => {
|
||||
it("prefers the structured connect error over the generic close reason", async () => {
|
||||
gatewayClientState.startMode = "connect-error-close";
|
||||
gatewayClientState.socketOpened = true;
|
||||
gatewayClientState.close = { code: 1008, reason: "connect failed" };
|
||||
|
||||
const result = await runTokenLightweightProbe({
|
||||
timeoutMs: 5_000,
|
||||
@@ -682,11 +683,25 @@ describe("probeGateway", () => {
|
||||
expectProbeResultFields(result, {
|
||||
ok: false,
|
||||
error: "scope upgrade pending approval (requestId: req-123)",
|
||||
close: { code: 1008, reason: "pairing required" },
|
||||
close: { code: 1008, reason: "connect failed" },
|
||||
});
|
||||
expectProbeAuthFields(result, { capability: "pairing_pending" });
|
||||
expect(result.connectLatencyMs).not.toBeNull();
|
||||
});
|
||||
|
||||
it("keeps probe capability unknown for temporary authentication lockouts", async () => {
|
||||
gatewayClientState.startMode = "connect-error-close";
|
||||
gatewayClientState.connectError =
|
||||
"unauthorized: too many failed authentication attempts (retry later)";
|
||||
gatewayClientState.connectErrorDetails = { code: "AUTH_RATE_LIMITED" };
|
||||
gatewayClientState.close = { code: 1008, reason: "connect failed" };
|
||||
|
||||
const result = await runTokenLightweightProbe({ timeoutMs: 5_000 });
|
||||
|
||||
expectProbeResultFields(result, { ok: false });
|
||||
expectProbeAuthFields(result, { capability: "unknown" });
|
||||
});
|
||||
|
||||
it("keeps latency unknown when the opened transport fails validation", async () => {
|
||||
gatewayClientState.startMode = "connect-error-close";
|
||||
gatewayClientState.socketOpened = true;
|
||||
|
||||
+12
-9
@@ -6,6 +6,7 @@ import {
|
||||
GATEWAY_CLIENT_MODES,
|
||||
GATEWAY_CLIENT_NAMES,
|
||||
} from "../../packages/gateway-protocol/src/client-info.js";
|
||||
import { classifyGatewayConnectFailure } from "../../packages/gateway-protocol/src/connect-error-details.js";
|
||||
import {
|
||||
readMissingScopeError,
|
||||
type MissingScopeErrorDetails,
|
||||
@@ -69,7 +70,6 @@ type GatewayProbeDetailLevel = "none" | "presence" | "config" | "full";
|
||||
|
||||
const MIN_PROBE_TIMEOUT_MS = 250;
|
||||
export const MAX_TIMER_DELAY_MS = MAX_SAFE_TIMEOUT_DELAY_MS;
|
||||
const PAIRING_REQUIRED_PATTERN = /\bpairing required\b/i;
|
||||
const OPERATOR_READ_SCOPE = "operator.read";
|
||||
const OPERATOR_WRITE_SCOPE = "operator.write";
|
||||
const OPERATOR_ADMIN_SCOPE = "operator.admin";
|
||||
@@ -188,6 +188,7 @@ function resolveProbeAuthSummary(params: {
|
||||
role?: string | null;
|
||||
scopes?: string[];
|
||||
authMetadataPresent?: boolean;
|
||||
connectErrorDetails?: unknown;
|
||||
error?: string | null;
|
||||
close?: GatewayProbeClose | null;
|
||||
verifiedRead?: boolean;
|
||||
@@ -200,6 +201,7 @@ function resolveProbeAuthSummary(params: {
|
||||
capability: resolveGatewayProbeCapability({
|
||||
auth: { scopes },
|
||||
authMetadataPresent: params.authMetadataPresent,
|
||||
connectErrorDetails: params.connectErrorDetails,
|
||||
error: params.error,
|
||||
close: params.close,
|
||||
verifiedRead: params.verifiedRead,
|
||||
@@ -208,22 +210,22 @@ function resolveProbeAuthSummary(params: {
|
||||
};
|
||||
}
|
||||
|
||||
function isPairingPendingProbeFailure(params: {
|
||||
error?: string | null;
|
||||
close?: GatewayProbeClose | null;
|
||||
}): boolean {
|
||||
return PAIRING_REQUIRED_PATTERN.test(params.close?.reason ?? params.error ?? "");
|
||||
}
|
||||
|
||||
function resolveGatewayProbeCapability(params: {
|
||||
auth?: Pick<GatewayProbeAuthSummary, "scopes"> | null;
|
||||
authMetadataPresent?: boolean;
|
||||
connectErrorDetails?: unknown;
|
||||
error?: string | null;
|
||||
close?: GatewayProbeClose | null;
|
||||
verifiedRead?: boolean;
|
||||
connectLatencyMs?: number | null;
|
||||
}): GatewayProbeCapability {
|
||||
if (isPairingPendingProbeFailure(params)) {
|
||||
if (
|
||||
classifyGatewayConnectFailure({
|
||||
details: params.connectErrorDetails,
|
||||
reason: params.close?.reason,
|
||||
message: params.error,
|
||||
}).kind === "pairing-required"
|
||||
) {
|
||||
return "pairing_pending";
|
||||
}
|
||||
const scopes = Array.isArray(params.auth?.scopes) ? params.auth.scopes : [];
|
||||
@@ -383,6 +385,7 @@ export async function probeGateway(opts: {
|
||||
role: auth.role,
|
||||
scopes: auth.scopes,
|
||||
authMetadataPresent,
|
||||
connectErrorDetails,
|
||||
error: params.error,
|
||||
close,
|
||||
verifiedRead: params.verifiedRead,
|
||||
|
||||
+39
-13
@@ -348,30 +348,56 @@ describe("resolveInitialTuiAgentId", () => {
|
||||
|
||||
describe("resolveGatewayDisconnectState", () => {
|
||||
it("returns scope-upgrade recovery guidance when disconnect reason requires pairing", () => {
|
||||
const state = resolveGatewayDisconnectState("gateway closed (1008): pairing required");
|
||||
const state = resolveGatewayDisconnectState({
|
||||
reason: "gateway closed (1008): pairing required",
|
||||
});
|
||||
expect(state.connectionStatus).toContain("pairing required");
|
||||
expect(state.activityStatus).toBe("device approval needed: preview latest request");
|
||||
expect(state.pairingHint).toContain("openclaw devices approve --latest");
|
||||
expect(state.pairingHint).toContain("openclaw devices approve <requestId>");
|
||||
expect(state.pairingHint).toContain("--token");
|
||||
expect(state.remediation).toContain("openclaw devices approve --latest");
|
||||
expect(state.remediation).toContain("openclaw devices approve <requestId>");
|
||||
expect(state.remediation).toContain("--url");
|
||||
expect(state.remediation).toContain("--token/--password");
|
||||
// Must steer users to `devices`, not the unrelated chat-DM `pairing` command.
|
||||
expect(state.pairingHint).not.toContain("openclaw pairing");
|
||||
expect(state.remediation).not.toContain("openclaw pairing");
|
||||
});
|
||||
|
||||
it("returns the same guidance when the gateway reports a pending scope upgrade", () => {
|
||||
const state = resolveGatewayDisconnectState(
|
||||
"gateway closed (1008): scope upgrade pending approval",
|
||||
);
|
||||
it("uses structured pairing details before the generic close reason", () => {
|
||||
const state = resolveGatewayDisconnectState({
|
||||
details: { code: "PAIRING_REQUIRED", reason: "scope-upgrade" },
|
||||
reason: "connect failed",
|
||||
});
|
||||
expect(state.activityStatus).toBe("device approval needed: preview latest request");
|
||||
expect(state.pairingHint).toContain("openclaw devices approve --latest");
|
||||
expect(state.pairingHint).toContain("openclaw devices approve <requestId>");
|
||||
expect(state.connectionStatus).toContain("scope upgrade pending approval");
|
||||
expect(state.remediation).toContain("openclaw devices approve --latest");
|
||||
});
|
||||
|
||||
it("shows the device-token rotation command for structured token mismatch", () => {
|
||||
const state = resolveGatewayDisconnectState({
|
||||
details: { code: "AUTH_DEVICE_TOKEN_MISMATCH" },
|
||||
reason: "device token mismatch",
|
||||
});
|
||||
expect(state.activityStatus).toBe("gateway authentication needs attention");
|
||||
expect(state.remediation).toContain(
|
||||
"openclaw devices rotate --device <deviceId> --role operator",
|
||||
);
|
||||
});
|
||||
|
||||
it("shows wait-and-retry guidance for a temporary authentication lockout", () => {
|
||||
const state = resolveGatewayDisconnectState({
|
||||
details: { code: "AUTH_RATE_LIMITED" },
|
||||
reason: "unauthorized: too many failed authentication attempts (retry later)",
|
||||
});
|
||||
expect(state.activityStatus).toBe("gateway authentication temporarily rate-limited");
|
||||
expect(state.remediation).toContain("temporary authentication lockout");
|
||||
expect(state.remediation).not.toContain("gateway.remote.token");
|
||||
expect(state.remediation).not.toContain("devices rotate");
|
||||
});
|
||||
|
||||
it("falls back to idle for generic disconnect reasons", () => {
|
||||
const state = resolveGatewayDisconnectState("network timeout");
|
||||
const state = resolveGatewayDisconnectState({ reason: "network timeout" });
|
||||
expect(state.connectionStatus).toBe("gateway disconnected: network timeout");
|
||||
expect(state.activityStatus).toBe("idle");
|
||||
expect(state.pairingHint).toBeUndefined();
|
||||
expect(state.remediation).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
+32
-19
@@ -10,6 +10,7 @@ import {
|
||||
Text,
|
||||
TUI,
|
||||
} from "@earendil-works/pi-tui";
|
||||
import { classifyGatewayConnectFailure } from "../../packages/gateway-protocol/src/connect-error-details.js";
|
||||
import type { CommandEntry } from "../../packages/gateway-protocol/src/index.js";
|
||||
import { resolveAgentIdByWorkspacePath, resolveDefaultAgentId } from "../agents/agent-scope.js";
|
||||
import { normalizeThinkLevel } from "../auto-reply/thinking.shared.js";
|
||||
@@ -204,26 +205,37 @@ export function resolveInitialTuiAgentId(params: {
|
||||
return normalizeAgentId(params.fallbackAgentId);
|
||||
}
|
||||
|
||||
export function resolveGatewayDisconnectState(reason?: string): {
|
||||
export function resolveGatewayDisconnectState(
|
||||
input: {
|
||||
details?: unknown;
|
||||
reason?: string | null;
|
||||
} = {},
|
||||
): {
|
||||
connectionStatus: string;
|
||||
activityStatus: string;
|
||||
pairingHint?: string;
|
||||
remediation?: string;
|
||||
} {
|
||||
const reasonLabel = reason?.trim() ? reason.trim() : "closed";
|
||||
// Covers both "pairing required" and a pending "scope upgrade" for a paired device.
|
||||
if (/pairing required|scope upgrade/i.test(reasonLabel)) {
|
||||
const failure = classifyGatewayConnectFailure(input);
|
||||
const reasonLabel =
|
||||
failure.userMessage === "gateway unreachable" ? "closed" : failure.userMessage;
|
||||
if (failure.kind === "pairing-required") {
|
||||
return {
|
||||
connectionStatus: `gateway disconnected: ${reasonLabel}`,
|
||||
activityStatus: "device approval needed: preview latest request",
|
||||
pairingHint:
|
||||
"Device approval needed. Run `openclaw devices approve --latest` to preview the pending request, " +
|
||||
"then rerun the printed `openclaw devices approve <requestId>` command " +
|
||||
"(reuse `--token` or other auth flags if needed), then reconnect.",
|
||||
remediation: failure.remediation,
|
||||
};
|
||||
}
|
||||
if (failure.kind === "rate-limited") {
|
||||
return {
|
||||
connectionStatus: `gateway disconnected: ${reasonLabel}`,
|
||||
activityStatus: "gateway authentication temporarily rate-limited",
|
||||
remediation: failure.remediation,
|
||||
};
|
||||
}
|
||||
return {
|
||||
connectionStatus: `gateway disconnected: ${reasonLabel}`,
|
||||
activityStatus: "idle",
|
||||
activityStatus: failure.remediation ? "gateway authentication needs attention" : "idle",
|
||||
remediation: failure.remediation,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -593,7 +605,7 @@ export async function runTui(opts: RunTuiOptions): Promise<TuiResult> {
|
||||
const sessionIds = new Map<string, string>();
|
||||
let connectionGeneration = 0;
|
||||
let wasDisconnected = false;
|
||||
let pairingHintShown = false;
|
||||
let remediationShown = false;
|
||||
const localRunIds = createTuiRunIdTracker();
|
||||
const localBtwRunIds = createTuiRunIdTracker();
|
||||
|
||||
@@ -1567,7 +1579,7 @@ export async function runTui(opts: RunTuiOptions): Promise<TuiResult> {
|
||||
const ownsConnection = () =>
|
||||
connectedGeneration === connectionGeneration && state.isConnected && !exitRequested;
|
||||
state.isConnected = true;
|
||||
pairingHintShown = false;
|
||||
remediationShown = false;
|
||||
const reconnected = wasDisconnected;
|
||||
wasDisconnected = false;
|
||||
if (reconnected) {
|
||||
@@ -1679,7 +1691,7 @@ export async function runTui(opts: RunTuiOptions): Promise<TuiResult> {
|
||||
});
|
||||
};
|
||||
|
||||
const handleBackendDisconnected = (reason: string) => {
|
||||
const handleBackendDisconnected = (reason: string, details?: unknown) => {
|
||||
if (exitRequested) {
|
||||
return;
|
||||
}
|
||||
@@ -1699,20 +1711,21 @@ export async function runTui(opts: RunTuiOptions): Promise<TuiResult> {
|
||||
? {
|
||||
connectionStatus: `local runtime stopped${reason ? `: ${reason}` : ""}`,
|
||||
activityStatus: "idle",
|
||||
pairingHint: undefined,
|
||||
remediation: undefined,
|
||||
}
|
||||
: resolveGatewayDisconnectState(reason);
|
||||
: resolveGatewayDisconnectState({ reason, details });
|
||||
setConnectionStatus(disconnectState.connectionStatus, 5000);
|
||||
setActivityStatus(disconnectState.activityStatus);
|
||||
if (disconnectState.pairingHint && !pairingHintShown) {
|
||||
pairingHintShown = true;
|
||||
chatLog.addSystem(disconnectState.pairingHint);
|
||||
if (disconnectState.remediation && !remediationShown) {
|
||||
remediationShown = true;
|
||||
chatLog.addSystem(disconnectState.remediation);
|
||||
}
|
||||
updateFooter();
|
||||
tui.requestRender();
|
||||
};
|
||||
client.onConnectError = (error) => {
|
||||
handleBackendDisconnected(formatTuiErrorMessage(error));
|
||||
const details = "details" in error ? (error as { details?: unknown }).details : undefined;
|
||||
handleBackendDisconnected(formatTuiErrorMessage(error), details);
|
||||
};
|
||||
client.onDisconnected = handleBackendDisconnected;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user