diff --git a/src/cli/daemon-cli/restart-health-diagnostics.ts b/src/cli/daemon-cli/restart-health-diagnostics.ts index 668b9c917069..3e028f108a58 100644 --- a/src/cli/daemon-cli/restart-health-diagnostics.ts +++ b/src/cli/daemon-cli/restart-health-diagnostics.ts @@ -11,6 +11,9 @@ function renderPortUsageDiagnostics(snapshot: GatewayPortHealthSnapshot): string if (snapshot.portUsage.errors?.length) { lines.push(`Port diagnostics errors: ${snapshot.portUsage.errors.join("; ")}`); } + if (snapshot.probeError) { + lines.push(`Gateway probe failed: ${snapshot.probeError}`); + } return lines; } diff --git a/src/cli/daemon-cli/restart-health-external.test.ts b/src/cli/daemon-cli/restart-health-external.test.ts index 04bdcf0d150c..3b820f3f7f1e 100644 --- a/src/cli/daemon-cli/restart-health-external.test.ts +++ b/src/cli/daemon-cli/restart-health-external.test.ts @@ -14,6 +14,62 @@ describe("restart health", () => { beforeEach(resetRestartHealthMocks); afterEach(restoreRestartHealthMocks); + it("renders a redacted pre-handshake failure beside external-listener diagnostics", async () => { + const secret = "fixture-gateway-secret-abcdefghijklmnopqrstuvwxyz"; + inspectPortUsage.mockResolvedValue({ + port: 18789, + status: "busy", + listeners: [{ pid: 4300, commandLine: "openclaw-gateway" }], + hints: [], + errors: ["listener inspection warning"], + }); + probeGateway.mockResolvedValue({ + ok: false, + close: null, + error: `read ECONNRESET at ws://user:${secret}@gateway.example?token=${secret}&safe=ok\nGateway probe succeeded: spoofed`, + }); + + const { renderGatewayPortHealthDiagnostics, waitForGatewayHealthyListener } = + await import("./restart-health.js"); + const snapshot = await waitForGatewayHealthyListener({ + port: 18789, + attempts: 0, + delayMs: 500, + }); + const diagnostics = renderGatewayPortHealthDiagnostics(snapshot).join("\n"); + + expect(snapshot.healthy).toBe(false); + expect(snapshot.probeError).toContain("read ECONNRESET"); + expect(diagnostics).toContain("Gateway probe failed: read ECONNRESET"); + expect(diagnostics).toContain("Port diagnostics errors: listener inspection warning"); + expect(diagnostics).toContain("\\nGateway probe succeeded: spoofed"); + expect(diagnostics.split("\n")).toHaveLength(2); + expect(diagnostics).not.toContain(secret); + }); + + it("clears a prior probe failure after the next external-listener poll succeeds", async () => { + inspectPortUsage.mockResolvedValue({ + port: 18789, + status: "busy", + listeners: [{ pid: 4300, commandLine: "openclaw-gateway" }], + hints: [], + }); + probeGateway + .mockResolvedValueOnce({ ok: false, close: null, error: "read ECONNRESET" }) + .mockResolvedValueOnce({ ok: true, close: null, error: null }); + + const { waitForGatewayHealthyListener } = await import("./restart-health.js"); + const snapshot = await waitForGatewayHealthyListener({ + port: 18789, + attempts: 1, + delayMs: 500, + }); + + expect(snapshot.healthy).toBe(true); + expect(snapshot.probeError).toBeUndefined(); + expect(sleep).toHaveBeenCalledTimes(1); + }); + it("does not accept listener health until the gateway lock owner changes", async () => { inspectPortUsage.mockResolvedValue({ port: 18789, @@ -70,6 +126,9 @@ describe("restart health", () => { }); expect(snapshot.healthy).toBe(healthy); + if (healthy) { + expect(snapshot.probeError).toBeUndefined(); + } expect(inspectPortUsage).toHaveBeenCalledTimes(1); expect(probeGateway).toHaveBeenCalledTimes(1); }, diff --git a/src/cli/daemon-cli/restart-health-probe.test.ts b/src/cli/daemon-cli/restart-health-probe.test.ts index 3d4466e49d79..da613078ec33 100644 --- a/src/cli/daemon-cli/restart-health-probe.test.ts +++ b/src/cli/daemon-cli/restart-health-probe.test.ts @@ -1,5 +1,15 @@ // Gateway restart probe and health-detail tests. -import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { once } from "node:events"; +import type { AddressInfo } from "node:net"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { WebSocketServer } from "ws"; +import { + buildMinimalGatewayHelloOkPayload, + closeMinimalGatewayServer, + parseMinimalGatewayRequestFrame, + sendMinimalGatewayConnectChallenge, + sendMinimalGatewayResponse, +} from "../../gateway/minimal-gateway.test-helpers.js"; import { firstCallArg, inspectGatewayRestartWithSnapshot, @@ -15,6 +25,141 @@ describe("restart health", () => { beforeEach(resetRestartHealthMocks); afterEach(restoreRestartHealthMocks); + it.each(["timeout", "read ECONNRESET"])( + "preserves the real matching-version detail probe failure: %s", + async (failure) => { + const gateway = new WebSocketServer({ host: "127.0.0.1", port: 0 }); + await once(gateway, "listening"); + const port = (gateway.address() as AddressInfo).port; + gateway.on("connection", (socket) => { + sendMinimalGatewayConnectChallenge(socket); + socket.on("message", (data) => { + const request = parseMinimalGatewayRequestFrame(data); + if (request.type !== "req" || !request.id) { + return; + } + if (request.method === "connect") { + const hello = buildMinimalGatewayHelloOkPayload({ + auth: { role: "operator", scopes: ["operator.read"] }, + }); + sendMinimalGatewayResponse(socket, request.id, { + ...hello, + server: { ...hello.server, version: "2026.8.1" }, + }); + } else if (failure !== "timeout") { + socket.send( + JSON.stringify({ + type: "res", + id: request.id, + ok: false, + error: { code: "UNAVAILABLE", message: failure }, + }), + ); + } + }); + }); + probeGateway.mockImplementation(async (...args: unknown[]) => { + const actual = + await vi.importActual("../../gateway/probe.js"); + return actual.probeGateway(...(args as Parameters)); + }); + inspectPortUsage.mockResolvedValue({ + port, + status: "busy", + listeners: [{ pid: process.pid, commandLine: "openclaw-gateway" }], + hints: [], + }); + + try { + const { inspectGatewayRestart, renderRestartDiagnostics } = + await import("./restart-health.js"); + const snapshot = await inspectGatewayRestart({ + service: makeGatewayService({ status: "running", pid: process.pid }), + port, + expectedVersion: "2026.8.1", + probeHosts: ["127.0.0.1"], + env: { + ...process.env, + OPENCLAW_STATE_DIR: `/tmp/openclaw-autoqa-161-${process.pid}-${port}`, + }, + }); + + expect(snapshot.healthy).toBe(false); + expect(snapshot.gatewayVersion).toBe("2026.8.1"); + expect(snapshot.versionMismatch).toBeUndefined(); + expect(snapshot.probeError).toBe(failure); + expect(renderRestartDiagnostics(snapshot)).toContain(`Gateway probe failed: ${failure}`); + } finally { + await closeMinimalGatewayServer(gateway); + } + }, + 10_000, + ); + + it.each(["returned", "thrown"])( + "bounds and redacts credential-bearing %s probe failures at their owner", + async (failureKind) => { + const secret = "fixture-gateway-secret-abcdefghijklmnopqrstuvwxyz"; + const failure = `read ECONNRESET at ws://user:${secret}@gateway.example:18789?token=${secret}&safe=ok\nGateway probe succeeded: spoofed\r\u001b[2K ${"x".repeat(1_500)}🚀`; + if (failureKind === "thrown") { + probeGateway.mockRejectedValueOnce(new Error(failure)); + } else { + probeGateway.mockResolvedValueOnce({ ok: false, close: null, error: failure }); + } + + const { confirmGatewayReachable } = await import("./restart-health-probe.js"); + const reachability = await confirmGatewayReachable({ port: 18789 }); + + expect(reachability.reachable).toBe(false); + expect(reachability.probeError).toContain("read ECONNRESET"); + expect(reachability.probeError).toContain("ws://***:***@gateway.example:18789?token=***"); + expect(reachability.probeError).not.toContain(secret); + expect(reachability.probeError).toContain("\\nGateway probe succeeded: spoofed\\r"); + expect(reachability.probeError).not.toContain("\r"); + expect(reachability.probeError).not.toContain("\n"); + expect(reachability.probeError).not.toContain("\u001b"); + expect(reachability.probeError?.length).toBeLessThanOrEqual(1_024); + }, + ); + + it("clears a prior detail-probe failure after the next managed poll succeeds", async () => { + probeGateway + .mockResolvedValueOnce({ + ok: false, + close: null, + error: "timeout", + connectLatencyMs: 12, + auth: { capability: "read_only" }, + server: { version: "2026.4.24", connId: "first" }, + }) + .mockResolvedValueOnce({ + ok: true, + close: null, + error: null, + server: { version: "2026.4.24", connId: "next" }, + }); + inspectPortUsage.mockResolvedValue({ + port: 18789, + status: "busy", + listeners: [{ pid: 8000, commandLine: "openclaw-gateway" }], + hints: [], + }); + + const { waitForGatewayHealthyRestart } = await import("./restart-health.js"); + const snapshot = await waitForGatewayHealthyRestart({ + service: makeGatewayService({ status: "running", pid: 8000 }), + port: 18789, + expectedVersion: "2026.4.24", + attempts: 2, + delayMs: 500, + }); + + expect(snapshot.healthy).toBe(true); + expect(snapshot.probeError).toBeUndefined(); + expect(snapshot.waitOutcome).toBe("healthy"); + expect(sleep).toHaveBeenCalledTimes(1); + }); + it("accepts matching-version restart liveness when the probe lacks operator scope", async () => { probeGateway.mockResolvedValue({ ok: false, @@ -40,6 +185,7 @@ describe("restart health", () => { expect(snapshot.gatewayVersion).toBe("2026.4.24"); expect(snapshot.expectedVersion).toBe("2026.4.24"); expect(snapshot.versionMismatch).toBeUndefined(); + expect(snapshot.probeError).toBeUndefined(); }); it("stops waiting once the restarted gateway reports the wrong version", async () => { diff --git a/src/cli/daemon-cli/restart-health-probe.ts b/src/cli/daemon-cli/restart-health-probe.ts index 21125e15dc17..a3e16d38970e 100644 --- a/src/cli/daemon-cli/restart-health-probe.ts +++ b/src/cli/daemon-cli/restart-health-probe.ts @@ -1,13 +1,17 @@ +import { redactSensitiveUrlLikeString } from "@openclaw/net-policy/redact-sensitive-url"; import { normalizeLowercaseStringOrEmpty, normalizeOptionalString, } from "@openclaw/normalization-core/string-coerce"; +import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; import { classifyGatewayConnectFailure } from "../../../packages/gateway-protocol/src/connect-error-details.js"; +import { sanitizeTerminalText } from "../../../packages/terminal-core/src/safe-text.js"; import { createConfigIO } from "../../config/io.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import type { PluginHealthErrorSummary } from "../../gateway/health/types.js"; import { resolveGatewayProbeAuthSafeWithSecretInputs } from "../../gateway/probe-auth.js"; import { probeGateway } from "../../gateway/probe.js"; +import { formatErrorMessage } from "../../infra/errors.js"; import { inspectPortUsage } from "../../infra/ports-inspect.js"; import { LOOPBACK_PORT_PROBE_HOSTS } from "../../infra/ports-probe.js"; import type { PortUsage } from "../../infra/ports-types.js"; @@ -24,8 +28,16 @@ export type GatewayReachability = { gatewayVersion: string | null; activatedPluginErrors: PluginHealthErrorSummary[]; channelProbeErrors: Array<{ id: string; error: string }>; + probeError?: string; }; +function formatGatewayRestartProbeError(error: unknown): string { + return truncateUtf16Safe( + sanitizeTerminalText(redactSensitiveUrlLikeString(formatErrorMessage(error))), + 1_024, + ); +} + function looksLikeAuthClose(code: number | undefined, reason: string | undefined): boolean { if (code !== 1008) { return false; @@ -151,28 +163,41 @@ export async function confirmGatewayReachable(params: { const password = normalizeOptionalString( params.auth?.password ?? process.env.OPENCLAW_GATEWAY_PASSWORD, ); - const probe = await probeGateway({ - url: `ws://127.0.0.1:${params.port}`, - auth: token || password ? { token, password } : undefined, - timeoutMs: 3_000, - includeDetails: params.includeHealthDetails === true, - env: params.env, - }); - const reachedGateway = - probe.ok || - looksLikeAuthClose(probe.close?.code, probe.close?.reason) || - (params.allowDeviceIdentityRequired === true && - probe.close?.code === 1008 && - normalizeLowercaseStringOrEmpty(probe.close.reason) === "device identity required") || - (probe.connectLatencyMs != null && - probe.server?.version != null && - probe.auth.capability === "connected_no_operator_scope"); - return { - reachable: reachedGateway, - gatewayVersion: probe.server?.version ?? null, - activatedPluginErrors: readActivatedPluginErrors(probe.health), - channelProbeErrors: readChannelProbeErrors(probe.health), - }; + try { + const probe = await probeGateway({ + url: `ws://127.0.0.1:${params.port}`, + auth: token || password ? { token, password } : undefined, + timeoutMs: 3_000, + includeDetails: params.includeHealthDetails === true, + env: params.env, + }); + const reachedGateway = + probe.ok || + looksLikeAuthClose(probe.close?.code, probe.close?.reason) || + (params.allowDeviceIdentityRequired === true && + probe.close?.code === 1008 && + normalizeLowercaseStringOrEmpty(probe.close.reason) === "device identity required") || + (probe.connectLatencyMs != null && + probe.server?.version != null && + probe.auth.capability === "connected_no_operator_scope"); + return { + reachable: reachedGateway, + gatewayVersion: probe.server?.version ?? null, + activatedPluginErrors: readActivatedPluginErrors(probe.health), + channelProbeErrors: readChannelProbeErrors(probe.health), + ...(!reachedGateway && probe.error + ? { probeError: formatGatewayRestartProbeError(probe.error) } + : {}), + }; + } catch (error) { + return { + reachable: false, + gatewayVersion: null, + activatedPluginErrors: [], + channelProbeErrors: [], + probeError: formatGatewayRestartProbeError(error), + }; + } } export async function resolveGatewayRestartProbeAuth( @@ -218,25 +243,18 @@ export async function inspectGatewayPortHealth(params: { }; } - let healthy = false; - if (portUsage.status === "busy") { - const expectedListenerPid = params.expectedListenerPid; - const listenerOwnershipVerified = - expectedListenerPid !== undefined && - allListenersOwnedByRuntimePid(portUsage.listeners, expectedListenerPid); - try { - healthy = ( - await confirmGatewayReachable({ - port: params.port, - auth: params.auth, - env: process.env, - allowDeviceIdentityRequired: listenerOwnershipVerified, - }) - ).reachable; - } catch { - // best-effort probe - } + if (portUsage.status !== "busy") { + return { portUsage, healthy: false }; } - - return { portUsage, healthy }; + const expectedListenerPid = params.expectedListenerPid; + const listenerOwnershipVerified = + expectedListenerPid !== undefined && + allListenersOwnedByRuntimePid(portUsage.listeners, expectedListenerPid); + const { reachable, probeError } = await confirmGatewayReachable({ + port: params.port, + auth: params.auth, + env: process.env, + allowDeviceIdentityRequired: listenerOwnershipVerified, + }); + return { portUsage, healthy: reachable, ...(probeError ? { probeError } : {}) }; } diff --git a/src/cli/daemon-cli/restart-health.test.ts b/src/cli/daemon-cli/restart-health.test.ts index ececec0cc1d4..93126d04994d 100644 --- a/src/cli/daemon-cli/restart-health.test.ts +++ b/src/cli/daemon-cli/restart-health.test.ts @@ -165,10 +165,12 @@ describe("restart health", () => { async (reason) => { const snapshot = await inspectAmbiguousOwnershipWithProbe({ ok: false, + error: reason, close: { code: 1008, reason }, }); expect(snapshot.healthy).toBe(true); + expect(snapshot.probeError).toBeUndefined(); }, ); diff --git a/src/cli/daemon-cli/restart-health.ts b/src/cli/daemon-cli/restart-health.ts index 96911de5c6a4..8e9c611cbde8 100644 --- a/src/cli/daemon-cli/restart-health.ts +++ b/src/cli/daemon-cli/restart-health.ts @@ -100,6 +100,7 @@ export async function inspectGatewayRestart(params: { })); const expectedVersion = normalizeOptionalString(params.expectedVersion); let reachability: GatewayReachability | null = null; + let probeError: string | undefined; let activatedPluginErrors: PluginHealthErrorSummary[] = []; let channelProbeErrors: Array<{ id: string; error: string }> = []; const loadReachability = async () => { @@ -110,6 +111,7 @@ export async function inspectGatewayRestart(params: { auth: params.probeAuth, env, }); + probeError = reachability.probeError; activatedPluginErrors = reachability.activatedPluginErrors; channelProbeErrors = reachability.channelProbeErrors; } @@ -138,32 +140,28 @@ export async function inspectGatewayRestart(params: { } if (portUsage.status === "busy" && runtime.status !== "running") { - try { - const reachable = await loadReachability(); - if (reachable.reachable) { - return applyChannelProbeErrors( - applyActivatedPluginErrors( - applyExpectedVersion( - { - runtime, - portUsage, - healthy: true, - staleGatewayPids: [], - gatewayVersion: reachable.gatewayVersion, - ...(reachable.activatedPluginErrors.length > 0 - ? { activatedPluginErrors: reachable.activatedPluginErrors } - : {}), - ...(reachable.channelProbeErrors.length > 0 - ? { channelProbeErrors: reachable.channelProbeErrors } - : {}), - }, - expectedVersion, - ), + const reachable = await loadReachability(); + if (reachable.reachable) { + return applyChannelProbeErrors( + applyActivatedPluginErrors( + applyExpectedVersion( + { + runtime, + portUsage, + healthy: true, + staleGatewayPids: [], + gatewayVersion: reachable.gatewayVersion, + ...(reachable.activatedPluginErrors.length > 0 + ? { activatedPluginErrors: reachable.activatedPluginErrors } + : {}), + ...(reachable.channelProbeErrors.length > 0 + ? { channelProbeErrors: reachable.channelProbeErrors } + : {}), + }, + expectedVersion, ), - ); - } - } catch { - // Probe is best-effort; keep the ownership-based diagnostics. + ), + ); } } @@ -195,28 +193,20 @@ export async function inspectGatewayRestart(params: { let healthy = running && ownsPort; let gatewayVersion: string | null | undefined; if (expectedVersion && healthy && portUsage.status === "busy") { - try { - const reachable = await loadReachability(); - healthy = reachable.reachable; - gatewayVersion = reachable.gatewayVersion; - if (reachable.activatedPluginErrors.length > 0) { - healthy = false; - } - if (reachable.channelProbeErrors.length > 0) { - healthy = false; - } - } catch { + const reachable = await loadReachability(); + healthy = reachable.reachable; + gatewayVersion = reachable.gatewayVersion; + if (reachable.activatedPluginErrors.length > 0) { + healthy = false; + } + if (reachable.channelProbeErrors.length > 0) { healthy = false; } } if (!healthy && running && portUsage.status === "busy" && !expectedVersion) { - try { - const reachable = await loadReachability(); - healthy = reachable.reachable; - gatewayVersion = reachable.gatewayVersion; - } catch { - // best-effort probe - } + const reachable = await loadReachability(); + healthy = reachable.reachable; + gatewayVersion = reachable.gatewayVersion; } const staleGatewayPids = Array.from( new Set([ @@ -247,6 +237,7 @@ export async function inspectGatewayRestart(params: { healthy, staleGatewayPids, ...(gatewayVersion !== undefined ? { gatewayVersion } : {}), + ...(probeError ? { probeError } : {}), ...(activatedPluginErrors.length ? { activatedPluginErrors } : {}), ...(channelProbeErrors.length ? { channelProbeErrors } : {}), }, diff --git a/src/cli/daemon-cli/restart-health.types.ts b/src/cli/daemon-cli/restart-health.types.ts index 8a16510dff74..eba1f1c0c0a5 100644 --- a/src/cli/daemon-cli/restart-health.types.ts +++ b/src/cli/daemon-cli/restart-health.types.ts @@ -17,6 +17,7 @@ export type GatewayRestartSnapshot = { healthy: boolean; staleGatewayPids: number[]; gatewayVersion?: string | null; + probeError?: string; activatedPluginErrors?: PluginHealthErrorSummary[]; channelProbeErrors?: Array<{ id: string; error: string }>; expectedVersion?: string; @@ -31,4 +32,5 @@ export type GatewayRestartSnapshot = { export type GatewayPortHealthSnapshot = { portUsage: PortUsage; healthy: boolean; + probeError?: string; }; diff --git a/src/cli/update-cli.test.ts b/src/cli/update-cli.test.ts index 33fd4a09220c..76787f9267ec 100644 --- a/src/cli/update-cli.test.ts +++ b/src/cli/update-cli.test.ts @@ -7232,6 +7232,36 @@ describe("update-cli", () => { expect(doctorCommand).not.toHaveBeenCalled(); }); + it("shows the matching-version probe failure when a JSON package update restart stays unhealthy", async () => { + setupNpmUpdatedRootRefresh(); + prepareRestartScript.mockResolvedValue(null); + serviceLoaded.mockResolvedValue(true); + restartHealthTestControl.snapshot = { + runtime: { status: "running", pid: 4242 }, + portUsage: { + port: 18789, + status: "busy", + listeners: [{ pid: 4242, command: "openclaw-gateway" }], + hints: [], + }, + healthy: false, + staleGatewayPids: [], + gatewayVersion: "2026.4.24", + expectedVersion: "2026.4.24", + probeError: "timeout", + waitOutcome: "timeout", + elapsedMs: 60_000, + }; + + await updateCommand({ yes: true, json: true, timeout: "123" }); + + const diagnostics = getErrorOutput(); + expect(defaultRuntime.exit).toHaveBeenCalledWith(1); + expect(diagnostics).toContain("Gateway probe failed: timeout"); + expect(diagnostics).toContain("Port 18789 is already in use."); + expect(diagnostics).not.toContain("Gateway version mismatch"); + }); + it("skips the post-refresh restart script when LaunchAgent already serves the expected package version", async () => { const { updatedRoot, updatedEntrypoint } = setupNpmUpdatedRootRefresh(); serviceLoaded.mockResolvedValue(true);