fix(gateway): preserve restart health errors (#126645)

src/cli/daemon-cli/restart-health-diagnostics.ts

@@ -11,6 +11,9 @@ function renderPortUsageDiagnostics(snapshot: GatewayPortHealthSnapshot): string
   if (snapshot.portUsage.errors?.length) {
     lines.push(`Port diagnostics errors: ${snapshot.portUsage.errors.join("; ")}`);
   }
+  if (snapshot.probeError) {
+    lines.push(`Gateway probe failed: ${snapshot.probeError}`);
+  }
   return lines;
 }
 

src/cli/daemon-cli/restart-health-external.test.ts

@@ -14,6 +14,62 @@ describe("restart health", () => {
   beforeEach(resetRestartHealthMocks);
   afterEach(restoreRestartHealthMocks);
 
+  it("renders a redacted pre-handshake failure beside external-listener diagnostics", async () => {
+    const secret = "fixture-gateway-secret-abcdefghijklmnopqrstuvwxyz";
+    inspectPortUsage.mockResolvedValue({
+      port: 18789,
+      status: "busy",
+      listeners: [{ pid: 4300, commandLine: "openclaw-gateway" }],
+      hints: [],
+      errors: ["listener inspection warning"],
+    });
+    probeGateway.mockResolvedValue({
+      ok: false,
+      close: null,
+      error: `read ECONNRESET at ws://user:${secret}@gateway.example?token=${secret}&safe=ok\nGateway probe succeeded: spoofed`,
+    });
+
+    const { renderGatewayPortHealthDiagnostics, waitForGatewayHealthyListener } =
+      await import("./restart-health.js");
+    const snapshot = await waitForGatewayHealthyListener({
+      port: 18789,
+      attempts: 0,
+      delayMs: 500,
+    });
+    const diagnostics = renderGatewayPortHealthDiagnostics(snapshot).join("\n");
+
+    expect(snapshot.healthy).toBe(false);
+    expect(snapshot.probeError).toContain("read ECONNRESET");
+    expect(diagnostics).toContain("Gateway probe failed: read ECONNRESET");
+    expect(diagnostics).toContain("Port diagnostics errors: listener inspection warning");
+    expect(diagnostics).toContain("\\nGateway probe succeeded: spoofed");
+    expect(diagnostics.split("\n")).toHaveLength(2);
+    expect(diagnostics).not.toContain(secret);
+  });
+
+  it("clears a prior probe failure after the next external-listener poll succeeds", async () => {
+    inspectPortUsage.mockResolvedValue({
+      port: 18789,
+      status: "busy",
+      listeners: [{ pid: 4300, commandLine: "openclaw-gateway" }],
+      hints: [],
+    });
+    probeGateway
+      .mockResolvedValueOnce({ ok: false, close: null, error: "read ECONNRESET" })
+      .mockResolvedValueOnce({ ok: true, close: null, error: null });
+
+    const { waitForGatewayHealthyListener } = await import("./restart-health.js");
+    const snapshot = await waitForGatewayHealthyListener({
+      port: 18789,
+      attempts: 1,
+      delayMs: 500,
+    });
+
+    expect(snapshot.healthy).toBe(true);
+    expect(snapshot.probeError).toBeUndefined();
+    expect(sleep).toHaveBeenCalledTimes(1);
+  });
+
   it("does not accept listener health until the gateway lock owner changes", async () => {
     inspectPortUsage.mockResolvedValue({
       port: 18789,
@@ -70,6 +126,9 @@ describe("restart health", () => {
       });
 
       expect(snapshot.healthy).toBe(healthy);
+      if (healthy) {
+        expect(snapshot.probeError).toBeUndefined();
+      }
       expect(inspectPortUsage).toHaveBeenCalledTimes(1);
       expect(probeGateway).toHaveBeenCalledTimes(1);
     },

src/cli/daemon-cli/restart-health-probe.test.ts

@@ -1,5 +1,15 @@
 // Gateway restart probe and health-detail tests.
-import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { once } from "node:events";
+import type { AddressInfo } from "node:net";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { WebSocketServer } from "ws";
+import {
+  buildMinimalGatewayHelloOkPayload,
+  closeMinimalGatewayServer,
+  parseMinimalGatewayRequestFrame,
+  sendMinimalGatewayConnectChallenge,
+  sendMinimalGatewayResponse,
+} from "../../gateway/minimal-gateway.test-helpers.js";
 import {
   firstCallArg,
   inspectGatewayRestartWithSnapshot,
@@ -15,6 +25,141 @@ describe("restart health", () => {
   beforeEach(resetRestartHealthMocks);
   afterEach(restoreRestartHealthMocks);
 
+  it.each(["timeout", "read ECONNRESET"])(
+    "preserves the real matching-version detail probe failure: %s",
+    async (failure) => {
+      const gateway = new WebSocketServer({ host: "127.0.0.1", port: 0 });
+      await once(gateway, "listening");
+      const port = (gateway.address() as AddressInfo).port;
+      gateway.on("connection", (socket) => {
+        sendMinimalGatewayConnectChallenge(socket);
+        socket.on("message", (data) => {
+          const request = parseMinimalGatewayRequestFrame(data);
+          if (request.type !== "req" || !request.id) {
+            return;
+          }
+          if (request.method === "connect") {
+            const hello = buildMinimalGatewayHelloOkPayload({
+              auth: { role: "operator", scopes: ["operator.read"] },
+            });
+            sendMinimalGatewayResponse(socket, request.id, {
+              ...hello,
+              server: { ...hello.server, version: "2026.8.1" },
+            });
+          } else if (failure !== "timeout") {
+            socket.send(
+              JSON.stringify({
+                type: "res",
+                id: request.id,
+                ok: false,
+                error: { code: "UNAVAILABLE", message: failure },
+              }),
+            );
+          }
+        });
+      });
+      probeGateway.mockImplementation(async (...args: unknown[]) => {
+        const actual =
+          await vi.importActual<typeof import("../../gateway/probe.js")>("../../gateway/probe.js");
+        return actual.probeGateway(...(args as Parameters<typeof actual.probeGateway>));
+      });
+      inspectPortUsage.mockResolvedValue({
+        port,
+        status: "busy",
+        listeners: [{ pid: process.pid, commandLine: "openclaw-gateway" }],
+        hints: [],
+      });
+
+      try {
+        const { inspectGatewayRestart, renderRestartDiagnostics } =
+          await import("./restart-health.js");
+        const snapshot = await inspectGatewayRestart({
+          service: makeGatewayService({ status: "running", pid: process.pid }),
+          port,
+          expectedVersion: "2026.8.1",
+          probeHosts: ["127.0.0.1"],
+          env: {
+            ...process.env,
+            OPENCLAW_STATE_DIR: `/tmp/openclaw-autoqa-161-${process.pid}-${port}`,
+          },
+        });
+
+        expect(snapshot.healthy).toBe(false);
+        expect(snapshot.gatewayVersion).toBe("2026.8.1");
+        expect(snapshot.versionMismatch).toBeUndefined();
+        expect(snapshot.probeError).toBe(failure);
+        expect(renderRestartDiagnostics(snapshot)).toContain(`Gateway probe failed: ${failure}`);
+      } finally {
+        await closeMinimalGatewayServer(gateway);
+      }
+    },
+    10_000,
+  );
+
+  it.each(["returned", "thrown"])(
+    "bounds and redacts credential-bearing %s probe failures at their owner",
+    async (failureKind) => {
+      const secret = "fixture-gateway-secret-abcdefghijklmnopqrstuvwxyz";
+      const failure = `read ECONNRESET at ws://user:${secret}@gateway.example:18789?token=${secret}&safe=ok\nGateway probe succeeded: spoofed\r\u001b[2K ${"x".repeat(1_500)}🚀`;
+      if (failureKind === "thrown") {
+        probeGateway.mockRejectedValueOnce(new Error(failure));
+      } else {
+        probeGateway.mockResolvedValueOnce({ ok: false, close: null, error: failure });
+      }
+
+      const { confirmGatewayReachable } = await import("./restart-health-probe.js");
+      const reachability = await confirmGatewayReachable({ port: 18789 });
+
+      expect(reachability.reachable).toBe(false);
+      expect(reachability.probeError).toContain("read ECONNRESET");
+      expect(reachability.probeError).toContain("ws://***:***@gateway.example:18789?token=***");
+      expect(reachability.probeError).not.toContain(secret);
+      expect(reachability.probeError).toContain("\\nGateway probe succeeded: spoofed\\r");
+      expect(reachability.probeError).not.toContain("\r");
+      expect(reachability.probeError).not.toContain("\n");
+      expect(reachability.probeError).not.toContain("\u001b");
+      expect(reachability.probeError?.length).toBeLessThanOrEqual(1_024);
+    },
+  );
+
+  it("clears a prior detail-probe failure after the next managed poll succeeds", async () => {
+    probeGateway
+      .mockResolvedValueOnce({
+        ok: false,
+        close: null,
+        error: "timeout",
+        connectLatencyMs: 12,
+        auth: { capability: "read_only" },
+        server: { version: "2026.4.24", connId: "first" },
+      })
+      .mockResolvedValueOnce({
+        ok: true,
+        close: null,
+        error: null,
+        server: { version: "2026.4.24", connId: "next" },
+      });
+    inspectPortUsage.mockResolvedValue({
+      port: 18789,
+      status: "busy",
+      listeners: [{ pid: 8000, commandLine: "openclaw-gateway" }],
+      hints: [],
+    });
+
+    const { waitForGatewayHealthyRestart } = await import("./restart-health.js");
+    const snapshot = await waitForGatewayHealthyRestart({
+      service: makeGatewayService({ status: "running", pid: 8000 }),
+      port: 18789,
+      expectedVersion: "2026.4.24",
+      attempts: 2,
+      delayMs: 500,
+    });
+
+    expect(snapshot.healthy).toBe(true);
+    expect(snapshot.probeError).toBeUndefined();
+    expect(snapshot.waitOutcome).toBe("healthy");
+    expect(sleep).toHaveBeenCalledTimes(1);
+  });
+
   it("accepts matching-version restart liveness when the probe lacks operator scope", async () => {
     probeGateway.mockResolvedValue({
       ok: false,
@@ -40,6 +185,7 @@ describe("restart health", () => {
     expect(snapshot.gatewayVersion).toBe("2026.4.24");
     expect(snapshot.expectedVersion).toBe("2026.4.24");
     expect(snapshot.versionMismatch).toBeUndefined();
+    expect(snapshot.probeError).toBeUndefined();
   });
 
   it("stops waiting once the restarted gateway reports the wrong version", async () => {

src/cli/daemon-cli/restart-health-probe.ts

@@ -1,13 +1,17 @@
+import { redactSensitiveUrlLikeString } from "@openclaw/net-policy/redact-sensitive-url";
 import {
   normalizeLowercaseStringOrEmpty,
   normalizeOptionalString,
 } from "@openclaw/normalization-core/string-coerce";
+import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
 import { classifyGatewayConnectFailure } from "../../../packages/gateway-protocol/src/connect-error-details.js";
+import { sanitizeTerminalText } from "../../../packages/terminal-core/src/safe-text.js";
 import { createConfigIO } from "../../config/io.js";
 import type { OpenClawConfig } from "../../config/types.openclaw.js";
 import type { PluginHealthErrorSummary } from "../../gateway/health/types.js";
 import { resolveGatewayProbeAuthSafeWithSecretInputs } from "../../gateway/probe-auth.js";
 import { probeGateway } from "../../gateway/probe.js";
+import { formatErrorMessage } from "../../infra/errors.js";
 import { inspectPortUsage } from "../../infra/ports-inspect.js";
 import { LOOPBACK_PORT_PROBE_HOSTS } from "../../infra/ports-probe.js";
 import type { PortUsage } from "../../infra/ports-types.js";
@@ -24,8 +28,16 @@ export type GatewayReachability = {
   gatewayVersion: string | null;
   activatedPluginErrors: PluginHealthErrorSummary[];
   channelProbeErrors: Array<{ id: string; error: string }>;
+  probeError?: string;
 };
 
+function formatGatewayRestartProbeError(error: unknown): string {
+  return truncateUtf16Safe(
+    sanitizeTerminalText(redactSensitiveUrlLikeString(formatErrorMessage(error))),
+    1_024,
+  );
+}
+
 function looksLikeAuthClose(code: number | undefined, reason: string | undefined): boolean {
   if (code !== 1008) {
     return false;
@@ -151,28 +163,41 @@ export async function confirmGatewayReachable(params: {
   const password = normalizeOptionalString(
     params.auth?.password ?? process.env.OPENCLAW_GATEWAY_PASSWORD,
   );
-  const probe = await probeGateway({
-    url: `ws://127.0.0.1:${params.port}`,
-    auth: token || password ? { token, password } : undefined,
-    timeoutMs: 3_000,
-    includeDetails: params.includeHealthDetails === true,
-    env: params.env,
-  });
-  const reachedGateway =
-    probe.ok ||
-    looksLikeAuthClose(probe.close?.code, probe.close?.reason) ||
-    (params.allowDeviceIdentityRequired === true &&
-      probe.close?.code === 1008 &&
-      normalizeLowercaseStringOrEmpty(probe.close.reason) === "device identity required") ||
-    (probe.connectLatencyMs != null &&
-      probe.server?.version != null &&
-      probe.auth.capability === "connected_no_operator_scope");
-  return {
-    reachable: reachedGateway,
-    gatewayVersion: probe.server?.version ?? null,
-    activatedPluginErrors: readActivatedPluginErrors(probe.health),
-    channelProbeErrors: readChannelProbeErrors(probe.health),
-  };
+  try {
+    const probe = await probeGateway({
+      url: `ws://127.0.0.1:${params.port}`,
+      auth: token || password ? { token, password } : undefined,
+      timeoutMs: 3_000,
+      includeDetails: params.includeHealthDetails === true,
+      env: params.env,
+    });
+    const reachedGateway =
+      probe.ok ||
+      looksLikeAuthClose(probe.close?.code, probe.close?.reason) ||
+      (params.allowDeviceIdentityRequired === true &&
+        probe.close?.code === 1008 &&
+        normalizeLowercaseStringOrEmpty(probe.close.reason) === "device identity required") ||
+      (probe.connectLatencyMs != null &&
+        probe.server?.version != null &&
+        probe.auth.capability === "connected_no_operator_scope");
+    return {
+      reachable: reachedGateway,
+      gatewayVersion: probe.server?.version ?? null,
+      activatedPluginErrors: readActivatedPluginErrors(probe.health),
+      channelProbeErrors: readChannelProbeErrors(probe.health),
+      ...(!reachedGateway && probe.error
+        ? { probeError: formatGatewayRestartProbeError(probe.error) }
+        : {}),
+    };
+  } catch (error) {
+    return {
+      reachable: false,
+      gatewayVersion: null,
+      activatedPluginErrors: [],
+      channelProbeErrors: [],
+      probeError: formatGatewayRestartProbeError(error),
+    };
+  }
 }
 
 export async function resolveGatewayRestartProbeAuth(
@@ -218,25 +243,18 @@ export async function inspectGatewayPortHealth(params: {
     };
   }
 
-  let healthy = false;
-  if (portUsage.status === "busy") {
-    const expectedListenerPid = params.expectedListenerPid;
-    const listenerOwnershipVerified =
-      expectedListenerPid !== undefined &&
-      allListenersOwnedByRuntimePid(portUsage.listeners, expectedListenerPid);
-    try {
-      healthy = (
-        await confirmGatewayReachable({
-          port: params.port,
-          auth: params.auth,
-          env: process.env,
-          allowDeviceIdentityRequired: listenerOwnershipVerified,
-        })
-      ).reachable;
-    } catch {
-      // best-effort probe
-    }
+  if (portUsage.status !== "busy") {
+    return { portUsage, healthy: false };
   }
-
-  return { portUsage, healthy };
+  const expectedListenerPid = params.expectedListenerPid;
+  const listenerOwnershipVerified =
+    expectedListenerPid !== undefined &&
+    allListenersOwnedByRuntimePid(portUsage.listeners, expectedListenerPid);
+  const { reachable, probeError } = await confirmGatewayReachable({
+    port: params.port,
+    auth: params.auth,
+    env: process.env,
+    allowDeviceIdentityRequired: listenerOwnershipVerified,
+  });
+  return { portUsage, healthy: reachable, ...(probeError ? { probeError } : {}) };
 }

src/cli/daemon-cli/restart-health.test.ts

@@ -165,10 +165,12 @@ describe("restart health", () => {
     async (reason) => {
       const snapshot = await inspectAmbiguousOwnershipWithProbe({
         ok: false,
+        error: reason,
         close: { code: 1008, reason },
       });
 
       expect(snapshot.healthy).toBe(true);
+      expect(snapshot.probeError).toBeUndefined();
     },
   );
 

src/cli/daemon-cli/restart-health.ts

@@ -100,6 +100,7 @@ export async function inspectGatewayRestart(params: {
     }));
   const expectedVersion = normalizeOptionalString(params.expectedVersion);
   let reachability: GatewayReachability | null = null;
+  let probeError: string | undefined;
   let activatedPluginErrors: PluginHealthErrorSummary[] = [];
   let channelProbeErrors: Array<{ id: string; error: string }> = [];
   const loadReachability = async () => {
@@ -110,6 +111,7 @@ export async function inspectGatewayRestart(params: {
         auth: params.probeAuth,
         env,
       });
+      probeError = reachability.probeError;
       activatedPluginErrors = reachability.activatedPluginErrors;
       channelProbeErrors = reachability.channelProbeErrors;
     }
@@ -138,32 +140,28 @@ export async function inspectGatewayRestart(params: {
   }
 
   if (portUsage.status === "busy" && runtime.status !== "running") {
-    try {
-      const reachable = await loadReachability();
-      if (reachable.reachable) {
-        return applyChannelProbeErrors(
-          applyActivatedPluginErrors(
-            applyExpectedVersion(
-              {
-                runtime,
-                portUsage,
-                healthy: true,
-                staleGatewayPids: [],
-                gatewayVersion: reachable.gatewayVersion,
-                ...(reachable.activatedPluginErrors.length > 0
-                  ? { activatedPluginErrors: reachable.activatedPluginErrors }
-                  : {}),
-                ...(reachable.channelProbeErrors.length > 0
-                  ? { channelProbeErrors: reachable.channelProbeErrors }
-                  : {}),
-              },
-              expectedVersion,
-            ),
+    const reachable = await loadReachability();
+    if (reachable.reachable) {
+      return applyChannelProbeErrors(
+        applyActivatedPluginErrors(
+          applyExpectedVersion(
+            {
+              runtime,
+              portUsage,
+              healthy: true,
+              staleGatewayPids: [],
+              gatewayVersion: reachable.gatewayVersion,
+              ...(reachable.activatedPluginErrors.length > 0
+                ? { activatedPluginErrors: reachable.activatedPluginErrors }
+                : {}),
+              ...(reachable.channelProbeErrors.length > 0
+                ? { channelProbeErrors: reachable.channelProbeErrors }
+                : {}),
+            },
+            expectedVersion,
           ),
-        );
-      }
-    } catch {
-      // Probe is best-effort; keep the ownership-based diagnostics.
+        ),
+      );
     }
   }
 
@@ -195,28 +193,20 @@ export async function inspectGatewayRestart(params: {
   let healthy = running && ownsPort;
   let gatewayVersion: string | null | undefined;
   if (expectedVersion && healthy && portUsage.status === "busy") {
-    try {
-      const reachable = await loadReachability();
-      healthy = reachable.reachable;
-      gatewayVersion = reachable.gatewayVersion;
-      if (reachable.activatedPluginErrors.length > 0) {
-        healthy = false;
-      }
-      if (reachable.channelProbeErrors.length > 0) {
-        healthy = false;
-      }
-    } catch {
+    const reachable = await loadReachability();
+    healthy = reachable.reachable;
+    gatewayVersion = reachable.gatewayVersion;
+    if (reachable.activatedPluginErrors.length > 0) {
+      healthy = false;
+    }
+    if (reachable.channelProbeErrors.length > 0) {
       healthy = false;
     }
   }
   if (!healthy && running && portUsage.status === "busy" && !expectedVersion) {
-    try {
-      const reachable = await loadReachability();
-      healthy = reachable.reachable;
-      gatewayVersion = reachable.gatewayVersion;
-    } catch {
-      // best-effort probe
-    }
+    const reachable = await loadReachability();
+    healthy = reachable.reachable;
+    gatewayVersion = reachable.gatewayVersion;
   }
   const staleGatewayPids = Array.from(
     new Set([
@@ -247,6 +237,7 @@ export async function inspectGatewayRestart(params: {
           healthy,
           staleGatewayPids,
           ...(gatewayVersion !== undefined ? { gatewayVersion } : {}),
+          ...(probeError ? { probeError } : {}),
           ...(activatedPluginErrors.length ? { activatedPluginErrors } : {}),
           ...(channelProbeErrors.length ? { channelProbeErrors } : {}),
         },

src/cli/daemon-cli/restart-health.types.ts

@@ -17,6 +17,7 @@ export type GatewayRestartSnapshot = {
   healthy: boolean;
   staleGatewayPids: number[];
   gatewayVersion?: string | null;
+  probeError?: string;
   activatedPluginErrors?: PluginHealthErrorSummary[];
   channelProbeErrors?: Array<{ id: string; error: string }>;
   expectedVersion?: string;
@@ -31,4 +32,5 @@ export type GatewayRestartSnapshot = {
 export type GatewayPortHealthSnapshot = {
   portUsage: PortUsage;
   healthy: boolean;
+  probeError?: string;
 };

src/cli/update-cli.test.ts

@@ -7232,6 +7232,36 @@ describe("update-cli", () => {
     expect(doctorCommand).not.toHaveBeenCalled();
   });
 
+  it("shows the matching-version probe failure when a JSON package update restart stays unhealthy", async () => {
+    setupNpmUpdatedRootRefresh();
+    prepareRestartScript.mockResolvedValue(null);
+    serviceLoaded.mockResolvedValue(true);
+    restartHealthTestControl.snapshot = {
+      runtime: { status: "running", pid: 4242 },
+      portUsage: {
+        port: 18789,
+        status: "busy",
+        listeners: [{ pid: 4242, command: "openclaw-gateway" }],
+        hints: [],
+      },
+      healthy: false,
+      staleGatewayPids: [],
+      gatewayVersion: "2026.4.24",
+      expectedVersion: "2026.4.24",
+      probeError: "timeout",
+      waitOutcome: "timeout",
+      elapsedMs: 60_000,
+    };
+
+    await updateCommand({ yes: true, json: true, timeout: "123" });
+
+    const diagnostics = getErrorOutput();
+    expect(defaultRuntime.exit).toHaveBeenCalledWith(1);
+    expect(diagnostics).toContain("Gateway probe failed: timeout");
+    expect(diagnostics).toContain("Port 18789 is already in use.");
+    expect(diagnostics).not.toContain("Gateway version mismatch");
+  });
+
   it("skips the post-refresh restart script when LaunchAgent already serves the expected package version", async () => {
     const { updatedRoot, updatedEntrypoint } = setupNpmUpdatedRootRefresh();
     serviceLoaded.mockResolvedValue(true);