diff --git a/CHANGELOG.md b/CHANGELOG.md index 712367f01264..711820fd8f5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ Docs: https://docs.openclaw.ai ### Fixes +- **External supervisor restart health:** accept device-identity policy closes only when the replacement gateway lock and listener PID agree, preventing OCM-managed restarts from timing out after a successful handoff. Thanks @shakkernerd. - **ACPX cleanup process inspection:** bound host process-table reads so stalled `ps` calls cannot hang gateway startup or session cleanup while retaining fail-closed ownership checks. Thanks @Alix-007. - **Cron lifecycle conflict retries:** preserve execution-phase retry decisions across scheduled, manual, and startup-recovered runs so post-execution claim conflicts cannot replay completed messages or tools. Fixes #108428. Thanks @yetval. - **Discord gateway metadata deadline:** carry the existing lookup deadline through DNS and proxy preflight, request headers, and response bodies so stalled gateway startup aborts cleanly. (#104580) Thanks @hugenshen. diff --git a/src/cli/daemon-cli/restart-health.test.ts b/src/cli/daemon-cli/restart-health.test.ts index 7b3b3294e6e0..f546aaf9d314 100644 --- a/src/cli/daemon-cli/restart-health.test.ts +++ b/src/cli/daemon-cli/restart-health.test.ts @@ -1025,6 +1025,49 @@ describe("inspectGatewayRestart", () => { expect(sleep).toHaveBeenCalledTimes(1); }); + it.each([ + { listenerPid: 4300, healthy: true }, + { listenerPid: 4400, healthy: false }, + ])( + "accepts device identity policy close only for the verified replacement listener", + async ({ listenerPid, healthy }) => { + inspectPortUsage.mockResolvedValue({ + port: 18789, + status: "busy", + listeners: [{ pid: listenerPid, commandLine: "openclaw-gateway" }], + hints: [], + }); + probeGateway.mockResolvedValue({ + ok: false, + close: { code: 1008, reason: "device identity required" }, + }); + const previousLockIdentity = { + pid: 4200, + ownerId: "gateway-owner-old", + createdAt: "2026-07-16T12:00:00.000Z", + port: 18789, + }; + readActiveGatewayLockIdentity.mockResolvedValueOnce(previousLockIdentity).mockResolvedValue({ + ...previousLockIdentity, + pid: 4300, + ownerId: "gateway-owner-new", + createdAt: "2026-07-16T12:00:01.000Z", + }); + + const { waitForGatewayHealthyListener } = await import("./restart-health.js"); + const snapshot = await waitForGatewayHealthyListener({ + port: 18789, + previousLockIdentity, + attempts: 1, + delayMs: 500, + }); + + expect(snapshot.healthy).toBe(healthy); + expect(inspectPortUsage).toHaveBeenCalledTimes(1); + expect(probeGateway).toHaveBeenCalledTimes(1); + }, + ); + it("bounds replacement health after an indefinite previous-owner wait", async () => { inspectPortUsage.mockResolvedValue({ port: 18789, diff --git a/src/cli/daemon-cli/restart-health.ts b/src/cli/daemon-cli/restart-health.ts index a0704d07cf6b..6845694b8d33 100644 --- a/src/cli/daemon-cli/restart-health.ts +++ b/src/cli/daemon-cli/restart-health.ts @@ -247,6 +247,7 @@ async function confirmGatewayReachable(params: { includeHealthDetails?: boolean; auth?: GatewayRestartProbeAuth; env?: NodeJS.ProcessEnv; + allowDeviceIdentityRequired?: boolean; }): Promise { const token = normalizeOptionalString(params.auth?.token ?? process.env.OPENCLAW_GATEWAY_TOKEN); const password = normalizeOptionalString( @@ -262,6 +263,9 @@ async function confirmGatewayReachable(params: { const reachedGateway = probe.ok || looksLikeAuthClose(probe.close?.code, probe.close?.reason) || + (params.allowDeviceIdentityRequired === true && + probe.close?.code === 1008 && + normalizeLowercaseStringOrEmpty(probe.close.reason) === "device identity required") || (probe.connectLatencyMs != null && probe.server?.version != null && probe.auth.capability === "connected_no_operator_scope"); @@ -298,6 +302,7 @@ async function resolveGatewayRestartProbeAuth( async function inspectGatewayPortHealth(params: { port: number; auth?: GatewayRestartProbeAuth; + expectedListenerPid?: number; }): Promise { let portUsage: PortUsage; try { @@ -314,12 +319,23 @@ async function inspectGatewayPortHealth(params: { let healthy = false; if (portUsage.status === "busy") { + const expectedListenerPid = params.expectedListenerPid; + const listenerOwnershipVerified = + expectedListenerPid !== undefined && + portUsage.listeners.length > 0 && + portUsage.listeners.every((listener) => + listenerOwnedByRuntimePid({ + listener, + runtimePid: expectedListenerPid, + }), + ); try { healthy = ( await confirmGatewayReachable({ port: params.port, auth: params.auth, env: process.env, + allowDeviceIdentityRequired: listenerOwnershipVerified, }) ).reachable; } catch { @@ -662,6 +678,7 @@ export async function waitForGatewayHealthyListener(params: { }); let attempt = 0; + let expectedListenerPid: number | undefined; if (previousLockIdentity) { const replacement = await waitForGatewayLockReplacement({ previousLockIdentity, @@ -673,9 +690,11 @@ export async function waitForGatewayHealthyListener(params: { return snapshot; } attempt = replacement.attemptsUsed; + expectedListenerPid = replacement.lockIdentity.pid; snapshot = await inspectGatewayPortHealth({ port: params.port, auth: probeAuth, + expectedListenerPid, }); } @@ -688,6 +707,7 @@ export async function waitForGatewayHealthyListener(params: { snapshot = await inspectGatewayPortHealth({ port: params.port, auth: probeAuth, + expectedListenerPid, }); if (snapshot.healthy) { return snapshot; diff --git a/src/cli/daemon-cli/restart-lock-replacement.test.ts b/src/cli/daemon-cli/restart-lock-replacement.test.ts index 839e0e3324bf..a4b1f6cf2278 100644 --- a/src/cli/daemon-cli/restart-lock-replacement.test.ts +++ b/src/cli/daemon-cli/restart-lock-replacement.test.ts @@ -33,12 +33,15 @@ describe("waitForGatewayLockReplacement", () => { }); it("checks for a replacement after the final bounded delay", async () => { - readActiveGatewayLockIdentity.mockResolvedValueOnce(previousLockIdentity).mockResolvedValue({ + const replacementLockIdentity = { ...previousLockIdentity, pid: 4300, ownerId: "gateway-owner-new", createdAt: "2026-07-16T12:00:01.000Z", - }); + }; + readActiveGatewayLockIdentity + .mockResolvedValueOnce(previousLockIdentity) + .mockResolvedValue(replacementLockIdentity); const { waitForGatewayLockReplacement } = await import("./restart-lock-replacement.js"); await expect( @@ -48,7 +51,11 @@ describe("waitForGatewayLockReplacement", () => { delayMs: 500, waitIndefinitelyForPreviousOwner: false, }), - ).resolves.toStrictEqual({ status: "replacement", attemptsUsed: 1 }); + ).resolves.toStrictEqual({ + status: "replacement", + attemptsUsed: 1, + lockIdentity: replacementLockIdentity, + }); expect(readActiveGatewayLockIdentity).toHaveBeenCalledTimes(2); expect(sleep).toHaveBeenCalledTimes(1); }); @@ -72,15 +79,16 @@ describe("waitForGatewayLockReplacement", () => { }); it("does not treat a transient lock read failure as owner release", async () => { + const replacementLockIdentity = { + ...previousLockIdentity, + pid: 4300, + ownerId: "gateway-owner-new", + createdAt: "2026-07-16T12:00:01.000Z", + }; readActiveGatewayLockIdentity .mockRejectedValueOnce(new Error("transient lock read failure")) .mockResolvedValueOnce(previousLockIdentity) - .mockResolvedValue({ - ...previousLockIdentity, - pid: 4300, - ownerId: "gateway-owner-new", - createdAt: "2026-07-16T12:00:01.000Z", - }); + .mockResolvedValue(replacementLockIdentity); const { waitForGatewayLockReplacement } = await import("./restart-lock-replacement.js"); await expect( @@ -90,7 +98,11 @@ describe("waitForGatewayLockReplacement", () => { delayMs: 500, waitIndefinitelyForPreviousOwner: true, }), - ).resolves.toStrictEqual({ status: "replacement", attemptsUsed: 0 }); + ).resolves.toStrictEqual({ + status: "replacement", + attemptsUsed: 0, + lockIdentity: replacementLockIdentity, + }); expect(readActiveGatewayLockIdentity).toHaveBeenCalledTimes(3); expect(sleep).toHaveBeenCalledTimes(2); }); diff --git a/src/cli/daemon-cli/restart-lock-replacement.ts b/src/cli/daemon-cli/restart-lock-replacement.ts index e96bec054776..7a9eb3848449 100644 --- a/src/cli/daemon-cli/restart-lock-replacement.ts +++ b/src/cli/daemon-cli/restart-lock-replacement.ts @@ -6,7 +6,11 @@ import { import { sleep } from "../../utils.js"; type GatewayLockReplacementWaitResult = - | { status: "replacement"; attemptsUsed: number } + | { + status: "replacement"; + attemptsUsed: number; + lockIdentity: GatewayLockIdentity; + } | { status: "timeout" }; export async function waitForGatewayLockReplacement(params: { @@ -57,7 +61,7 @@ export async function waitForGatewayLockReplacement(params: { currentLockIdentity && !isSameGatewayLockIdentity(params.previousLockIdentity, currentLockIdentity) ) { - return { status: "replacement", attemptsUsed }; + return { status: "replacement", attemptsUsed, lockIdentity: currentLockIdentity }; } if (attemptsUsed >= params.attempts) {