fix: prevent external gateway restart timeouts (#109273)

This commit is contained in:
Shakker
2026-07-16 19:12:48 +01:00
committed by Shakker
parent 6bcf071120
commit f084ab2240
5 changed files with 92 additions and 12 deletions
+1
View File
@@ -42,6 +42,7 @@ Docs: https://docs.openclaw.ai
### Fixes
- **External supervisor restart health:** accept device-identity policy closes only when the replacement gateway lock and listener PID agree, preventing OCM-managed restarts from timing out after a successful handoff. Thanks @shakkernerd.
- **ACPX cleanup process inspection:** bound host process-table reads so stalled `ps` calls cannot hang gateway startup or session cleanup while retaining fail-closed ownership checks. Thanks @Alix-007.
- **Cron lifecycle conflict retries:** preserve execution-phase retry decisions across scheduled, manual, and startup-recovered runs so post-execution claim conflicts cannot replay completed messages or tools. Fixes #108428. Thanks @yetval.
- **Discord gateway metadata deadline:** carry the existing lookup deadline through DNS and proxy preflight, request headers, and response bodies so stalled gateway startup aborts cleanly. (#104580) Thanks @hugenshen.
+43
View File
@@ -1025,6 +1025,49 @@ describe("inspectGatewayRestart", () => {
expect(sleep).toHaveBeenCalledTimes(1);
});
it.each([
{ listenerPid: 4300, healthy: true },
{ listenerPid: 4400, healthy: false },
])(
"accepts device identity policy close only for the verified replacement listener",
async ({ listenerPid, healthy }) => {
inspectPortUsage.mockResolvedValue({
port: 18789,
status: "busy",
listeners: [{ pid: listenerPid, commandLine: "openclaw-gateway" }],
hints: [],
});
probeGateway.mockResolvedValue({
ok: false,
close: { code: 1008, reason: "device identity required" },
});
const previousLockIdentity = {
pid: 4200,
ownerId: "gateway-owner-old",
createdAt: "2026-07-16T12:00:00.000Z",
port: 18789,
};
readActiveGatewayLockIdentity.mockResolvedValueOnce(previousLockIdentity).mockResolvedValue({
...previousLockIdentity,
pid: 4300,
ownerId: "gateway-owner-new",
createdAt: "2026-07-16T12:00:01.000Z",
});
const { waitForGatewayHealthyListener } = await import("./restart-health.js");
const snapshot = await waitForGatewayHealthyListener({
port: 18789,
previousLockIdentity,
attempts: 1,
delayMs: 500,
});
expect(snapshot.healthy).toBe(healthy);
expect(inspectPortUsage).toHaveBeenCalledTimes(1);
expect(probeGateway).toHaveBeenCalledTimes(1);
},
);
it("bounds replacement health after an indefinite previous-owner wait", async () => {
inspectPortUsage.mockResolvedValue({
port: 18789,
+20
View File
@@ -247,6 +247,7 @@ async function confirmGatewayReachable(params: {
includeHealthDetails?: boolean;
auth?: GatewayRestartProbeAuth;
env?: NodeJS.ProcessEnv;
allowDeviceIdentityRequired?: boolean;
}): Promise<GatewayReachability> {
const token = normalizeOptionalString(params.auth?.token ?? process.env.OPENCLAW_GATEWAY_TOKEN);
const password = normalizeOptionalString(
@@ -262,6 +263,9 @@ async function confirmGatewayReachable(params: {
const reachedGateway =
probe.ok ||
looksLikeAuthClose(probe.close?.code, probe.close?.reason) ||
(params.allowDeviceIdentityRequired === true &&
probe.close?.code === 1008 &&
normalizeLowercaseStringOrEmpty(probe.close.reason) === "device identity required") ||
(probe.connectLatencyMs != null &&
probe.server?.version != null &&
probe.auth.capability === "connected_no_operator_scope");
@@ -298,6 +302,7 @@ async function resolveGatewayRestartProbeAuth(
async function inspectGatewayPortHealth(params: {
port: number;
auth?: GatewayRestartProbeAuth;
expectedListenerPid?: number;
}): Promise<GatewayPortHealthSnapshot> {
let portUsage: PortUsage;
try {
@@ -314,12 +319,23 @@ async function inspectGatewayPortHealth(params: {
let healthy = false;
if (portUsage.status === "busy") {
const expectedListenerPid = params.expectedListenerPid;
const listenerOwnershipVerified =
expectedListenerPid !== undefined &&
portUsage.listeners.length > 0 &&
portUsage.listeners.every((listener) =>
listenerOwnedByRuntimePid({
listener,
runtimePid: expectedListenerPid,
}),
);
try {
healthy = (
await confirmGatewayReachable({
port: params.port,
auth: params.auth,
env: process.env,
allowDeviceIdentityRequired: listenerOwnershipVerified,
})
).reachable;
} catch {
@@ -662,6 +678,7 @@ export async function waitForGatewayHealthyListener(params: {
});
let attempt = 0;
let expectedListenerPid: number | undefined;
if (previousLockIdentity) {
const replacement = await waitForGatewayLockReplacement({
previousLockIdentity,
@@ -673,9 +690,11 @@ export async function waitForGatewayHealthyListener(params: {
return snapshot;
}
attempt = replacement.attemptsUsed;
expectedListenerPid = replacement.lockIdentity.pid;
snapshot = await inspectGatewayPortHealth({
port: params.port,
auth: probeAuth,
expectedListenerPid,
});
}
@@ -688,6 +707,7 @@ export async function waitForGatewayHealthyListener(params: {
snapshot = await inspectGatewayPortHealth({
port: params.port,
auth: probeAuth,
expectedListenerPid,
});
if (snapshot.healthy) {
return snapshot;
@@ -33,12 +33,15 @@ describe("waitForGatewayLockReplacement", () => {
});
it("checks for a replacement after the final bounded delay", async () => {
readActiveGatewayLockIdentity.mockResolvedValueOnce(previousLockIdentity).mockResolvedValue({
const replacementLockIdentity = {
...previousLockIdentity,
pid: 4300,
ownerId: "gateway-owner-new",
createdAt: "2026-07-16T12:00:01.000Z",
});
};
readActiveGatewayLockIdentity
.mockResolvedValueOnce(previousLockIdentity)
.mockResolvedValue(replacementLockIdentity);
const { waitForGatewayLockReplacement } = await import("./restart-lock-replacement.js");
await expect(
@@ -48,7 +51,11 @@ describe("waitForGatewayLockReplacement", () => {
delayMs: 500,
waitIndefinitelyForPreviousOwner: false,
}),
).resolves.toStrictEqual({ status: "replacement", attemptsUsed: 1 });
).resolves.toStrictEqual({
status: "replacement",
attemptsUsed: 1,
lockIdentity: replacementLockIdentity,
});
expect(readActiveGatewayLockIdentity).toHaveBeenCalledTimes(2);
expect(sleep).toHaveBeenCalledTimes(1);
});
@@ -72,15 +79,16 @@ describe("waitForGatewayLockReplacement", () => {
});
it("does not treat a transient lock read failure as owner release", async () => {
const replacementLockIdentity = {
...previousLockIdentity,
pid: 4300,
ownerId: "gateway-owner-new",
createdAt: "2026-07-16T12:00:01.000Z",
};
readActiveGatewayLockIdentity
.mockRejectedValueOnce(new Error("transient lock read failure"))
.mockResolvedValueOnce(previousLockIdentity)
.mockResolvedValue({
...previousLockIdentity,
pid: 4300,
ownerId: "gateway-owner-new",
createdAt: "2026-07-16T12:00:01.000Z",
});
.mockResolvedValue(replacementLockIdentity);
const { waitForGatewayLockReplacement } = await import("./restart-lock-replacement.js");
await expect(
@@ -90,7 +98,11 @@ describe("waitForGatewayLockReplacement", () => {
delayMs: 500,
waitIndefinitelyForPreviousOwner: true,
}),
).resolves.toStrictEqual({ status: "replacement", attemptsUsed: 0 });
).resolves.toStrictEqual({
status: "replacement",
attemptsUsed: 0,
lockIdentity: replacementLockIdentity,
});
expect(readActiveGatewayLockIdentity).toHaveBeenCalledTimes(3);
expect(sleep).toHaveBeenCalledTimes(2);
});
@@ -6,7 +6,11 @@ import {
import { sleep } from "../../utils.js";
type GatewayLockReplacementWaitResult =
| { status: "replacement"; attemptsUsed: number }
| {
status: "replacement";
attemptsUsed: number;
lockIdentity: GatewayLockIdentity;
}
| { status: "timeout" };
export async function waitForGatewayLockReplacement(params: {
@@ -57,7 +61,7 @@ export async function waitForGatewayLockReplacement(params: {
currentLockIdentity &&
!isSameGatewayLockIdentity(params.previousLockIdentity, currentLockIdentity)
) {
return { status: "replacement", attemptsUsed };
return { status: "replacement", attemptsUsed, lockIdentity: currentLockIdentity };
}
if (attemptsUsed >= params.attempts) {