mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
fix: prevent external gateway restart timeouts (#109273)
This commit is contained in:
@@ -42,6 +42,7 @@ Docs: https://docs.openclaw.ai
|
||||
|
||||
### Fixes
|
||||
|
||||
- **External supervisor restart health:** accept device-identity policy closes only when the replacement gateway lock and listener PID agree, preventing OCM-managed restarts from timing out after a successful handoff. Thanks @shakkernerd.
|
||||
- **ACPX cleanup process inspection:** bound host process-table reads so stalled `ps` calls cannot hang gateway startup or session cleanup while retaining fail-closed ownership checks. Thanks @Alix-007.
|
||||
- **Cron lifecycle conflict retries:** preserve execution-phase retry decisions across scheduled, manual, and startup-recovered runs so post-execution claim conflicts cannot replay completed messages or tools. Fixes #108428. Thanks @yetval.
|
||||
- **Discord gateway metadata deadline:** carry the existing lookup deadline through DNS and proxy preflight, request headers, and response bodies so stalled gateway startup aborts cleanly. (#104580) Thanks @hugenshen.
|
||||
|
||||
@@ -1025,6 +1025,49 @@ describe("inspectGatewayRestart", () => {
|
||||
expect(sleep).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ listenerPid: 4300, healthy: true },
|
||||
{ listenerPid: 4400, healthy: false },
|
||||
])(
|
||||
"accepts device identity policy close only for the verified replacement listener",
|
||||
async ({ listenerPid, healthy }) => {
|
||||
inspectPortUsage.mockResolvedValue({
|
||||
port: 18789,
|
||||
status: "busy",
|
||||
listeners: [{ pid: listenerPid, commandLine: "openclaw-gateway" }],
|
||||
hints: [],
|
||||
});
|
||||
probeGateway.mockResolvedValue({
|
||||
ok: false,
|
||||
close: { code: 1008, reason: "device identity required" },
|
||||
});
|
||||
const previousLockIdentity = {
|
||||
pid: 4200,
|
||||
ownerId: "gateway-owner-old",
|
||||
createdAt: "2026-07-16T12:00:00.000Z",
|
||||
port: 18789,
|
||||
};
|
||||
readActiveGatewayLockIdentity.mockResolvedValueOnce(previousLockIdentity).mockResolvedValue({
|
||||
...previousLockIdentity,
|
||||
pid: 4300,
|
||||
ownerId: "gateway-owner-new",
|
||||
createdAt: "2026-07-16T12:00:01.000Z",
|
||||
});
|
||||
|
||||
const { waitForGatewayHealthyListener } = await import("./restart-health.js");
|
||||
const snapshot = await waitForGatewayHealthyListener({
|
||||
port: 18789,
|
||||
previousLockIdentity,
|
||||
attempts: 1,
|
||||
delayMs: 500,
|
||||
});
|
||||
|
||||
expect(snapshot.healthy).toBe(healthy);
|
||||
expect(inspectPortUsage).toHaveBeenCalledTimes(1);
|
||||
expect(probeGateway).toHaveBeenCalledTimes(1);
|
||||
},
|
||||
);
|
||||
|
||||
it("bounds replacement health after an indefinite previous-owner wait", async () => {
|
||||
inspectPortUsage.mockResolvedValue({
|
||||
port: 18789,
|
||||
|
||||
@@ -247,6 +247,7 @@ async function confirmGatewayReachable(params: {
|
||||
includeHealthDetails?: boolean;
|
||||
auth?: GatewayRestartProbeAuth;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
allowDeviceIdentityRequired?: boolean;
|
||||
}): Promise<GatewayReachability> {
|
||||
const token = normalizeOptionalString(params.auth?.token ?? process.env.OPENCLAW_GATEWAY_TOKEN);
|
||||
const password = normalizeOptionalString(
|
||||
@@ -262,6 +263,9 @@ async function confirmGatewayReachable(params: {
|
||||
const reachedGateway =
|
||||
probe.ok ||
|
||||
looksLikeAuthClose(probe.close?.code, probe.close?.reason) ||
|
||||
(params.allowDeviceIdentityRequired === true &&
|
||||
probe.close?.code === 1008 &&
|
||||
normalizeLowercaseStringOrEmpty(probe.close.reason) === "device identity required") ||
|
||||
(probe.connectLatencyMs != null &&
|
||||
probe.server?.version != null &&
|
||||
probe.auth.capability === "connected_no_operator_scope");
|
||||
@@ -298,6 +302,7 @@ async function resolveGatewayRestartProbeAuth(
|
||||
async function inspectGatewayPortHealth(params: {
|
||||
port: number;
|
||||
auth?: GatewayRestartProbeAuth;
|
||||
expectedListenerPid?: number;
|
||||
}): Promise<GatewayPortHealthSnapshot> {
|
||||
let portUsage: PortUsage;
|
||||
try {
|
||||
@@ -314,12 +319,23 @@ async function inspectGatewayPortHealth(params: {
|
||||
|
||||
let healthy = false;
|
||||
if (portUsage.status === "busy") {
|
||||
const expectedListenerPid = params.expectedListenerPid;
|
||||
const listenerOwnershipVerified =
|
||||
expectedListenerPid !== undefined &&
|
||||
portUsage.listeners.length > 0 &&
|
||||
portUsage.listeners.every((listener) =>
|
||||
listenerOwnedByRuntimePid({
|
||||
listener,
|
||||
runtimePid: expectedListenerPid,
|
||||
}),
|
||||
);
|
||||
try {
|
||||
healthy = (
|
||||
await confirmGatewayReachable({
|
||||
port: params.port,
|
||||
auth: params.auth,
|
||||
env: process.env,
|
||||
allowDeviceIdentityRequired: listenerOwnershipVerified,
|
||||
})
|
||||
).reachable;
|
||||
} catch {
|
||||
@@ -662,6 +678,7 @@ export async function waitForGatewayHealthyListener(params: {
|
||||
});
|
||||
|
||||
let attempt = 0;
|
||||
let expectedListenerPid: number | undefined;
|
||||
if (previousLockIdentity) {
|
||||
const replacement = await waitForGatewayLockReplacement({
|
||||
previousLockIdentity,
|
||||
@@ -673,9 +690,11 @@ export async function waitForGatewayHealthyListener(params: {
|
||||
return snapshot;
|
||||
}
|
||||
attempt = replacement.attemptsUsed;
|
||||
expectedListenerPid = replacement.lockIdentity.pid;
|
||||
snapshot = await inspectGatewayPortHealth({
|
||||
port: params.port,
|
||||
auth: probeAuth,
|
||||
expectedListenerPid,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -688,6 +707,7 @@ export async function waitForGatewayHealthyListener(params: {
|
||||
snapshot = await inspectGatewayPortHealth({
|
||||
port: params.port,
|
||||
auth: probeAuth,
|
||||
expectedListenerPid,
|
||||
});
|
||||
if (snapshot.healthy) {
|
||||
return snapshot;
|
||||
|
||||
@@ -33,12 +33,15 @@ describe("waitForGatewayLockReplacement", () => {
|
||||
});
|
||||
|
||||
it("checks for a replacement after the final bounded delay", async () => {
|
||||
readActiveGatewayLockIdentity.mockResolvedValueOnce(previousLockIdentity).mockResolvedValue({
|
||||
const replacementLockIdentity = {
|
||||
...previousLockIdentity,
|
||||
pid: 4300,
|
||||
ownerId: "gateway-owner-new",
|
||||
createdAt: "2026-07-16T12:00:01.000Z",
|
||||
});
|
||||
};
|
||||
readActiveGatewayLockIdentity
|
||||
.mockResolvedValueOnce(previousLockIdentity)
|
||||
.mockResolvedValue(replacementLockIdentity);
|
||||
|
||||
const { waitForGatewayLockReplacement } = await import("./restart-lock-replacement.js");
|
||||
await expect(
|
||||
@@ -48,7 +51,11 @@ describe("waitForGatewayLockReplacement", () => {
|
||||
delayMs: 500,
|
||||
waitIndefinitelyForPreviousOwner: false,
|
||||
}),
|
||||
).resolves.toStrictEqual({ status: "replacement", attemptsUsed: 1 });
|
||||
).resolves.toStrictEqual({
|
||||
status: "replacement",
|
||||
attemptsUsed: 1,
|
||||
lockIdentity: replacementLockIdentity,
|
||||
});
|
||||
expect(readActiveGatewayLockIdentity).toHaveBeenCalledTimes(2);
|
||||
expect(sleep).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
@@ -72,15 +79,16 @@ describe("waitForGatewayLockReplacement", () => {
|
||||
});
|
||||
|
||||
it("does not treat a transient lock read failure as owner release", async () => {
|
||||
const replacementLockIdentity = {
|
||||
...previousLockIdentity,
|
||||
pid: 4300,
|
||||
ownerId: "gateway-owner-new",
|
||||
createdAt: "2026-07-16T12:00:01.000Z",
|
||||
};
|
||||
readActiveGatewayLockIdentity
|
||||
.mockRejectedValueOnce(new Error("transient lock read failure"))
|
||||
.mockResolvedValueOnce(previousLockIdentity)
|
||||
.mockResolvedValue({
|
||||
...previousLockIdentity,
|
||||
pid: 4300,
|
||||
ownerId: "gateway-owner-new",
|
||||
createdAt: "2026-07-16T12:00:01.000Z",
|
||||
});
|
||||
.mockResolvedValue(replacementLockIdentity);
|
||||
|
||||
const { waitForGatewayLockReplacement } = await import("./restart-lock-replacement.js");
|
||||
await expect(
|
||||
@@ -90,7 +98,11 @@ describe("waitForGatewayLockReplacement", () => {
|
||||
delayMs: 500,
|
||||
waitIndefinitelyForPreviousOwner: true,
|
||||
}),
|
||||
).resolves.toStrictEqual({ status: "replacement", attemptsUsed: 0 });
|
||||
).resolves.toStrictEqual({
|
||||
status: "replacement",
|
||||
attemptsUsed: 0,
|
||||
lockIdentity: replacementLockIdentity,
|
||||
});
|
||||
expect(readActiveGatewayLockIdentity).toHaveBeenCalledTimes(3);
|
||||
expect(sleep).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
@@ -6,7 +6,11 @@ import {
|
||||
import { sleep } from "../../utils.js";
|
||||
|
||||
type GatewayLockReplacementWaitResult =
|
||||
| { status: "replacement"; attemptsUsed: number }
|
||||
| {
|
||||
status: "replacement";
|
||||
attemptsUsed: number;
|
||||
lockIdentity: GatewayLockIdentity;
|
||||
}
|
||||
| { status: "timeout" };
|
||||
|
||||
export async function waitForGatewayLockReplacement(params: {
|
||||
@@ -57,7 +61,7 @@ export async function waitForGatewayLockReplacement(params: {
|
||||
currentLockIdentity &&
|
||||
!isSameGatewayLockIdentity(params.previousLockIdentity, currentLockIdentity)
|
||||
) {
|
||||
return { status: "replacement", attemptsUsed };
|
||||
return { status: "replacement", attemptsUsed, lockIdentity: currentLockIdentity };
|
||||
}
|
||||
|
||||
if (attemptsUsed >= params.attempts) {
|
||||
|
||||
Reference in New Issue
Block a user