fix(gateway): never leave the restart admission fence closed without a restart

A failed, refused, superseded, or thrown restart emission could leave the
reversible restart-signal admission fence closed forever: concurrent emitters
could overwrite the live rollback lease with a dead stand-in, the fenced body
had no try/finally, and the outer catch swallowed errors precisely because the
stuck fence made isGatewayRestartDraining() true. The gateway then rejected
every new task with GatewayDrainingError - silently - until an operator
restarted the process.

beginGatewayRestartSignalAdmission now returns null instead of stand-in
leases (single fence owner), emitPreparedGatewayRestart reopens the fence on
every non-delivery path via try/finally while preserving it whenever a queued
SIGUSR1 is unconsumed, refused-signal cleanup force-clears orphaned fences,
and admission close/reopen transitions are logged with their reason. The
self-contained SQLite restart-intent persistence moves to restart-intent.ts
to keep restart.ts within the LOC ratchet.

Fixes #107322
This commit is contained in:
Ayaan Zaidi
2026-07-14 14:59:48 +05:30
parent 26fe6e8ea3
commit 319a796079
14 changed files with 528 additions and 242 deletions
+1 -1
View File
@@ -7,7 +7,7 @@ import path from "node:path";
import { performance } from "node:perf_hooks";
import { pathToFileURL } from "node:url";
import { expectDefined } from "../packages/normalization-core/src/expect.js";
import { writeGatewayRestartIntentSync } from "../src/infra/restart.js";
import { writeGatewayRestartIntentSync } from "../src/infra/restart-intent.js";
import { parseStrictIntegerOption } from "./lib/dev-tooling-safety.ts";
import { delay, stopChild, type StopChildResult } from "./lib/gateway-bench-child.ts";
import {
+1 -1
View File
@@ -32,7 +32,7 @@ vi.mock("../../runtime.js", () => ({
defaultRuntime,
}));
vi.mock("../../infra/restart.js", () => ({
vi.mock("../../infra/restart-intent.js", () => ({
clearGatewayRestartIntentSync: () => clearGatewayRestartIntentSync(),
writeGatewayRestartIntentSync: (opts: unknown) => writeGatewayRestartIntentSync(opts),
}));
+1 -1
View File
@@ -21,7 +21,7 @@ import {
clearGatewayRestartIntentSync,
type GatewayRestartIntent,
writeGatewayRestartIntentSync,
} from "../../infra/restart.js";
} from "../../infra/restart-intent.js";
import { isWSL } from "../../infra/wsl.js";
import { defaultRuntime } from "../../runtime.js";
import { formatCliCommand } from "../command-format.js";
+4 -1
View File
@@ -19,7 +19,10 @@ import {
signalVerifiedGatewayPidSync,
} from "../../infra/gateway-processes.js";
import type { SafeGatewayRestartRequestResult } from "../../infra/restart-coordinator.js";
import { type GatewayRestartIntent, writeGatewayRestartIntentSync } from "../../infra/restart.js";
import {
type GatewayRestartIntent,
writeGatewayRestartIntentSync,
} from "../../infra/restart-intent.js";
import { defaultRuntime } from "../../runtime.js";
import { formatCliCommand } from "../command-format.js";
import { parseDurationMs } from "../parse-duration.js";
+4 -2
View File
@@ -14,9 +14,7 @@ export {
} from "../../infra/process-respawn.js";
export {
resolveGatewayRestartDeferralTimeoutMs,
consumeGatewayRestartIntentPayloadSync,
consumeGatewaySigusr1RestartIntent,
consumeGatewayRestartIntentSync,
consumeGatewaySigusr1RestartAuthorization,
isGatewaySigusr1RestartExternallyAllowed,
markGatewaySigusr1RestartHandled,
@@ -26,6 +24,10 @@ export {
rollbackGatewayRestartSignalAdmission,
scheduleGatewaySigusr1Restart,
} from "../../infra/restart.js";
export {
consumeGatewayRestartIntentPayloadSync,
consumeGatewayRestartIntentSync,
} from "../../infra/restart-intent.js";
export { writeGatewayRestartHandoffSync } from "../../infra/restart-handoff.js";
export { resetGatewaySuspendCoordinatorForLifecycleRestart } from "../../infra/gateway-suspend-coordinator.js";
export { rotateAgentEventLifecycleGeneration } from "../../infra/agent-events.js";
+5 -2
View File
@@ -129,10 +129,8 @@ vi.mock("../../infra/gateway-lock.js", () => ({
}));
vi.mock("../../infra/restart.js", () => ({
consumeGatewayRestartIntentPayloadSync: () => consumeGatewayRestartIntentPayloadSync(),
consumeGatewaySigusr1RestartIntent: () => consumeGatewaySigusr1RestartIntent(),
consumeGatewaySigusr1RestartAuthorization: () => consumeGatewaySigusr1RestartAuthorization(),
consumeGatewayRestartIntentSync: () => consumeGatewayRestartIntentSync(),
isGatewaySigusr1RestartExternallyAllowed: () => isGatewaySigusr1RestartExternallyAllowed(),
markGatewaySigusr1RestartHandled: () => markGatewaySigusr1RestartHandled(),
peekGatewaySigusr1RestartReason: () => peekGatewaySigusr1RestartReason(),
@@ -152,6 +150,11 @@ vi.mock("../../infra/restart.js", () => ({
scheduleGatewaySigusr1Restart(opts),
}));
vi.mock("../../infra/restart-intent.js", () => ({
consumeGatewayRestartIntentPayloadSync: () => consumeGatewayRestartIntentPayloadSync(),
consumeGatewayRestartIntentSync: () => consumeGatewayRestartIntentSync(),
}));
vi.mock("../../infra/gateway-suspend-coordinator.js", () => ({
resetGatewaySuspendCoordinatorForLifecycleRestart: () =>
resetGatewaySuspendCoordinatorForLifecycleRestart(),
+2 -1
View File
@@ -244,11 +244,12 @@ describe("CronService interval/cron jobs fire on time", () => {
});
const pendingSignal = beginGatewayRestartSignalAdmission();
expect(pendingSignal).not.toBeNull();
const finishedRun = finished.waitForOk(job.id);
await vi.advanceTimersByTimeAsync(10_005);
expect(enqueueSystemEvent).not.toHaveBeenCalled();
expect(pendingSignal.rollback()).toBe(true);
expect(pendingSignal?.rollback()).toBe(true);
await finishedRun;
expectMainSystemEvent(enqueueSystemEvent, "rollback-tick", job.id);
} finally {
+1 -1
View File
@@ -24,10 +24,10 @@ import { isTruthyEnvValue } from "../infra/env.js";
import { formatErrorMessage } from "../infra/errors.js";
import type { HeartbeatRunner } from "../infra/heartbeat-runner.js";
import { resetDirectoryCache } from "../infra/outbound/target-resolver.js";
import type { GatewayRestartIntent } from "../infra/restart-intent.js";
import {
deferGatewayRestartUntilIdle,
type GatewayRestartEmitter,
type GatewayRestartIntent,
type RestartDeferralHandle,
resolveGatewayRestartDeferralTimeoutMs,
setGatewaySigusr1RestartPolicy,
+125
View File
@@ -3,6 +3,7 @@ import { importFreshModule } from "openclaw/plugin-sdk/test-fixtures";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { clearRuntimeConfigSnapshot, setRuntimeConfigSnapshot } from "../config/config.js";
import {
beginGatewayRestartSignalAdmission,
isGatewayWorkAdmissionClosed,
resetGatewayWorkAdmission,
tryBeginGatewayRootWorkAdmission,
@@ -16,6 +17,7 @@ let isGatewaySigusr1RestartExternallyAllowed: RestartModule["isGatewaySigusr1Res
let markGatewaySigusr1RestartHandled: RestartModule["markGatewaySigusr1RestartHandled"];
let peekGatewaySigusr1RestartReason: RestartModule["peekGatewaySigusr1RestartReason"];
let requestGatewayRestartWithSignalAdmission: RestartModule["requestGatewayRestartWithSignalAdmission"];
let rollbackGatewayRestartSignalAdmission: RestartModule["rollbackGatewayRestartSignalAdmission"];
let scheduleGatewaySigusr1Restart: RestartModule["scheduleGatewaySigusr1Restart"];
let setGatewaySigusr1RestartPolicy: RestartModule["setGatewaySigusr1RestartPolicy"];
let setPreRestartDeferralCheck: RestartModule["setPreRestartDeferralCheck"];
@@ -106,6 +108,7 @@ describe("infra runtime", () => {
markGatewaySigusr1RestartHandled,
peekGatewaySigusr1RestartReason,
requestGatewayRestartWithSignalAdmission,
rollbackGatewayRestartSignalAdmission,
scheduleGatewaySigusr1Restart,
setGatewaySigusr1RestartPolicy,
setPreRestartDeferralCheck,
@@ -172,6 +175,128 @@ describe("infra runtime", () => {
}
});
it("reopens admission when refused-handler rollback finds no live emission lease", () => {
// Fence closed outside restart.ts ownership (lost/overwritten lease).
const orphanLease = beginGatewayRestartSignalAdmission();
expect(orphanLease).not.toBeNull();
expect(isGatewayWorkAdmissionClosed()).toBe(true);
// Run-loop refused path: mark handled / explicit rollback with no stored lease.
expect(rollbackGatewayRestartSignalAdmission()).toBe(true);
expect(isGatewayWorkAdmissionClosed()).toBe(false);
expect(orphanLease?.rollback()).toBe(false);
const root = tryBeginGatewayRootWorkAdmission();
expect(root).not.toBeNull();
root?.release();
});
it("does not leave admission closed when a deferred emission is cancelled mid-prepare", async () => {
let releasePrepare: (() => void) | undefined;
const prepareGate = new Promise<void>((resolve) => {
releasePrepare = resolve;
});
const handle = deferGatewayRestartUntilIdle({
getPendingCount: () => 0,
reason: "config.reload.cancelled",
emitHooks: {
beforeEmit: async () => {
await prepareGate;
},
},
});
await Promise.resolve();
expect(isGatewayWorkAdmissionClosed()).toBe(true);
handle.cancel();
releasePrepare?.();
await vi.advanceTimersByTimeAsync(0);
await Promise.resolve();
await Promise.resolve();
expect(isGatewayWorkAdmissionClosed()).toBe(false);
const root = tryBeginGatewayRootWorkAdmission();
expect(root).not.toBeNull();
root?.release();
});
it("keeps admission open when a deferred restart emission races config supersession", async () => {
let pending = 1;
let releasePrepare: (() => void) | undefined;
const prepareGate = new Promise<void>((resolve) => {
releasePrepare = resolve;
});
const handle = deferGatewayRestartUntilIdle({
getPendingCount: () => pending,
reason: "config.reload.superseded",
emitHooks: {
beforeEmit: async () => {
await prepareGate;
},
emitRestart: () => ({ status: "coalesced" as const }),
},
});
expect(isGatewayWorkAdmissionClosed()).toBe(false);
pending = 0;
await vi.advanceTimersByTimeAsync(500);
await Promise.resolve();
expect(isGatewayWorkAdmissionClosed()).toBe(true);
// Superseding reload cancels the in-flight emission before signal delivery.
handle.cancel();
releasePrepare?.();
await Promise.resolve();
await Promise.resolve();
expect(isGatewayWorkAdmissionClosed()).toBe(false);
});
it("keeps the signal fence closed when cancel races a concurrent emitted SIGUSR1", async () => {
let releasePrepare: (() => void) | undefined;
const prepareGate = new Promise<void>((resolve) => {
releasePrepare = resolve;
});
const handler = () => {};
process.on("SIGUSR1", handler);
try {
const handle = deferGatewayRestartUntilIdle({
getPendingCount: () => 0,
reason: "config.reload.shared-fence",
emitHooks: {
beforeEmit: async () => {
await prepareGate;
},
},
});
await Promise.resolve();
expect(isGatewayWorkAdmissionClosed()).toBe(true);
// Concurrent path reuses the deferred prepare lease and queues SIGUSR1.
expect(requestGatewayRestartWithSignalAdmission("concurrent.emit")).toEqual({
status: "emitted",
});
expect(isGatewayWorkAdmissionClosed()).toBe(true);
handle.cancel();
expect(isGatewayWorkAdmissionClosed()).toBe(true);
expect(tryBeginGatewayRootWorkAdmission()).toBeNull();
releasePrepare?.();
await Promise.resolve();
await Promise.resolve();
// In-flight signal still owns the fence until the handled path reopens it.
expect(isGatewayWorkAdmissionClosed()).toBe(true);
expect(tryBeginGatewayRootWorkAdmission()).toBeNull();
markGatewaySigusr1RestartHandled();
expect(isGatewayWorkAdmissionClosed()).toBe(false);
} finally {
process.removeListener("SIGUSR1", handler);
}
});
it("backs off before an emoji that crosses the restart reason limit", () => {
const restart = scheduleGatewaySigusr1Restart({
delayMs: 0,
+1 -1
View File
@@ -17,7 +17,7 @@ import {
consumeGatewayRestartIntentPayloadSync,
consumeGatewayRestartIntentSync,
writeGatewayRestartIntentSync,
} from "./restart.js";
} from "./restart-intent.js";
const tempDirs: string[] = [];
type GatewayRestartIntentDatabase = Pick<OpenClawStateKyselyDatabase, "gateway_restart_intent">;
+192
View File
@@ -0,0 +1,192 @@
// Persists short-lived gateway restart intent for supervisor SIGTERM handoff.
import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
import { createSubsystemLogger } from "../logging/subsystem.js";
import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js";
import {
openOpenClawStateDatabase,
runOpenClawStateWriteTransaction,
} from "../state/openclaw-state-db.js";
import {
executeSqliteQuerySync,
executeSqliteQueryTakeFirstSync,
getNodeSqliteKysely,
} from "./kysely-sync.js";
const GATEWAY_RESTART_INTENT_KEY = "gateway-restart";
const GATEWAY_RESTART_INTENT_TTL_MS = 60_000;
const restartLog = createSubsystemLogger("restart");
type GatewayRestartIntentDatabase = Pick<OpenClawStateKyselyDatabase, "gateway_restart_intent">;
type GatewayRestartIntentPayload = {
kind: "gateway-restart";
pid: number;
createdAt: number;
reason?: string;
force?: boolean;
waitMs?: number;
};
export type GatewayRestartIntent = {
reason?: string;
force?: boolean;
waitMs?: number;
};
function normalizeRestartIntentPid(pid: number | undefined): number | null {
return typeof pid === "number" && Number.isSafeInteger(pid) && pid > 0 ? pid : null;
}
export function normalizeRestartIntentReason(reason: string | undefined): string | undefined {
const normalized = reason?.trim();
return normalized ? truncateUtf16Safe(normalized, 200) : undefined;
}
export function writeGatewayRestartIntentSync(opts: {
env?: NodeJS.ProcessEnv;
targetPid?: number;
intent?: GatewayRestartIntent;
reason?: string;
}): boolean {
const targetPid = normalizeRestartIntentPid(opts.targetPid);
if (targetPid === null) {
return false;
}
const env = opts.env ?? process.env;
try {
const reason = normalizeRestartIntentReason(opts.reason ?? opts.intent?.reason);
const waitMs =
typeof opts.intent?.waitMs === "number" &&
Number.isFinite(opts.intent.waitMs) &&
opts.intent.waitMs >= 0
? Math.floor(opts.intent.waitMs)
: null;
const createdAt = Date.now();
runOpenClawStateWriteTransaction(
({ db }) => {
const stateDb = getNodeSqliteKysely<GatewayRestartIntentDatabase>(db);
executeSqliteQuerySync(
db,
stateDb
.insertInto("gateway_restart_intent")
.values({
intent_key: GATEWAY_RESTART_INTENT_KEY,
kind: "gateway-restart",
pid: targetPid,
created_at: createdAt,
reason: reason ?? null,
force: opts.intent?.force ? 1 : null,
wait_ms: waitMs,
updated_at_ms: createdAt,
})
.onConflict((conflict) =>
conflict.column("intent_key").doUpdateSet({
kind: (eb) => eb.ref("excluded.kind"),
pid: (eb) => eb.ref("excluded.pid"),
created_at: (eb) => eb.ref("excluded.created_at"),
reason: (eb) => eb.ref("excluded.reason"),
force: (eb) => eb.ref("excluded.force"),
wait_ms: (eb) => eb.ref("excluded.wait_ms"),
updated_at_ms: (eb) => eb.ref("excluded.updated_at_ms"),
}),
),
);
},
{ env },
);
return true;
} catch (err) {
restartLog.warn(`failed to write gateway restart intent: ${String(err)}`);
return false;
}
}
export function clearGatewayRestartIntentSync(env: NodeJS.ProcessEnv = process.env): void {
try {
runOpenClawStateWriteTransaction(
({ db }) => {
const stateDb = getNodeSqliteKysely<GatewayRestartIntentDatabase>(db);
executeSqliteQuerySync(
db,
stateDb
.deleteFrom("gateway_restart_intent")
.where("intent_key", "=", GATEWAY_RESTART_INTENT_KEY),
);
},
{ env },
);
} catch {}
}
function readGatewayRestartIntentPayloadSync(
env: NodeJS.ProcessEnv,
): GatewayRestartIntentPayload | null {
try {
const { db } = openOpenClawStateDatabase({ env });
const stateDb = getNodeSqliteKysely<GatewayRestartIntentDatabase>(db);
const parsed = executeSqliteQueryTakeFirstSync(
db,
stateDb
.selectFrom("gateway_restart_intent")
.select(["kind", "pid", "created_at", "reason", "force", "wait_ms"])
.where("intent_key", "=", GATEWAY_RESTART_INTENT_KEY),
);
if (
parsed?.kind === "gateway-restart" &&
typeof parsed.pid === "number" &&
Number.isFinite(parsed.pid) &&
typeof parsed.created_at === "number" &&
Number.isFinite(parsed.created_at) &&
(parsed.reason === null || typeof parsed.reason === "string") &&
(parsed.force === null ||
(typeof parsed.force === "number" && Number.isFinite(parsed.force))) &&
(parsed.wait_ms === null ||
(typeof parsed.wait_ms === "number" &&
Number.isFinite(parsed.wait_ms) &&
parsed.wait_ms >= 0))
) {
const reason = normalizeRestartIntentReason(parsed.reason ?? undefined);
return {
kind: "gateway-restart",
pid: parsed.pid,
createdAt: parsed.created_at,
...(reason ? { reason } : {}),
...(parsed.force ? { force: true } : {}),
...(typeof parsed.wait_ms === "number" ? { waitMs: Math.floor(parsed.wait_ms) } : {}),
};
}
} catch {
return null;
}
return null;
}
export function consumeGatewayRestartIntentPayloadSync(
env: NodeJS.ProcessEnv = process.env,
now = Date.now(),
): GatewayRestartIntent | null {
const payload = readGatewayRestartIntentPayloadSync(env);
clearGatewayRestartIntentSync(env);
if (!payload) {
return null;
}
if (payload.pid !== process.pid) {
return null;
}
const ageMs = now - payload.createdAt;
if (ageMs < 0 || ageMs > GATEWAY_RESTART_INTENT_TTL_MS) {
return null;
}
return {
...(payload.reason ? { reason: payload.reason } : {}),
...(payload.force ? { force: true } : {}),
...(typeof payload.waitMs === "number" ? { waitMs: payload.waitMs } : {}),
};
}
export function consumeGatewayRestartIntentSync(
env: NodeJS.ProcessEnv = process.env,
now = Date.now(),
): boolean {
return consumeGatewayRestartIntentPayloadSync(env, now) !== null;
}
+87 -221
View File
@@ -2,7 +2,6 @@
import { spawnSync } from "node:child_process";
import os from "node:os";
import path from "node:path";
import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
import { getRuntimeConfig } from "../config/config.js";
import {
resolveGatewayLaunchAgentLabel,
@@ -13,20 +12,12 @@ import {
beginGatewayRestartSignalAdmission,
getActiveGatewayRootWorkCount,
isGatewayRestartDraining,
rollbackGatewayRestartSignalFence,
runWithGatewayIndependentRootWorkAdmission,
type GatewayRestartSignalAdmissionLease,
} from "../process/gateway-work-admission.js";
import { resolveTimerTimeoutMs } from "../shared/number-coercion.js";
import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js";
import {
openOpenClawStateDatabase,
runOpenClawStateWriteTransaction,
} from "../state/openclaw-state-db.js";
import {
executeSqliteQuerySync,
executeSqliteQueryTakeFirstSync,
getNodeSqliteKysely,
} from "./kysely-sync.js";
import { type GatewayRestartIntent, normalizeRestartIntentReason } from "./restart-intent.js";
import { cleanStaleGatewayProcessesSync } from "./restart-stale-pids.js";
import type { RestartAttempt } from "./restart.types.js";
import { relaunchGatewayScheduledTask } from "./windows-task-restart.js";
@@ -38,11 +29,8 @@ const DEFAULT_DEFERRAL_STILL_PENDING_WARN_MS = 30_000;
const DEFAULT_RESTART_DEFERRAL_TIMEOUT_MS = 300_000;
const RESTART_COOLDOWN_MS = 30_000;
const LAUNCHCTL_ALREADY_LOADED_EXIT_CODE = 37;
const GATEWAY_RESTART_INTENT_KEY = "gateway-restart";
const GATEWAY_RESTART_INTENT_TTL_MS = 60_000;
const restartLog = createSubsystemLogger("restart");
type GatewayRestartIntentDatabase = Pick<OpenClawStateKyselyDatabase, "gateway_restart_intent">;
let sigusr1AuthorizedCount = 0;
let sigusr1AuthorizedUntil = 0;
@@ -88,9 +76,15 @@ function clearPendingScheduledRestart(): void {
}
function clearPendingRestartSignalAdmission(): boolean {
const rolledBack = pendingRestartSignalAdmission?.rollback() ?? false;
const lease = pendingRestartSignalAdmission;
pendingRestartSignalAdmission = null;
return rolledBack;
if (lease?.rollback()) {
return true;
}
// A concurrent emission must never replace a live lease with a dead handle.
// If that still happens, reopen the reversible fence directly so refused or
// abandoned signals cannot wedge process admission forever.
return rollbackGatewayRestartSignalFence();
}
/** Releases a signal fence when the run loop rejects or fails to handle the signal. */
@@ -170,179 +164,6 @@ type RestartAuditInfo = {
changedPaths?: string[];
};
type GatewayRestartIntentPayload = {
kind: "gateway-restart";
pid: number;
createdAt: number;
reason?: string;
force?: boolean;
waitMs?: number;
};
export type GatewayRestartIntent = {
reason?: string;
force?: boolean;
waitMs?: number;
};
function normalizeRestartIntentPid(pid: number | undefined): number | null {
return typeof pid === "number" && Number.isSafeInteger(pid) && pid > 0 ? pid : null;
}
export function writeGatewayRestartIntentSync(opts: {
env?: NodeJS.ProcessEnv;
targetPid?: number;
intent?: GatewayRestartIntent;
reason?: string;
}): boolean {
const targetPid = normalizeRestartIntentPid(opts.targetPid);
if (targetPid === null) {
return false;
}
const env = opts.env ?? process.env;
try {
const reason = normalizeRestartIntentReason(opts.reason ?? opts.intent?.reason);
const waitMs =
typeof opts.intent?.waitMs === "number" &&
Number.isFinite(opts.intent.waitMs) &&
opts.intent.waitMs >= 0
? Math.floor(opts.intent.waitMs)
: null;
const createdAt = Date.now();
runOpenClawStateWriteTransaction(
({ db }) => {
const stateDb = getNodeSqliteKysely<GatewayRestartIntentDatabase>(db);
executeSqliteQuerySync(
db,
stateDb
.insertInto("gateway_restart_intent")
.values({
intent_key: GATEWAY_RESTART_INTENT_KEY,
kind: "gateway-restart",
pid: targetPid,
created_at: createdAt,
reason: reason ?? null,
force: opts.intent?.force ? 1 : null,
wait_ms: waitMs,
updated_at_ms: createdAt,
})
.onConflict((conflict) =>
conflict.column("intent_key").doUpdateSet({
kind: (eb) => eb.ref("excluded.kind"),
pid: (eb) => eb.ref("excluded.pid"),
created_at: (eb) => eb.ref("excluded.created_at"),
reason: (eb) => eb.ref("excluded.reason"),
force: (eb) => eb.ref("excluded.force"),
wait_ms: (eb) => eb.ref("excluded.wait_ms"),
updated_at_ms: (eb) => eb.ref("excluded.updated_at_ms"),
}),
),
);
},
{ env },
);
return true;
} catch (err) {
restartLog.warn(`failed to write gateway restart intent: ${String(err)}`);
return false;
}
}
export function clearGatewayRestartIntentSync(env: NodeJS.ProcessEnv = process.env): void {
try {
runOpenClawStateWriteTransaction(
({ db }) => {
const stateDb = getNodeSqliteKysely<GatewayRestartIntentDatabase>(db);
executeSqliteQuerySync(
db,
stateDb
.deleteFrom("gateway_restart_intent")
.where("intent_key", "=", GATEWAY_RESTART_INTENT_KEY),
);
},
{ env },
);
} catch {}
}
function readGatewayRestartIntentPayloadSync(
env: NodeJS.ProcessEnv,
): GatewayRestartIntentPayload | null {
try {
const { db } = openOpenClawStateDatabase({ env });
const stateDb = getNodeSqliteKysely<GatewayRestartIntentDatabase>(db);
const parsed = executeSqliteQueryTakeFirstSync(
db,
stateDb
.selectFrom("gateway_restart_intent")
.select(["kind", "pid", "created_at", "reason", "force", "wait_ms"])
.where("intent_key", "=", GATEWAY_RESTART_INTENT_KEY),
);
if (
parsed?.kind === "gateway-restart" &&
typeof parsed.pid === "number" &&
Number.isFinite(parsed.pid) &&
typeof parsed.created_at === "number" &&
Number.isFinite(parsed.created_at) &&
(parsed.reason === null || typeof parsed.reason === "string") &&
(parsed.force === null ||
(typeof parsed.force === "number" && Number.isFinite(parsed.force))) &&
(parsed.wait_ms === null ||
(typeof parsed.wait_ms === "number" &&
Number.isFinite(parsed.wait_ms) &&
parsed.wait_ms >= 0))
) {
const reason = normalizeRestartIntentReason(parsed.reason ?? undefined);
return {
kind: "gateway-restart",
pid: parsed.pid,
createdAt: parsed.created_at,
...(reason ? { reason } : {}),
...(parsed.force ? { force: true } : {}),
...(typeof parsed.wait_ms === "number" ? { waitMs: Math.floor(parsed.wait_ms) } : {}),
};
}
} catch {
return null;
}
return null;
}
function normalizeRestartIntentReason(reason: string | undefined): string | undefined {
const normalized = reason?.trim();
return normalized ? truncateUtf16Safe(normalized, 200) : undefined;
}
export function consumeGatewayRestartIntentPayloadSync(
env: NodeJS.ProcessEnv = process.env,
now = Date.now(),
): GatewayRestartIntent | null {
const payload = readGatewayRestartIntentPayloadSync(env);
clearGatewayRestartIntentSync(env);
if (!payload) {
return null;
}
if (payload.pid !== process.pid) {
return null;
}
const ageMs = now - payload.createdAt;
if (ageMs < 0 || ageMs > GATEWAY_RESTART_INTENT_TTL_MS) {
return null;
}
return {
...(payload.reason ? { reason: payload.reason } : {}),
...(payload.force ? { force: true } : {}),
...(typeof payload.waitMs === "number" ? { waitMs: payload.waitMs } : {}),
};
}
export function consumeGatewayRestartIntentSync(
env: NodeJS.ProcessEnv = process.env,
now = Date.now(),
): boolean {
return consumeGatewayRestartIntentPayloadSync(env, now) !== null;
}
function summarizeChangedPaths(paths: string[] | undefined, maxPaths = 6): string | null {
if (!Array.isArray(paths) || paths.length === 0) {
return null;
@@ -444,8 +265,20 @@ function emitGatewayRestartWithSignalAdmission(
reasonOverride?: string,
intent?: GatewayRestartIntent,
): boolean {
const signalAdmission = pendingRestartSignalAdmission ?? beginGatewayRestartSignalAdmission();
pendingRestartSignalAdmission = signalAdmission;
let signalAdmission = pendingRestartSignalAdmission;
if (!signalAdmission) {
// Orphan fence: pending without a lease and without a delivered signal.
// Reopen before acquiring so a lost lease cannot block all future emissions.
if (!hasUnconsumedRestartSignal()) {
rollbackGatewayRestartSignalFence();
}
signalAdmission = beginGatewayRestartSignalAdmission();
if (!signalAdmission) {
// Another emission owns the fence, or one-way drain already closed admission.
return false;
}
pendingRestartSignalAdmission = signalAdmission;
}
const hadUnconsumedRestartSignal = hasUnconsumedRestartSignal();
const emitted = emitGatewayRestart(reasonOverride, intent);
if (!emitted && !hadUnconsumedRestartSignal) {
@@ -773,12 +606,39 @@ async function emitPreparedGatewayRestart(
if (transientGeneration !== restartTransientGeneration) {
return false;
}
// Close new roots before the final synchronous idle check. The independent
// emission owner is excluded; any other admitted root makes this attempt retry.
const signalAdmission = beginGatewayRestartSignalAdmission();
pendingRestartSignalAdmission = signalAdmission;
// SIGUSR1 already queued: coalesce. Run loop owns reopen-or-drain.
if (hasUnconsumedRestartSignal()) {
return false;
}
// Single live lease, multiple attempts may share it (deferred prepare →
// concurrent emit / retry). Never invent a dead stand-in lease.
let signalAdmission = pendingRestartSignalAdmission;
let ownsFenceLease = false;
if (!signalAdmission) {
// Orphan fence: pending without a lease and without a delivered signal.
rollbackGatewayRestartSignalFence();
signalAdmission = beginGatewayRestartSignalAdmission();
if (!signalAdmission) {
return false;
}
pendingRestartSignalAdmission = signalAdmission;
ownsFenceLease = true;
}
let fenceActive = true;
let keepFenceForRunLoop = false;
const rollbackFence = () => {
// A concurrent emitter may queue SIGUSR1 on this shared lease while we
// await beforeEmit. Cancel/finally must not reopen over an in-flight
// signal — the run loop owns reopen-or-drain from here.
if (keepFenceForRunLoop || hasUnconsumedRestartSignal()) {
return;
}
// Adopters share the lease with a still-active prepare/deferral owner.
// Only the creator may reopen on abandon; stop this attempt's canEmit.
if (!ownsFenceLease) {
fenceActive = false;
return;
}
fenceActive = false;
signalAdmission.rollback();
if (pendingRestartSignalAdmission === signalAdmission) {
@@ -786,37 +646,37 @@ async function emitPreparedGatewayRestart(
}
};
setFenceRollback?.(rollbackFence);
let isIdle: boolean;
try {
isIdle = finalIdleCheck
const isIdle = finalIdleCheck
? finalIdleCheck() && getActiveGatewayRootWorkCount({ excludeCurrent: true }) === 0
: true;
} catch (err) {
rollbackFence();
if (!isIdle) {
return false;
}
const emitResult = await emitPreparedGatewayRestartUnderAdmission(
hooks,
reasonOverride,
intent,
transientGeneration,
() => fenceActive,
);
if (
emitResult &&
(emitResult.status === "emitted" ||
(emitResult.status === "coalesced" && hasUnconsumedRestartSignal()))
) {
// Delivered or already-in-flight signal: run loop owns reopen-or-drain.
keepFenceForRunLoop = true;
return true;
}
return emitResult !== null;
} finally {
// Creator non-delivery reopens; adopters leave the live prepare lease.
if (!keepFenceForRunLoop) {
rollbackFence();
}
setFenceRollback?.(null);
throw err;
}
if (!isIdle) {
rollbackFence();
setFenceRollback?.(null);
return false;
}
const emitResult = await emitPreparedGatewayRestartUnderAdmission(
hooks,
reasonOverride,
intent,
transientGeneration,
() => fenceActive,
);
if (
!emitResult ||
emitResult.status === "failed" ||
(emitResult.status === "coalesced" && !hasUnconsumedRestartSignal())
) {
rollbackFence();
}
setFenceRollback?.(null);
return emitResult !== null;
});
} catch (err) {
if (!isGatewayRestartDraining()) {
@@ -887,6 +747,8 @@ export function deferGatewayRestartUntilIdle(opts: {
)
.then((attempted) => {
attemptingEmission = false;
// Successful delivery clears the cancel hook after the fence is owned by
// the run loop. Failed attempts already reopened via emitPrepared finally.
cancelEmissionFence = null;
if (cancelled || !attempted) {
return;
@@ -898,6 +760,10 @@ export function deferGatewayRestartUntilIdle(opts: {
})
.catch((err: unknown) => {
attemptingEmission = false;
// Invoke before clearing: a thrown emission must reopen the fence even
// when emitPreparedGatewayRestart's finally did not run (for example a
// rejection from the independent-root wrapper after cancel raced).
cancelEmissionFence?.();
cancelEmissionFence = null;
stopPoll();
opts.hooks?.onCheckError?.(err);
+41 -2
View File
@@ -2,6 +2,7 @@
import { afterEach, beforeEach, expect, it, vi } from "vitest";
import {
beginGatewayRestartSignalAdmission,
beginGatewayRootWorkAdmissionWhenOpen,
GatewayDrainingError,
getActiveGatewayRootWorkCount,
isGatewaySubordinateWorkAdmissionClosed,
@@ -9,6 +10,7 @@ import {
markGatewayRestartDraining,
retainGatewayRootWorkAdmissionContinuation,
resetGatewayWorkAdmission,
rollbackGatewayRestartSignalFence,
runWithGatewayIndependentRootWorkContinuation,
tryBeginGatewayRootWorkAdmission,
tryBeginGatewaySuspendAdmission,
@@ -176,25 +178,62 @@ it("does not let a stale suspension release clear restart drain", () => {
it("blocks suspension while restart signal handling is pending", () => {
const pendingSignal = beginGatewayRestartSignalAdmission();
expect(pendingSignal).not.toBeNull();
expect(isGatewayWorkAdmissionClosed()).toBe(true);
expect(tryBeginGatewayRootWorkAdmission()).toBeNull();
expect(tryBeginGatewaySuspendAdmission(() => {})).toBeNull();
expect(pendingSignal.rollback()).toBe(true);
expect(beginGatewayRestartSignalAdmission()).toBeNull();
expect(pendingSignal?.rollback()).toBe(true);
expect(isGatewayWorkAdmissionClosed()).toBe(false);
expect(tryBeginGatewaySuspendAdmission(() => {})?.rollback()).toBe(true);
});
it("promotes a pending restart signal to one-way drain", () => {
const pendingSignal = beginGatewayRestartSignalAdmission();
expect(pendingSignal).not.toBeNull();
markGatewayRestartDraining();
expect(pendingSignal.rollback()).toBe(false);
expect(pendingSignal?.rollback()).toBe(false);
expect(isGatewayWorkAdmissionClosed()).toBe(true);
expect(tryBeginGatewayRootWorkAdmission()).toBeNull();
});
it("force-rolls back an orphan restart-signal fence without a live lease", () => {
const pendingSignal = beginGatewayRestartSignalAdmission();
expect(pendingSignal).not.toBeNull();
expect(isGatewayWorkAdmissionClosed()).toBe(true);
// Drop the lease the way a concurrent emission overwrite used to: the fence
// stays closed with no handle that can reopen it.
expect(rollbackGatewayRestartSignalFence()).toBe(true);
expect(pendingSignal?.rollback()).toBe(false);
expect(isGatewayWorkAdmissionClosed()).toBe(false);
const root = tryBeginGatewayRootWorkAdmission();
expect(root).not.toBeNull();
root?.release();
});
it("wakes beginGatewayRootWorkAdmissionWhenOpen waiters when the signal fence rolls back", async () => {
const pendingSignal = beginGatewayRestartSignalAdmission();
expect(pendingSignal).not.toBeNull();
const waiting = beginGatewayRootWorkAdmissionWhenOpen();
let resolved = false;
void waiting.then(() => {
resolved = true;
});
await Promise.resolve();
expect(resolved).toBe(false);
expect(pendingSignal?.rollback()).toBe(true);
const admission = await waiting;
expect(resolved).toBe(true);
expect(admission.ownsRoot).toBe(true);
admission.release();
});
it("defers required internal root work until suspension reopens", async () => {
const suspension = tryBeginGatewaySuspendAdmission(() => {});
expect(suspension?.commit()).toBe(true);
+63 -8
View File
@@ -1,9 +1,13 @@
// Coordinates process-wide root work admission with reversible host suspension.
import { AsyncLocalStorage } from "node:async_hooks";
import { createSubsystemLogger } from "../logging/subsystem.js";
import { resolveGlobalSingleton } from "../shared/global-singleton.js";
type GatewaySuspendAdmissionPhase = "accepting" | "preparing" | "prepared";
type AdmissionCloseReason = "restart-signal fence" | "restart drain" | "suspend phase";
type AdmissionReopenReason = "restart-signal fence" | "suspend phase";
export class GatewayDrainingError extends Error {
constructor() {
super("Gateway is draining; new tasks are not accepted");
@@ -29,6 +33,8 @@ type GatewayWorkAdmissionState = {
suspendOpenWaiters: Set<() => void>;
};
const admissionLog = createSubsystemLogger("gateway/admission");
const GATEWAY_WORK_ADMISSION_STATE = resolveGlobalSingleton(
Symbol.for("openclaw.gatewayWorkAdmissionState"),
(): GatewayWorkAdmissionState => ({
@@ -44,6 +50,14 @@ const GATEWAY_WORK_ADMISSION_STATE = resolveGlobalSingleton(
}),
);
function logAdmissionClosed(reason: AdmissionCloseReason): void {
admissionLog.info(`admission closed: ${reason}`);
}
function logAdmissionReopened(reason: AdmissionReopenReason): void {
admissionLog.info(`admission reopened: ${reason}`);
}
type GatewayRootWorkAdmissionLease = {
ownsRoot: boolean;
release: () => void;
@@ -105,13 +119,32 @@ function resolveRootDrainWaiters(): void {
function invalidateSuspendAdmission(): void {
const callback = GATEWAY_WORK_ADMISSION_STATE.suspendInvalidated;
const wasClosed = GATEWAY_WORK_ADMISSION_STATE.suspendPhase !== "accepting";
GATEWAY_WORK_ADMISSION_STATE.suspendInvalidated = undefined;
GATEWAY_WORK_ADMISSION_STATE.suspendPhase = "accepting";
GATEWAY_WORK_ADMISSION_STATE.suspendGeneration += 1;
resolveSuspendOpenWaiters();
// Restart drain supersedes suspension without reopening process admission.
if (wasClosed && !GATEWAY_WORK_ADMISSION_STATE.restartDraining) {
logAdmissionReopened("suspend phase");
}
callback?.();
}
function clearRestartSignalFence(): boolean {
if (
GATEWAY_WORK_ADMISSION_STATE.restartDraining ||
!GATEWAY_WORK_ADMISSION_STATE.restartSignalPending
) {
return false;
}
GATEWAY_WORK_ADMISSION_STATE.restartSignalPending = false;
GATEWAY_WORK_ADMISSION_STATE.restartSignalGeneration += 1;
resolveSuspendOpenWaiters();
logAdmissionReopened("restart-signal fence");
return true;
}
function resolveSuspendOpenWaiters(): void {
const waiters = Array.from(GATEWAY_WORK_ADMISSION_STATE.suspendOpenWaiters);
GATEWAY_WORK_ADMISSION_STATE.suspendOpenWaiters.clear();
@@ -160,10 +193,16 @@ export function isGatewayRestartDraining(): boolean {
/** Restart drain is one-way until the in-process restart resets runtime state. */
export function markGatewayRestartDraining(): void {
if (GATEWAY_WORK_ADMISSION_STATE.restartDraining) {
return;
}
// Drain supersedes the reversible signal fence; do not reopen before the
// one-way close, or waiters could briefly admit work into a dying process.
GATEWAY_WORK_ADMISSION_STATE.restartSignalPending = false;
GATEWAY_WORK_ADMISSION_STATE.restartSignalGeneration += 1;
GATEWAY_WORK_ADMISSION_STATE.restartDraining = true;
resolveSuspendOpenWaiters();
logAdmissionClosed("restart drain");
if (GATEWAY_WORK_ADMISSION_STATE.suspendPhase !== "accepting") {
// A restart supersedes a reversible suspension. The coordinator callback
// drops its timer/token without reopening the scheduler being shut down.
@@ -171,13 +210,22 @@ export function markGatewayRestartDraining(): void {
}
}
/** Blocks suspension across signal emission until the run loop starts restart drain. */
export function beginGatewayRestartSignalAdmission(): GatewayRestartSignalAdmissionLease {
if (GATEWAY_WORK_ADMISSION_STATE.restartSignalPending) {
return { rollback: () => false };
/**
* Blocks suspension across signal emission until the run loop starts restart drain.
* Returns null when another owner already holds the fence or one-way drain is active.
* Callers must not invent a stand-in lease: a dead rollback handle is how the fence
* can stay closed after the real owner is lost.
*/
export function beginGatewayRestartSignalAdmission(): GatewayRestartSignalAdmissionLease | null {
if (
GATEWAY_WORK_ADMISSION_STATE.restartDraining ||
GATEWAY_WORK_ADMISSION_STATE.restartSignalPending
) {
return null;
}
GATEWAY_WORK_ADMISSION_STATE.restartSignalPending = true;
const generation = ++GATEWAY_WORK_ADMISSION_STATE.restartSignalGeneration;
logAdmissionClosed("restart-signal fence");
return {
rollback: () => {
if (
@@ -186,14 +234,19 @@ export function beginGatewayRestartSignalAdmission(): GatewayRestartSignalAdmiss
) {
return false;
}
GATEWAY_WORK_ADMISSION_STATE.restartSignalPending = false;
GATEWAY_WORK_ADMISSION_STATE.restartSignalGeneration += 1;
resolveSuspendOpenWaiters();
return true;
return clearRestartSignalFence();
},
};
}
/**
* Reopens a reversible restart-signal fence that no longer has a live lease.
* No-op while one-way restart drain owns admission.
*/
export function rollbackGatewayRestartSignalFence(): boolean {
return clearRestartSignalFence();
}
/** Root RPC/timer admission. Nested work in the same async chain counts once. */
export function tryBeginGatewayRootWorkAdmission(): GatewayRootWorkAdmissionLease | null {
const current = GATEWAY_WORK_ADMISSION_STATE.currentRootWork.getStore();
@@ -355,6 +408,7 @@ export function tryBeginGatewaySuspendAdmission(
GATEWAY_WORK_ADMISSION_STATE.suspendPhase = "preparing";
const generation = ++GATEWAY_WORK_ADMISSION_STATE.suspendGeneration;
GATEWAY_WORK_ADMISSION_STATE.suspendInvalidated = onInvalidated;
logAdmissionClosed("suspend phase");
const transition = (
expected: GatewaySuspendAdmissionPhase,
@@ -370,6 +424,7 @@ export function tryBeginGatewaySuspendAdmission(
if (next === "accepting") {
GATEWAY_WORK_ADMISSION_STATE.suspendInvalidated = undefined;
resolveSuspendOpenWaiters();
logAdmissionReopened("suspend phase");
}
return true;
};