fix(update): keep managed handoff as successor owner (#128212)

* fix(update): make managed handoff own successor

Park the exact native service before Gateway close and coalesce same-root handoff requests so only the detached updater can activate and restart.

* refactor(update): reuse restart ownership helpers

* test(gateway): isolate supervisor restart fixtures

* fix(update): cancel handoff before restart fallback

* fix(update): harden managed service handoff

* fix(update): use canonical SQLite opener

* fix(update): preserve lifecycle runtime boundaries
This commit is contained in:
Peter Steinberger
2026-08-24 11:56:20 -07:00
committed by GitHub
parent b0e8a985ee
commit e1a700840a
27 changed files with 4701 additions and 2359 deletions
+6
View File
@@ -30,6 +30,12 @@ export {
consumeGatewayRestartIntentSync,
} from "../../infra/restart-intent.js";
export { writeGatewayRestartHandoffSync } from "../../infra/restart-handoff.js";
export {
cancelManagedServiceUpdateHandoff,
claimManagedServiceUpdateHandoff,
commitManagedServiceUpdateHandoff,
requestManagedServiceUpdateHandoffPark,
} from "../../infra/update-managed-service-handoff.js";
export { resetGatewaySuspendCoordinatorForLifecycleRestart } from "../../infra/gateway-suspend-coordinator.js";
export { rotateAgentEventLifecycleGeneration } from "../../infra/agent-events.js";
export { markUpdateRestartSentinelFailure } from "../../infra/restart-sentinel.js";
+266 -93
View File
@@ -1,13 +1,16 @@
// Gateway run loop tests cover foreground gateway lifecycle and restart behavior.
import { beforeEach, describe, expect, it, vi } from "vitest";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { GatewayServer } from "../../gateway/server-public.js";
import type { GatewayBonjourBeacon } from "../../infra/bonjour-discovery.js";
import type { GatewayActiveWorkSnapshot } from "../../infra/gateway-active-work.js";
import type { GatewayRestartIntent } from "../../infra/restart-intent.js";
import { SUPERVISOR_HINT_ENV_VARS } from "../../infra/supervisor-markers.js";
import { resolveGlobalMap } from "../../shared/global-singleton.js";
import {
GATEWAY_AGENT_MEDIA_MIGRATION_REQUIRED_REASON,
OpenClawAgentDatabaseMediaMigrationRequiredError,
} from "../../state/openclaw-agent-db-migration-required.js";
import { captureEnv, deleteTestEnvValue } from "../../test-utils/env.js";
import { pickBeaconHost, pickGatewayPort } from "./discover.js";
const acquireGatewayLock = vi.fn(async (_opts?: { port?: number }) => ({
@@ -16,9 +19,21 @@ const acquireGatewayLock = vi.fn(async (_opts?: { port?: number }) => ({
const consumeGatewayRestartIntentPayloadSync = vi.fn<
() => { reason?: string; force?: boolean; waitMs?: number } | null
>(() => null);
const consumeGatewaySigusr1RestartIntent = vi.fn<
() => { reason?: string; force?: boolean; waitMs?: number } | null
>(() => null);
const consumeGatewaySigusr1RestartIntent = vi.fn<() => GatewayRestartIntent | null>(() => null);
const managedUpdateSuccessorOwner = {
kind: "managed-update-handoff",
handoffId: "handoff-under-test",
installRoot: "/openclaw/install",
} as const;
type ManagedUpdateOwner = NonNullable<GatewayRestartIntent["successorOwner"]>;
const cancelManagedServiceUpdateHandoff = vi.fn<
(_identity: ManagedUpdateOwner) => Promise<false | "restored-in-process" | "restart-after-exit">
>(async () => "restored-in-process");
const claimManagedServiceUpdateHandoff = vi.fn((_identity: ManagedUpdateOwner) => true);
const requestManagedServiceUpdateHandoffPark = vi.fn(async (_identity: ManagedUpdateOwner) => true);
const commitManagedServiceUpdateHandoff = vi.fn(
async (_identity: ManagedUpdateOwner, _outcome?: "update" | "restore") => true,
);
const consumeGatewaySigusr1RestartAuthorization = vi.fn(() => true);
const consumeGatewayRestartIntentSync = vi.fn(() => false);
const isGatewaySigusr1RestartExternallyAllowed = vi.fn(() => false);
@@ -117,19 +132,17 @@ const reloadTaskRuntimeStateFromStore = vi.fn();
const clearRuntimeConfigSnapshot = vi.fn();
const restartGatewayProcessWithFreshPid = vi.fn<
(_opts?: { env?: NodeJS.ProcessEnv }) => {
mode: "spawned" | "supervised" | "disabled" | "failed";
pid?: number;
mode: "supervised" | "disabled" | "failed";
detail?: string;
handoffSpawned?: Promise<boolean>;
}
>(() => ({ mode: "disabled" }));
const respawnGatewayProcessForUpdate = vi.fn<
(_opts?: { env?: NodeJS.ProcessEnv }) => {
mode: "spawned" | "supervised" | "disabled" | "failed";
mode: "spawned" | "disabled" | "failed";
pid?: number;
detail?: string;
child?: { kill: () => void };
handoffSpawned?: Promise<boolean>;
}
>(() => ({ mode: "disabled", detail: "OPENCLAW_NO_RESPAWN" }));
const markUpdateRestartSentinelFailure = vi.fn<(reason: string) => Promise<null>>(
@@ -188,6 +201,19 @@ vi.mock("../../infra/restart-intent.js", () => ({
consumeGatewayRestartIntentSync: () => consumeGatewayRestartIntentSync(),
}));
vi.mock("../../infra/update-managed-service-handoff.js", () => ({
cancelManagedServiceUpdateHandoff: (identity: ManagedUpdateOwner) =>
cancelManagedServiceUpdateHandoff(identity),
claimManagedServiceUpdateHandoff: (identity: ManagedUpdateOwner) =>
claimManagedServiceUpdateHandoff(identity),
requestManagedServiceUpdateHandoffPark: (identity: ManagedUpdateOwner) =>
requestManagedServiceUpdateHandoffPark(identity),
commitManagedServiceUpdateHandoff: (
identity: ManagedUpdateOwner,
outcome?: "update" | "restore",
) => commitManagedServiceUpdateHandoff(identity, outcome),
}));
vi.mock("../../infra/gateway-suspend-coordinator.js", () => ({
resetGatewaySuspendCoordinatorForLifecycleRestart: () =>
resetGatewaySuspendCoordinatorForLifecycleRestart(),
@@ -473,8 +499,29 @@ function expectRestartHandoffCall(expected: {
}
let gatewayWorkAdmissionActual: typeof import("../../process/gateway-work-admission.js");
let supervisorEnvSnapshot: ReturnType<typeof captureEnv> | undefined;
beforeEach(async () => {
vi.useRealTimers();
supervisorEnvSnapshot = captureEnv([...SUPERVISOR_HINT_ENV_VARS]);
for (const key of SUPERVISOR_HINT_ENV_VARS) {
deleteTestEnvValue(key);
}
// clearAllMocks preserves queued one-shot results. A skipped lifecycle branch
// must not shift a stale supervisor or respawn decision into the next case.
consumeGatewaySigusr1RestartIntent.mockReset();
consumeGatewaySigusr1RestartIntent.mockReturnValue(null);
peekGatewaySigusr1RestartReason.mockReset();
peekGatewaySigusr1RestartReason.mockReturnValue(undefined);
restartGatewayProcessWithFreshPid.mockReset();
restartGatewayProcessWithFreshPid.mockReturnValue({ mode: "disabled" });
respawnGatewayProcessForUpdate.mockReset();
respawnGatewayProcessForUpdate.mockReturnValue({
mode: "disabled",
detail: "OPENCLAW_NO_RESPAWN",
});
gatewayWorkAdmissionActual = await vi.importActual("../../process/gateway-work-admission.js");
gatewayWorkAdmissionActual.resetGatewayWorkAdmission();
createGatewayActiveWorkSnapshot.mockReset();
@@ -485,6 +532,23 @@ beforeEach(async () => {
options?.onSnapshot?.(snapshot);
return { drained: snapshot.idle, snapshot };
});
cancelManagedServiceUpdateHandoff.mockReset();
cancelManagedServiceUpdateHandoff.mockResolvedValue("restored-in-process");
claimManagedServiceUpdateHandoff.mockReset();
claimManagedServiceUpdateHandoff.mockReturnValue(true);
requestManagedServiceUpdateHandoffPark.mockReset();
requestManagedServiceUpdateHandoffPark.mockResolvedValue(true);
commitManagedServiceUpdateHandoff.mockReset();
commitManagedServiceUpdateHandoff.mockResolvedValue(true);
});
afterEach(() => {
supervisorEnvSnapshot?.restore();
supervisorEnvSnapshot = undefined;
vi.useRealTimers();
if (originalPlatformDescriptor) {
Object.defineProperty(process, "platform", originalPlatformDescriptor);
}
});
describe("runGatewayLoop", () => {
@@ -1501,9 +1565,14 @@ describe("runGatewayLoop", () => {
await withIsolatedSignals(async ({ captureSignal }) => {
const close = vi.fn(async () => {});
const startupNeverReturns = new Promise<void>(() => {});
let markStartupEntered: () => void = () => {};
const startupEntered = new Promise<void>((resolve) => {
markStartupEntered = resolve;
});
const { runtime, exited } = createRuntimeWithExitSignal();
const completeBoot = vi.fn();
const start = vi.fn(async () => {
markStartupEntered();
await startupNeverReturns;
return createGatewayServer(close);
});
@@ -1515,6 +1584,7 @@ describe("runGatewayLoop", () => {
completeBoot,
});
await vi.advanceTimersByTimeAsync(0);
await startupEntered;
const sigusr1 = captureSignal("SIGUSR1");
sigusr1();
@@ -2033,11 +2103,12 @@ describe("runGatewayLoop", () => {
});
});
it("releases the lock before exiting on spawned restart", async () => {
it("releases the lock before exiting on supervised restart", async () => {
vi.clearAllMocks();
peekGatewaySigusr1RestartReason.mockReturnValue(undefined);
const originalTraceEnv = process.env.OPENCLAW_GATEWAY_RESTART_TRACE;
process.env.OPENCLAW_GATEWAY_RESTART_TRACE = "1";
process.env.OPENCLAW_SUPERVISOR_MODE = "external";
try {
await withIsolatedSignals(async ({ captureSignal }) => {
@@ -2046,11 +2117,7 @@ describe("runGatewayLoop", () => {
release: lockRelease,
});
// Override process-respawn to return "spawned" mode
restartGatewayProcessWithFreshPid.mockReturnValueOnce({
mode: "spawned",
pid: 9999,
});
restartGatewayProcessWithFreshPid.mockReturnValueOnce({ mode: "supervised" });
const exitCallOrder: string[] = [];
const { runtime, exited } = await createSignaledLoopHarness(exitCallOrder);
@@ -2068,9 +2135,10 @@ describe("runGatewayLoop", () => {
const [respawnOpts] = restartGatewayProcessWithFreshPid.mock.calls[0] ?? [];
expect(respawnOpts?.env?.OPENCLAW_GATEWAY_RESTART_TRACE_STARTED_AT_MS).toMatch(/^\d/u);
expect(respawnOpts?.env?.OPENCLAW_GATEWAY_RESTART_TRACE_LAST_AT_MS).toMatch(/^\d/u);
expect(writeGatewayRestartHandoffSync).not.toHaveBeenCalled();
expect(writeGatewayRestartHandoffSync).toHaveBeenCalledOnce();
});
} finally {
delete process.env.OPENCLAW_SUPERVISOR_MODE;
if (originalTraceEnv === undefined) {
delete process.env.OPENCLAW_GATEWAY_RESTART_TRACE;
} else {
@@ -2363,11 +2431,12 @@ describe("runGatewayLoop", () => {
async (reason) => {
vi.clearAllMocks();
peekGatewaySigusr1RestartReason.mockReturnValue(reason);
respawnGatewayProcessForUpdate.mockReturnValueOnce({
restartGatewayProcessWithFreshPid.mockReturnValueOnce({
mode: "supervised",
});
try {
setPlatform("freebsd");
process.env.OPENCLAW_SUPERVISOR_MODE = "external";
await withIsolatedSignals(async ({ captureSignal }) => {
const { runtime, exited } = await createSignaledLoopHarness();
const sigusr1 = captureSignal("SIGUSR1");
@@ -2381,8 +2450,10 @@ describe("runGatewayLoop", () => {
reason,
supervisorMode: "external",
});
expect(respawnGatewayProcessForUpdate).not.toHaveBeenCalled();
});
} finally {
delete process.env.OPENCLAW_SUPERVISOR_MODE;
if (originalPlatformDescriptor) {
Object.defineProperty(process, "platform", originalPlatformDescriptor);
}
@@ -2393,7 +2464,7 @@ describe("runGatewayLoop", () => {
it("falls back in-process when a launchd update handoff fails to spawn", async () => {
vi.clearAllMocks();
peekGatewaySigusr1RestartReason.mockReturnValue("update.run");
respawnGatewayProcessForUpdate.mockReturnValueOnce({
restartGatewayProcessWithFreshPid.mockReturnValueOnce({
mode: "supervised",
handoffSpawned: Promise.resolve(false),
});
@@ -2431,7 +2502,7 @@ describe("runGatewayLoop", () => {
vi.clearAllMocks();
peekGatewaySigusr1RestartReason.mockReturnValue("update.run");
process.env.OPENCLAW_SUPERVISOR_MODE = "external";
respawnGatewayProcessForUpdate.mockReturnValueOnce({
restartGatewayProcessWithFreshPid.mockReturnValueOnce({
mode: "supervised",
});
writeGatewayRestartHandoffSync.mockReturnValueOnce(null);
@@ -2461,19 +2532,12 @@ describe("runGatewayLoop", () => {
}
});
it("upgrades an accepted restart when a managed update arrives during shutdown", async () => {
it("upgrades an accepted restart when an update arrives during shutdown", async () => {
vi.clearAllMocks();
consumeGatewayRestartIntentPayloadSync.mockReset();
consumeGatewayRestartIntentPayloadSync.mockReturnValue(null);
consumeGatewaySigusr1RestartIntent.mockReset();
consumeGatewaySigusr1RestartIntent.mockReturnValue(null);
consumeGatewaySigusr1RestartAuthorization.mockReset();
consumeGatewaySigusr1RestartAuthorization.mockReturnValue(true);
peekGatewaySigusr1RestartReason.mockReset();
peekGatewaySigusr1RestartReason
.mockReturnValueOnce("config.patch")
.mockReturnValueOnce("update.auto");
respawnGatewayProcessForUpdate.mockReturnValueOnce({ mode: "supervised" });
restartGatewayProcessWithFreshPid.mockReturnValueOnce({ mode: "supervised" });
let releaseClose: () => void = () => {};
const close = vi.fn<GatewayCloseFn>(
@@ -2482,9 +2546,9 @@ describe("runGatewayLoop", () => {
releaseClose = resolve;
}),
);
setPlatform("freebsd");
process.env.OPENCLAW_SUPERVISOR_MODE = "external";
try {
setPlatform("freebsd");
await withIsolatedSignals(async ({ captureSignal }) => {
const { start, started } = createSignaledStart(close);
const { runtime, exited } = createRuntimeWithExitSignal();
@@ -2492,58 +2556,41 @@ describe("runGatewayLoop", () => {
await waitForStart(started);
const sigusr1 = captureSignal("SIGUSR1");
try {
sigusr1();
await waitForLoopCondition(
() => close.mock.calls.length === 1,
"restart close did not start",
);
expect(armShutdownHardExitWatchdog).toHaveBeenCalledTimes(1);
sigusr1();
await waitForLoopCondition(
() =>
gatewayLog.info.mock.calls.some(([message]) =>
String(message).includes("upgrading to update.auto"),
),
"accepted restart was not upgraded",
);
expect(cancelShutdownHardExitWatchdog).toHaveBeenCalledTimes(1);
expect(armShutdownHardExitWatchdog).toHaveBeenCalledTimes(2);
sigusr1();
await waitForLoopCondition(
() => close.mock.calls.length === 1,
"restart close did not start",
);
sigusr1();
await waitForLoopCondition(
() =>
gatewayLog.info.mock.calls.some(([message]) =>
String(message).includes("upgrading to update.auto"),
),
"accepted restart was not upgraded",
);
releaseClose();
await expect(exited).resolves.toBe(0);
expect(cancelShutdownHardExitWatchdog).toHaveBeenCalledTimes(2);
expect(respawnGatewayProcessForUpdate).toHaveBeenCalledTimes(1);
expectRestartHandoffCall({
restartKind: "update-process",
reason: "update.auto",
supervisorMode: "external",
});
} finally {
releaseClose();
await exited;
}
releaseClose();
await expect(exited).resolves.toBe(0);
expect(restartGatewayProcessWithFreshPid).toHaveBeenCalledOnce();
expectRestartHandoffCall({
restartKind: "update-process",
reason: "update.auto",
supervisorMode: "external",
});
});
} finally {
if (originalPlatformDescriptor) {
Object.defineProperty(process, "platform", originalPlatformDescriptor);
}
releaseClose();
delete process.env.OPENCLAW_SUPERVISOR_MODE;
}
});
it("reads a managed update upgrade after asynchronous lock release", async () => {
it("reads an update upgrade after asynchronous lock release", async () => {
vi.clearAllMocks();
consumeGatewayRestartIntentPayloadSync.mockReset();
consumeGatewayRestartIntentPayloadSync.mockReturnValue(null);
consumeGatewaySigusr1RestartIntent.mockReset();
consumeGatewaySigusr1RestartIntent.mockReturnValue(null);
consumeGatewaySigusr1RestartAuthorization.mockReset();
consumeGatewaySigusr1RestartAuthorization.mockReturnValue(true);
peekGatewaySigusr1RestartReason.mockReset();
peekGatewaySigusr1RestartReason
.mockReturnValueOnce("config.patch")
.mockReturnValueOnce("update.auto");
respawnGatewayProcessForUpdate.mockReturnValueOnce({ mode: "supervised" });
restartGatewayProcessWithFreshPid.mockReturnValueOnce({ mode: "supervised" });
let releaseLock: () => void = () => {};
const lockReleaseBlocked = new Promise<void>((resolve) => {
@@ -2553,31 +2600,157 @@ describe("runGatewayLoop", () => {
await lockReleaseBlocked;
});
acquireGatewayLock.mockResolvedValueOnce({ release: lockRelease });
process.env.OPENCLAW_SUPERVISOR_MODE = "external";
try {
await withIsolatedSignals(async ({ captureSignal }) => {
const { runtime, exited } = await createSignaledLoopHarness();
const sigusr1 = captureSignal("SIGUSR1");
sigusr1();
await waitForLoopCondition(
() => lockRelease.mock.calls.length === 1,
"restart did not reach lock release",
);
sigusr1();
await waitForLoopCondition(
() =>
gatewayLog.info.mock.calls.some(([message]) =>
String(message).includes("upgrading to update.auto"),
),
"lock-release restart was not upgraded",
);
await withIsolatedSignals(async ({ captureSignal }) => {
const { runtime, exited } = await createSignaledLoopHarness();
const sigusr1 = captureSignal("SIGUSR1");
sigusr1();
await waitForLoopCondition(
() => lockRelease.mock.calls.length === 1,
"restart did not reach lock release",
);
sigusr1();
await waitForLoopCondition(
() =>
gatewayLog.info.mock.calls.some(([message]) =>
String(message).includes("upgrading to update.auto"),
),
"lock-release restart was not upgraded",
);
releaseLock();
await expect(exited).resolves.toBe(0);
expect(restartGatewayProcessWithFreshPid).toHaveBeenCalledOnce();
expect(runtime.exit).toHaveBeenCalledWith(0);
});
} finally {
releaseLock();
await expect(exited).resolves.toBe(0);
expect(respawnGatewayProcessForUpdate).toHaveBeenCalledTimes(1);
expect(restartGatewayProcessWithFreshPid).not.toHaveBeenCalled();
expect(runtime.exit).toHaveBeenCalledWith(0);
delete process.env.OPENCLAW_SUPERVISOR_MODE;
}
});
it("recovers in process after exactly cancelling a replacement managed owner before exit", async () => {
vi.clearAllMocks();
const replacementOwner = { ...managedUpdateSuccessorOwner, handoffId: "replacement-handoff" };
consumeGatewaySigusr1RestartIntent
.mockReturnValueOnce({ reason: "update.run", successorOwner: managedUpdateSuccessorOwner })
.mockReturnValueOnce({ reason: "update.auto", successorOwner: replacementOwner });
cancelManagedServiceUpdateHandoff
.mockResolvedValueOnce("restored-in-process")
.mockResolvedValueOnce("restored-in-process");
let releaseCommit: () => void = () => {};
const commitBlocked = new Promise<void>((resolve) => {
releaseCommit = resolve;
});
commitManagedServiceUpdateHandoff.mockImplementationOnce(async () => {
await commitBlocked;
return true;
});
setPlatform("linux");
process.env.OPENCLAW_SERVICE_MARKER = "openclaw";
process.env.OPENCLAW_SERVICE_KIND = "gateway";
try {
await withIsolatedSignals(async ({ captureSignal }) => {
const { start, runtime, exited } = await createSignaledLoopHarness();
const sigusr1 = captureSignal("SIGUSR1");
const sigint = captureSignal("SIGINT");
sigusr1();
await waitForLoopCondition(
() => commitManagedServiceUpdateHandoff.mock.calls.length === 1,
"managed owner did not reach its final helper commit",
);
sigusr1();
await waitForLoopCondition(
() => consumeGatewaySigusr1RestartIntent.mock.calls.length === 2,
"replacement owner was not admitted before exit",
);
releaseCommit();
await waitForLoopCondition(
() => start.mock.calls.length === 2,
"replacement managed owner cancellation did not reopen gateway admission",
);
expect(requestManagedServiceUpdateHandoffPark).toHaveBeenCalledExactlyOnceWith(
managedUpdateSuccessorOwner,
);
expect(cancelManagedServiceUpdateHandoff).toHaveBeenNthCalledWith(
1,
managedUpdateSuccessorOwner,
);
expect(cancelManagedServiceUpdateHandoff).toHaveBeenNthCalledWith(2, replacementOwner);
expect(commitManagedServiceUpdateHandoff).toHaveBeenCalledExactlyOnceWith(
managedUpdateSuccessorOwner,
"update",
);
expect(runtime.exit).not.toHaveBeenCalled();
expect(start).toHaveBeenCalledTimes(2);
expect(gatewayWorkAdmissionActual.isGatewayWorkAdmissionClosed()).toBe(false);
sigint();
await expect(exited).resolves.toBe(0);
});
} finally {
releaseCommit();
delete process.env.OPENCLAW_SERVICE_MARKER;
delete process.env.OPENCLAW_SERVICE_KIND;
}
});
it("reopens admission after a broken control pipe waits for the exact helper to exit", async () => {
vi.clearAllMocks();
consumeGatewaySigusr1RestartIntent.mockReturnValueOnce({
reason: "update.run",
successorOwner: managedUpdateSuccessorOwner,
});
requestManagedServiceUpdateHandoffPark.mockResolvedValueOnce(false);
let releaseHelperExit: () => void = () => {};
const helperExit = new Promise<void>((resolve) => {
releaseHelperExit = resolve;
});
cancelManagedServiceUpdateHandoff.mockImplementationOnce(async () => {
await helperExit;
return "restored-in-process";
});
setPlatform("linux");
process.env.OPENCLAW_SERVICE_MARKER = "openclaw";
process.env.OPENCLAW_SERVICE_KIND = "gateway";
try {
await withIsolatedSignals(async ({ captureSignal }) => {
const { start, runtime, exited } = await createSignaledLoopHarness();
const sigusr1 = captureSignal("SIGUSR1");
const sigint = captureSignal("SIGINT");
sigusr1();
await waitForLoopCondition(
() => cancelManagedServiceUpdateHandoff.mock.calls.length === 1,
"broken helper control pipe did not begin exact-owner cancellation",
);
expect(start).toHaveBeenCalledOnce();
expect(gatewayWorkAdmissionActual.isGatewayWorkAdmissionClosed()).toBe(true);
releaseHelperExit();
await waitForLoopCondition(
() => start.mock.calls.length === 2,
"broken helper control pipe left the gateway permanently draining",
);
expect(cancelManagedServiceUpdateHandoff).toHaveBeenCalledExactlyOnceWith(
managedUpdateSuccessorOwner,
);
expect(commitManagedServiceUpdateHandoff).not.toHaveBeenCalled();
expect(runtime.exit).not.toHaveBeenCalled();
expect(gatewayWorkAdmissionActual.isGatewayWorkAdmissionClosed()).toBe(false);
sigint();
await expect(exited).resolves.toBe(0);
});
} finally {
releaseHelperExit();
delete process.env.OPENCLAW_SERVICE_MARKER;
delete process.env.OPENCLAW_SERVICE_KIND;
}
});
it("probes the configured gateway host for update respawn health", async () => {
+370 -324
View File
@@ -1,4 +1,5 @@
// In-process gateway run loop, restart signaling, drain, and update respawn handling.
import type { ChildProcess } from "node:child_process";
import { randomUUID } from "node:crypto";
import net from "node:net";
import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
@@ -17,6 +18,7 @@ import {
type GatewayBootLifecycleCompletion,
} from "../../infra/gateway-boot-lifecycle.js";
import { acquireGatewayLock } from "../../infra/gateway-lock.js";
import type { GatewayRestartIntent } from "../../infra/restart-intent.js";
import type { GatewayRestartEmitter } from "../../infra/restart.js";
import { flushLogger } from "../../logging/logger.js";
import { createSubsystemLogger } from "../../logging/subsystem.js";
@@ -42,17 +44,11 @@ const LOG_FLUSH_EXIT_TIMEOUT_MS = 4_000;
const HARD_EXIT_WATCHDOG_GRACE_MS = 2_000;
type GatewayRunSignalAction = "stop" | "restart";
type RestartDrainTimeoutMs = number | undefined;
type RestartIntentOptions = {
reason?: string;
force?: boolean;
waitMs?: number;
};
type GatewayRunSignalRequest = {
action: GatewayRunSignalAction;
signal: string;
restartReason?: string;
restartIntent?: RestartIntentOptions;
restartIntent?: GatewayRestartIntent;
};
type GatewayLifecycleRuntimeModule = typeof import("./lifecycle.runtime.js");
@@ -70,20 +66,11 @@ const loadGatewayLifecycleRuntimeModule = () => gatewayLifecycleRuntimeLoader.lo
async function waitForGatewayPortReady(host: string, port: number): Promise<boolean> {
return await new Promise<boolean>((resolve) => {
const socket = net.createConnection({ host, port });
let settled = false;
const finish = (value: boolean) => {
if (settled) {
return;
}
settled = true;
clearTimeout(timer);
socket.removeAllListeners();
socket.destroy();
resolve(value);
};
const timer = setTimeout(() => {
finish(false);
}, UPDATE_RESPAWN_HEALTH_POLL_MS);
socket.setTimeout(UPDATE_RESPAWN_HEALTH_POLL_MS, () => finish(false));
socket.once("connect", () => finish(true));
socket.once("error", () => finish(false));
});
@@ -141,6 +128,11 @@ export async function runGatewayLoop(params: {
// here pulls the lifecycle re-export graph into memory, immune to later disk
// rotation.
const eagerLifecycleRuntime = await loadGatewayLifecycleRuntimeModule();
const supervisorMode = eagerLifecycleRuntime.detectGatewayRespawnSupervisor(
process.env,
process.platform,
{ includeLinuxOpenClawGatewayServiceMarker: true },
);
let lock = await acquireGatewayLock({ port: params.lockPort });
let server: Awaited<ReturnType<typeof startGatewayServer>> | null = null;
let shuttingDown = false;
@@ -149,12 +141,22 @@ export async function runGatewayLoop(params: {
// Defer lifecycle signals from that window until the loop can close and advance.
let pendingStartupRequest: GatewayRunSignalRequest | null = null;
let activeRestartRequest: GatewayRunSignalRequest | null = null;
let committedGenericSuccessor: ChildProcess | true | null = null;
let forceActiveRestartExit: (() => void) | null = null;
let pendingStartupForceExitTimer: ReturnType<typeof setTimeout> | null = null;
let restartDrainingMarked = false;
let startupFailedWithoutServerHandle = false;
const processInstanceId = randomUUID();
const waitForHealthyChild = params.waitForHealthyChild ?? waitForHealthyGatewayChild;
const getManagedUpdateOwner = () =>
(pendingStartupRequest ?? activeRestartRequest)?.restartIntent?.successorOwner;
const sameManagedUpdateOwner = (
left: GatewayRestartIntent["successorOwner"],
right: GatewayRestartIntent["successorOwner"],
) =>
Boolean(
left && right && left.handoffId === right.handoffId && left.installRoot === right.installRoot,
);
const cleanupSignals = () => {
process.removeListener("SIGTERM", onSigterm);
@@ -165,7 +167,13 @@ export async function runGatewayLoop(params: {
cleanupSignals();
params.runtime.exit(code);
};
const exitProcessAfterLogFlush = async (code: number) => {
const exitProcessAfterLogFlush = async (
code: number,
initialOwner?: GatewayRestartIntent["successorOwner"],
initialOutcome: "update" | "restore" = "update",
): Promise<void> => {
let ownerToCommit = initialOwner;
let commitOutcome = initialOutcome;
// Graceful signal/restart paths call process.exit(), which skips beforeExit.
let flushTimer: ReturnType<typeof setTimeout> | undefined;
const flushed = await Promise.race([
@@ -174,69 +182,179 @@ export async function runGatewayLoop(params: {
flushTimer = setTimeout(() => resolve(false), LOG_FLUSH_EXIT_TIMEOUT_MS);
}),
]);
if (flushTimer) {
clearTimeout(flushTimer);
}
clearTimeout(flushTimer);
if (!flushed) {
gatewayLog.warn(
`log flush did not settle within ${LOG_FLUSH_EXIT_TIMEOUT_MS}ms; continuing shutdown`,
);
}
exitProcess(code);
for (;;) {
const owner = getManagedUpdateOwner();
if (!owner) {
if (!ownerToCommit) {
exitProcess(code);
}
return;
}
if (
sameManagedUpdateOwner(owner, ownerToCommit) &&
eagerLifecycleRuntime.claimManagedServiceUpdateHandoff(owner) &&
(await eagerLifecycleRuntime.commitManagedServiceUpdateHandoff(owner, commitOutcome)) &&
sameManagedUpdateOwner(getManagedUpdateOwner(), owner) &&
eagerLifecycleRuntime.claimManagedServiceUpdateHandoff(owner)
) {
// Keep exact request ownership live through the synchronous exit call.
exitProcess(code);
return;
}
await markRestartHandoffUnavailable();
const ownerToCancel = ownerToCommit ?? owner;
const restoration = await cancelManagedUpdateHandoffBeforeRecovery(ownerToCancel);
if (!restoration) {
const child = committedGenericSuccessor === true ? null : committedGenericSuccessor;
if (child && child.exitCode === null && child.signalCode === null) {
const exited = new Promise<void>((resolve) => {
child.once("exit", () => {
resolve();
});
});
try {
child.kill("SIGKILL");
await exited;
} catch {}
}
return;
}
if (restoration === "restart-after-exit") {
ownerToCommit = ownerToCancel;
commitOutcome = "restore";
const currentRequest = pendingStartupRequest ?? activeRestartRequest;
if (
currentRequest &&
!sameManagedUpdateOwner(currentRequest.restartIntent?.successorOwner, ownerToCancel)
) {
currentRequest.restartIntent = {
...currentRequest.restartIntent,
successorOwner: ownerToCancel,
};
}
continue;
}
if (!committedGenericSuccessor && initialOwner) {
return reacquireAndResumeInProcessRestart(getManagedUpdateOwner() ?? owner);
}
exitProcess(code);
return;
}
};
const completeForcedStop = (reason: string) => {
params.completeBoot?.({ outcome: "forced_stop", reason });
};
const writeStabilityBundle = async (reason: string, error?: unknown) => {
const { writeDiagnosticStabilityBundleForFailureSync } =
await loadGatewayLifecycleRuntimeModule();
const result = writeDiagnosticStabilityBundleForFailureSync(reason, error);
const writeStabilityBundle = (reason: string, error?: unknown) => {
const result = eagerLifecycleRuntime.writeDiagnosticStabilityBundleForFailureSync(
reason,
error,
);
if ("message" in result) {
gatewayLog.warn(result.message);
}
};
const releaseLockIfHeld = async (): Promise<boolean> => {
if (!lock) {
return false;
}
await lock.release();
const releaseLockIfHeld = async (): Promise<void> => {
await lock?.release();
lock = null;
return true;
};
const reacquireLockForInProcessRestart = async (): Promise<boolean> => {
const cancelManagedUpdateHandoffBeforeRecovery = async (
initialOwner = getManagedUpdateOwner(),
): Promise<false | "restored-in-process" | "restart-after-exit"> => {
let owner = initialOwner;
let requiresParentExit = false;
try {
lock = await acquireGatewayLock({ port: params.lockPort });
return true;
for (;;) {
if (!owner) {
return requiresParentExit ? "restart-after-exit" : "restored-in-process";
}
const restoration = await eagerLifecycleRuntime.cancelManagedServiceUpdateHandoff(owner);
if (!restoration) {
gatewayLog.error("managed update handoff cancellation unconfirmed; remaining draining");
return false;
}
requiresParentExit ||= restoration === "restart-after-exit";
const replacement = getManagedUpdateOwner();
if (!replacement || sameManagedUpdateOwner(owner, replacement)) {
return requiresParentExit ? "restart-after-exit" : "restored-in-process";
}
owner = replacement;
}
} catch (err) {
gatewayLog.error(`failed to reacquire gateway lock for in-process restart: ${String(err)}`);
exitProcess(1);
gatewayLog.error(`managed update handoff cancellation failed: ${formatErrorMessage(err)}`);
return false;
}
};
const confirmLaunchdHandoff = async (respawn: {
handoffSpawned?: Promise<boolean>;
}): Promise<boolean> => {
const delay = new Promise<void>((resolve) => {
setTimeout(resolve, LAUNCHD_SUPERVISED_RESTART_EXIT_DELAY_MS);
});
const spawned = respawn.handoffSpawned
? await Promise.race([respawn.handoffSpawned, delay.then(() => true)])
: false;
// Preserve the crash-loop throttle window even when spawn settles early.
await delay;
return spawned;
const forceExitAfterStabilityBundle = async (reason: string) => {
try {
writeStabilityBundle(reason);
} finally {
const owner = getManagedUpdateOwner();
if (owner) {
forceActiveRestartExit?.();
}
const restoration = await cancelManagedUpdateHandoffBeforeRecovery(owner);
if (restoration) {
params.completeBoot?.({ outcome: "forced_stop", reason });
if (restoration === "restart-after-exit") {
await exitProcessAfterLogFlush(1, owner, "restore");
} else {
exitProcess(1);
}
}
}
};
const handleRestartAfterServerClose = async () => {
const reacquireAndResumeInProcessRestart = async (
alreadyCancelledOwner?: GatewayRestartIntent["successorOwner"],
): Promise<void> => {
for (;;) {
const restartRequest = activeRestartRequest;
const restartOwner = restartRequest?.restartIntent?.successorOwner;
const restoration = sameManagedUpdateOwner(restartOwner, alreadyCancelledOwner)
? "restored-in-process"
: await cancelManagedUpdateHandoffBeforeRecovery(restartOwner);
if (!restoration) {
return;
}
if (restoration === "restart-after-exit") {
await releaseLockIfHeld();
return exitProcessAfterLogFlush(0, restartOwner, "restore");
}
if (activeRestartRequest !== restartRequest) {
continue;
}
try {
lock = await acquireGatewayLock({ port: params.lockPort });
} catch (err) {
if (activeRestartRequest !== restartRequest) {
continue;
}
gatewayLog.error(`failed to reacquire gateway lock for in-process restart: ${String(err)}`);
exitProcess(1);
return;
}
if (activeRestartRequest === restartRequest) {
activeRestartRequest = null;
shuttingDown = false;
restartResolver?.();
return;
}
await releaseLockIfHeld();
}
};
const markRestartHandoffUnavailable = async (reason = "restart-handoff-unavailable") => {
await eagerLifecycleRuntime.markUpdateRestartSentinelFailure(reason).catch((err: unknown) => {
gatewayLog.warn(`failed to mark update restart ${reason}: ${String(err)}`);
});
};
const handleRestartAfterServerClose = async (
expectedOwner?: GatewayRestartIntent["successorOwner"],
cancelled = false,
): Promise<void> => {
await releaseLockIfHeld();
const {
detectGatewayRespawnSupervisor,
markUpdateRestartSentinelFailure,
respawnGatewayProcessForUpdate,
restartGatewayProcessWithFreshPid,
writeGatewayRestartHandoffSync,
} = await loadGatewayLifecycleRuntimeModule();
// Lock release and lazy lifecycle loading may yield while a managed update
// upgrades this restart. Keep the request live until a restart path commits.
// Lock release may yield while a managed update upgrades this restart.
const restartReason = activeRestartRequest?.restartReason;
params.completeBoot?.({
outcome: "planned_restart",
@@ -244,214 +362,122 @@ export async function runGatewayLoop(params: {
});
const isUpdateRestart = isUpdateProcessRestartReason(restartReason);
if (isUpdateRestart) {
const restartTraceHandoff = captureGatewayRestartTraceHandoff();
const respawn = respawnGatewayProcessForUpdate({
env: createGatewayRestartTraceHandoffEnv(restartTraceHandoff),
});
if (respawn.mode === "spawned") {
const port = params.lockPort;
const healthy =
typeof port === "number"
? await waitForHealthyChild(port, respawn.pid, params.healthHost ?? "127.0.0.1")
: false;
if (healthy) {
activeRestartRequest = null;
gatewayLog.info(
`restart mode: update process respawn (spawned pid ${respawn.pid ?? "unknown"})`,
);
await exitProcessAfterLogFlush(0);
return;
}
gatewayLog.warn(
`update respawn child did not become healthy (${respawn.pid ?? "unknown"}); falling back to in-process restart`,
);
try {
respawn.child?.kill();
} catch {
// Best-effort; parent fallback keeps the gateway reachable for recovery.
}
await markUpdateRestartSentinelFailure("restart-unhealthy").catch((err: unknown) => {
gatewayLog.warn(`failed to mark update restart sentinel unhealthy: ${String(err)}`);
});
if (!(await reacquireLockForInProcessRestart())) {
return;
}
shuttingDown = false;
restartResolver?.();
return;
if (cancelled) {
return reacquireAndResumeInProcessRestart(expectedOwner);
}
if (activeRestartRequest?.restartIntent?.successorOwner) {
if (!expectedOwner) {
gatewayLog.error("managed update handoff arrived after successor parking closed");
await markRestartHandoffUnavailable();
return reacquireAndResumeInProcessRestart();
}
if (respawn.mode === "supervised") {
const supervisorMode = detectGatewayRespawnSupervisor(process.env, process.platform);
markGatewayRestartTrace("restart.full-process-handoff", [
["kind", "update-process"],
["mode", respawn.mode],
["supervisorMode", supervisorMode ?? "external"],
]);
const handoff = writeGatewayRestartHandoffSync({
restartKind: "update-process",
reason: restartReason,
processInstanceId,
supervisorMode: supervisorMode ?? "external",
restartTrace: captureGatewayRestartTraceHandoff(),
});
if (supervisorMode === "external" && !handoff) {
gatewayLog.warn(
"external supervisor restart handoff could not be persisted; falling back to in-process restart",
);
await markUpdateRestartSentinelFailure("restart-handoff-unavailable").catch(
(err: unknown) => {
gatewayLog.warn(`failed to mark update restart handoff unavailable: ${String(err)}`);
},
);
if (!(await reacquireLockForInProcessRestart())) {
return;
}
activeRestartRequest = null;
shuttingDown = false;
restartResolver?.();
return;
gatewayLog.info("restart mode: managed update handoff owns successor");
return exitProcessAfterLogFlush(0, expectedOwner);
}
const respawnOptions = {
env: createGatewayRestartTraceHandoffEnv(captureGatewayRestartTraceHandoff()),
};
const isStandaloneUpdate = isUpdateRestart && !supervisorMode;
const respawn = isStandaloneUpdate
? eagerLifecycleRuntime.respawnGatewayProcessForUpdate(respawnOptions)
: eagerLifecycleRuntime.restartGatewayProcessWithFreshPid(respawnOptions);
if (respawn.mode === "spawned") {
const port = params.lockPort;
const healthy =
typeof port === "number"
? await waitForHealthyChild(port, respawn.pid, params.healthHost ?? "127.0.0.1")
: false;
if (healthy) {
committedGenericSuccessor = respawn.child ?? true;
gatewayLog.info(
`restart mode: update process respawn (spawned pid ${respawn.pid ?? "unknown"})`,
);
return exitProcessAfterLogFlush(0);
}
gatewayLog.warn(
`update respawn child did not become healthy (${respawn.pid ?? "unknown"}); falling back to in-process restart`,
);
try {
respawn.child?.kill();
} catch {
// Best-effort; parent fallback keeps the gateway reachable for recovery.
}
await markRestartHandoffUnavailable("restart-unhealthy");
return reacquireAndResumeInProcessRestart();
}
if (respawn.mode === "supervised") {
const restartKind = isUpdateRestart ? "update-process" : "full-process";
markGatewayRestartTrace("restart.full-process-handoff", [
["kind", restartKind],
["mode", respawn.mode],
["pid", "none"],
["supervisorMode", supervisorMode ?? "none"],
]);
const handoff = eagerLifecycleRuntime.writeGatewayRestartHandoffSync({
restartKind,
reason: restartReason,
processInstanceId,
supervisorMode: supervisorMode ?? "external",
restartTrace: captureGatewayRestartTraceHandoff(),
});
if (supervisorMode === "external" && !handoff) {
gatewayLog.warn(
"external supervisor restart handoff could not be persisted; falling back to in-process restart",
);
if (isUpdateRestart) {
await markRestartHandoffUnavailable();
}
gatewayLog.info("restart mode: update process respawn (supervisor restart)");
if (supervisorMode === "launchd" && !(await confirmLaunchdHandoff(respawn))) {
return reacquireAndResumeInProcessRestart();
}
gatewayLog.info("restart mode: full process restart (supervisor restart)");
if (supervisorMode === "launchd") {
const delay = new Promise<void>((resolve) => {
setTimeout(resolve, LAUNCHD_SUPERVISED_RESTART_EXIT_DELAY_MS);
});
const spawned = respawn.handoffSpawned
? await Promise.race([respawn.handoffSpawned, delay.then(() => true)])
: false;
// Preserve the crash-loop throttle window even when spawn settles early.
await delay;
if (!spawned) {
writeStabilityBundle("gateway.restart_handoff_spawn_failed");
gatewayLog.warn(
"launchd restart handoff failed to spawn; falling back to in-process restart",
);
await markUpdateRestartSentinelFailure("restart-handoff-unavailable").catch(
(err: unknown) => {
gatewayLog.warn(`failed to mark update restart handoff unavailable: ${String(err)}`);
},
);
if (!(await reacquireLockForInProcessRestart())) {
return;
if (isUpdateRestart) {
await markRestartHandoffUnavailable();
}
activeRestartRequest = null;
shuttingDown = false;
restartResolver?.();
return;
}
activeRestartRequest = null;
await exitProcessAfterLogFlush(0);
return;
}
if (respawn.mode === "failed") {
gatewayLog.warn(
`update respawn failed (${respawn.detail ?? "unknown error"}); falling back to in-process restart`,
);
await markUpdateRestartSentinelFailure("restart-unhealthy").catch((err: unknown) => {
gatewayLog.warn(`failed to mark update restart sentinel unhealthy: ${String(err)}`);
});
} else {
gatewayLog.info(
`restart mode: in-process restart (${respawn.detail ?? "OPENCLAW_NO_RESPAWN"})`,
);
}
if (!(await reacquireLockForInProcessRestart())) {
return;
}
activeRestartRequest = null;
shuttingDown = false;
restartResolver?.();
return;
}
// Release the lock BEFORE spawning so the child can acquire it immediately.
const restartTraceHandoff = captureGatewayRestartTraceHandoff();
const respawn = restartGatewayProcessWithFreshPid({
env: createGatewayRestartTraceHandoffEnv(restartTraceHandoff),
});
if (respawn.mode === "spawned" || respawn.mode === "supervised") {
const supervisorMode =
respawn.mode === "supervised"
? detectGatewayRespawnSupervisor(process.env, process.platform)
: null;
const modeLabel =
respawn.mode === "spawned"
? `spawned pid ${respawn.pid ?? "unknown"}`
: "supervisor restart";
markGatewayRestartTrace("restart.full-process-handoff", [
["kind", "full-process"],
["mode", respawn.mode],
["pid", respawn.mode === "spawned" ? (respawn.pid ?? "unknown") : "none"],
["supervisorMode", supervisorMode ?? "none"],
]);
if (respawn.mode === "supervised") {
const handoff = writeGatewayRestartHandoffSync({
restartKind: "full-process",
reason: restartReason,
processInstanceId,
supervisorMode: supervisorMode ?? "external",
restartTrace: captureGatewayRestartTraceHandoff(),
});
if (supervisorMode === "external" && !handoff) {
gatewayLog.warn(
"external supervisor restart handoff could not be persisted; falling back to in-process restart",
);
if (!(await reacquireLockForInProcessRestart())) {
return;
}
activeRestartRequest = null;
shuttingDown = false;
restartResolver?.();
return;
return reacquireAndResumeInProcessRestart();
}
}
gatewayLog.info(`restart mode: full process restart (${modeLabel})`);
if (supervisorMode === "launchd" && !(await confirmLaunchdHandoff(respawn))) {
await writeStabilityBundle("gateway.restart_handoff_spawn_failed");
gatewayLog.warn(
"launchd restart handoff failed to spawn; falling back to in-process restart",
);
if (!(await reacquireLockForInProcessRestart())) {
return;
}
activeRestartRequest = null;
shuttingDown = false;
restartResolver?.();
return;
}
activeRestartRequest = null;
await exitProcessAfterLogFlush(0);
return;
committedGenericSuccessor = true;
return exitProcessAfterLogFlush(0);
}
if (respawn.mode === "failed") {
await writeStabilityBundle("gateway.restart_respawn_failed");
if (!isStandaloneUpdate) {
writeStabilityBundle("gateway.restart_respawn_failed");
}
gatewayLog.warn(
`full process restart failed (${respawn.detail ?? "unknown error"}); falling back to in-process restart`,
`${isStandaloneUpdate ? "update respawn" : "full process restart"} failed (${respawn.detail ?? "unknown error"}); falling back to in-process restart`,
);
if (isUpdateRestart) {
await markRestartHandoffUnavailable("restart-unhealthy");
}
} else {
gatewayLog.info(
`restart mode: in-process restart (${respawn.detail ?? "OPENCLAW_NO_RESPAWN"})`,
);
}
if (isUpdateProcessRestartReason(activeRestartRequest?.restartReason)) {
await handleRestartAfterServerClose();
return;
if (!isUpdateRestart && isUpdateProcessRestartReason(activeRestartRequest?.restartReason)) {
return handleRestartAfterServerClose();
}
if (!(await reacquireLockForInProcessRestart())) {
return;
}
if (isUpdateProcessRestartReason(activeRestartRequest?.restartReason)) {
await handleRestartAfterServerClose();
return;
}
activeRestartRequest = null;
shuttingDown = false;
restartResolver?.();
return reacquireAndResumeInProcessRestart();
};
const handleStopAfterServerClose = async () => {
params.completeBoot?.({ outcome: "clean_stop", reason: "gateway.stop" });
await releaseLockIfHeld();
await exitProcessAfterLogFlush(0);
};
const SUPERVISOR_STOP_TIMEOUT_MS = 30_000;
const SHUTDOWN_TIMEOUT_MS = SUPERVISOR_STOP_TIMEOUT_MS - 5_000;
const clearPendingStartupForceExitTimer = () => {
if (!pendingStartupForceExitTimer) {
return;
}
clearTimeout(pendingStartupForceExitTimer);
clearTimeout(pendingStartupForceExitTimer ?? undefined);
pendingStartupForceExitTimer = null;
};
const armPendingStartupForceExitTimer = () => {
@@ -463,20 +489,13 @@ export async function runGatewayLoop(params: {
gatewayLog.error(
"startup restart request timed out before gateway returned a close handle; exiting for supervisor recovery",
);
void (async () => {
try {
await writeStabilityBundle("gateway.restart_startup_request_timeout");
} finally {
completeForcedStop("gateway.restart_startup_request_timeout");
exitProcess(1);
}
})();
void forceExitAfterStabilityBundle("gateway.restart_startup_request_timeout");
}, SHUTDOWN_TIMEOUT_MS);
pendingStartupForceExitTimer.unref?.();
};
const resolveRestartDrainTimeoutMs = async (
restartIntent?: RestartIntentOptions,
): Promise<RestartDrainTimeoutMs> => {
const resolveRestartDrainTimeoutMs = (
restartIntent?: GatewayRestartIntent,
): number | undefined => {
if (restartIntent?.force) {
return 0;
}
@@ -484,8 +503,7 @@ export async function runGatewayLoop(params: {
return restartIntent.waitMs > 0 ? Math.floor(restartIntent.waitMs) : undefined;
}
try {
const { resolveGatewayRestartDeferralTimeoutMs } = await loadGatewayLifecycleRuntimeModule();
return resolveGatewayRestartDeferralTimeoutMs();
return eagerLifecycleRuntime.resolveGatewayRestartDeferralTimeoutMs();
} catch {
return DEFAULT_RESTART_DRAIN_TIMEOUT_MS;
}
@@ -515,21 +533,9 @@ export async function runGatewayLoop(params: {
}
forceExitTimer = setTimeout(() => {
gatewayLog.error("shutdown timed out; exiting without full cleanup");
void (async () => {
try {
await writeStabilityBundle(
isRestart ? "gateway.restart_shutdown_timeout" : "gateway.stop_shutdown_timeout",
);
} finally {
// Keep the in-process watchdog below the supervisor stop budget so this
// path wins before launchd/systemd escalates to a hard kill. Exit
// non-zero on any timeout so supervised installs restart cleanly.
completeForcedStop(
isRestart ? "gateway.restart_shutdown_timeout" : "gateway.stop_shutdown_timeout",
);
exitProcess(1);
}
})();
void forceExitAfterStabilityBundle(
isRestart ? "gateway.restart_shutdown_timeout" : "gateway.stop_shutdown_timeout",
);
}, forceExitMs);
if (params.ownsProcessLifecycle === true) {
hardExitWatchdog = armShutdownHardExitWatchdog({
@@ -543,31 +549,36 @@ export async function runGatewayLoop(params: {
}
};
const clearForceExitTimer = () => {
if (forceExitTimer) {
clearTimeout(forceExitTimer);
forceExitTimer = null;
}
clearTimeout(forceExitTimer ?? undefined);
forceExitTimer = null;
hardExitWatchdog?.cancel();
hardExitWatchdog = null;
};
if (isRestart) {
forceActiveRestartExit = () => {
clearForceExitTimer();
armForceExitTimer(SHUTDOWN_TIMEOUT_MS);
if (!getManagedUpdateOwner()) {
armForceExitTimer(SHUTDOWN_TIMEOUT_MS);
}
};
}
void (async () => {
const restartDrainTimeoutMs = isRestart
? await resolveRestartDrainTimeoutMs(restartIntent)
: 0;
let managedUpdateOwner: GatewayRestartIntent["successorOwner"];
let managedUpdateCancellation:
| false
| "restored-in-process"
| "restart-after-exit"
| undefined;
const restartDrainTimeoutMs = isRestart ? resolveRestartDrainTimeoutMs(restartIntent) : 0;
const restartDrainDeadlineAt =
isRestart && restartDrainTimeoutMs !== undefined
? Date.now() + restartDrainTimeoutMs
: undefined;
// Managed helpers must reach native parking before either exit watchdog can arm.
if (!isRestart) {
armForceExitTimer(SHUTDOWN_TIMEOUT_MS);
} else if (restartDrainTimeoutMs !== undefined) {
} else if (restartDrainTimeoutMs !== undefined && !getManagedUpdateOwner()) {
// Allow extra time for draining active turns on explicitly capped restarts.
armForceExitTimer(restartDrainTimeoutMs + SHUTDOWN_TIMEOUT_MS);
}
@@ -576,21 +587,6 @@ export async function runGatewayLoop(params: {
restartDrainTimeoutMs === undefined
? "without a timeout"
: `with timeout ${restartDrainTimeoutMs}ms`;
const armCloseForceExitTimerForIndefiniteRestart = () => {
if (isRestart && restartDrainTimeoutMs === undefined) {
armForceExitTimer(SHUTDOWN_TIMEOUT_MS);
}
};
const resolveRestartCloseDrainTimeoutMs = () => {
if (!isRestart) {
return null;
}
if (restartDrainTimeoutMs === undefined) {
return Math.max(0, SHUTDOWN_TIMEOUT_MS - RESTART_CLOSE_REPLY_DRAIN_SHUTDOWN_RESERVE_MS);
}
return Math.max(0, (restartDrainDeadlineAt ?? Date.now()) - Date.now());
};
try {
// On restart, wait for the canonical process activity inventory before
// tearing down the server so active work can settle.
@@ -610,12 +606,6 @@ export async function runGatewayLoop(params: {
markRestartAbortedMainSessions,
waitForGatewayActiveWork,
} = await loadGatewayLifecycleRuntimeModule();
const collectActiveRestartSessionKeys = () => {
return new Set<string>(listActiveEmbeddedRunSessionKeys());
};
const collectActiveRestartSessionIds = () => {
return new Set<string>(listActiveEmbeddedRunSessionIds());
};
let activeRestartSessionKeysAtDrainStart = new Set<string>();
let activeRestartSessionIdsAtDrainStart = new Set<string>();
let hasMarkedActiveMainSessionsForRestart = false;
@@ -627,11 +617,11 @@ export async function runGatewayLoop(params: {
}
const sessionKeys = new Set<string>([
...activeRestartSessionKeysAtDrainStart,
...collectActiveRestartSessionKeys(),
...listActiveEmbeddedRunSessionKeys(),
]);
const sessionIds = new Set<string>([
...activeRestartSessionIdsAtDrainStart,
...collectActiveRestartSessionIds(),
...listActiveEmbeddedRunSessionIds(),
]);
if (sessionKeys.size === 0 && sessionIds.size === 0) {
return;
@@ -643,9 +633,7 @@ export async function runGatewayLoop(params: {
sessionIds,
reason,
});
if (result.marked > 0) {
hasMarkedActiveMainSessionsForRestart = true;
}
hasMarkedActiveMainSessionsForRestart = result.marked > 0;
} catch (err) {
gatewayLog.warn(
`failed to mark interrupted main sessions for restart recovery: ${String(err)}`,
@@ -662,8 +650,8 @@ export async function runGatewayLoop(params: {
const initialSnapshot = createGatewayActiveWorkSnapshot();
activeWorkAtDrainStart = initialSnapshot.counts.totalActive;
activeRunsAtDrainStart = initialSnapshot.counts.embeddedRuns;
activeRestartSessionKeysAtDrainStart = collectActiveRestartSessionKeys();
activeRestartSessionIdsAtDrainStart = collectActiveRestartSessionIds();
activeRestartSessionKeysAtDrainStart = new Set(listActiveEmbeddedRunSessionKeys());
activeRestartSessionIdsAtDrainStart = new Set(listActiveEmbeddedRunSessionIds());
// Best-effort abort for compacting runs so transcript settlement does
// not remain pending across restart boundaries.
@@ -725,9 +713,7 @@ export async function runGatewayLoop(params: {
["force", restartIntent?.force === true],
],
);
}
if (!isRestart) {
} else {
// Keep all process-owned work alive without spending the shutdown reserve
// that server teardown and the supervisor watchdog need.
try {
@@ -746,8 +732,48 @@ export async function runGatewayLoop(params: {
}
}
armCloseForceExitTimerForIndefiniteRestart();
const closeDrainTimeoutMs = resolveRestartCloseDrainTimeoutMs();
if (isRestart && activeRestartRequest?.restartIntent?.successorOwner) {
const owner = activeRestartRequest.restartIntent.successorOwner;
managedUpdateOwner = owner;
try {
if (
!sameManagedUpdateOwner(getManagedUpdateOwner(), owner) ||
!(await eagerLifecycleRuntime.requestManagedServiceUpdateHandoffPark(owner)) ||
!sameManagedUpdateOwner(getManagedUpdateOwner(), owner) ||
!eagerLifecycleRuntime.claimManagedServiceUpdateHandoff(owner)
) {
throw new Error("managed update helper lost exact ownership during service parking");
}
} catch (err) {
clearForceExitTimer();
gatewayLog.error(
`managed update handoff could not park ${supervisorMode}: ${String(err)}`,
);
await markRestartHandoffUnavailable();
managedUpdateCancellation = await cancelManagedUpdateHandoffBeforeRecovery(owner);
if (!managedUpdateCancellation) {
return;
}
if (managedUpdateCancellation === "restart-after-exit") {
await releaseLockIfHeld();
await exitProcessAfterLogFlush(0, owner, "restore");
return;
}
}
}
if (
isRestart &&
!forceExitTimer &&
(!managedUpdateOwner || managedUpdateCancellation === "restored-in-process")
) {
armForceExitTimer(SHUTDOWN_TIMEOUT_MS);
}
const closeDrainTimeoutMs = !isRestart
? null
: restartDrainTimeoutMs === undefined
? SHUTDOWN_TIMEOUT_MS - RESTART_CLOSE_REPLY_DRAIN_SHUTDOWN_RESERVE_MS
: Math.max(0, (restartDrainDeadlineAt ?? Date.now()) - Date.now());
await server?.close({
reason: isRestart ? "gateway restarting" : "gateway stopping",
restartExpectedMs: isRestart ? 1500 : null,
@@ -756,17 +782,28 @@ export async function runGatewayLoop(params: {
} catch (err) {
gatewayLog.error(`shutdown step failed (gateway server close): ${formatErrorMessage(err)}`);
} finally {
server = null;
const handoffClosed =
managedUpdateCancellation !== false && managedUpdateCancellation !== "restart-after-exit";
if (handoffClosed) {
server = null;
}
if (isRestart) {
try {
await handleRestartAfterServerClose();
if (handoffClosed) {
await handleRestartAfterServerClose(
managedUpdateOwner,
managedUpdateCancellation === "restored-in-process",
);
}
} finally {
clearForceExitTimer();
forceActiveRestartExit = null;
}
} else {
clearForceExitTimer();
await handleStopAfterServerClose();
params.completeBoot?.({ outcome: "clean_stop", reason: "gateway.stop" });
await releaseLockIfHeld();
await exitProcessAfterLogFlush(0);
}
}
})();
@@ -788,7 +825,7 @@ export async function runGatewayLoop(params: {
action: GatewayRunSignalAction,
signal: string,
restartReason?: string,
restartIntent?: RestartIntentOptions,
restartIntent?: GatewayRestartIntent,
) => {
const acceptedRequest = { action, signal, restartReason, restartIntent };
if (shuttingDown) {
@@ -797,7 +834,12 @@ export async function runGatewayLoop(params: {
action === "restart" &&
isUpdateProcessRestartReason(restartReason) &&
currentRestartRequest?.action === "restart" &&
!isUpdateProcessRestartReason(currentRestartRequest.restartReason)
(!isUpdateProcessRestartReason(currentRestartRequest.restartReason) ||
(restartIntent?.successorOwner &&
!sameManagedUpdateOwner(
restartIntent.successorOwner,
currentRestartRequest.restartIntent?.successorOwner,
)))
) {
const upgradedRequest = {
...currentRestartRequest,
@@ -900,6 +942,10 @@ export async function runGatewayLoop(params: {
if (restartIntent) {
abortPendingChannelReloads();
const authorized = consumeGatewaySigusr1RestartAuthorization();
const processLocalIntent = authorized ? consumeGatewaySigusr1RestartIntent() : null;
if (processLocalIntent?.successorOwner) {
Object.assign(restartIntent, processLocalIntent);
}
markRestartDraining();
if (authorized) {
markGatewaySigusr1RestartHandled();
@@ -1088,7 +1134,7 @@ export async function runGatewayLoop(params: {
}
const errMsg = formatErrorMessage(err);
const errStack = err instanceof Error && err.stack ? `\n${err.stack}` : "";
await writeStabilityBundle("gateway.restart_startup_failed", err);
writeStabilityBundle("gateway.restart_startup_failed", err);
gatewayLog.error(
`gateway startup failed: ${errMsg}. ` +
`Process will stay alive; fix the issue and restart.${errStack}`,
@@ -48,6 +48,7 @@ const startManagedServiceUpdateHandoffMock = vi.fn<
command: "openclaw update --yes --timeout 1800",
logPath: "/tmp/openclaw-update-run-handoff/handoff.log",
handoffId: "handoff-1",
installRoot: "/tmp/openclaw",
}));
const scheduleGatewaySigusr1RestartMock = vi.fn(() => ({ scheduled: true }));
const logGatewayInfoMock = vi.fn();
@@ -87,8 +88,8 @@ vi.mock("../../infra/restart-sentinel.js", async () => {
};
});
vi.mock("../../infra/restart.js", () => ({
resolveGatewayRestartDeferralTimeoutMs: () => 300_000,
vi.mock("../../infra/restart.js", async () => ({
...(await vi.importActual<typeof import("../../infra/restart.js")>("../../infra/restart.js")),
scheduleGatewaySigusr1Restart: scheduleGatewaySigusr1RestartMock,
}));
+65 -29
View File
@@ -55,13 +55,14 @@ type ManagedServiceUpdateHandoffResult = Awaited<
>
>;
const startManagedServiceUpdateHandoffMock = vi.fn<
(params?: { handoffId?: string }) => Promise<ManagedServiceUpdateHandoffResult>
(params?: { handoffId?: string; root?: string }) => Promise<ManagedServiceUpdateHandoffResult>
>(async (params) => ({
status: "started",
pid: 12345,
command: "openclaw update --yes --timeout 1800",
logPath: "/tmp/openclaw-update-run-handoff/handoff.log",
handoffId: params?.handoffId,
handoffId: params?.handoffId ?? "handoff-default",
installRoot: params?.root ?? "/tmp/openclaw",
}));
const scheduleGatewaySigusr1RestartMock = vi.fn(() => ({ scheduled: true }));
@@ -88,9 +89,7 @@ vi.mock("../../config/config.js", () => ({
readConfigFileSnapshot: readConfigFileSnapshotMock,
}));
vi.mock("../../config/commands.flags.js", () => ({
isRestartEnabled: isRestartEnabledMock,
}));
vi.mock("../../config/commands.flags.js", () => ({ isRestartEnabled: isRestartEnabledMock }));
vi.mock("../../config/sessions.js", () => ({
extractDeliveryInfo: (sessionKey: string | undefined) => {
@@ -124,19 +123,12 @@ vi.mock("../../infra/restart-sentinel.js", async () => {
};
});
vi.mock("../../infra/restart.js", () => ({
resolveGatewayRestartDeferralTimeoutMs: (timeoutMs: unknown) => {
if (typeof timeoutMs !== "number" || !Number.isFinite(timeoutMs)) {
return 300_000;
}
return timeoutMs <= 0 ? undefined : Math.floor(timeoutMs);
},
vi.mock("../../infra/restart.js", async () => ({
...(await vi.importActual<typeof import("../../infra/restart.js")>("../../infra/restart.js")),
scheduleGatewaySigusr1Restart: scheduleGatewaySigusr1RestartMock,
}));
vi.mock("../../infra/package-json.js", () => ({
readPackageVersion: readPackageVersionMock,
}));
vi.mock("../../infra/package-json.js", () => ({ readPackageVersion: readPackageVersionMock }));
vi.mock("../../version.js", () => ({
get VERSION() {
@@ -291,12 +283,13 @@ beforeEach(() => {
recordLatestUpdateRestartSentinelMock.mockClear();
startManagedServiceUpdateHandoffMock.mockClear();
startManagedServiceUpdateHandoffMock.mockImplementation(
async (params?: { handoffId?: string }) => ({
async (params?: { handoffId?: string; root?: string }) => ({
status: "started" as const,
pid: 12345,
command: "openclaw update --yes --timeout 1800",
logPath: "/tmp/openclaw-update-run-handoff/handoff.log",
handoffId: params?.handoffId,
handoffId: params?.handoffId ?? "handoff-default",
installRoot: params?.root ?? "/tmp/openclaw",
}),
);
scheduleGatewaySigusr1RestartMock.mockClear();
@@ -550,8 +543,21 @@ describe("update.run restart scheduling", () => {
const [restartParams] = firstMockCall(
scheduleGatewaySigusr1RestartMock,
"gateway restart schedule",
) as [{ delayMs?: number; reason?: string; skipCooldown?: boolean; skipDeferral?: boolean }];
) as [
{
delayMs?: number;
reason?: string;
successorOwner?: unknown;
skipCooldown?: boolean;
skipDeferral?: boolean;
},
];
expect(restartParams?.reason).toBe("update.run");
expect(restartParams?.successorOwner).toEqual({
kind: "managed-update-handoff",
handoffId: handoffParams.handoffId,
installRoot: "/tmp/openclaw-global",
});
expect(restartParams?.skipCooldown).toBe(true);
expect(restartParams?.skipDeferral).toBe(true);
expect(payload?.ok).toBe(true);
@@ -673,6 +679,7 @@ describe("update.run restart scheduling", () => {
expect(startManagedServiceUpdateHandoffMock).toHaveBeenCalledWith(
expect.objectContaining({
supervisor: "systemd",
restartDrainTimeoutMs: 300_000,
restartDelayMs: 2000,
}),
);
@@ -680,12 +687,34 @@ describe("update.run restart scheduling", () => {
expect.objectContaining({
delayMs: 2000,
reason: "update.run",
successorOwner: {
kind: "managed-update-handoff",
handoffId: expect.any(String),
installRoot: "/tmp/openclaw-global",
},
skipCooldown: true,
skipDeferral: true,
}),
);
});
it.each(["launchd", "systemd"] as const)(
"normalizes overflow-sized %s handoff and restart delays together",
async (supervisor) => {
detectRespawnSupervisorMock.mockReturnValueOnce(supervisor);
mockGlobalInstallSurface();
await invokeUpdateRun({ restartDelayMs: 2_147_153_648 });
expect(startManagedServiceUpdateHandoffMock).toHaveBeenCalledWith(
expect.objectContaining({ supervisor, restartDelayMs: 60_000 }),
);
expect(scheduleGatewaySigusr1RestartMock).toHaveBeenCalledWith(
expect.objectContaining({ delayMs: 60_000 }),
);
},
);
it("starts managed package handoff when the gateway cwd is unavailable", async () => {
detectRespawnSupervisorMock.mockReturnValueOnce("launchd");
mockGlobalInstallSurface();
@@ -902,27 +931,29 @@ describe("update.run restart scheduling", () => {
expect(payload?.handoff?.status).toBe("started");
});
it("does not hand off systemd-supervised git/dev updates from generic systemd markers alone", async () => {
it("hands marker-only systemd git/dev updates to the helper for exact ownership verification", async () => {
detectRespawnSupervisorMock.mockReturnValueOnce("systemd");
mockGitInstallSurface("/tmp/openclaw-git");
const payload = await withProcessEnv(
{
OPENCLAW_SYSTEMD_UNIT: undefined,
INVOCATION_ID: "8a77e69a8f604bf0b7984879b9f17a7c",
OPENCLAW_SERVICE_MARKER: "openclaw",
OPENCLAW_SERVICE_KIND: "gateway",
},
() => captureUpdateRunPayload(),
);
expect(runGatewayUpdateMock).not.toHaveBeenCalled();
expect(startManagedServiceUpdateHandoffMock).not.toHaveBeenCalled();
expect(scheduleGatewaySigusr1RestartMock).not.toHaveBeenCalled();
expect(payload?.ok).toBe(false);
expect(payload?.restart).toBeNull();
expect(startManagedServiceUpdateHandoffMock).toHaveBeenCalledWith(
expect.objectContaining({ root: "/tmp/openclaw-git", supervisor: "systemd" }),
);
expect(scheduleGatewaySigusr1RestartMock).toHaveBeenCalledOnce();
expect(payload?.ok).toBe(true);
expect(payload?.result?.status).toBe("skipped");
expect(payload?.result?.reason).toBe("managed-service-handoff-unavailable");
expect(payload?.result?.reason).toBe("managed-service-handoff-started");
expect(payload?.result?.mode).toBe("git");
expect(payload?.handoff?.status).toBe("unavailable");
expect(payload?.handoff?.status).toBe("started");
});
it("returns a safe command when package updates cannot be handed off", async () => {
@@ -961,11 +992,16 @@ describe("update.run restart scheduling", () => {
expect(payload?.result?.mode).toBe("npm");
});
it("delegates update.run without mutating or restarting under external supervision", async () => {
it("keeps external update supervision authoritative even with native systemd markers", async () => {
mockGlobalInstallSurface();
detectRespawnSupervisorMock.mockReturnValue("systemd");
const payload = await withProcessEnv({ OPENCLAW_SUPERVISOR_MODE: "external" }, () =>
captureUpdateRunPayload(),
const payload = await withProcessEnv(
{
OPENCLAW_SUPERVISOR_MODE: "external",
OPENCLAW_SYSTEMD_UNIT: "openclaw-gateway.service",
},
() => captureUpdateRunPayload(),
);
expect(runGatewayUpdateMock).not.toHaveBeenCalled();
+19 -54
View File
@@ -13,7 +13,6 @@ import { isRestartEnabled } from "../../config/commands.flags.js";
import { readConfigFileSnapshot } from "../../config/config.js";
import { extractDeliveryInfo } from "../../config/sessions.js";
import type { OpenClawConfig } from "../../config/types.openclaw.js";
import { GATEWAY_SERVICE_KIND, GATEWAY_SERVICE_MARKER } from "../../daemon/constants.js";
import {
EXTERNAL_SUPERVISOR_UPDATE_REQUIRED_REASON,
isGatewayExternallySupervised,
@@ -21,6 +20,7 @@ import {
import { readPackageVersion } from "../../infra/package-json.js";
import { type RestartSentinelPayload, writeRestartSentinel } from "../../infra/restart-sentinel.js";
import {
normalizeGatewayRestartDelayMs,
resolveGatewayRestartDeferralTimeoutMs,
scheduleGatewaySigusr1Restart,
} from "../../infra/restart.js";
@@ -114,47 +114,6 @@ async function readPreUpdateConfigForPostCoreFinalize(): Promise<
};
}
function resolveManagedServiceHandoffRestartDelayMs(
restartDelayMs: number | undefined,
supervisor: ReturnType<typeof detectRespawnSupervisor>,
): number {
const resolvedDelayMs = restartDelayMs ?? MANAGED_HANDOFF_RESTART_DELAY_MS;
if (supervisor !== "systemd") {
return resolvedDelayMs;
}
// systemd needs a short grace period after the handoff process starts before
// the gateway exits, otherwise the service can restart before handoff state is durable.
return Math.max(resolvedDelayMs, MANAGED_HANDOFF_RESTART_DELAY_MS);
}
function hasManagedServiceHandoffContext(
env: NodeJS.ProcessEnv,
supervisor: ReturnType<typeof detectRespawnSupervisor>,
): boolean {
if (supervisor === "launchd") {
return Boolean(
env.OPENCLAW_LAUNCHD_LABEL?.trim() ||
env.LAUNCH_JOB_LABEL?.trim() ||
env.LAUNCH_JOB_NAME?.trim() ||
env.XPC_SERVICE_NAME?.trim(),
);
}
if (supervisor === "systemd") {
// Ambient systemd markers only prove that a service manager started this
// process. The detached CLI needs the durable unit name to stop the same
// gateway before mutating the install root.
return Boolean(env.OPENCLAW_SYSTEMD_UNIT?.trim());
}
if (supervisor === "schtasks") {
return Boolean(
env.OPENCLAW_WINDOWS_TASK_NAME?.trim() ||
(env.OPENCLAW_SERVICE_MARKER?.trim() === GATEWAY_SERVICE_MARKER &&
env.OPENCLAW_SERVICE_KIND?.trim() === GATEWAY_SERVICE_KIND),
);
}
return false;
}
export const updateHandlers: GatewayRequestHandlers = {
"update.status": async ({ params, respond, context }) => {
if (!assertValidParams(params, validateUpdateStatusParams, "update.status", respond)) {
@@ -269,8 +228,9 @@ export const updateHandlers: GatewayRequestHandlers = {
threadId: requestedThreadId,
note,
continuationMessage,
restartDelayMs,
restartDelayMs: requestedRestartDelayMs,
} = parseRestartRequestParams(params);
const restartDelayMs = normalizeGatewayRestartDelayMs(requestedRestartDelayMs);
const { deliveryContext: sessionDeliveryContext, threadId: sessionThreadId } =
extractDeliveryInfo(sessionKey);
const deliveryContext = requestedDeliveryContext ?? sessionDeliveryContext;
@@ -312,16 +272,15 @@ export const updateHandlers: GatewayRequestHandlers = {
installKind: status.installKind,
git: status.git,
}).channel;
const supervisor = detectRespawnSupervisor(process.env, process.platform);
const hasHandoffContext = supervisor
? hasManagedServiceHandoffContext(process.env, supervisor)
: false;
const supervisor = detectRespawnSupervisor(process.env, process.platform, {
includeLinuxOpenClawGatewayServiceMarker: true,
});
const requiresManagedServiceHandoff =
installSurface.kind === "global" || (installSurface.kind === "git" && supervisor !== null);
const managedGitPreflightFailure =
installSurface.kind === "git" &&
effectiveChannel === "dev" &&
hasHandoffContext &&
supervisor &&
!isGatewayExternallySupervised()
? await runGatewayUpdatePreflight(installRoot, timeoutMs, adoptedDevTarget)
: undefined;
@@ -386,15 +345,16 @@ export const updateHandlers: GatewayRequestHandlers = {
...(handoffChannel ? { channel: handoffChannel } : {}),
...(adoptedPackageTargetVersion ? { tag: adoptedPackageTargetVersion } : {}),
});
if (supervisor && hasHandoffContext) {
if (supervisor) {
try {
const beforeVersion = await readPackageVersion(installRoot);
const startedAt = Date.now();
const handoffId = randomUUID();
const managedRestartDelayMs = resolveManagedServiceHandoffRestartDelayMs(
restartDelayMs,
supervisor,
);
// systemd needs startup grace before the Gateway exits and its state becomes durable.
const managedRestartDelayMs =
supervisor === "systemd"
? Math.max(restartDelayMs, MANAGED_HANDOFF_RESTART_DELAY_MS)
: restartDelayMs;
sentinelMeta.handoffId = handoffId;
sentinelMeta.root = resolveUpdateInstallRoot(installRoot);
// Managed services update from a detached helper so the running
@@ -416,7 +376,7 @@ export const updateHandlers: GatewayRequestHandlers = {
sentinelMeta.handoffId = started.handoffId ?? handoffId;
// The owner pairs helper creation with parent exit before any
// persistence can fail. Joiners leave both to the active owner.
if (ownsManagedServiceHandoff) {
if (started.status === "started") {
handoff = {
status: "started",
...(started.pid ? { pid: started.pid } : {}),
@@ -425,6 +385,11 @@ export const updateHandlers: GatewayRequestHandlers = {
managedHandoffRestart = scheduleGatewaySigusr1Restart({
delayMs: managedRestartDelayMs,
reason: "update.run",
successorOwner: {
kind: "managed-update-handoff",
handoffId: started.handoffId,
installRoot: started.installRoot,
},
skipDeferral: true,
skipCooldown: true,
audit: {
+76 -8
View File
@@ -9,8 +9,14 @@ import {
tryBeginGatewayRootWorkAdmission,
} from "../process/gateway-work-admission.js";
type RestartModule = typeof import("./restart.js");
const managedSuccessorOwner = {
kind: "managed-update-handoff",
handoffId: "managed-handoff",
installRoot: "/canonical/install",
} as const;
let consumeGatewaySigusr1RestartAuthorization: RestartModule["consumeGatewaySigusr1RestartAuthorization"];
let consumeGatewaySigusr1RestartIntent: RestartModule["consumeGatewaySigusr1RestartIntent"];
let deferGatewayRestartUntilIdle: RestartModule["deferGatewayRestartUntilIdle"];
let isGatewaySigusr1RestartExternallyAllowed: RestartModule["isGatewaySigusr1RestartExternallyAllowed"];
let markGatewaySigusr1RestartHandled: RestartModule["markGatewaySigusr1RestartHandled"];
@@ -114,6 +120,7 @@ describe("infra runtime", () => {
);
({
consumeGatewaySigusr1RestartAuthorization,
consumeGatewaySigusr1RestartIntent,
deferGatewayRestartUntilIdle,
isGatewaySigusr1RestartExternallyAllowed,
markGatewaySigusr1RestartHandled,
@@ -445,17 +452,16 @@ describe("infra runtime", () => {
const beforeEmit = vi.fn(async () => {
await preparationBlocked;
});
let resolveSignal: () => void = () => {};
const signalEmitted = new Promise<void>((resolve) => {
resolveSignal = resolve;
});
const handler = () => resolveSignal();
const staleEmitRestart = vi.fn(() => ({ status: "failed" as const }));
const emitSpy = vi.spyOn(process, "emit");
const handler = () => {};
process.on("SIGUSR1", handler);
try {
scheduleGatewaySigusr1Restart({
delayMs: 0,
reason: "config.patch",
emitHooks: { beforeEmit },
sessionKey: "agent:main:session-A",
emitHooks: { beforeEmit, emitRestart: staleEmitRestart },
});
await vi.advanceTimersByTimeAsync(0);
await Promise.resolve();
@@ -464,14 +470,71 @@ describe("infra runtime", () => {
const update = scheduleGatewaySigusr1Restart({
delayMs: 0,
reason: "update.auto",
successorOwner: managedSuccessorOwner,
skipDeferral: true,
});
expect(update.coalesced).toBe(true);
releasePreparation();
await signalEmitted;
await vi.advanceTimersByTimeAsync(0);
expect(staleEmitRestart).not.toHaveBeenCalled();
expect(emitSpy).toHaveBeenCalledWith("SIGUSR1");
expect(peekGatewaySigusr1RestartReason()).toBe("update.auto");
expect(consumeGatewaySigusr1RestartIntent()).toEqual({
reason: "update.auto",
successorOwner: managedSuccessorOwner,
});
} finally {
process.removeListener("SIGUSR1", handler);
}
});
it("retains managed successor ownership when an ordinary restart pulls the timer earlier", async () => {
const handler = () => {};
process.on("SIGUSR1", handler);
try {
scheduleGatewaySigusr1Restart({
delayMs: 1_000,
reason: "update.auto",
successorOwner: managedSuccessorOwner,
});
scheduleGatewaySigusr1Restart({ delayMs: 0, reason: "config.patch" });
await vi.advanceTimersByTimeAsync(0);
expect(peekGatewaySigusr1RestartReason()).toBe("update.auto");
expect(consumeGatewaySigusr1RestartIntent()).toEqual({
reason: "update.auto",
successorOwner: managedSuccessorOwner,
});
} finally {
process.removeListener("SIGUSR1", handler);
}
});
it("replaces stale managed successor ownership when its replacement coalesces", async () => {
const replacementOwner = { ...managedSuccessorOwner, handoffId: "replacement-handoff" };
const handler = () => {};
process.on("SIGUSR1", handler);
try {
scheduleGatewaySigusr1Restart({
delayMs: 1_000,
reason: "update.auto",
successorOwner: managedSuccessorOwner,
});
const replacement = scheduleGatewaySigusr1Restart({
delayMs: 1_000,
reason: "update.auto",
successorOwner: replacementOwner,
});
expect(replacement.coalesced).toBe(true);
await vi.advanceTimersByTimeAsync(1_000);
expect(consumeGatewaySigusr1RestartIntent()).toEqual({
reason: "update.auto",
successorOwner: replacementOwner,
});
} finally {
process.removeListener("SIGUSR1", handler);
}
@@ -651,7 +714,7 @@ describe("infra runtime", () => {
it("rejects coalesced emit hooks from a different session while preparation is in flight (#86742)", async () => {
// Pins the CWE-200 in-flight preparation race: pendingRestartSessionKey
// must stay alive through await beforeEmit(), otherwise a coalesced
// different-session caller slips past updatePendingRestartEmitHooks
// different-session caller slips past canReplacePendingRestartEmitHooks
// and chains its own hooks while preparation runs.
let releaseSessionAPrep: () => void = () => {};
const sessionAPrepBlocked = new Promise<void>((resolve) => {
@@ -1219,6 +1282,7 @@ describe("infra runtime", () => {
const forced = scheduleGatewaySigusr1Restart({
delayMs: 0,
reason: "update.run",
successorOwner: managedSuccessorOwner,
skipDeferral: true,
});
@@ -1226,6 +1290,10 @@ describe("infra runtime", () => {
expect(emitSpy).toHaveBeenCalledWith("SIGUSR1");
expect(staleBeforeEmit).not.toHaveBeenCalled();
expect(peekGatewaySigusr1RestartReason()).toBe("update.run");
expect(consumeGatewaySigusr1RestartIntent()).toEqual({
reason: "update.run",
successorOwner: managedSuccessorOwner,
});
} finally {
process.removeListener("SIGUSR1", handler);
}
-41
View File
@@ -354,22 +354,6 @@ describe("respawnGatewayProcessForUpdate", () => {
expect(spawnMock).not.toHaveBeenCalled();
});
it("actively schedules launchd update relaunch before exiting", () => {
clearSupervisorHints();
setPlatform("darwin");
process.env.OPENCLAW_LAUNCHD_LABEL = "ai.openclaw.gateway";
process.env.OPENCLAW_NO_RESPAWN = "1";
const result = respawnGatewayProcessForUpdate();
expect(result.mode).toBe("supervised");
expect(result.handoffSpawned).toBeInstanceOf(Promise);
expect(scheduleLaunchdHandoffMock).toHaveBeenCalledWith({
mode: "start-after-exit",
waitForPid: process.pid,
});
});
it("allows detached respawn on unmanaged Windows during updates", () => {
clearSupervisorHints();
setPlatform("win32");
@@ -397,18 +381,6 @@ describe("respawnGatewayProcessForUpdate", () => {
);
});
it("delegates update restarts to external supervision without spawning", () => {
clearSupervisorHints();
setPlatform("linux");
process.env.OPENCLAW_SUPERVISOR_MODE = "external";
const result = respawnGatewayProcessForUpdate();
expect(result).toEqual({ mode: "supervised" });
expect(spawnMock).not.toHaveBeenCalled();
expect(triggerOpenClawRestartMock).not.toHaveBeenCalled();
});
it("rewrites a pnpm-versioned OpenClaw entry before detached update respawn", () => {
clearSupervisorHints();
setPlatform("linux");
@@ -497,19 +469,6 @@ describe("respawnGatewayProcessForUpdate", () => {
expect(onCallOrder).toBeLessThan(unrefCallOrder);
});
it("exits to a managed supervisor for updates even when respawn is disabled", () => {
clearSupervisorHints();
setPlatform("linux");
process.env.OPENCLAW_NO_RESPAWN = "1";
process.env.OPENCLAW_SERVICE_MARKER = "openclaw";
process.env.OPENCLAW_SERVICE_KIND = "gateway";
const result = respawnGatewayProcessForUpdate();
expect(result).toEqual({ mode: "supervised" });
expect(spawnMock).not.toHaveBeenCalled();
});
it("returns failed when update detached respawn throws", () => {
delete process.env.OPENCLAW_NO_RESPAWN;
clearSupervisorHints();
+38 -87
View File
@@ -7,16 +7,16 @@ import { formatErrorMessage } from "./errors.js";
import { triggerOpenClawRestart } from "./restart.js";
import { detectGatewayRespawnSupervisor } from "./supervisor-markers.js";
type RespawnMode = "spawned" | "supervised" | "disabled" | "failed";
type GatewayRespawnResult = {
mode: RespawnMode;
pid?: number;
mode: "supervised" | "disabled" | "failed";
detail?: string;
handoffSpawned?: Promise<boolean>;
};
type GatewayUpdateRespawnResult = GatewayRespawnResult & {
type GatewayUpdateRespawnResult = {
mode: "spawned" | "disabled" | "failed";
pid?: number;
detail?: string;
child?: ChildProcess;
};
type GatewayRespawnOptions = {
@@ -35,39 +35,6 @@ function rewritePnpmVersionedOpenClawEntryPath(entryPath: string): string {
);
}
function spawnDetachedGatewayProcess(opts: GatewayRespawnOptions = {}): {
child: ChildProcess;
pid?: number;
} {
const [entryArg, ...entryArgs] = process.argv.slice(1);
const args = [
...process.execArgv,
...(entryArg ? [rewritePnpmVersionedOpenClawEntryPath(entryArg)] : []),
...entryArgs,
];
const child = spawn(process.execPath, args, {
env: opts.env ? { ...process.env, ...opts.env } : process.env,
detached: true,
stdio: "inherit",
});
// Detached spawn failures can arrive asynchronously after spawn() returns.
// Keep this listener before unref() so the parent does not crash during handoff.
child.on("error", () => {});
child.unref();
return { child, pid: child.pid ?? undefined };
}
function scheduleLaunchdRestartAfterExit(): GatewayRespawnResult {
const handoff = scheduleDetachedLaunchdRestartHandoff({
mode: "start-after-exit",
waitForPid: process.pid,
});
if (!handoff.ok) {
return { mode: "failed", detail: handoff.error };
}
return { mode: "supervised", handoffSpawned: handoff.value };
}
/**
* Attempt to restart this process with a fresh PID.
* - supervised environments (launchd/systemd/schtasks): caller should exit and let supervisor restart
@@ -84,7 +51,13 @@ export function restartGatewayProcessWithFreshPid(
const supervisor = detectGatewayRespawnSupervisor(process.env);
if (supervisor) {
if (supervisor === "launchd") {
return scheduleLaunchdRestartAfterExit();
const handoff = scheduleDetachedLaunchdRestartHandoff({
mode: "start-after-exit",
waitForPid: process.pid,
});
return handoff.ok
? { mode: "supervised", handoffSpawned: handoff.value }
: { mode: "failed", detail: handoff.error };
}
if (supervisor === "schtasks") {
const restart = triggerOpenClawRestart();
@@ -97,68 +70,46 @@ export function restartGatewayProcessWithFreshPid(
}
return { mode: "supervised" };
}
if (process.platform === "win32") {
// Detached respawn is unsafe on Windows without an identified Scheduled Task:
// the child becomes orphaned if the original process exits.
return {
mode: "disabled",
detail: "win32: detached respawn unsupported without Scheduled Task markers",
};
}
if (isContainerEnvironment()) {
return {
mode: "disabled",
detail: "container: use in-process restart to keep PID 1 alive",
};
}
return {
mode: "disabled",
detail: "unmanaged: use in-process restart to keep custom supervisor PID tracking stable",
};
// Unmanaged Windows or containers cannot safely surrender their tracked process.
const detail =
process.platform === "win32"
? "win32: detached respawn unsupported without Scheduled Task markers"
: isContainerEnvironment()
? "container: use in-process restart to keep PID 1 alive"
: "unmanaged: use in-process restart to keep custom supervisor PID tracking stable";
return { mode: "disabled", detail };
}
/**
* Update restarts must replace the OS process so the new code runs from a
* fresh module graph after package files have changed on disk.
*
* Unlike the generic restart path, update mode allows detached respawn on
* unmanaged Windows installs because there is no safe in-process fallback once
* the installed package contents have been replaced.
* The caller resolves supervisor ownership first; this path is only for an
* unmanaged process whose installed package contents have been replaced.
*/
export function respawnGatewayProcessForUpdate(
opts: GatewayRespawnOptions = {},
): GatewayUpdateRespawnResult {
const supervisor = detectGatewayRespawnSupervisor(process.env, process.platform, {
includeLinuxOpenClawGatewayServiceMarker: true,
});
if (supervisor) {
// Managed update handoffs require the original PID to exit before the
// detached helper can mutate the install, even when respawn is disabled.
if (supervisor === "launchd") {
return scheduleLaunchdRestartAfterExit();
}
if (supervisor === "schtasks") {
const restart = triggerOpenClawRestart();
if (!restart.ok) {
return {
mode: "failed",
detail: restart.detail ?? `${restart.method} restart failed`,
};
}
}
return { mode: "supervised" };
}
if (isTruthyEnvValue(process.env.OPENCLAW_NO_RESPAWN)) {
return { mode: "disabled", detail: "OPENCLAW_NO_RESPAWN" };
}
try {
const { child, pid } = spawnDetachedGatewayProcess(opts);
return { mode: "spawned", pid, child };
const [entryArg, ...entryArgs] = process.argv.slice(1);
const args = [
...process.execArgv,
...(entryArg ? [rewritePnpmVersionedOpenClawEntryPath(entryArg)] : []),
...entryArgs,
];
const child = spawn(process.execPath, args, {
env: opts.env ? { ...process.env, ...opts.env } : process.env,
detached: true,
stdio: "inherit",
});
// Register before unref: late detached-spawn failures must not crash the parent.
child.on("error", () => {});
child.unref();
return { mode: "spawned", pid: child.pid ?? undefined, child };
} catch (err) {
return {
mode: "failed",
detail: formatErrorMessage(err),
};
return { mode: "failed", detail: formatErrorMessage(err) };
}
}
+10 -2
View File
@@ -120,7 +120,7 @@ describe("gateway restart intent", () => {
expect(readIntentRow(env)).toBeUndefined();
});
it("round-trips restart reason, force, and wait options", () => {
it("round-trips restart options without persisting process-local successor identity", () => {
const env = createIntentEnv();
expect(
@@ -128,7 +128,15 @@ describe("gateway restart intent", () => {
env,
targetPid: process.pid,
reason: "gateway.restart",
intent: { force: true, waitMs: 12_345 },
intent: {
force: true,
waitMs: 12_345,
successorOwner: {
kind: "managed-update-handoff",
handoffId: "private-handoff",
installRoot: "/private/install",
},
},
}),
).toBe(true);
+7 -5
View File
@@ -32,12 +32,14 @@ export type GatewayRestartIntent = {
reason?: string;
force?: boolean;
waitMs?: number;
// Process-local only: persisted restart requests cannot delegate successor ownership.
successorOwner?: {
kind: "managed-update-handoff";
handoffId: string;
installRoot: string;
};
};
function normalizeRestartIntentPid(pid: number | undefined): number | null {
return asPositiveSafeInteger(pid) ?? null;
}
export function normalizeRestartIntentReason(reason: string | undefined): string | undefined {
const normalized = reason?.trim();
return normalized ? truncateUtf16Safe(normalized, 200) : undefined;
@@ -49,7 +51,7 @@ export function writeGatewayRestartIntentSync(opts: {
intent?: GatewayRestartIntent;
reason?: string;
}): boolean {
const targetPid = normalizeRestartIntentPid(opts.targetPid);
const targetPid = asPositiveSafeInteger(opts.targetPid) ?? null;
if (targetPid === null) {
return false;
}
+37 -1
View File
@@ -36,7 +36,12 @@ vi.mock("../config/paths.js", () => ({
const { cleanStaleGatewayProcessesSync, findGatewayPidsOnPortSync } =
await import("./restart-stale-pids.js");
const { triggerOpenClawRestart } = await import("./restart.js");
const {
normalizeGatewayRestartDelayMs,
resetGatewayRestartStateForInProcessRestart,
scheduleGatewaySigusr1Restart,
triggerOpenClawRestart,
} = await import("./restart.js");
const envSnapshot = captureFullEnv();
@@ -262,3 +267,34 @@ describe("triggerOpenClawRestart", () => {
);
});
});
describe("gateway restart delay normalization", () => {
it.each([
{ requested: undefined, effective: 2000 },
{ requested: Number.NaN, effective: 2000 },
{ requested: Number.POSITIVE_INFINITY, effective: 2000 },
{ requested: -1, effective: 0 },
{ requested: 1500.8, effective: 1500 },
{ requested: 2_147_153_648, effective: 60_000 },
])("normalizes $requested to $effective ms", ({ requested, effective }) => {
expect(normalizeGatewayRestartDelayMs(requested)).toBe(effective);
});
it("does not emit an overflow-sized restart before its effective 60-second delay", async () => {
vi.useFakeTimers();
const killSpy = vi.spyOn(process, "kill").mockImplementation(() => true);
try {
const restart = scheduleGatewaySigusr1Restart({
delayMs: 2_147_153_648,
skipCooldown: true,
});
expect(restart.delayMs).toBe(60_000);
await vi.advanceTimersByTimeAsync(59_999);
expect(killSpy).not.toHaveBeenCalled();
} finally {
resetGatewayRestartStateForInProcessRestart();
vi.useRealTimers();
}
});
});
+119 -140
View File
@@ -45,6 +45,7 @@ let lastRestartEmittedAt = 0;
let pendingRestartTimer: ReturnType<typeof setTimeout> | null = null;
let pendingRestartDueAt = 0;
let pendingRestartReason: string | undefined;
let pendingRestartSuccessorOwner: GatewayRestartIntent["successorOwner"];
let pendingRestartEmitHooks: RestartEmitHooks | undefined;
let pendingRestartSessionKey: string | undefined;
let pendingRestartSkipDeferral = false;
@@ -63,12 +64,11 @@ function hasUnconsumedRestartSignal(): boolean {
}
function clearPendingScheduledRestart(): void {
if (pendingRestartTimer) {
clearTimeout(pendingRestartTimer);
}
clearTimeout(pendingRestartTimer ?? undefined);
pendingRestartTimer = null;
pendingRestartDueAt = 0;
pendingRestartReason = undefined;
pendingRestartSuccessorOwner = undefined;
pendingRestartEmitHooks = undefined;
pendingRestartSessionKey = undefined;
pendingRestartSkipDeferral = false;
@@ -179,19 +179,12 @@ function formatRestartAudit(audit: RestartAuditInfo | undefined): string {
const clientIp =
typeof audit?.clientIp === "string" && audit.clientIp.trim() ? audit.clientIp.trim() : null;
const changed = summarizeChangedPaths(audit?.changedPaths);
const fields = [];
if (actor) {
fields.push(`actor=${actor}`);
}
if (deviceId) {
fields.push(`device=${deviceId}`);
}
if (clientIp) {
fields.push(`ip=${clientIp}`);
}
if (changed) {
fields.push(`changedPaths=${changed}`);
}
const fields = [
actor && `actor=${actor}`,
deviceId && `device=${deviceId}`,
clientIp && `ip=${clientIp}`,
changed && `changedPaths=${changed}`,
].filter(Boolean);
return fields.length > 0 ? fields.join(" ") : "actor=<unknown>";
}
@@ -297,10 +290,7 @@ export function requestGatewayRestartWithSignalAdmission(
}
function resetSigusr1AuthorizationIfExpired(now = Date.now()) {
if (sigusr1AuthorizedCount <= 0) {
return;
}
if (now <= sigusr1AuthorizedUntil) {
if (sigusr1AuthorizedCount <= 0 || now <= sigusr1AuthorizedUntil) {
return;
}
sigusr1AuthorizedCount = 0;
@@ -315,9 +305,8 @@ export function isGatewaySigusr1RestartExternallyAllowed() {
return sigusr1ExternalAllowed;
}
function authorizeGatewaySigusr1Restart(delayMs = 0) {
const delay = Math.max(0, Math.floor(delayMs));
const expiresAt = Date.now() + delay + SIGUSR1_AUTH_GRACE_MS;
function authorizeGatewaySigusr1Restart() {
const expiresAt = Date.now() + SIGUSR1_AUTH_GRACE_MS;
sigusr1AuthorizedCount += 1;
if (expiresAt > sigusr1AuthorizedUntil) {
sigusr1AuthorizedUntil = expiresAt;
@@ -406,44 +395,22 @@ type GatewayRestartEmitResult =
| { status: "coalesced" }
| { status: "failed" };
export function resolveGatewayRestartDeferralTimeoutMs(): number;
export function resolveGatewayRestartDeferralTimeoutMs(timeoutMs: unknown): number | undefined;
export function resolveGatewayRestartDeferralTimeoutMs(timeoutMs?: unknown): number | undefined {
if (typeof timeoutMs !== "number" || !Number.isFinite(timeoutMs)) {
return DEFAULT_RESTART_DEFERRAL_TIMEOUT_MS;
}
if (timeoutMs <= 0) {
return undefined;
}
return Math.floor(timeoutMs);
return timeoutMs > 0 ? Math.floor(timeoutMs) : undefined;
}
function canReplacePendingRestartEmitHooks(
hooks: RestartEmitHooks | undefined,
sessionKey: string | undefined,
): boolean {
if (!hooks) {
return true;
}
return pendingRestartSessionKey === undefined || pendingRestartSessionKey === sessionKey;
}
// Returns true when the new hooks took ownership of the pending restart slot.
// Coalesced callers from a different sessionKey are rejected to prevent the
// cross-session continuation overwrite documented in #86742 (CWE-200).
function updatePendingRestartEmitHooks(
hooks: RestartEmitHooks | undefined,
sessionKey: string | undefined,
): boolean {
if (!canReplacePendingRestartEmitHooks(hooks, sessionKey)) {
return false;
}
if (!hooks) {
return false;
}
pendingRestartEmitHooks = hooks;
if (sessionKey !== undefined) {
pendingRestartSessionKey = sessionKey;
}
return true;
return (
!hooks || pendingRestartSessionKey === undefined || pendingRestartSessionKey === sessionKey
);
}
async function rejectPreparedRestartHook(hooks: RestartEmitHooks | undefined): Promise<void> {
@@ -556,24 +523,26 @@ async function emitPreparedGatewayRestartUnderAdmission(
pendingRestartEmitHooks = undefined;
}
// Slot settled and no awaits remain before emission — release ownership for
// every emission attempt, not only hookless ones, so a later session can
// claim continuation hooks for the next restart cycle.
pendingRestartSessionKey = undefined;
// Track every successfully prepared hook set (parked + caller) so non-emitted
// outcomes can roll back both the gateway-tool sentinel and reload preflight.
const preparedHooksList: RestartEmitHooks[] = [];
if (preparedParked) {
preparedHooksList.push(preparedParked);
}
const preparedHooksList: RestartEmitHooks[] = preparedParked ? [preparedParked] : [];
if (hooks && callerPrepared) {
preparedHooksList.push(hooks);
}
// With caller hooks, emission stays the caller's (or falls back to the core
// signal path if its preparation failed); parked hooks never own emission
// when a caller is present.
const emitOwner = hooks ? (callerPrepared ? hooks : undefined) : preparedParked;
const emitOwner =
hooks && callerPrepared
? hooks
: hooks || (pendingRestartSuccessorOwner && pendingRestartSessionKey === undefined)
? undefined
: preparedParked;
// Slot settled and no awaits remain before emission — release ownership for
// every emission attempt, not only hookless ones, so a later session can
// claim continuation hooks for the next restart cycle.
pendingRestartSessionKey = undefined;
if (!isCurrent()) {
await rejectPreparedRestartHooks(preparedHooksList);
@@ -586,8 +555,15 @@ async function emitPreparedGatewayRestartUnderAdmission(
? pendingRestartReason
: undefined;
const resolvedReason = preferredReason ?? reasonOverride;
const successorOwner = pendingRestartSuccessorOwner ?? intent?.successorOwner;
const resolvedIntent =
preferredReason && intent ? { ...intent, reason: preferredReason } : intent;
preferredReason || successorOwner
? {
...intent,
...(resolvedReason ? { reason: resolvedReason } : {}),
...(successorOwner ? { successorOwner } : {}),
}
: intent;
const emitResult = emitOwner?.emitRestart
? emitOwner.emitRestart(resolvedReason, resolvedIntent)
: requestGatewayRestartWithSignalAdmission(resolvedReason, resolvedIntent);
@@ -862,18 +838,11 @@ function formatSpawnDetail(result: {
return "unknown error";
}
}
const stderr = clean(result.stderr);
if (stderr) {
return stderr;
}
const stdout = clean(result.stdout);
if (stdout) {
return stdout;
}
if (typeof result.status === "number") {
return `exit ${result.status}`;
}
return "unknown error";
return (
clean(result.stderr) ||
clean(result.stdout) ||
(typeof result.status === "number" ? `exit ${result.status}` : "unknown error")
);
}
function normalizeSystemdUnit(raw?: string, profile?: string): string {
@@ -1009,6 +978,12 @@ export type ScheduledRestart = {
emitHooksQueued: boolean;
};
export function normalizeGatewayRestartDelayMs(delayMs?: number): number {
return typeof delayMs === "number" && Number.isFinite(delayMs)
? Math.min(Math.max(Math.floor(delayMs), 0), 60_000)
: 2000;
}
export function scheduleGatewaySigusr1Restart(opts?: {
delayMs?: number;
reason?: string;
@@ -1018,24 +993,35 @@ export function scheduleGatewaySigusr1Restart(opts?: {
sessionKey?: string;
skipDeferral?: boolean;
skipCooldown?: boolean;
successorOwner?: GatewayRestartIntent["successorOwner"];
}): ScheduledRestart {
const delayMsRaw =
typeof opts?.delayMs === "number" && Number.isFinite(opts.delayMs)
? Math.floor(opts.delayMs)
: 2000;
const delayMs = Math.min(Math.max(delayMsRaw, 0), 60_000);
const delayMs = normalizeGatewayRestartDelayMs(opts?.delayMs);
const reason = normalizeRestartIntentReason(opts?.reason);
const hasSigusr1Listener = process.listenerCount("SIGUSR1") > 0;
const mode = hasSigusr1Listener ? "emit" : process.platform === "win32" ? "supervisor" : "signal";
const mode: ScheduledRestart["mode"] =
process.listenerCount("SIGUSR1") > 0
? "emit"
: process.platform === "win32"
? "supervisor"
: "signal";
const nowMs = Date.now();
const skipCooldown = opts?.skipCooldown === true;
const cooldownMsApplied = skipCooldown
? 0
: Math.max(0, lastRestartEmittedAt + RESTART_COOLDOWN_MS - nowMs);
const cooldownMsApplied =
opts?.skipCooldown === true
? 0
: Math.max(0, lastRestartEmittedAt + RESTART_COOLDOWN_MS - nowMs);
const restartResultBase = {
ok: true,
pid: process.pid,
signal: "SIGUSR1" as const,
reason,
mode,
cooldownMsApplied,
};
const requestedDueAt = nowMs + delayMs + cooldownMsApplied;
const skipDeferral = opts?.skipDeferral === true;
let nextPendingEmitHooks = opts?.emitHooks;
let nextPendingSessionKey = opts?.sessionKey;
let nextPendingReason = reason;
let nextPendingSuccessorOwner = opts?.successorOwner;
if (hasUnconsumedRestartSignal()) {
if (shouldPreferRestartReason(reason, emittedRestartReason)) {
@@ -1045,18 +1031,20 @@ export function scheduleGatewaySigusr1Restart(opts?: {
emittedRestartIntent = { ...emittedRestartIntent, reason };
}
}
if (opts?.successorOwner) {
emittedRestartIntent = {
...emittedRestartIntent,
...(emittedRestartReason ? { reason: emittedRestartReason } : {}),
successorOwner: opts.successorOwner,
};
}
restartLog.warn(
`restart request coalesced (already in-flight) reason=${reason ?? "unspecified"} ${formatRestartAudit(opts?.audit)}`,
);
return {
ok: true,
pid: process.pid,
signal: "SIGUSR1",
...restartResultBase,
delayMs: 0,
reason,
mode,
coalesced: true,
cooldownMsApplied,
// SIGUSR1 already emitted; the new caller's hooks cannot run for this cycle.
emitHooksQueued: false,
};
@@ -1064,44 +1052,39 @@ export function scheduleGatewaySigusr1Restart(opts?: {
if (pendingRestartTimer || pendingRestartPreparing) {
const remainingMs = pendingRestartPreparing ? 0 : Math.max(0, pendingRestartDueAt - nowMs);
// Hookless forced restarts that own no sentinel may preserve an accepted
// pending hook; update/handoff callers rely on the default clear path.
const preservePendingHooks =
opts?.preservePendingEmitHooksOnDeferralBypass === true &&
opts?.emitHooks === undefined &&
pendingRestartSessionKey !== undefined;
if (pendingRestartPreparing && skipDeferral && activeDeferralPolls.size > 0) {
restartLog.warn(
`restart request bypassed active deferral reason=${reason ?? "unspecified"} pendingReason=${pendingRestartReason ?? "unspecified"} ${formatRestartAudit(opts?.audit)}`,
);
clearActiveDeferralPolls();
pendingRestartReason = reason;
// Hookless forced restarts that own no sentinel may preserve an accepted
// pending hook; update/handoff callers rely on the default clear path.
const preservePendingHooks =
opts?.preservePendingEmitHooksOnDeferralBypass === true &&
opts?.emitHooks === undefined &&
pendingRestartSessionKey !== undefined;
pendingRestartSuccessorOwner = opts?.successorOwner ?? pendingRestartSuccessorOwner;
if (!preservePendingHooks) {
pendingRestartEmitHooks = opts?.emitHooks;
pendingRestartSessionKey = opts?.sessionKey;
}
void emitPreparedGatewayRestart(undefined, reason);
return {
ok: true,
pid: process.pid,
signal: "SIGUSR1",
...restartResultBase,
delayMs: 0,
reason,
mode,
coalesced: false,
cooldownMsApplied,
emitHooksQueued: opts?.emitHooks !== undefined,
};
}
const shouldUpgradeToSkipDeferral = skipDeferral && !pendingRestartSkipDeferral;
const shouldPullEarlier =
!pendingRestartPreparing &&
(requestedDueAt < pendingRestartDueAt || shouldUpgradeToSkipDeferral);
(requestedDueAt < pendingRestartDueAt || (skipDeferral && !pendingRestartSkipDeferral));
if (shouldPullEarlier) {
const preservePendingHooks =
opts?.preservePendingEmitHooksOnDeferralBypass === true &&
opts?.emitHooks === undefined &&
pendingRestartSessionKey !== undefined;
if (shouldPreferRestartReason(pendingRestartReason, reason)) {
nextPendingReason = pendingRestartReason;
}
nextPendingSuccessorOwner ??= pendingRestartSuccessorOwner;
if (
!preservePendingHooks &&
!canReplacePendingRestartEmitHooks(opts?.emitHooks, opts?.sessionKey)
@@ -1109,79 +1092,75 @@ export function scheduleGatewaySigusr1Restart(opts?: {
restartLog.warn(
`restart continuation dropped: another session owns the pending restart (callerSessionKey=${opts?.sessionKey ?? "unspecified"} pendingSessionKey=${pendingRestartSessionKey ?? "unspecified"})`,
);
if (pendingRestartTimer) {
clearTimeout(pendingRestartTimer);
}
clearTimeout(pendingRestartTimer ?? undefined);
pendingRestartTimer = null;
pendingRestartDueAt = requestedDueAt;
pendingRestartReason = reason;
pendingRestartReason = nextPendingReason;
pendingRestartSuccessorOwner = nextPendingSuccessorOwner;
pendingRestartSkipDeferral = pendingRestartSkipDeferral || skipDeferral;
armPendingRestartTimer(requestedDueAt, nowMs);
return {
ok: true,
pid: process.pid,
signal: "SIGUSR1",
...restartResultBase,
delayMs: Math.max(0, requestedDueAt - nowMs),
reason,
mode,
coalesced: true,
cooldownMsApplied,
emitHooksQueued: false,
};
}
const preservedEmitHooks = preservePendingHooks ? pendingRestartEmitHooks : undefined;
const preservedSessionKey = preservePendingHooks ? pendingRestartSessionKey : undefined;
if (preservePendingHooks) {
nextPendingEmitHooks = pendingRestartEmitHooks;
nextPendingSessionKey = pendingRestartSessionKey;
}
restartLog.warn(
`restart request rescheduled earlier reason=${reason ?? "unspecified"} pendingReason=${pendingRestartReason ?? "unspecified"} oldDelayMs=${remainingMs} newDelayMs=${Math.max(0, requestedDueAt - nowMs)} ${formatRestartAudit(opts?.audit)}`,
);
clearPendingScheduledRestart();
if (preservePendingHooks) {
nextPendingEmitHooks = preservedEmitHooks;
nextPendingSessionKey = preservedSessionKey;
}
} else {
if (shouldPreferRestartReason(reason, pendingRestartReason)) {
const restartReasonPromoted = shouldPreferRestartReason(reason, pendingRestartReason);
if (restartReasonPromoted) {
pendingRestartReason = reason;
}
pendingRestartSuccessorOwner = opts?.successorOwner ?? pendingRestartSuccessorOwner;
pendingRestartSkipDeferral = pendingRestartSkipDeferral || skipDeferral;
restartLog.warn(
`restart request coalesced (already scheduled) reason=${reason ?? "unspecified"} pendingReason=${pendingRestartReason ?? "unspecified"} delayMs=${remainingMs} ${formatRestartAudit(opts?.audit)}`,
);
const emitHooksQueued = updatePendingRestartEmitHooks(opts?.emitHooks, opts?.sessionKey);
const emitHooksQueued =
opts?.emitHooks !== undefined &&
canReplacePendingRestartEmitHooks(opts.emitHooks, opts.sessionKey);
if (
emitHooksQueued ||
(!preservePendingHooks &&
opts?.emitHooks === undefined &&
(restartReasonPromoted || opts?.successorOwner !== undefined))
) {
pendingRestartEmitHooks = opts?.emitHooks;
pendingRestartSessionKey = opts?.emitHooks ? opts.sessionKey : undefined;
}
if (opts?.emitHooks && !emitHooksQueued) {
restartLog.warn(
`restart continuation dropped: another session owns the pending restart (callerSessionKey=${opts.sessionKey ?? "unspecified"} pendingSessionKey=${pendingRestartSessionKey ?? "unspecified"})`,
);
}
return {
ok: true,
pid: process.pid,
signal: "SIGUSR1",
...restartResultBase,
delayMs: remainingMs,
reason,
mode,
coalesced: true,
cooldownMsApplied,
emitHooksQueued,
};
}
}
pendingRestartDueAt = requestedDueAt;
pendingRestartReason = reason;
pendingRestartReason = nextPendingReason;
pendingRestartSuccessorOwner = nextPendingSuccessorOwner;
pendingRestartEmitHooks = nextPendingEmitHooks;
pendingRestartSessionKey = nextPendingSessionKey;
pendingRestartSkipDeferral = skipDeferral;
armPendingRestartTimer(requestedDueAt, nowMs);
return {
ok: true,
pid: process.pid,
signal: "SIGUSR1",
...restartResultBase,
delayMs: Math.max(0, requestedDueAt - nowMs),
reason,
mode,
coalesced: false,
cooldownMsApplied,
emitHooksQueued: opts?.emitHooks !== undefined,
};
}
+9
View File
@@ -106,6 +106,15 @@ describe("detectRespawnSupervisor", () => {
"win32",
),
).toBeNull();
expect(
detectRespawnSupervisor(
{
OPENCLAW_SERVICE_MARKER: "other",
OPENCLAW_SERVICE_KIND: "gateway",
},
"win32",
),
).toBeNull();
});
it("ignores service markers on non-Windows platforms and unknown platforms", () => {
+1 -3
View File
@@ -75,9 +75,7 @@ export function detectRespawnSupervisor(
if (hasAnyHint(env, SUPERVISOR_HINTS.schtasks)) {
return "schtasks";
}
const marker = env.OPENCLAW_SERVICE_MARKER?.trim();
const serviceKind = env.OPENCLAW_SERVICE_KIND?.trim();
return marker && serviceKind === "gateway" ? "schtasks" : null;
return hasOpenClawGatewayServiceMarker(env) ? "schtasks" : null;
}
return null;
}
@@ -1,24 +1,33 @@
// Managed-service handoff command tests cover immutable update target serialization.
import { EventEmitter } from "node:events";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { PassThrough } from "node:stream";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { parseDevUpdateTargetEnv, type DevUpdateTarget } from "./update-dev-target.js";
import { signalMockManagedUpdateHandoffReady } from "./update-managed-service-handoff.test-support.js";
const spawnMock = vi.hoisted(() => vi.fn());
const tempDirs = new Set<string>();
const mockedHandoffLeaseCleanups = new Set<() => void>();
const MOCK_INSTALL_ROOT = path.join(os.tmpdir(), `openclaw-handoff-command-${process.pid}`);
function createReadyChild() {
function createReadyChild(_command: string, args: string[]) {
const child = Object.assign(new EventEmitter(), {
pid: 24680,
pid: process.pid,
exitCode: null,
signalCode: null,
stdin: new PassThrough(),
stdout: new PassThrough(),
unref: vi.fn(),
});
process.nextTick(() => {
child.stdout.write("OPENCLAW_UPDATE_HANDOFF_READY\n");
signalMockManagedUpdateHandoffReady({
child,
paramsPath: args.at(-1) ?? "",
cleanups: mockedHandoffLeaseCleanups,
});
});
return child;
}
@@ -37,6 +46,9 @@ beforeEach(() => {
});
afterEach(async () => {
for (const cleanup of mockedHandoffLeaseCleanups) {
cleanup();
}
await Promise.all([...tempDirs].map((dir) => fs.rm(dir, { recursive: true, force: true })));
tempDirs.clear();
vi.resetModules();
@@ -47,18 +59,23 @@ async function startHandoffAndReadCommand(params: {
tag?: string;
devTarget?: DevUpdateTarget;
env?: NodeJS.ProcessEnv;
restartDelayMs?: number;
restartDrainTimeoutMs?: number;
}): Promise<{
command: string;
commandArgv: string[] | undefined;
parentExitTimeoutMs: number;
parentExitDeadlineAt: number;
spawnEnv: NodeJS.ProcessEnv | undefined;
}> {
const { startManagedServiceUpdateHandoff } = await import("./update-managed-service-handoff.js");
const result = await startManagedServiceUpdateHandoff({
root: "/tmp/openclaw",
restartDrainTimeoutMs: 300_000,
root: MOCK_INSTALL_ROOT,
restartDrainTimeoutMs: params.restartDrainTimeoutMs ?? 300_000,
...(params.restartDelayMs === undefined ? {} : { restartDelayMs: params.restartDelayMs }),
channel: params.channel,
...(params.tag ? { tag: params.tag } : {}),
parentPid: 12345,
parentPid: process.pid,
execPath: "/usr/local/bin/node",
argv1: "/opt/openclaw/openclaw.mjs",
meta: {},
@@ -75,22 +92,45 @@ async function startHandoffAndReadCommand(params: {
tempDirs.add(path.dirname(paramsPath));
const helperParams = JSON.parse(await fs.readFile(paramsPath, "utf-8")) as {
commandArgv?: string[];
parentExitTimeoutMs: number;
parentExitDeadlineAt: number;
};
const metaPath = path.join(path.dirname(paramsPath), "sentinel-meta.json");
const metaFile = JSON.parse(await fs.readFile(metaPath, "utf-8")) as {
meta?: { root?: string };
};
expect(metaFile.meta?.root).toBe(
await fs.realpath("/tmp/openclaw").catch(() => path.resolve("/tmp/openclaw")),
await fs.realpath(MOCK_INSTALL_ROOT).catch(() => path.resolve(MOCK_INSTALL_ROOT)),
);
return {
command: result.command,
commandArgv: helperParams.commandArgv,
parentExitTimeoutMs: helperParams.parentExitTimeoutMs,
parentExitDeadlineAt: helperParams.parentExitDeadlineAt,
spawnEnv: spawnCall?.[2]?.env,
};
}
describe("managed service update handoff command", () => {
it.each([
{ drain: 300_000, expected: 390_000 },
{ drain: Number.MAX_SAFE_INTEGER, expected: 2_147_483_647 },
])(
"serializes a bounded timer-safe restart deadline for drain $drain",
async ({ drain, expected }) => {
const startedAt = Date.now();
const result = await startHandoffAndReadCommand({
channel: "beta",
restartDelayMs: 60_000,
restartDrainTimeoutMs: drain,
});
expect(result.parentExitTimeoutMs).toBe(expected);
expect(result.parentExitDeadlineAt).toBeGreaterThanOrEqual(startedAt + expected);
expect(result.parentExitDeadlineAt).toBeLessThanOrEqual(Date.now() + expected);
},
);
it("serializes extended-stable into the detached CLI command", async () => {
const result = await startHandoffAndReadCommand({ channel: "extended-stable" });
@@ -5,20 +5,27 @@ import os from "node:os";
import path from "node:path";
import { PassThrough } from "node:stream";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { isPidAlive } from "../shared/pid-alive.js";
import { signalMockManagedUpdateHandoffReady } from "./update-managed-service-handoff.test-support.js";
const spawnMock = vi.hoisted(() => vi.fn());
const tempDirs = new Set<string>();
const handoffParents = new Map<string, import("node:child_process").ChildProcess>();
const mockedHandoffLeaseCleanups = new Set<() => void>();
function createReadyChild() {
function createReadyChild(_command: string, args: string[]) {
const child = Object.assign(new EventEmitter(), {
pid: 24680,
pid: process.pid,
exitCode: null,
signalCode: null,
stdin: new PassThrough(),
stdout: new PassThrough(),
unref: vi.fn(),
});
process.nextTick(() => {
child.stdout.write("OPENCLAW_UPDATE_HANDOFF_READY\n");
process.nextTick(signalMockManagedUpdateHandoffReady, {
child,
paramsPath: args.at(-1) ?? "",
cleanups: mockedHandoffLeaseCleanups,
});
return child;
}
@@ -31,12 +38,24 @@ vi.mock("node:child_process", async () => {
});
});
vi.mock("../daemon/systemd-scope.js", async (importOriginal) => ({
...(await importOriginal<typeof import("../daemon/systemd-scope.js")>()),
findInstalledSystemdGatewayScope: vi.fn(async () => null),
}));
beforeEach(() => {
spawnMock.mockReset();
spawnMock.mockImplementation(createReadyChild);
});
afterEach(async () => {
for (const parent of handoffParents.values()) {
parent.stdin?.end();
}
handoffParents.clear();
for (const cleanup of mockedHandoffLeaseCleanups) {
cleanup();
}
await Promise.all([...tempDirs].map((dir) => fs.rm(dir, { recursive: true, force: true })));
tempDirs.clear();
vi.resetModules();
@@ -51,20 +70,12 @@ async function pathExists(filePath: string): Promise<boolean> {
}
}
function processIsAlive(pid: number): boolean {
try {
process.kill(pid, 0);
return true;
} catch {
return false;
}
}
async function prepareConcurrentHandoffHelper(): Promise<{
tmpDir: string;
helperScriptPath: string;
baseParams: Record<string, unknown>;
}> {
const { DatabaseSync } = await import("node:sqlite");
const { startManagedServiceUpdateHandoff } = await import("./update-managed-service-handoff.js");
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-handoff-concurrent-test-"));
tempDirs.add(tmpDir);
@@ -84,12 +95,20 @@ async function prepareConcurrentHandoffHelper(): Promise<{
const [, args] = spawnMock.mock.calls.at(-1) as unknown as [string, string[]];
const helperScriptPath = args[0] ?? "";
const baseParams = JSON.parse(await fs.readFile(args[1] ?? "", "utf-8")) as Record<
string,
unknown
>;
const database = new DatabaseSync(String(baseParams.updateLeaseDatabasePath));
try {
database
.prepare("DELETE FROM managed_update_handoffs WHERE install_root = ? AND owner = ?")
.run(String(baseParams.updateLeaseKey), String(baseParams.updateLeaseOwner));
} finally {
database.close();
}
tempDirs.add(path.dirname(helperScriptPath));
return {
tmpDir,
helperScriptPath,
baseParams: JSON.parse(await fs.readFile(args[1] ?? "", "utf-8")) as Record<string, unknown>,
};
return { tmpDir, helperScriptPath, baseParams };
}
async function writeConcurrentHandoffParams(params: {
@@ -101,13 +120,27 @@ async function writeConcurrentHandoffParams(params: {
stateDatabasePath?: string;
leaseDatabasePath?: string;
}): Promise<string> {
const { spawn } =
await vi.importActual<typeof import("node:child_process")>("node:child_process");
const { getFileLockProcessStartTime } = await import("../shared/pid-alive.js");
const parent = spawn(process.execPath, ["-e", "process.stdin.resume()"], {
stdio: ["pipe", "ignore", "ignore"],
});
const parentPid = parent.pid;
const startIdentity = parentPid ? getFileLockProcessStartTime(parentPid) : null;
if (!parentPid || startIdentity === null) {
parent.kill("SIGKILL");
throw new Error("expected a parent process with a stable start identity");
}
const paramsPath = path.join(params.tmpDir, `${params.name}.json`);
handoffParents.set(paramsPath, parent);
await fs.writeFile(
paramsPath,
`${JSON.stringify(
{
...params.baseParams,
parentPid: 0,
parentPid,
parentStartIdentity: String(startIdentity),
parentExitTimeoutMs: 5_000,
handoffId: params.owner,
updateLeaseOwner: params.owner,
@@ -128,18 +161,42 @@ async function writeConcurrentHandoffParams(params: {
return paramsPath;
}
function driveHandoffProtocol(
child: import("node:child_process").ChildProcess,
paramsPath: string,
): void {
let buffered = "";
child.stdout?.on("data", (chunk: Buffer | string) => {
buffered += chunk.toString();
let newline: number;
while ((newline = buffered.indexOf("\n")) >= 0) {
const line = buffered.slice(0, newline);
buffered = buffered.slice(newline + 1);
if (line === "OPENCLAW_UPDATE_HANDOFF_READY") {
child.stdin?.write("park\n");
} else if (line === "parked") {
child.stdin?.write("commit\n");
} else if (line === "committed") {
handoffParents.get(paramsPath)?.stdin?.end();
}
}
});
}
async function runHelper(params: {
execFile: typeof import("node:child_process").execFile;
helperScriptPath: string;
paramsPath: string;
cwd: string;
env?: NodeJS.ProcessEnv;
}): Promise<{ code: number | null; stdout: string; stderr: string }> {
return await new Promise((resolve) => {
params.execFile(
const child = params.execFile(
process.execPath,
[params.helperScriptPath, params.paramsPath],
{ cwd: params.cwd, encoding: "utf8" },
{ cwd: params.cwd, encoding: "utf8", ...(params.env ? { env: params.env } : {}) },
(error, stdout, stderr) => {
handoffParents.get(params.paramsPath)?.stdin?.end();
const childError = error as NodeJS.ErrnoException | null;
resolve({
code: typeof childError?.code === "number" ? childError.code : 0,
@@ -148,15 +205,17 @@ async function runHelper(params: {
});
},
);
driveHandoffProtocol(child, params.paramsPath);
});
}
describe("managed service update handoff cross-process lease", () => {
it("joins the durable owner reported by a replacement helper", async () => {
const child = Object.assign(new EventEmitter(), {
pid: 24681,
pid: process.pid,
exitCode: null,
signalCode: null,
stdin: new PassThrough(),
stdout: new PassThrough(),
unref: vi.fn(),
});
@@ -171,7 +230,7 @@ describe("managed service update handoff cross-process lease", () => {
const result = await startManagedServiceUpdateHandoff({
root: "/tmp/openclaw",
restartDrainTimeoutMs: 300_000,
parentPid: 12345,
parentPid: process.pid,
execPath: "/usr/local/bin/node",
argv1: "/opt/openclaw/openclaw.mjs",
handoffId: "replacement-handoff",
@@ -187,6 +246,488 @@ describe("managed service update handoff cross-process lease", () => {
expect(result).not.toHaveProperty("pid");
});
it.runIf(process.platform !== "win32").each([
{ label: "its exact dead helper", replacement: "exact", reclaimed: true },
{ label: "a mismatched owner", replacement: "owner", reclaimed: false },
{ label: "a mismatched helper identity", replacement: "identity", reclaimed: false },
{ label: "another live process", replacement: "live", reclaimed: false },
{ label: "an unknown malformed process identity", replacement: "unknown", reclaimed: false },
] as const)(
"claims a separate systemd scope helper and fences cancellation against $label",
async ({ replacement, reclaimed }) => {
const { DatabaseSync } = await import("node:sqlite");
const { getFileLockProcessStartTime } = await import("../shared/pid-alive.js");
const { spawn } =
await vi.importActual<typeof import("node:child_process")>("node:child_process");
const {
cancelManagedServiceUpdateHandoff,
claimManagedServiceUpdateHandoff,
startManagedServiceUpdateHandoff,
} = await import("./update-managed-service-handoff.js");
const tmpDir = await fs.realpath(
await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-handoff-scope-wrapper-")),
);
tempDirs.add(tmpDir);
const helperExitPath = path.join(tmpDir, "nested-helper-exited");
const launcherPath = path.join(tmpDir, "systemd-run");
await fs.writeFile(
launcherPath,
`#!${process.execPath}
const fs = require("node:fs");
const { spawn } = require("node:child_process");
const [command, scriptPath, paramsPath] = process.argv.slice(-3);
const helper = spawn(command, [scriptPath, paramsPath], { stdio: ["pipe", "pipe", "ignore"] });
let helperAlive = true;
helper.stdin.on("error", () => {});
helper.stdout.pipe(process.stdout, { end: false });
helper.once("exit", () => {
helperAlive = false;
fs.writeFileSync(${JSON.stringify(helperExitPath)}, String(helper.pid));
});
process.stdin.on("data", (chunk) => {
if (helperAlive) {
helper.stdin.write(chunk);
} else if (chunk.toString().includes("cancel\\n")) {
process.stdout.write("cancelled\\n", () => process.exit(0));
}
});
`,
{ mode: 0o700 },
);
spawnMock.mockImplementationOnce(spawn);
const handoffId = `scope-wrapper-${replacement}-${path.basename(tmpDir)}`;
let launcher: import("node:child_process").ChildProcess | undefined;
let helperPid = 0;
let leaseDatabasePath: string | undefined;
let leaseOwner = handoffId;
try {
const started = await startManagedServiceUpdateHandoff({
root: tmpDir,
restartDrainTimeoutMs: 5_000,
restartDelayMs: 0,
parentPid: process.pid,
execPath: process.execPath,
argv1: "/opt/openclaw/openclaw.mjs",
supervisor: "systemd",
handoffId,
meta: { handoffId },
env: {
OPENCLAW_STATE_DIR: tmpDir,
PATH: `${tmpDir}${path.delimiter}${process.env.PATH ?? ""}`,
},
});
if (started.status !== "started" || !started.pid) {
throw new Error("expected the systemd scope wrapper to start a real handoff");
}
launcher = spawnMock.mock.results.at(-1)?.value as
| import("node:child_process").ChildProcess
| undefined;
tempDirs.add(path.dirname(started.logPath));
const [, args] = spawnMock.mock.calls.at(-1) as unknown as [string, string[]];
const helperParams = JSON.parse(await fs.readFile(args.at(-1) ?? "", "utf8")) as {
updateLeaseDatabasePath: string;
updateLeaseKey: string;
};
leaseDatabasePath = helperParams.updateLeaseDatabasePath;
const identity = {
kind: "managed-update-handoff" as const,
handoffId,
installRoot: started.installRoot,
};
const leaseDatabase = new DatabaseSync(leaseDatabasePath);
let originalRow: { owner: string; payload_json: string };
try {
const current = leaseDatabase
.prepare(
"SELECT owner, payload_json FROM managed_update_handoffs WHERE install_root = ?",
)
.get(helperParams.updateLeaseKey) as
| { owner: string; payload_json: string }
| undefined;
if (!current) {
throw new Error("expected the nested handoff helper to own its durable lease");
}
originalRow = current;
const helperIdentity = JSON.parse(current.payload_json) as {
pid: number;
startIdentity: string;
};
helperPid = helperIdentity.pid;
expect(helperPid).not.toBe(started.pid);
expect(launcher?.pid).toBe(started.pid);
expect(isPidAlive(helperPid)).toBe(true);
expect(claimManagedServiceUpdateHandoff(identity)).toBe(true);
process.kill(helperPid, "SIGKILL");
await vi.waitFor(
async () => {
await expect(fs.readFile(helperExitPath, "utf8")).resolves.toBe(String(helperPid));
expect(isPidAlive(helperPid)).toBe(false);
expect(isPidAlive(started.pid!)).toBe(true);
},
{ interval: 10, timeout: 5_000 },
);
expect(claimManagedServiceUpdateHandoff(identity)).toBe(false);
let payload = current.payload_json;
if (replacement === "owner") {
leaseOwner = `${handoffId}-foreign`;
} else if (replacement === "identity") {
payload = JSON.stringify({
version: 1,
pid: helperPid,
startIdentity: `${helperIdentity.startIdentity}-mismatched`,
});
} else if (replacement === "live") {
const liveStartIdentity = getFileLockProcessStartTime(process.pid);
if (liveStartIdentity === null) {
throw new Error("expected the live replacement to have a stable process identity");
}
payload = JSON.stringify({
version: 1,
pid: process.pid,
startIdentity: String(liveStartIdentity),
});
} else if (replacement === "unknown") {
payload = JSON.stringify({ version: 1, pid: helperPid, startIdentity: null });
}
if (replacement !== "exact") {
leaseDatabase
.prepare(
"UPDATE managed_update_handoffs SET owner = ?, payload_json = ? WHERE install_root = ? AND owner = ?",
)
.run(leaseOwner, payload, helperParams.updateLeaseKey, handoffId);
originalRow = { owner: leaseOwner, payload_json: payload };
}
} finally {
leaseDatabase.close();
}
await expect(cancelManagedServiceUpdateHandoff(identity)).resolves.toBe(
reclaimed ? "restored-in-process" : false,
);
const retained = new DatabaseSync(leaseDatabasePath, { readOnly: true });
try {
expect(
retained
.prepare(
"SELECT owner, payload_json FROM managed_update_handoffs WHERE install_root = ?",
)
.get(helperParams.updateLeaseKey),
).toEqual(reclaimed ? undefined : originalRow!);
} finally {
retained.close();
}
} finally {
if (helperPid > 0 && isPidAlive(helperPid)) {
process.kill(helperPid, "SIGKILL");
}
if (launcher?.pid && isPidAlive(launcher.pid)) {
launcher.kill("SIGKILL");
}
if (leaseDatabasePath) {
const cleanup = new DatabaseSync(leaseDatabasePath);
try {
cleanup
.prepare("DELETE FROM managed_update_handoffs WHERE install_root = ? AND owner = ?")
.run(tmpDir, leaseOwner);
} finally {
cleanup.close();
}
}
}
},
);
it.each([
["an unavailable parent start identity", null, false],
["a reused parent start identity", "0", false],
["an existing live owner with a null start identity", "current", true],
])("rejects %s before announcing ownership", async (_label, identity, invalidOwner) => {
const { execFile } =
await vi.importActual<typeof import("node:child_process")>("node:child_process");
const { getFileLockProcessStartTime } = await import("../shared/pid-alive.js");
const { DatabaseSync } = await import("node:sqlite");
const { tmpDir, helperScriptPath, baseParams } = await prepareConcurrentHandoffHelper();
const markerPath = path.join(tmpDir, "invalid-parent-update-ran");
const paramsPath = await writeConcurrentHandoffParams({
tmpDir,
baseParams,
name: "invalid-parent",
owner: "invalid-parent-owner",
commandArgv: [
process.execPath,
"-e",
`require("node:fs").writeFileSync(${JSON.stringify(markerPath)}, "ran")`,
],
});
const params = JSON.parse(await fs.readFile(paramsPath, "utf8")) as Record<string, unknown>;
const currentStartIdentity = getFileLockProcessStartTime(process.pid);
if (currentStartIdentity === null) {
throw new Error("expected the live parent to have a stable start identity");
}
await fs.writeFile(
paramsPath,
JSON.stringify({
...params,
parentPid: process.pid,
parentStartIdentity: identity === "current" ? String(currentStartIdentity) : identity,
}),
);
const leaseDatabasePath = String(params.updateLeaseDatabasePath);
const leaseKey = String(params.updateLeaseKey);
const invalidPayload = JSON.stringify({
version: 1,
pid: process.pid,
startIdentity: null,
});
const invalidUpdatedAt = Date.now();
if (invalidOwner) {
await fs.mkdir(path.dirname(leaseDatabasePath), { recursive: true, mode: 0o700 });
const db = new DatabaseSync(leaseDatabasePath);
try {
db.exec(
"CREATE TABLE IF NOT EXISTS managed_update_handoffs (install_root TEXT NOT NULL PRIMARY KEY, owner TEXT NOT NULL, payload_json TEXT NOT NULL, updated_at INTEGER NOT NULL) STRICT;",
);
db.prepare(
"INSERT INTO managed_update_handoffs (install_root, owner, payload_json, updated_at) VALUES (?, ?, ?, ?)",
).run(leaseKey, "unverifiable-owner", invalidPayload, invalidUpdatedAt);
} finally {
db.close();
}
if (process.platform !== "win32") {
await fs.chmod(leaseDatabasePath, 0o600);
}
}
try {
const result = await runHelper({ execFile, helperScriptPath, paramsPath, cwd: tmpDir });
expect(result.code).toBe(1);
expect(result.stdout).not.toContain("OPENCLAW_UPDATE_HANDOFF_READY");
await expect(pathExists(markerPath)).resolves.toBe(false);
if (await pathExists(leaseDatabasePath)) {
const db = new DatabaseSync(leaseDatabasePath, { readOnly: true });
try {
const row = db
.prepare(
"SELECT owner, payload_json, updated_at FROM managed_update_handoffs WHERE install_root = ?",
)
.get(leaseKey);
expect(row).toEqual(
invalidOwner
? {
owner: "unverifiable-owner",
payload_json: invalidPayload,
updated_at: invalidUpdatedAt,
}
: undefined,
);
} finally {
db.close();
}
}
} finally {
if (invalidOwner) {
const db = new DatabaseSync(leaseDatabasePath);
db.prepare("DELETE FROM managed_update_handoffs WHERE install_root = ?").run(leaseKey);
db.close();
}
}
});
it("preserves a live durable owner when its current start identity cannot be observed", async () => {
const { execFile } =
await vi.importActual<typeof import("node:child_process")>("node:child_process");
const { DatabaseSync } = await import("node:sqlite");
const { getFileLockProcessStartTime } = await import("../shared/pid-alive.js");
const { tmpDir, helperScriptPath, baseParams } = await prepareConcurrentHandoffHelper();
const ownerStartIdentity = getFileLockProcessStartTime(process.pid);
if (ownerStartIdentity === null) {
throw new Error("expected the live lease owner to have a stable process identity");
}
const leaseDatabasePath = String(baseParams.updateLeaseDatabasePath);
const leaseKey = String(baseParams.updateLeaseKey);
const owner = "identity-probe-unavailable-owner";
const payload = JSON.stringify({
version: 1,
pid: process.pid,
startIdentity: String(ownerStartIdentity),
});
await fs.mkdir(path.dirname(leaseDatabasePath), { recursive: true, mode: 0o700 });
const database = new DatabaseSync(leaseDatabasePath);
try {
database.exec(
"CREATE TABLE IF NOT EXISTS managed_update_handoffs (install_root TEXT NOT NULL PRIMARY KEY, owner TEXT NOT NULL, payload_json TEXT NOT NULL, updated_at INTEGER NOT NULL) STRICT;",
);
database
.prepare(
"INSERT INTO managed_update_handoffs (install_root, owner, payload_json, updated_at) VALUES (?, ?, ?, ?)",
)
.run(leaseKey, owner, payload, Date.now());
} finally {
database.close();
}
if (process.platform !== "win32") {
await fs.chmod(leaseDatabasePath, 0o600);
}
const preloadPath = path.join(tmpDir, "unavailable-owner-identity.cjs");
await fs.writeFile(
preloadPath,
`const fs = require("node:fs");
const childProcess = require("node:child_process");
const ownerPid = ${process.pid};
const readFileSync = fs.readFileSync;
fs.readFileSync = function(filePath, ...args) {
if (filePath === "/proc/" + ownerPid + "/stat") {
throw Object.assign(new Error("identity probe unavailable"), { code: "EIO" });
}
return readFileSync.call(this, filePath, ...args);
};
const spawnSync = childProcess.spawnSync;
childProcess.spawnSync = function(command, args, options) {
if (args.some((argument) => String(argument).includes(String(ownerPid)))) {
return { status: null, stdout: "", error: new Error("identity probe unavailable") };
}
return spawnSync.call(this, command, args, options);
};
`,
);
const markerPath = path.join(tmpDir, "identity-probe-unavailable-updater-ran");
const paramsPath = await writeConcurrentHandoffParams({
tmpDir,
baseParams,
name: "identity-probe-unavailable",
owner: "competing-owner",
commandArgv: [
process.execPath,
"-e",
`require("node:fs").writeFileSync(${JSON.stringify(markerPath)}, "ran")`,
],
});
try {
const result = await runHelper({
execFile,
helperScriptPath,
paramsPath,
cwd: tmpDir,
env: { ...process.env, NODE_OPTIONS: `--require ${preloadPath}` },
});
expect(result, result.stderr).toMatchObject({
code: 0,
stdout: expect.stringContaining(`HANDOFF_BUSY ${owner}`),
});
const retained = new DatabaseSync(leaseDatabasePath, { readOnly: true });
try {
expect(
retained
.prepare(
"SELECT owner, payload_json FROM managed_update_handoffs WHERE install_root = ?",
)
.get(leaseKey),
).toEqual({ owner, payload_json: payload });
} finally {
retained.close();
}
await expect(pathExists(markerPath)).resolves.toBe(false);
} finally {
const cleanup = new DatabaseSync(leaseDatabasePath);
cleanup
.prepare("DELETE FROM managed_update_handoffs WHERE install_root = ? AND owner = ?")
.run(leaseKey, owner);
cleanup.close();
}
});
it("releases exact helper ownership when cancellation cannot record its terminal sentinel", async () => {
const { spawn } =
await vi.importActual<typeof import("node:child_process")>("node:child_process");
const { DatabaseSync } = await import("node:sqlite");
const { tmpDir, helperScriptPath, baseParams } = await prepareConcurrentHandoffHelper();
const markerPath = path.join(tmpDir, "sentinel-failure-updater-ran");
const owner = "sentinel-failure-owner";
const paramsPath = await writeConcurrentHandoffParams({
tmpDir,
baseParams,
name: "sentinel-failure",
owner,
commandArgv: [
process.execPath,
"-e",
`require("node:fs").writeFileSync(${JSON.stringify(markerPath)}, "ran")`,
],
});
const stateDatabasePath = String(baseParams.stateDatabasePath);
await fs.mkdir(path.dirname(stateDatabasePath), { recursive: true });
const stateDb = new DatabaseSync(stateDatabasePath);
try {
stateDb.exec(
[
"CREATE TABLE gateway_restart_sentinel (",
"sentinel_key TEXT NOT NULL PRIMARY KEY, version INTEGER NOT NULL,",
"kind TEXT NOT NULL, status TEXT NOT NULL, ts INTEGER NOT NULL,",
"session_key TEXT, thread_id TEXT, delivery_channel TEXT, delivery_to TEXT,",
"delivery_account_id TEXT, message TEXT, continuation_json TEXT, doctor_hint TEXT,",
"stats_json TEXT, payload_json TEXT NOT NULL, updated_at_ms INTEGER NOT NULL",
") STRICT;",
"CREATE TRIGGER reject_update_sentinel_write",
"BEFORE INSERT ON gateway_restart_sentinel",
"WHEN NEW.sentinel_key = 'current'",
"BEGIN SELECT RAISE(ABORT, 'sentinel write rejected'); END;",
].join(" "),
);
} finally {
stateDb.close();
}
const leaseDatabasePath = String(baseParams.updateLeaseDatabasePath);
const leaseKey = String(baseParams.updateLeaseKey);
const helper = spawn(process.execPath, [helperScriptPath, paramsPath], {
cwd: tmpDir,
stdio: ["pipe", "pipe", "pipe"],
});
let output = "";
helper.stdout.on("data", (chunk: Buffer | string) => {
output += chunk.toString();
if (output.includes("OPENCLAW_UPDATE_HANDOFF_READY") && !output.includes("cancelled")) {
helper.stdin.write("cancel\n");
}
});
const exited = new Promise<number | null>((resolve) => {
helper.once("close", resolve);
});
try {
await expect(exited).resolves.toBe(0);
expect(output).toContain("cancelled");
const db = new DatabaseSync(leaseDatabasePath, { readOnly: true });
try {
expect(
db
.prepare("SELECT owner FROM managed_update_handoffs WHERE install_root = ?")
.get(leaseKey),
).toBeUndefined();
} finally {
db.close();
}
const log = await fs.readFile(path.join(tmpDir, "sentinel-failure.log"), "utf8");
expect(log).toContain("failed to write update sentinel failure");
expect(log).toContain("sentinel write rejected");
await expect(pathExists(markerPath)).resolves.toBe(false);
} finally {
if (helper.exitCode === null && helper.signalCode === null) {
helper.kill("SIGKILL");
}
const db = new DatabaseSync(leaseDatabasePath);
db.prepare("DELETE FROM managed_update_handoffs WHERE install_root = ? AND owner = ?").run(
leaseKey,
owner,
);
db.close();
}
});
it.runIf(process.platform === "win32")(
"reclaims a reused Windows PID with a different creation identity",
async () => {
@@ -330,8 +871,9 @@ describe("managed service update handoff cross-process lease", () => {
const first = spawn(process.execPath, [helperScriptPath, firstParamsPath], {
cwd: tmpDir,
stdio: ["ignore", "pipe", "pipe"],
stdio: ["pipe", "pipe", "pipe"],
});
driveHandoffProtocol(first, firstParamsPath);
let firstStdout = "";
first.stdout.on("data", (chunk) => (firstStdout += chunk));
let orphanPid = 0;
@@ -344,7 +886,7 @@ describe("managed service update handoff cross-process lease", () => {
{ interval: 10, timeout: 5_000 },
);
orphanPid = Number(await fs.readFile(orphanPidPath, "utf8"));
expect(processIsAlive(orphanPid)).toBe(true);
expect(isPidAlive(orphanPid)).toBe(true);
first.kill("SIGKILL");
await new Promise<void>((resolve) => {
first.once("close", () => resolve());
@@ -363,7 +905,7 @@ describe("managed service update handoff cross-process lease", () => {
await expect(pathExists(secondStartedPath)).resolves.toBe(false);
await fs.writeFile(releaseOrphanPath, "release");
await vi.waitFor(() => expect(processIsAlive(orphanPid)).toBe(false), {
await vi.waitFor(() => expect(isPidAlive(orphanPid)).toBe(false), {
interval: 20,
timeout: 5_000,
});
@@ -384,7 +926,7 @@ describe("managed service update handoff cross-process lease", () => {
if (first.exitCode === null) {
first.kill("SIGKILL");
}
if (orphanPid > 0 && processIsAlive(orphanPid)) {
if (orphanPid > 0 && isPidAlive(orphanPid)) {
process.kill(orphanPid, "SIGKILL");
}
}
@@ -437,8 +979,9 @@ describe("managed service update handoff cross-process lease", () => {
const first = spawn(process.execPath, [helperScriptPath, firstParamsPath], {
cwd: tmpDir,
stdio: ["ignore", "pipe", "pipe"],
stdio: ["pipe", "pipe", "pipe"],
});
driveHandoffProtocol(first, firstParamsPath);
let firstStdout = "";
let firstStderr = "";
first.stdout.on("data", (chunk) => (firstStdout += chunk));
@@ -0,0 +1,145 @@
export type ManagedServiceManagerBoundaryOptions = {
cancelAfterPark?: boolean;
parentExitTimeoutMs?: number;
launchdFault?: "wrong-parent" | "missing-restored-pid" | "dead-restored-pid";
launchdTeardown?: {
bootoutDelayMs?: number;
clockEachCommandMs?: number;
loadedPrints?: number;
pendingBootstrapFailures?: number;
pendingOperationInProgress?: number;
};
lateCommand?: "park" | "commit";
overdueCommit?: boolean;
systemdFault?: "start-failed" | "dead-restored-pid";
};
export type ManagedServiceCommandTiming = {
action: string;
startedAtMs: number;
timeoutMs: number;
};
export function createManagedServiceManagerFixtureScript(params: {
kind: "systemd" | "launchd";
parentPid: number;
statePath: string;
commandsPath: string;
options?: ManagedServiceManagerBoundaryOptions;
}): string {
const { commandsPath, kind, options, parentPid, statePath } = params;
return `#!${process.execPath}
const fs = require("node:fs");
const args = process.argv.slice(2);
const statePath = ${JSON.stringify(statePath)};
const state = fs.existsSync(statePath) ? JSON.parse(fs.readFileSync(statePath, "utf8")) : {};
fs.appendFileSync(${JSON.stringify(commandsPath)}, args.join(" ") + "\\n");
const action = args.find((arg) => ["show", "stop", "reset-failed", "start", "print", "disable", "bootout", "enable", "bootstrap", "kickstart"].includes(arg));
if (${JSON.stringify(kind)} === "systemd") {
if (action === "stop") state.parked = true;
if (action === "stop" && ${Boolean(options?.lateCommand)}) setTimeout(() => {}, 200);
if (action === "reset-failed") state.reset = true;
if (action === "start" && ${JSON.stringify(options?.systemdFault)} === "start-failed") {
state.startFailed = true;
process.stderr.write("start limit hit\\n");
process.exitCode = 1;
} else if (action === "start") state.restored = true;
if (action === "show") {
const active = !state.parked || state.restored;
const restoredPid = ${JSON.stringify(options?.systemdFault)} === "dead-restored-pid" ? 2147483647 : ${process.pid};
process.stdout.write([
"Id=openclaw-gateway.service",
"LoadState=loaded",
"ActiveState=" + (active ? "active" : "inactive"),
"MainPID=" + (state.restored ? restoredPid : active ? ${parentPid} : 0),
"ExecMainStartTimestampMonotonic=" + (state.restored ? "222" : "111"),
].join("\\n") + "\\n");
}
} else {
if (action === "disable") state.disabled = true;
if (action === "bootout") {
state.parked = true;
state.loadedPrintsRemaining = ${options?.launchdTeardown?.loadedPrints ?? 0};
state.pendingBootstrapFailures = ${options?.launchdTeardown?.pendingBootstrapFailures ?? 0};
state.pendingOperationInProgress = ${options?.launchdTeardown?.pendingOperationInProgress ?? 0};
const delay = ${options?.launchdTeardown?.bootoutDelayMs ?? 0};
if (delay) setTimeout(() => {
state.bootoutCompleted = true;
fs.writeFileSync(statePath, JSON.stringify(state));
}, delay);
}
if (action === "enable") state.disabled = false;
if (action === "bootstrap" || action === "kickstart") {
state.bootstrapAttempts = (state.bootstrapAttempts || 0) + 1;
if (state.pendingOperationInProgress > 0) {
state.pendingOperationInProgress -= 1;
state.operationInProgressObserved = (state.operationInProgressObserved || 0) + 1;
process.stderr.write("Bootstrap failed: 37: Operation already in progress\\n");
process.exitCode = 37;
} else if (!state.unloaded) {
process.stderr.write("Bootstrap failed: 37: Operation already in progress\\n");
process.exitCode = 37;
} else if (action === "bootstrap" && state.pendingBootstrapFailures > 0) {
state.pendingBootstrapFailures -= 1;
process.stderr.write("Bootstrap failed: 5: Input/output error\\n");
process.exitCode = 5;
} else state.restored = true;
}
if (action === "print") {
let parentAlive = false;
try { process.kill(${parentPid}, 0); parentAlive = true; } catch {}
if (state.parked && !state.restored && !parentAlive) {
if (state.loadedPrintsRemaining > 0) {
state.loadedPrintsRemaining -= 1;
state.loadedPrintsObserved = (state.loadedPrintsObserved || 0) + 1;
} else {
state.unloaded = true;
process.stderr.write("Could not find service\\n");
fs.writeFileSync(statePath, JSON.stringify(state));
process.exit(113);
}
}
const fault = ${JSON.stringify(options?.launchdFault)};
if (state.restored && fault === "missing-restored-pid") {
process.stdout.write("state = running\\n");
} else {
const restoredPid = fault === "dead-restored-pid" ? 2147483647 : ${process.pid};
const currentPid = fault === "wrong-parent" ? ${process.pid} : ${parentPid};
process.stdout.write("state = running\\npid = " + (state.restored ? restoredPid : currentPid) + "\\n");
}
}
}
fs.writeFileSync(statePath, JSON.stringify(state));
`;
}
export function createManagedServiceLaunchdClockPreload(params: {
commandTimingsPath: string;
clockEachCommandMs: number;
}): string {
return [
'const fs = require("node:fs");',
'const children = require("node:child_process");',
"const actualSpawn = children.spawn;",
"const actualSetTimeout = global.setTimeout;",
"const startedAt = Date.now();",
"let elapsed = 0;",
"Date.now = () => startedAt + elapsed;",
"global.setTimeout = (callback, delay, ...args) => {",
" if (delay === 500) {",
" elapsed += delay;",
" return actualSetTimeout(callback, 0, ...args);",
" }",
" return actualSetTimeout(callback, delay, ...args);",
"};",
"children.spawn = (command, args, options) => {",
' if (command === "launchctl") {',
" const timeoutMs = options.timeout;",
" const startedAtMs = Date.now();",
` fs.appendFileSync(${JSON.stringify(params.commandTimingsPath)}, JSON.stringify({ action: args[0], startedAtMs, timeoutMs }) + "\\n");`,
` elapsed += Math.min(${params.clockEachCommandMs}, timeoutMs);`,
" }",
" return actualSpawn(command, args, options);",
"};",
].join("\n");
}
File diff suppressed because it is too large Load Diff
@@ -5,7 +5,7 @@ import { EventEmitter } from "node:events";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { PassThrough } from "node:stream";
import { PassThrough, type Readable } from "node:stream";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js";
import {
@@ -14,20 +14,40 @@ import {
} from "../state/openclaw-state-db.js";
import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js";
import { claimOpenClawStateOwnership } from "../state/openclaw-state-ownership-operations.js";
import { executeSqliteQuerySync, getNodeSqliteKysely } from "./kysely-sync.js";
import {
executeSqliteQuerySync,
executeSqliteQueryTakeFirstSync,
getNodeSqliteKysely,
} from "./kysely-sync.js";
import { signalMockManagedUpdateHandoffReady } from "./update-managed-service-handoff.test-support.js";
const { spawnMock } = vi.hoisted(() => ({ spawnMock: vi.fn() }));
function createSpawnMock() {
return Object.assign(new EventEmitter(), {
pid: 24680,
pid: process.pid,
exitCode: null,
signalCode: null,
stdin: new PassThrough(),
stdout: new PassThrough(),
unref: vi.fn(),
});
}
async function waitForHandoffLine(output: Readable | null, expected: string): Promise<void> {
await new Promise<void>((resolve, reject) => {
const onData = (chunk: Buffer | string) => {
if (!chunk.toString().includes(`${expected}\n`)) {
return;
}
output?.removeListener("data", onData);
resolve();
};
output?.on("data", onData);
output?.once("end", () => reject(new Error(`helper exited before ${expected}`)));
});
}
vi.mock("node:child_process", async () => {
const { mockNodeChildProcessModule } =
await import("../gateway/server-methods/node-child-process.test-support.js");
@@ -37,82 +57,170 @@ vi.mock("node:child_process", async () => {
});
const tempDirs = new Set<string>();
const mockedHandoffLeaseCleanups = new Set<() => void>();
type GatewayRestartSentinelDatabase = Pick<OpenClawStateKyselyDatabase, "gateway_restart_sentinel">;
beforeEach(() => {
spawnMock.mockReset();
spawnMock.mockImplementation(() => {
spawnMock.mockImplementation((_command: string, args: string[]) => {
const child = createSpawnMock();
process.nextTick(() => {
child.stdout.write("OPENCLAW_UPDATE_HANDOFF_READY\n");
signalMockManagedUpdateHandoffReady({
child,
paramsPath: args.at(-1) ?? "",
cleanups: mockedHandoffLeaseCleanups,
});
});
return child;
});
});
afterEach(async () => {
for (const cleanup of mockedHandoffLeaseCleanups) {
cleanup();
}
closeOpenClawStateDatabaseForTest();
await Promise.all([...tempDirs].map((dir) => fs.rm(dir, { recursive: true, force: true })));
tempDirs.clear();
vi.resetModules();
});
function writeRestartSentinelRow(
env: NodeJS.ProcessEnv,
sentinel: {
version: 1;
revision: number;
payload: {
kind: string;
status: string;
ts: number;
stats: Record<string, unknown>;
};
},
): void {
function writeRestartSentinelRow(env: NodeJS.ProcessEnv, sentinel: unknown): void {
const { db } = openOpenClawStateDatabase({ env });
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
const payload =
sentinel && typeof sentinel === "object" && (sentinel as { version?: unknown }).version === 1
? (sentinel as { payload?: unknown }).payload
: null;
if (!payload || typeof payload !== "object") {
throw new Error("expected versioned restart sentinel payload");
}
const record = payload as {
kind?: unknown;
status?: unknown;
ts?: unknown;
sessionKey?: unknown;
threadId?: unknown;
deliveryContext?: { channel?: unknown; to?: unknown; accountId?: unknown };
message?: unknown;
continuation?: unknown;
doctorHint?: unknown;
stats?: unknown;
};
const revision =
typeof (sentinel as { revision?: unknown }).revision === "number"
? (sentinel as { revision: number }).revision
: Date.now();
executeSqliteQuerySync(
db,
stateDb.insertInto("gateway_restart_sentinel").values({
sentinel_key: record.kind === "revision-floor" ? "revision-floor" : "current",
version: 1,
kind: typeof record.kind === "string" ? record.kind : "update",
status: typeof record.status === "string" ? record.status : "skipped",
ts: typeof record.ts === "number" ? record.ts : Date.now(),
session_key: typeof record.sessionKey === "string" ? record.sessionKey : null,
thread_id: typeof record.threadId === "string" ? record.threadId : null,
delivery_channel:
typeof record.deliveryContext?.channel === "string" ? record.deliveryContext.channel : null,
delivery_to:
typeof record.deliveryContext?.to === "string" ? record.deliveryContext.to : null,
delivery_account_id:
typeof record.deliveryContext?.accountId === "string"
? record.deliveryContext.accountId
: null,
message: typeof record.message === "string" ? record.message : null,
continuation_json: record.continuation ? JSON.stringify(record.continuation) : null,
doctor_hint: typeof record.doctorHint === "string" ? record.doctorHint : null,
stats_json: record.stats ? JSON.stringify(record.stats) : null,
payload_json: JSON.stringify(payload),
updated_at_ms: revision,
}),
);
}
function replaceRestartSentinelRow(env: NodeJS.ProcessEnv, sentinel: unknown): void {
const { db } = openOpenClawStateDatabase({ env });
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
executeSqliteQuerySync(
db,
stateDb.insertInto("gateway_restart_sentinel").values({
sentinel_key: "current",
version: sentinel.version,
kind: sentinel.payload.kind,
status: sentinel.payload.status,
ts: sentinel.payload.ts,
session_key: null,
thread_id: null,
delivery_channel: null,
delivery_to: null,
delivery_account_id: null,
message: null,
continuation_json: null,
doctor_hint: null,
stats_json: JSON.stringify(sentinel.payload.stats),
payload_json: JSON.stringify(sentinel.payload),
updated_at_ms: sentinel.revision,
}),
stateDb.deleteFrom("gateway_restart_sentinel").where("sentinel_key", "=", "current"),
);
writeRestartSentinelRow(env, sentinel);
}
function readRestartSentinelPayload(env: NodeJS.ProcessEnv, key = "current"): unknown {
const { db } = openOpenClawStateDatabase({ env });
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
const row = executeSqliteQueryTakeFirstSync(
db,
stateDb
.selectFrom("gateway_restart_sentinel")
.select(["version", "payload_json", "updated_at_ms"])
.where("sentinel_key", "=", key),
);
return row
? { version: row.version, payload: JSON.parse(row.payload_json), revision: row.updated_at_ms }
: null;
}
async function createLegacyRestartSentinelTable(env: NodeJS.ProcessEnv): Promise<void> {
const sqlite = await import("node:sqlite");
const stateDatabasePath = resolveOpenClawStateSqlitePath(env);
await fs.mkdir(path.dirname(stateDatabasePath), { recursive: true });
const db = new sqlite.DatabaseSync(stateDatabasePath);
try {
db.exec(`
CREATE TABLE gateway_restart_sentinel (
sentinel_key TEXT NOT NULL PRIMARY KEY,
version INTEGER NOT NULL,
kind TEXT NOT NULL,
status TEXT NOT NULL,
ts INTEGER NOT NULL,
session_key TEXT,
thread_id TEXT,
payload_json TEXT NOT NULL,
updated_at_ms INTEGER NOT NULL
);
`);
} finally {
db.close();
}
}
async function runOwnershipHelper(params: {
handoffId?: string;
metaHandoffId?: string;
prepareStateDatabase: (env: NodeJS.ProcessEnv) => Promise<void> | void;
whileHelperRunning?: (context: { logPath: string }) => Promise<void> | void;
prepareStateDatabase?: (env: NodeJS.ProcessEnv) => Promise<void> | void;
sentinel?: unknown;
deepStatePath?: boolean;
commandDelayMs?: number;
commandExitCode?: number;
runnerFault?: "closed-stdin";
whileHelperRunning?: (context: {
env: NodeJS.ProcessEnv;
logPath: string;
}) => Promise<void> | void;
}) {
const { execFile } =
const { spawn } =
await vi.importActual<typeof import("node:child_process")>("node:child_process");
const { getFileLockProcessStartTime } = await import("../shared/pid-alive.js");
const { startManagedServiceUpdateHandoff } = await import("./update-managed-service-handoff.js");
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-handoff-ownership-test-"));
tempDirs.add(tmpDir);
const env = { OPENCLAW_STATE_DIR: tmpDir } as NodeJS.ProcessEnv;
let stateDir = tmpDir;
while (
params.deepStatePath &&
resolveOpenClawStateSqlitePath({ OPENCLAW_STATE_DIR: stateDir }).length <= 260
) {
stateDir = path.join(stateDir, `segment-${"x".repeat(24)}`);
}
const env = { OPENCLAW_STATE_DIR: stateDir } as NodeJS.ProcessEnv;
await startManagedServiceUpdateHandoff({
root: tmpDir,
timeoutMs: 1_800_000,
restartDrainTimeoutMs: 300_000,
restartDelayMs: 500,
parentPid: process.pid,
execPath: "/usr/local/bin/node",
argv1: "/opt/openclaw/openclaw.mjs",
@@ -124,6 +232,8 @@ async function runOwnershipHelper(params: {
continuationMessage: "continue after restart",
},
});
const mockedLauncher = spawnMock.mock.results.at(-1)?.value as ReturnType<typeof createSpawnMock>;
mockedLauncher.emit("exit", 0, null);
const [, args, spawnOptions] = spawnMock.mock.calls.at(-1) as unknown as [
string,
@@ -136,17 +246,63 @@ async function runOwnershipHelper(params: {
string,
unknown
>;
await params.prepareStateDatabase(env);
await params.prepareStateDatabase?.(env);
if (params.sentinel !== undefined) {
writeRestartSentinelRow(env, params.sentinel);
}
const parent = spawn(process.execPath, ["-e", "process.stdin.resume()"], {
stdio: ["pipe", "ignore", "ignore"],
});
const parentPid = parent.pid;
const startIdentity = parentPid ? getFileLockProcessStartTime(parentPid) : null;
if (!parentPid || startIdentity === null) {
parent.kill("SIGKILL");
throw new Error("expected a parent process with a stable start identity");
}
const helperParamsPath = path.join(tmpDir, "helper-params.json");
const logPath = path.join(tmpDir, "handoff.log");
const updaterPath = path.join(tmpDir, "updater-ran");
const runnerClosedPath = path.join(tmpDir, "runner-closed-stdin");
const preloadPath = path.join(tmpDir, "runner-fault-preload.cjs");
if (params.runnerFault === "closed-stdin") {
await fs.writeFile(
preloadPath,
`const fs = require("node:fs");
const childProcess = require("node:child_process");
const originalSpawn = childProcess.spawn;
const closedPath = ${JSON.stringify(runnerClosedPath)};
childProcess.spawn = function(command, args, options) {
if (command !== process.execPath || args[0] !== "-e" ||
!args[1].includes('process.stdin.once("data"')) {
return originalSpawn.apply(this, arguments);
}
const runnerScript = 'const fs=require("node:fs");fs.closeSync(0);' +
'fs.writeFileSync(' + JSON.stringify(closedPath) + ',String(process.pid));' +
'setTimeout(() => process.exit(0),5000)';
const child = originalSpawn.call(this, command, ["-e", runnerScript, args[2]], options);
const deadline = Date.now() + 5000;
const waitWord = new Int32Array(new SharedArrayBuffer(4));
while (!fs.existsSync(closedPath) && Date.now() < deadline) {
Atomics.wait(waitWord, 0, 0, 10);
}
if (!fs.existsSync(closedPath)) throw new Error("runner did not close stdin");
return child;
};
`,
);
}
await fs.writeFile(
helperParamsPath,
`${JSON.stringify(
{
...helperParams,
parentPid: 0,
parentExitTimeoutMs: 5_000,
commandArgv: [process.execPath, "-e", "process.exit(7)"],
parentPid,
parentStartIdentity: String(startIdentity),
commandArgv: [
process.execPath,
"-e",
`require("node:fs").writeFileSync(${JSON.stringify(updaterPath)},"ran");setTimeout(() => process.exit(${params.commandExitCode ?? 1}), ${params.commandDelayMs ?? 0})`,
],
logPath,
sensitivePaths: [],
},
@@ -155,27 +311,55 @@ async function runOwnershipHelper(params: {
)}\n`,
);
const helper = spawn(process.execPath, [helperScriptPath, helperParamsPath], {
cwd: tmpDir,
env: {
...spawnOptions.env,
...(params.runnerFault ? { NODE_OPTIONS: `--require ${preloadPath}` } : {}),
},
stdio: ["pipe", "pipe", params.runnerFault ? "pipe" : "ignore"],
});
const helperInput = helper.stdin;
if (!helperInput) {
throw new Error("expected the managed update helper to expose its control pipe");
}
let stderr = "";
helper.stderr?.on("data", (chunk: Buffer | string) => {
stderr += chunk.toString();
});
const resultPromise = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>(
(resolve) => {
execFile(
process.execPath,
[helperScriptPath, helperParamsPath],
{ cwd: tmpDir, env: spawnOptions.env },
(err) => {
const childError = err as (NodeJS.ErrnoException & { signal?: NodeJS.Signals }) | null;
resolve({
code: typeof childError?.code === "number" ? childError.code : 0,
signal: childError?.signal ?? null,
});
},
);
helper.once("close", (code, signal) => resolve({ code, signal }));
},
);
await params.whileHelperRunning?.({ logPath });
return { result: await resultPromise, env, logPath };
await waitForHandoffLine(helper.stdout, "OPENCLAW_UPDATE_HANDOFF_READY");
const parked = waitForHandoffLine(helper.stdout, "parked");
helperInput.write("park\n");
await parked;
const committed = waitForHandoffLine(helper.stdout, "committed");
helperInput.write("commit\n");
await committed;
parent.stdin.end();
await params.whileHelperRunning?.({ env, logPath });
const result = await resultPromise;
if (params.runnerFault) {
const runnerPid = Number(await fs.readFile(runnerClosedPath, "utf8"));
try {
process.kill(runnerPid, "SIGKILL");
} catch {}
}
return {
result,
env,
logPath,
stderr,
updaterPath,
leaseDatabasePath: String(helperParams.updateLeaseDatabasePath),
leaseKey: String(helperParams.updateLeaseKey),
};
}
describe("managed service update handoff external ownership", () => {
describe("managed service update handoff state ownership and sentinel persistence", () => {
it("refuses fallback writes to externally owned state without the supervisor marker", async () => {
let before:
| {
@@ -188,6 +372,7 @@ describe("managed service update handoff external ownership", () => {
}
| undefined;
const { result, env, logPath } = await runOwnershipHelper({
commandExitCode: 7,
prepareStateDatabase: async (stateEnv) => {
const externalEnv = { ...stateEnv, OPENCLAW_SUPERVISOR_MODE: "external" };
claimOpenClawStateOwnership("gateway-supervisor", { env: externalEnv });
@@ -247,6 +432,7 @@ describe("managed service update handoff external ownership", () => {
let helperResult: Awaited<ReturnType<typeof runOwnershipHelper>> | undefined;
try {
helperResult = await runOwnershipHelper({
commandExitCode: 7,
handoffId: "handoff-ownership-race",
metaHandoffId: "handoff-ownership-race",
prepareStateDatabase: async (stateEnv) => {
@@ -321,4 +507,249 @@ describe("managed service update handoff external ownership", () => {
/race-supervisor.*OPENCLAW_SUPERVISOR_MODE=external/u,
);
});
it("writes a fallback update failure when no restart sentinel row exists", async () => {
const { result, env } = await runOwnershipHelper({
handoffId: "handoff-123",
metaHandoffId: "handoff-123",
});
expect(result).toEqual({ code: 1, signal: null });
expect(readRestartSentinelPayload(env)).toMatchObject({
version: 1,
payload: {
kind: "update",
status: "error",
sessionKey: "agent:test:webchat:dm:user-123",
stats: {
handoffId: "handoff-123",
reason: "managed-service-handoff-failed",
},
},
});
if (process.platform !== "win32") {
const mode = (await fs.stat(resolveOpenClawStateSqlitePath(env))).mode & 0o777;
expect(mode).toBe(0o600);
}
});
it("reclaims the exact runner lease and records failure when its stdin closes before the update gate", async () => {
const { DatabaseSync } = await import("node:sqlite");
const { result, env, stderr, updaterPath, leaseDatabasePath, leaseKey } =
await runOwnershipHelper({
handoffId: "handoff-runner-closed-stdin",
metaHandoffId: "handoff-runner-closed-stdin",
runnerFault: "closed-stdin",
});
expect(result).toEqual({ code: 1, signal: null });
expect(stderr).not.toMatch(/Unhandled 'error' event|Error: write EPIPE/u);
await expect(fs.access(updaterPath)).rejects.toMatchObject({ code: "ENOENT" });
const leaseDatabase = new DatabaseSync(leaseDatabasePath, { readOnly: true });
try {
expect(
leaseDatabase
.prepare("SELECT owner FROM managed_update_handoffs WHERE install_root = ?")
.get(leaseKey),
).toBeUndefined();
} finally {
leaseDatabase.close();
}
expect(readRestartSentinelPayload(env)).toMatchObject({
payload: {
kind: "update",
status: "error",
stats: {
handoffId: "handoff-runner-closed-stdin",
reason: expect.stringMatching(/^managed-service-handoff-(?:helper|spawn)-failed$/u),
},
},
});
});
it.runIf(process.platform === "win32")(
"writes fallback state through the detached helper beyond MAX_PATH",
async () => {
const { result, env } = await runOwnershipHelper({
deepStatePath: true,
handoffId: "handoff-windows-long-path",
metaHandoffId: "handoff-windows-long-path",
});
const statePath = resolveOpenClawStateSqlitePath(env);
expect(statePath.startsWith("\\\\?\\")).toBe(false);
expect(statePath.length).toBeGreaterThan(260);
expect(result).toEqual({ code: 1, signal: null });
expect(readRestartSentinelPayload(env)).toMatchObject({
payload: { status: "error" },
});
},
);
it("waits for a concurrent state writer before persisting the fallback failure", async () => {
let lockReleased: Promise<void> | undefined;
const { result, env } = await runOwnershipHelper({
handoffId: "handoff-locked",
metaHandoffId: "handoff-locked",
prepareStateDatabase: async (stateEnv) => {
openOpenClawStateDatabase({ env: stateEnv });
closeOpenClawStateDatabaseForTest();
const sqlite = await import("node:sqlite");
const lock = new sqlite.DatabaseSync(resolveOpenClawStateSqlitePath(stateEnv));
lock.exec("BEGIN IMMEDIATE;");
lockReleased = new Promise((resolve) => {
setTimeout(() => {
lock.exec("COMMIT;");
lock.close();
resolve();
}, 200);
});
},
});
await lockReleased;
expect(result).toEqual({ code: 1, signal: null });
expect(readRestartSentinelPayload(env)).toMatchObject({
version: 1,
payload: {
status: "error",
stats: {
handoffId: "handoff-locked",
reason: "managed-service-handoff-failed",
},
},
});
});
it("repairs legacy restart sentinel columns before writing fallback failures", async () => {
const { result, env } = await runOwnershipHelper({
handoffId: "handoff-123",
metaHandoffId: "handoff-123",
prepareStateDatabase: createLegacyRestartSentinelTable,
});
expect(result).toEqual({ code: 1, signal: null });
expect(readRestartSentinelPayload(env)).toMatchObject({
version: 1,
payload: {
kind: "update",
status: "error",
stats: {
reason: "managed-service-handoff-failed",
},
},
});
});
it("does not overwrite a restart sentinel owned by another startup task", async () => {
const unrelatedSentinel = {
version: 1,
payload: {
kind: "config",
status: "skipped",
message: "preserve this restart task",
stats: { reason: "config-restart-pending" },
},
};
const { result, env } = await runOwnershipHelper({
sentinel: unrelatedSentinel,
});
expect(result).toEqual({ code: 1, signal: null });
expect(readRestartSentinelPayload(env)).toMatchObject(unrelatedSentinel);
});
it("does not overwrite a newer pending update handoff sentinel", async () => {
const newerSentinel = {
version: 1,
payload: {
kind: "update",
status: "skipped",
message: "new handoff still pending",
stats: {
mode: "npm",
handoffId: "newer-handoff",
reason: "managed-service-handoff-started",
steps: [],
durationMs: 0,
},
},
};
const { result, env } = await runOwnershipHelper({
handoffId: "old-handoff",
metaHandoffId: "old-handoff",
sentinel: newerSentinel,
});
expect(result).toEqual({ code: 1, signal: null });
expect(readRestartSentinelPayload(env)).toMatchObject(newerSentinel);
});
it("preserves a newer sentinel written while the detached helper is active", async () => {
const oldSentinel = {
version: 1,
revision: 100,
payload: {
kind: "update",
status: "skipped",
ts: 100,
stats: {
handoffId: "old-handoff",
reason: "managed-service-handoff-started",
},
},
};
const newerSentinel = {
version: 1,
revision: 200,
payload: {
kind: "restart",
status: "ok",
ts: 200,
},
};
const { env } = await runOwnershipHelper({
handoffId: "old-handoff",
metaHandoffId: "old-handoff",
sentinel: oldSentinel,
commandDelayMs: 200,
whileHelperRunning: async ({ env: stateEnv }) => {
await new Promise<void>((resolve) => {
setTimeout(resolve, 50);
});
replaceRestartSentinelRow(stateEnv, newerSentinel);
},
});
expect(readRestartSentinelPayload(env)).toMatchObject({
payload: newerSentinel.payload,
revision: 200,
});
});
it("advances the durable revision floor even when it is ahead of the clock", async () => {
const futureRevision = Date.now() + 60_000;
const { env } = await runOwnershipHelper({
handoffId: "handoff-future-revision",
metaHandoffId: "handoff-future-revision",
sentinel: {
version: 1,
revision: futureRevision,
payload: {
kind: "revision-floor",
status: "skipped",
ts: 123,
stats: {
handoffId: "handoff-future-revision",
reason: "managed-service-handoff-started",
},
},
},
});
expect(readRestartSentinelPayload(env, "revision-floor")).toMatchObject({
revision: futureRevision + 1,
});
});
});
@@ -0,0 +1,775 @@
// Process-local handoff sharing complements the durable cross-process lease.
import { EventEmitter } from "node:events";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { PassThrough } from "node:stream";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
import {
signalMockManagedUpdateHandoffReady,
type MockManagedUpdateHandoffLeaseFailure,
} from "./update-managed-service-handoff.test-support.js";
const spawnMock = vi.hoisted(() => vi.fn());
const forceKillChildProcessTreeMock = vi.hoisted(() => vi.fn());
const findInstalledSystemdGatewayScopeMock = vi.hoisted(() =>
vi.fn(
async (_env: NodeJS.ProcessEnv) =>
null as {
scope: "user" | "system";
unitName: string;
unitPath: string;
} | null,
),
);
const tempRoots = useAutoCleanupTempDirTracker(afterEach);
const mockedHandoffLeaseCleanups = new Set<() => void>();
const MOCK_INSTALL_ROOT = path.join(os.tmpdir(), `openclaw-handoff-single-flight-${process.pid}`);
function createReadyChild(
pid: number,
paramsPath: string,
failure?: MockManagedUpdateHandoffLeaseFailure,
) {
const child = Object.assign(new EventEmitter(), {
pid,
exitCode: null,
signalCode: null,
stdin: new PassThrough(),
stdout: new PassThrough(),
unref: vi.fn(),
});
process.nextTick(() => {
signalMockManagedUpdateHandoffReady({
child,
paramsPath,
cleanups: mockedHandoffLeaseCleanups,
...(child.pid === process.pid ? {} : { startIdentity: 17 }),
failure,
});
});
return child;
}
vi.mock("node:child_process", async () => {
const { mockNodeChildProcessModule } =
await import("../gateway/server-methods/node-child-process.test-support.js");
return mockNodeChildProcessModule({
spawn: spawnMock as unknown as typeof import("node:child_process").spawn,
});
});
vi.mock("../daemon/systemd-scope.js", async (importOriginal) => ({
...(await importOriginal<typeof import("../daemon/systemd-scope.js")>()),
findInstalledSystemdGatewayScope: findInstalledSystemdGatewayScopeMock,
}));
vi.mock("../process/child-process-tree.js", async (importOriginal) => ({
...(await importOriginal<typeof import("../process/child-process-tree.js")>()),
forceKillChildProcessTree: forceKillChildProcessTreeMock,
}));
beforeEach(async () => {
let pid = 24680;
const liveChildren = new Set<number>();
const processIdentity = await import("../shared/pid-alive.js");
const parentStartIdentity = processIdentity.getFileLockProcessStartTime(process.pid);
vi.spyOn(processIdentity, "getFileLockProcessStartTime").mockImplementation((targetPid) =>
targetPid === process.pid ? parentStartIdentity : liveChildren.has(targetPid) ? 17 : null,
);
vi.spyOn(processIdentity, "isPidAlive").mockImplementation(
(targetPid) => targetPid === process.pid || liveChildren.has(targetPid),
);
forceKillChildProcessTreeMock.mockReset();
forceKillChildProcessTreeMock.mockImplementation((child: ReturnType<typeof createReadyChild>) => {
child.stdout.destroy();
});
findInstalledSystemdGatewayScopeMock.mockReset();
findInstalledSystemdGatewayScopeMock.mockResolvedValue(null);
spawnMock.mockReset();
spawnMock.mockImplementation((_command: string, args: string[]) => {
const child = createReadyChild(pid++, args.at(-1) ?? "");
liveChildren.add(child.pid);
child.once("exit", () => liveChildren.delete(child.pid));
return child;
});
});
afterEach(async () => {
for (const cleanup of mockedHandoffLeaseCleanups) {
cleanup();
}
const handoffDirs = spawnMock.mock.calls.flatMap((call) => {
const args = call[1] as string[] | undefined;
const scriptPath = args?.[0];
return scriptPath ? [path.dirname(scriptPath)] : [];
});
await Promise.all(handoffDirs.map((dir) => fs.rm(dir, { recursive: true, force: true })));
vi.restoreAllMocks();
vi.resetModules();
});
const baseParams = {
restartDrainTimeoutMs: 300_000,
parentPid: process.pid,
execPath: "/usr/local/bin/node",
argv1: "/opt/openclaw/openclaw.mjs",
};
describe("managed service update handoff single-flight", () => {
it.each([
["does not exist", "absent"],
["has malformed helper identity", "malformed"],
["belongs to a different owner", "wrong-owner"],
["identifies a dead helper", "dead-helper"],
] as const)("rejects READY when the durable helper lease %s", async (_label, failure) => {
spawnMock.mockImplementationOnce((_command: string, args: string[]) =>
createReadyChild(process.pid, args.at(-1) ?? "", failure),
);
const { claimManagedServiceUpdateHandoff, startManagedServiceUpdateHandoff } =
await import("./update-managed-service-handoff.js");
const identity = {
kind: "managed-update-handoff" as const,
handoffId: `invalid-ready-${failure}`,
installRoot: `${MOCK_INSTALL_ROOT}-${failure}`,
};
await expect(
startManagedServiceUpdateHandoff({
...baseParams,
root: identity.installRoot,
handoffId: identity.handoffId,
meta: {},
}),
).rejects.toThrow(/lease|helper|identity/u);
const child = spawnMock.mock.results[0]?.value as ReturnType<typeof createReadyChild>;
expect(forceKillChildProcessTreeMock).toHaveBeenCalledExactlyOnceWith(child);
expect(child.unref).not.toHaveBeenCalled();
expect(claimManagedServiceUpdateHandoff(identity)).toBe(false);
});
it("rejects system-scope systemd before spawning or reserving handoff ownership", async () => {
findInstalledSystemdGatewayScopeMock.mockResolvedValueOnce({
scope: "system",
unitName: "openclaw-gateway.service",
unitPath: "/etc/systemd/system/openclaw-gateway.service",
});
const { claimManagedServiceUpdateHandoff, startManagedServiceUpdateHandoff } =
await import("./update-managed-service-handoff.js");
const root = `${MOCK_INSTALL_ROOT}-system-scope`;
await expect(
startManagedServiceUpdateHandoff({
...baseParams,
root,
handoffId: "system-handoff",
supervisor: "systemd",
env: { OPENCLAW_SYSTEMD_UNIT: "openclaw-gateway.service" },
meta: {},
}),
).rejects.toThrow(/user-scope systemd unit.*manual system-service update/);
expect(spawnMock).not.toHaveBeenCalled();
expect(
claimManagedServiceUpdateHandoff({
kind: "managed-update-handoff",
handoffId: "system-handoff",
installRoot: root,
}),
).toBe(false);
await expect(
startManagedServiceUpdateHandoff({ ...baseParams, root, meta: {} }),
).resolves.toMatchObject({ status: "started" });
expect(spawnMock).toHaveBeenCalledOnce();
const owner = spawnMock.mock.results[0]?.value as ReturnType<typeof createReadyChild>;
owner.emit("exit", 0, null);
});
it("shares one same-root helper until its lifecycle ends", async () => {
const { startManagedServiceUpdateHandoff } =
await import("./update-managed-service-handoff.js");
const first = startManagedServiceUpdateHandoff({
...baseParams,
root: MOCK_INSTALL_ROOT,
handoffId: "handoff-first",
meta: { handoffId: "handoff-first" },
});
const second = startManagedServiceUpdateHandoff({
...baseParams,
root: MOCK_INSTALL_ROOT,
handoffId: "handoff-second",
meta: { handoffId: "handoff-second" },
});
const outcomes = await Promise.all([first, second]);
expect(outcomes).toEqual([
expect.objectContaining({ status: "started", handoffId: "handoff-first" }),
expect.objectContaining({ status: "joined", handoffId: "handoff-first" }),
]);
expect(outcomes[1]).not.toHaveProperty("installRoot");
expect(spawnMock).toHaveBeenCalledOnce();
const owner = spawnMock.mock.results[0]?.value as ReturnType<typeof createReadyChild>;
owner.emit("exit", 0, null);
const next = startManagedServiceUpdateHandoff({
...baseParams,
root: MOCK_INSTALL_ROOT,
handoffId: "handoff-next",
meta: { handoffId: "handoff-next" },
});
await expect(next).resolves.toMatchObject({
status: "started",
handoffId: "handoff-next",
});
expect(spawnMock).toHaveBeenCalledTimes(2);
const nextOwner = spawnMock.mock.results[1]?.value as ReturnType<typeof createReadyChild>;
nextOwner.emit("exit", 0, null);
});
it.each([
["has exited before its ChildProcess notification", "dead"],
["reuses its PID for another process", "reused"],
["no longer exposes its process start identity", "unknown"],
])("rejects a helper claim when its operating-system process %s", async (_label, failure) => {
vi.restoreAllMocks();
const { spawn } =
await vi.importActual<typeof import("node:child_process")>("node:child_process");
spawnMock.mockImplementation(spawn);
const processIdentity = await import("../shared/pid-alive.js");
const root = await fs.realpath(tempRoots.make("openclaw-helper-process-identity-"));
const parent = spawn(process.execPath, ["-e", "process.stdin.resume()"], {
stdio: ["pipe", "ignore", "ignore"],
});
try {
const {
cancelManagedServiceUpdateHandoff,
claimManagedServiceUpdateHandoff,
startManagedServiceUpdateHandoff,
} = await import("./update-managed-service-handoff.js");
const started = await startManagedServiceUpdateHandoff({
root,
restartDrainTimeoutMs: 300_000,
parentPid: parent.pid,
execPath: process.execPath,
argv1: process.argv[1],
env: { ...process.env, OPENCLAW_STATE_DIR: root },
meta: {},
});
if (started.status !== "started" || !started.pid) {
throw new Error("expected real detached handoff ownership");
}
const identity = { kind: "managed-update-handoff" as const, ...started };
expect(claimManagedServiceUpdateHandoff(identity)).toBe(true);
if (failure === "dead") {
vi.spyOn(processIdentity, "isPidAlive").mockReturnValue(false);
} else {
const helperStartIdentity = processIdentity.getFileLockProcessStartTime(started.pid);
if (helperStartIdentity === null) {
throw new Error("expected the real detached helper to have a process identity");
}
vi.spyOn(processIdentity, "getFileLockProcessStartTime").mockReturnValue(
failure === "reused" ? helperStartIdentity + 1 : null,
);
}
expect(claimManagedServiceUpdateHandoff(identity)).toBe(false);
vi.restoreAllMocks();
await expect(cancelManagedServiceUpdateHandoff(identity)).resolves.toBe(
"restored-in-process",
);
} finally {
vi.restoreAllMocks();
parent.stdin?.end();
}
});
it("terminates the exact helper when its initial start identity is unavailable", async () => {
const processIdentity = await import("../shared/pid-alive.js");
vi.mocked(processIdentity.getFileLockProcessStartTime)
.mockReturnValueOnce(17)
.mockReturnValueOnce(null);
const { claimManagedServiceUpdateHandoff, startManagedServiceUpdateHandoff } =
await import("./update-managed-service-handoff.js");
const identity = {
kind: "managed-update-handoff" as const,
handoffId: "identity-unavailable",
installRoot: `${MOCK_INSTALL_ROOT}-identity-unavailable`,
};
await expect(
startManagedServiceUpdateHandoff({
...baseParams,
root: identity.installRoot,
handoffId: identity.handoffId,
meta: {},
}),
).rejects.toThrow("process start identity is unavailable");
const child = spawnMock.mock.results[0]?.value as ReturnType<typeof createReadyChild>;
expect(forceKillChildProcessTreeMock).toHaveBeenCalledExactlyOnceWith(child);
expect(child.unref).not.toHaveBeenCalled();
expect(claimManagedServiceUpdateHandoff(identity)).toBe(false);
});
it("reclaims only its exact dead detached helper before reopening the install root", async () => {
vi.restoreAllMocks();
const { spawn } =
await vi.importActual<typeof import("node:child_process")>("node:child_process");
const { DatabaseSync } = await import("node:sqlite");
const processIdentity = await import("../shared/pid-alive.js");
const { getFileLockProcessStartTime } = processIdentity;
spawnMock.mockImplementation(spawn);
const root = await fs.realpath(tempRoots.make("openclaw-dead-handoff-owner-"));
const markerPath = path.join(root, "updater-ran");
const updaterPath = path.join(root, "updater.cjs");
await fs.writeFile(
updaterPath,
`require("node:fs").writeFileSync(${JSON.stringify(markerPath)}, "ran")`,
);
const parent = spawn(process.execPath, ["-e", "process.stdin.resume()"], {
stdio: ["pipe", "ignore", "ignore"],
});
const { cancelManagedServiceUpdateHandoff, startManagedServiceUpdateHandoff } =
await import("./update-managed-service-handoff.js");
let leaseDatabasePath: string | undefined;
let deadOwner: string | undefined;
let replacement: Awaited<ReturnType<typeof startManagedServiceUpdateHandoff>> | undefined;
try {
const start = () =>
startManagedServiceUpdateHandoff({
root,
restartDrainTimeoutMs: 300_000,
parentPid: parent.pid,
execPath: process.execPath,
argv1: updaterPath,
env: { ...process.env, OPENCLAW_STATE_DIR: root },
meta: {},
});
const started = await start();
if (started.status !== "started" || !started.pid) {
throw new Error("expected real detached handoff ownership");
}
deadOwner = started.handoffId;
const helperStartIdentity = getFileLockProcessStartTime(started.pid);
if (helperStartIdentity === null) {
throw new Error("expected the detached helper to have a stable process identity");
}
const helper = spawnMock.mock.results[0]?.value as import("node:child_process").ChildProcess;
const [, args] = spawnMock.mock.calls[0] as [string, string[]];
leaseDatabasePath = (
JSON.parse(await fs.readFile(args[1] ?? "", "utf8")) as {
updateLeaseDatabasePath: string;
}
).updateLeaseDatabasePath;
const readLease = () => {
const db = new DatabaseSync(leaseDatabasePath!, { readOnly: true });
try {
return db
.prepare(
"SELECT owner, payload_json FROM managed_update_handoffs WHERE install_root = ?",
)
.get(root) as { owner: string; payload_json: string } | undefined;
} finally {
db.close();
}
};
const initialLease = readLease();
expect(initialLease?.owner).toBe(started.handoffId);
expect(JSON.parse(initialLease?.payload_json ?? "null")).toEqual({
version: 1,
pid: started.pid,
startIdentity: String(helperStartIdentity),
});
const helperExited = new Promise<void>((resolve) => {
helper.once("exit", () => resolve());
});
helper.kill("SIGKILL");
await helperExited;
expect(readLease()).toEqual(initialLease);
const ownStartIdentity = getFileLockProcessStartTime(process.pid);
if (ownStartIdentity === null || !initialLease) {
throw new Error("expected complete live-parent and dead-helper lease identities");
}
const originalPayload = {
version: 1,
pid: started.pid,
startIdentity: String(helperStartIdentity),
};
const rejectedOwners = [
{
label: "replacement owner",
owner: "replacement-owner",
payload: originalPayload,
},
{
label: "different live process",
owner: started.handoffId,
payload: {
...originalPayload,
pid: process.pid,
startIdentity: String(ownStartIdentity),
},
},
{
label: "different recorded start identity",
owner: started.handoffId,
payload: { ...originalPayload, startIdentity: `${helperStartIdentity}-reused` },
},
{
label: "malformed process identity",
owner: started.handoffId,
payload: { ...originalPayload, startIdentity: null },
},
{
label: "noncanonical process identity",
owner: started.handoffId,
payload: { ...originalPayload, unexpected: true },
},
];
const writeLease = (owner: string, payload: unknown) => {
if (!leaseDatabasePath) {
throw new Error("expected the detached helper lease database path");
}
const db = new DatabaseSync(leaseDatabasePath);
try {
db.prepare(
"UPDATE managed_update_handoffs SET owner = ?, payload_json = ? WHERE install_root = ?",
).run(owner, JSON.stringify(payload), root);
} finally {
db.close();
}
};
const identity = { kind: "managed-update-handoff" as const, ...started };
for (const rejected of rejectedOwners) {
writeLease(rejected.owner, rejected.payload);
await expect(cancelManagedServiceUpdateHandoff(identity), rejected.label).resolves.toBe(
false,
);
expect(readLease(), rejected.label).toEqual({
owner: rejected.owner,
payload_json: JSON.stringify(rejected.payload),
});
}
writeLease(started.handoffId, originalPayload);
const unknownDeath = vi.spyOn(processIdentity, "isPidDefinitelyDead").mockReturnValue(false);
await expect(cancelManagedServiceUpdateHandoff(identity)).resolves.toBe(false);
expect(readLease()).toEqual(initialLease);
unknownDeath.mockRestore();
await expect(cancelManagedServiceUpdateHandoff(identity)).resolves.toBe(
"restored-in-process",
);
expect(readLease()).toBeUndefined();
await expect(fs.access(markerPath)).rejects.toThrow();
replacement = await start();
if (replacement.status !== "started") {
throw new Error("expected cancellation to reopen the install root");
}
expect(replacement.installRoot).toBe(root);
expect(replacement.handoffId).not.toBe(started.handoffId);
await expect(
cancelManagedServiceUpdateHandoff({ kind: "managed-update-handoff", ...replacement }),
).resolves.toBe("restored-in-process");
replacement = undefined;
expect(readLease()).toBeUndefined();
await expect(fs.access(markerPath)).rejects.toThrow();
} finally {
if (replacement?.status === "started") {
await cancelManagedServiceUpdateHandoff({
kind: "managed-update-handoff",
...replacement,
});
}
if (leaseDatabasePath && deadOwner) {
const db = new DatabaseSync(leaseDatabasePath);
db.prepare("DELETE FROM managed_update_handoffs WHERE install_root = ? AND owner = ?").run(
root,
deadOwner,
);
db.close();
}
parent.stdin?.end();
}
});
it("waits for the exact helper to release its lease after an immediate control-pipe EPIPE", async () => {
vi.restoreAllMocks();
const { spawn } =
await vi.importActual<typeof import("node:child_process")>("node:child_process");
const { DatabaseSync } = await import("node:sqlite");
spawnMock.mockImplementation(spawn);
const root = await fs.realpath(tempRoots.make("openclaw-handoff-control-epipe-"));
const parent = spawn(process.execPath, ["-e", "process.stdin.resume()"], {
stdio: ["pipe", "ignore", "ignore"],
});
const {
cancelManagedServiceUpdateHandoff,
requestManagedServiceUpdateHandoffPark,
startManagedServiceUpdateHandoff,
} = await import("./update-managed-service-handoff.js");
const started = await startManagedServiceUpdateHandoff({
root,
restartDrainTimeoutMs: 300_000,
parentPid: parent.pid,
execPath: process.execPath,
argv1: process.argv[1],
env: { ...process.env, OPENCLAW_STATE_DIR: root },
meta: {},
});
if (started.status !== "started") {
throw new Error("expected real detached helper ownership");
}
const identity = { kind: "managed-update-handoff" as const, ...started };
const helper = spawnMock.mock.results[0]?.value as import("node:child_process").ChildProcess;
const input = helper.stdin;
if (!input) {
throw new Error("expected the detached helper control pipe");
}
const [, args] = spawnMock.mock.calls[0] as [string, string[]];
const { updateLeaseDatabasePath } = JSON.parse(await fs.readFile(args[1] ?? "", "utf8")) as {
updateLeaseDatabasePath: string;
};
let controlDestroyed = false;
const destroyed = vi
.spyOn(input, "destroyed", "get")
.mockImplementation(() => controlDestroyed);
const write = vi.spyOn(input, "write").mockImplementation(((
_chunk: unknown,
callback: unknown,
) => {
controlDestroyed = true;
const error = Object.assign(new Error("write EPIPE"), { code: "EPIPE" });
if (typeof callback === "function") {
callback(error);
}
input.emit("error", error);
return false;
}) as typeof input.write);
try {
await expect(requestManagedServiceUpdateHandoffPark(identity)).resolves.toBe(false);
expect(helper.exitCode).toBeNull();
let cancellationSettled = false;
const cancellation = cancelManagedServiceUpdateHandoff(identity).then((result) => {
cancellationSettled = true;
return result;
});
await new Promise<void>((resolve) => {
setImmediate(resolve);
});
expect(cancellationSettled).toBe(false);
expect(helper.exitCode).toBeNull();
write.mockRestore();
destroyed.mockRestore();
input.end();
await expect(cancellation).resolves.toBe("restored-in-process");
expect(helper.exitCode !== null || helper.signalCode !== null).toBe(true);
const database = new DatabaseSync(updateLeaseDatabasePath, { readOnly: true });
try {
expect(
database
.prepare("SELECT owner FROM managed_update_handoffs WHERE install_root = ?")
.get(root),
).toBeUndefined();
} finally {
database.close();
}
} finally {
write.mockRestore();
destroyed.mockRestore();
input.end();
parent.stdin?.end();
}
});
it("joins canonical aliases while distinct install roots remain independent", async () => {
const tempDir = tempRoots.make("openclaw-handoff-root-");
const root = path.join(tempDir, "install");
const alias = path.join(tempDir, "install-alias");
const otherRoot = path.join(tempDir, "other");
await fs.mkdir(root);
await fs.mkdir(otherRoot);
await fs.symlink(root, alias, "dir");
const { startManagedServiceUpdateHandoff } =
await import("./update-managed-service-handoff.js");
const owner = await startManagedServiceUpdateHandoff({
...baseParams,
root,
handoffId: "handoff-root",
meta: {},
});
await expect(
startManagedServiceUpdateHandoff({ ...baseParams, root: alias, meta: {} }),
).resolves.toMatchObject({ status: "joined", handoffId: "handoff-root" });
const other = await startManagedServiceUpdateHandoff({
...baseParams,
root: otherRoot,
handoffId: "handoff-other",
meta: {},
});
expect(owner).toMatchObject({
status: "started",
handoffId: "handoff-root",
installRoot: await fs.realpath(root),
});
expect(other).toMatchObject({ status: "started", handoffId: "handoff-other" });
expect(spawnMock).toHaveBeenCalledTimes(2);
for (const result of spawnMock.mock.results) {
(result.value as ReturnType<typeof createReadyChild>).emit("exit", 0, null);
}
});
it.each([
["releases the root for another owner", false, false, false, false],
["retains a claimed exited owner until release is confirmed", false, false, true, false],
["treats control-pipe disconnect as cancellation", false, false, true, true],
["wins a concurrent parent exit without running the updater", true, false, false, false],
["refuses recovery when another owner replaces the completed helper", false, true, true, false],
])("cancellation %s", async (_label, exitParent, replaceOwner, claimBeforeExit, disconnect) => {
vi.restoreAllMocks();
const { spawn } =
await vi.importActual<typeof import("node:child_process")>("node:child_process");
const { DatabaseSync } = await import("node:sqlite");
const { getFileLockProcessStartTime } = await import("../shared/pid-alive.js");
const replacementStartIdentity = getFileLockProcessStartTime(process.pid);
if (replacementStartIdentity === null) {
throw new Error("expected the replacement owner to have a stable process identity");
}
spawnMock.mockImplementation(spawn);
const root = await fs.realpath(tempRoots.make("openclaw-cancel-owner-"));
const markerPath = path.join(root, "updater-ran");
const updaterPath = path.join(root, "updater.cjs");
await fs.writeFile(
updaterPath,
`require("node:fs").writeFileSync(${JSON.stringify(markerPath)}, "ran")`,
);
const parent = spawn(process.execPath, ["-e", "process.stdin.resume()"], {
stdio: ["pipe", "ignore", "ignore"],
});
const {
cancelManagedServiceUpdateHandoff,
claimManagedServiceUpdateHandoff,
startManagedServiceUpdateHandoff,
} = await import("./update-managed-service-handoff.js");
const start = () =>
startManagedServiceUpdateHandoff({
root,
restartDrainTimeoutMs: 300_000,
parentPid: parent.pid,
execPath: process.execPath,
argv1: updaterPath,
env: { ...process.env, OPENCLAW_STATE_DIR: root },
meta: {},
});
const started = await start();
if (started.status !== "started") {
throw new Error("expected handoff ownership");
}
const identity = { kind: "managed-update-handoff" as const, ...started };
if (claimBeforeExit) {
expect(claimManagedServiceUpdateHandoff(identity)).toBe(true);
expect(claimManagedServiceUpdateHandoff(identity)).toBe(true);
}
await expect(
cancelManagedServiceUpdateHandoff({ ...identity, handoffId: "joined" }),
).resolves.toBe(false);
const child = spawnMock.mock.results[0]?.value as import("node:child_process").ChildProcess;
const [, args] = spawnMock.mock.calls[0] as [string, string[]];
const helper = JSON.parse(await fs.readFile(args[1] ?? "", "utf8")) as {
updateLeaseDatabasePath: string;
stateDatabasePath: string;
};
let joinedAfterExit: ReturnType<typeof start> | undefined;
child.once("exit", () => {
if (!exitParent) {
joinedAfterExit = start();
}
if (replaceOwner) {
const replacement = new DatabaseSync(helper.updateLeaseDatabasePath);
replacement
.prepare(
"INSERT INTO managed_update_handoffs (install_root, owner, payload_json, updated_at) VALUES (?, ?, ?, ?)",
)
.run(
root,
"replacement",
JSON.stringify({
version: 1,
pid: process.pid,
startIdentity: String(replacementStartIdentity),
}),
Date.now(),
);
replacement.close();
}
});
let cancellation: ReturnType<typeof cancelManagedServiceUpdateHandoff>;
if (claimBeforeExit) {
const exited = new Promise<void>((resolve) => {
child.once("exit", () => resolve());
});
if (disconnect) {
child.stdin?.end();
} else {
child.stdin?.write("cancel\n");
}
await exited;
expect(claimManagedServiceUpdateHandoff(identity)).toBe(false);
expect(joinedAfterExit).toBeDefined();
cancellation = cancelManagedServiceUpdateHandoff(identity);
} else {
cancellation = cancelManagedServiceUpdateHandoff(identity);
if (exitParent) {
parent.stdin?.end();
}
}
await expect(cancellation).resolves.toBe(replaceOwner ? false : "restored-in-process");
expect(child.exitCode !== null || child.signalCode !== null).toBe(true);
const sentinel = new DatabaseSync(helper.stateDatabasePath, { readOnly: true });
const terminal = sentinel
.prepare("SELECT payload_json FROM gateway_restart_sentinel WHERE sentinel_key = 'current'")
.get() as { payload_json: string };
sentinel.close();
expect(JSON.parse(terminal.payload_json)).toMatchObject({
status: "error",
stats: { reason: "managed-service-handoff-cancelled" },
});
if (joinedAfterExit) {
await expect(joinedAfterExit).resolves.toMatchObject({
status: "joined",
handoffId: started.handoffId,
});
expect(spawnMock).toHaveBeenCalledOnce();
}
await expect(fs.access(markerPath)).rejects.toThrow();
if (!replaceOwner && !exitParent) {
const priorSentinel = new DatabaseSync(helper.stateDatabasePath);
priorSentinel
.prepare("DELETE FROM gateway_restart_sentinel WHERE sentinel_key = 'current'")
.run();
priorSentinel.close();
const next = await start();
if (next.status !== "started") {
throw new Error("expected replacement ownership");
}
await expect(
cancelManagedServiceUpdateHandoff({ kind: "managed-update-handoff", ...next }),
).resolves.toBe("restored-in-process");
}
if (replaceOwner) {
const replacement = new DatabaseSync(helper.updateLeaseDatabasePath);
replacement.prepare("DELETE FROM managed_update_handoffs WHERE install_root = ?").run(root);
replacement.close();
}
parent.stdin?.end();
});
});
@@ -0,0 +1,84 @@
import type { EventEmitter } from "node:events";
import fs from "node:fs";
import path from "node:path";
import { DatabaseSync } from "node:sqlite";
import type { Writable } from "node:stream";
import { getFileLockProcessStartTime } from "../shared/pid-alive.js";
export type MockManagedUpdateHandoffLeaseFailure =
| "absent"
| "malformed"
| "wrong-owner"
| "dead-helper";
export function signalMockManagedUpdateHandoffReady(params: {
child: EventEmitter & { pid: number; stdout: Pick<Writable, "destroyed" | "write"> };
paramsPath: string;
cleanups: Set<() => void>;
startIdentity?: number;
failure?: MockManagedUpdateHandoffLeaseFailure;
}): void {
const { child, cleanups, failure } = params;
if (child.stdout.destroyed) {
return;
}
const lease = JSON.parse(fs.readFileSync(params.paramsPath, "utf8")) as {
updateLeaseDatabasePath: string;
updateLeaseKey: string;
updateLeaseOwner: string;
};
const startIdentity = params.startIdentity ?? getFileLockProcessStartTime(child.pid);
if (startIdentity === null) {
throw new Error("expected the mocked handoff child to have a live process identity");
}
fs.mkdirSync(path.dirname(lease.updateLeaseDatabasePath), { recursive: true, mode: 0o700 });
const owner =
failure === "wrong-owner" ? `${lease.updateLeaseOwner}-replacement` : lease.updateLeaseOwner;
const payload = JSON.stringify({
version: 1,
pid: failure === "dead-helper" ? child.pid + 1_000_000 : child.pid,
startIdentity: failure === "malformed" ? null : String(startIdentity),
});
const db = new DatabaseSync(lease.updateLeaseDatabasePath);
try {
if (process.platform !== "win32") {
fs.chmodSync(lease.updateLeaseDatabasePath, 0o600);
}
db.exec("PRAGMA busy_timeout = 5000;");
db.exec(
"CREATE TABLE IF NOT EXISTS managed_update_handoffs " +
"(install_root TEXT NOT NULL PRIMARY KEY, owner TEXT NOT NULL, " +
"payload_json TEXT NOT NULL, updated_at INTEGER NOT NULL) STRICT",
);
if (failure !== "absent") {
db.prepare(
"INSERT INTO managed_update_handoffs " +
"(install_root, owner, payload_json, updated_at) VALUES (?, ?, ?, ?) " +
"ON CONFLICT(install_root) DO UPDATE SET updated_at = excluded.updated_at " +
"WHERE owner = excluded.owner AND payload_json = excluded.payload_json",
).run(lease.updateLeaseKey, owner, payload, Date.now());
}
} finally {
db.close();
}
if (failure !== "absent") {
const cleanup = () => {
cleanups.delete(cleanup);
const cleanupDb = new DatabaseSync(lease.updateLeaseDatabasePath);
try {
cleanupDb.exec("PRAGMA busy_timeout = 5000;");
cleanupDb
.prepare(
"DELETE FROM managed_update_handoffs " +
"WHERE install_root = ? AND owner = ? AND payload_json = ?",
)
.run(lease.updateLeaseKey, owner, payload);
} finally {
cleanupDb.close();
}
};
cleanups.add(cleanup);
child.once("exit", cleanup);
}
child.stdout.write("OPENCLAW_UPDATE_HANDOFF_READY\n");
}
File diff suppressed because it is too large Load Diff
+22 -8
View File
@@ -54,6 +54,8 @@ const {
pid: 12345,
command: "openclaw update --yes --channel beta --timeout 2700",
logPath: "/tmp/openclaw-handoff.log",
handoffId: "auto-handoff-id",
installRoot: "/opt/openclaw",
})),
versionMock: { value: "1.0.0" },
}));
@@ -77,13 +79,8 @@ vi.mock("./openclaw-root.js", async () => {
};
});
vi.mock("./restart.js", () => ({
resolveGatewayRestartDeferralTimeoutMs: (timeoutMs: unknown) => {
if (typeof timeoutMs !== "number" || !Number.isFinite(timeoutMs)) {
return 300_000;
}
return timeoutMs <= 0 ? undefined : Math.floor(timeoutMs);
},
vi.mock("./restart.js", async () => ({
...(await vi.importActual<typeof import("./restart.js")>("./restart.js")),
scheduleGatewaySigusr1Restart: scheduleGatewaySigusr1RestartMock,
}));
@@ -313,6 +310,8 @@ describe("update-startup", () => {
pid: 12345,
command: "openclaw update --yes --channel beta --timeout 2700",
logPath: "/tmp/openclaw-handoff.log",
handoffId: "auto-handoff-id",
installRoot: "/opt/openclaw",
});
resetUpdateAvailableStateForTest();
});
@@ -1911,9 +1910,11 @@ describe("update-startup", () => {
expect(log.info).not.toHaveBeenCalled();
});
it("delegates configured auto-updates to an external supervisor", async () => {
it("keeps external auto-update supervision authoritative over native systemd markers", async () => {
mockPackageUpdateStatus("beta", "2.0.0-beta.1");
process.env.OPENCLAW_SUPERVISOR_MODE = "external";
process.env.OPENCLAW_SYSTEMD_UNIT = "openclaw-gateway.service";
detectRespawnSupervisorMock.mockReturnValue("systemd");
const log = { info: vi.fn() };
const runAutoUpdate = createAutoUpdateSuccessMock();
@@ -1994,6 +1995,8 @@ describe("update-startup", () => {
pid: 12345,
command: "openclaw update --yes --channel beta --tag 2.0.0-beta.1 --timeout 2700",
logPath: "/tmp/openclaw-handoff.log",
handoffId: "started-auto-handoff-id",
installRoot: await fs.realpath(installRoot),
});
const log = { info: vi.fn() };
@@ -2036,6 +2039,11 @@ describe("update-startup", () => {
expect(scheduleGatewaySigusr1RestartMock).toHaveBeenCalledWith({
delayMs: 0,
reason: "update.auto",
successorOwner: {
kind: "managed-update-handoff",
handoffId: "started-auto-handoff-id",
installRoot: await fs.realpath(installRoot),
},
skipCooldown: true,
skipDeferral: true,
});
@@ -2130,6 +2138,7 @@ describe("update-startup", () => {
expect.objectContaining({
root: "/opt/openclaw",
timeoutMs: 45 * 60 * 1000,
restartDrainTimeoutMs: 300_000,
channel: "beta",
tag: "2.0.0-beta.1",
restartDelayMs: 2000,
@@ -2139,6 +2148,11 @@ describe("update-startup", () => {
expect(scheduleGatewaySigusr1RestartMock).toHaveBeenCalledWith({
delayMs: 2000,
reason: "update.auto",
successorOwner: {
kind: "managed-update-handoff",
handoffId: "auto-handoff-id",
installRoot: "/opt/openclaw",
},
skipCooldown: true,
skipDeferral: true,
});
+11 -23
View File
@@ -39,6 +39,7 @@ import {
import { resolveOpenClawPackageRoot } from "./openclaw-root.js";
import { readVerifiedGitUpdateReceipt, type VerifiedGitUpdateReceipt } from "./restart-sentinel.js";
import {
normalizeGatewayRestartDelayMs,
resolveGatewayRestartDeferralTimeoutMs,
scheduleGatewaySigusr1Restart,
} from "./restart.js";
@@ -154,7 +155,6 @@ const AUTO_UPDATE_COMMAND_TIMEOUT_MS = 45 * 60 * 1000;
const AUTO_STABLE_DELAY_HOURS_DEFAULT = 6;
const AUTO_STABLE_JITTER_HOURS_DEFAULT = 12;
const AUTO_BETA_CHECK_INTERVAL_HOURS_DEFAULT = 1;
const MANAGED_AUTO_UPDATE_SYSTEMD_RESTART_GRACE_MS = 2000;
const DEV_COMMIT_LIMIT = 5;
const DEV_COMMIT_SUBJECT_MAX_LENGTH = 120;
const DEV_COMMIT_LOG_MAX_OUTPUT_BYTES = 8 * 1024;
@@ -162,13 +162,7 @@ const DEV_COMMIT_LOG_MAX_OUTPUT_BYTES = 8 * 1024;
type UpdateCheckStateDatabase = Pick<OpenClawStateKyselyDatabase, "update_check_state">;
function shouldSkipCheck(allowInTests: boolean): boolean {
if (allowInTests) {
return false;
}
if (process.env.VITEST || process.env.NODE_ENV === "test") {
return true;
}
return false;
return !allowInTests && Boolean(process.env.VITEST || process.env.NODE_ENV === "test");
}
function resolveAutoUpdatePolicy(cfg: OpenClawConfig): AutoUpdatePolicy {
@@ -438,14 +432,12 @@ function resolveStableAutoApplyAtMs(params: {
return firstSeenMs + baseDelayMs + jitterMs;
}
function resolveManagedAutoUpdateRestartDelayMs(supervisor: RespawnSupervisor): number {
return supervisor === "systemd" ? MANAGED_AUTO_UPDATE_SYSTEMD_RESTART_GRACE_MS : 0;
}
async function startManagedServiceAutoUpdateHandoff(
params: AutoUpdateRunParams & { supervisor: RespawnSupervisor },
): Promise<AutoUpdateRunResult> {
const restartDelayMs = resolveManagedAutoUpdateRestartDelayMs(params.supervisor);
const restartDelayMs = normalizeGatewayRestartDelayMs(
params.supervisor === "systemd" ? undefined : 0,
);
const handoffId = randomUUID();
try {
if (!params.root?.trim()) {
@@ -454,7 +446,9 @@ async function startManagedServiceAutoUpdateHandoff(
const started = await startManagedServiceUpdateHandoff({
root: params.root,
timeoutMs: params.timeoutMs,
restartDrainTimeoutMs: params.restartDrainTimeoutMs,
restartDrainTimeoutMs:
resolveGatewayRestartDeferralTimeoutMs(params.restartDrainTimeoutMs) ??
resolveGatewayRestartDeferralTimeoutMs(),
channel: params.channel,
...(params.packageTargetVersion ? { tag: params.packageTargetVersion } : {}),
restartDelayMs,
@@ -469,9 +463,11 @@ async function startManagedServiceAutoUpdateHandoff(
// Pair helper creation with restart scheduling before any state persistence
// can fail and leave an indefinite handoff waiting on a live parent.
if (started.status === "started") {
const { handoffId: ownerId, installRoot } = started;
scheduleGatewaySigusr1Restart({
delayMs: restartDelayMs,
reason: "update.auto",
successorOwner: { kind: "managed-update-handoff", handoffId: ownerId, installRoot },
skipCooldown: true,
skipDeferral: true,
});
@@ -514,15 +510,7 @@ async function runAutoUpdateCommand(params: AutoUpdateRunParams): Promise<AutoUp
}
}
if (supervisor) {
return await startManagedServiceAutoUpdateHandoff({
channel: params.channel,
timeoutMs: params.timeoutMs,
restartDrainTimeoutMs: params.restartDrainTimeoutMs,
root: params.root,
...(params.packageTargetVersion ? { packageTargetVersion: params.packageTargetVersion } : {}),
...(params.devTarget ? { devTarget: params.devTarget } : {}),
supervisor,
});
return await startManagedServiceAutoUpdateHandoff({ ...params, supervisor });
}
const targetArgs = [
+24 -1
View File
@@ -223,8 +223,31 @@ describe("process start times", () => {
});
});
it("returns null on unsupported platforms", () => {
it("reads a bounded Windows PowerShell process start identity", () => {
const execSpy = vi
.spyOn(childProcess, "execFileSync")
.mockReturnValue("2026-07-06T12:34:56.7890000Z\n");
return withMockedPlatform("win32", async () => {
expect(getProcessStartTime(42)).toBeNull();
expect(getFileLockProcessStartTime(42)).toBe(Date.UTC(2026, 6, 6, 12, 34, 56, 789));
expect(execSpy).toHaveBeenCalledWith(
"powershell.exe",
[
"-NoProfile",
"-NonInteractive",
"-Command",
"(Get-Process -Id 42).StartTime.ToString('o')",
],
expect.objectContaining({ timeout: 1000, windowsHide: true }),
);
execSpy.mockReturnValueOnce("invalid\n");
expect(getFileLockProcessStartTime(42)).toBeNull();
});
});
it("returns null on unsupported platforms", () => {
return withMockedPlatform("freebsd", async () => {
expect(getProcessStartTime(process.pid)).toBeNull();
expect(getFileLockProcessStartTime(process.pid)).toBeNull();
});
+18 -15
View File
@@ -2,7 +2,7 @@
import childProcess from "node:child_process";
import fsSync from "node:fs";
const DARWIN_PS_TIMEOUT_MS = 1000;
const PROCESS_START_TIMEOUT_MS = 1000;
function isValidPid(pid: number): boolean {
return Number.isInteger(pid) && pid > 0;
@@ -40,10 +40,7 @@ export function isPidAlive(pid: number): boolean {
return false;
}
}
if (isZombieProcess(pid)) {
return false;
}
return true;
return !isZombieProcess(pid);
}
/** Returns true only when the PID is invalid, missing, or known to be a Linux zombie. */
@@ -59,20 +56,27 @@ export function isPidDefinitelyDead(pid: number): boolean {
return isZombieProcess(pid);
}
function getDarwinProcessStartTime(pid: number): number | null {
function getPlatformProcessStartTime(pid: number): number | null {
try {
const windows = process.platform === "win32";
const command = windows ? `(Get-Process -Id ${pid}).StartTime.ToString('o')` : String(pid);
const args = windows
? ["-NoProfile", "-NonInteractive", "-Command", command]
: ["-o", "lstart=", "-p", command];
const startedAt = childProcess
.execFileSync("/bin/ps", ["-o", "lstart=", "-p", String(pid)], {
.execFileSync(windows ? "powershell.exe" : "/bin/ps", args, {
encoding: "utf8",
env: { ...process.env, LC_ALL: "C", TZ: "UTC" },
stdio: ["ignore", "pipe", "ignore"],
timeout: DARWIN_PS_TIMEOUT_MS,
timeout: PROCESS_START_TIMEOUT_MS,
killSignal: "SIGKILL",
windowsHide: windows,
})
.trim();
// Darwin's lstart output has no timezone. Force UTC for both ps and parsing so
// a system timezone change cannot make a live lock owner look like PID reuse.
const startedAtMs = Date.parse(`${startedAt} UTC`);
return Number.isFinite(startedAtMs) ? Math.floor(startedAtMs / 1000) : null;
const startedAtMs = Date.parse(windows ? startedAt : `${startedAt} UTC`);
return Number.isFinite(startedAtMs) ? Math.floor(startedAtMs / (windows ? 1 : 1000)) : null;
} catch {
return null;
}
@@ -80,10 +84,7 @@ function getDarwinProcessStartTime(pid: number): number | null {
/** Read the Linux procfs start identity used by Linux-owned runtime state. */
export function getProcessStartTime(pid: number): number | null {
if (!isValidPid(pid)) {
return null;
}
if (process.platform !== "linux") {
if (!isValidPid(pid) || process.platform !== "linux") {
return null;
}
try {
@@ -109,5 +110,7 @@ export function getFileLockProcessStartTime(pid: number): number | null {
if (!isValidPid(pid)) {
return null;
}
return process.platform === "darwin" ? getDarwinProcessStartTime(pid) : getProcessStartTime(pid);
return process.platform === "darwin" || process.platform === "win32"
? getPlatformProcessStartTime(pid)
: getProcessStartTime(pid);
}