diff --git a/src/cli/gateway-cli/lifecycle.runtime.ts b/src/cli/gateway-cli/lifecycle.runtime.ts index 89ec6de9c1d1..7b5dc7bc4c86 100644 --- a/src/cli/gateway-cli/lifecycle.runtime.ts +++ b/src/cli/gateway-cli/lifecycle.runtime.ts @@ -30,6 +30,12 @@ export { consumeGatewayRestartIntentSync, } from "../../infra/restart-intent.js"; export { writeGatewayRestartHandoffSync } from "../../infra/restart-handoff.js"; +export { + cancelManagedServiceUpdateHandoff, + claimManagedServiceUpdateHandoff, + commitManagedServiceUpdateHandoff, + requestManagedServiceUpdateHandoffPark, +} from "../../infra/update-managed-service-handoff.js"; export { resetGatewaySuspendCoordinatorForLifecycleRestart } from "../../infra/gateway-suspend-coordinator.js"; export { rotateAgentEventLifecycleGeneration } from "../../infra/agent-events.js"; export { markUpdateRestartSentinelFailure } from "../../infra/restart-sentinel.js"; diff --git a/src/cli/gateway-cli/run-loop.test.ts b/src/cli/gateway-cli/run-loop.test.ts index abf4f9d2902a..5a93b2d48e76 100644 --- a/src/cli/gateway-cli/run-loop.test.ts +++ b/src/cli/gateway-cli/run-loop.test.ts @@ -1,13 +1,16 @@ // Gateway run loop tests cover foreground gateway lifecycle and restart behavior. -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { GatewayServer } from "../../gateway/server-public.js"; import type { GatewayBonjourBeacon } from "../../infra/bonjour-discovery.js"; import type { GatewayActiveWorkSnapshot } from "../../infra/gateway-active-work.js"; +import type { GatewayRestartIntent } from "../../infra/restart-intent.js"; +import { SUPERVISOR_HINT_ENV_VARS } from "../../infra/supervisor-markers.js"; import { resolveGlobalMap } from "../../shared/global-singleton.js"; import { GATEWAY_AGENT_MEDIA_MIGRATION_REQUIRED_REASON, OpenClawAgentDatabaseMediaMigrationRequiredError, } from "../../state/openclaw-agent-db-migration-required.js"; +import { captureEnv, deleteTestEnvValue } from "../../test-utils/env.js"; import { pickBeaconHost, pickGatewayPort } from "./discover.js"; const acquireGatewayLock = vi.fn(async (_opts?: { port?: number }) => ({ @@ -16,9 +19,21 @@ const acquireGatewayLock = vi.fn(async (_opts?: { port?: number }) => ({ const consumeGatewayRestartIntentPayloadSync = vi.fn< () => { reason?: string; force?: boolean; waitMs?: number } | null >(() => null); -const consumeGatewaySigusr1RestartIntent = vi.fn< - () => { reason?: string; force?: boolean; waitMs?: number } | null ->(() => null); +const consumeGatewaySigusr1RestartIntent = vi.fn<() => GatewayRestartIntent | null>(() => null); +const managedUpdateSuccessorOwner = { + kind: "managed-update-handoff", + handoffId: "handoff-under-test", + installRoot: "/openclaw/install", +} as const; +type ManagedUpdateOwner = NonNullable; +const cancelManagedServiceUpdateHandoff = vi.fn< + (_identity: ManagedUpdateOwner) => Promise +>(async () => "restored-in-process"); +const claimManagedServiceUpdateHandoff = vi.fn((_identity: ManagedUpdateOwner) => true); +const requestManagedServiceUpdateHandoffPark = vi.fn(async (_identity: ManagedUpdateOwner) => true); +const commitManagedServiceUpdateHandoff = vi.fn( + async (_identity: ManagedUpdateOwner, _outcome?: "update" | "restore") => true, +); const consumeGatewaySigusr1RestartAuthorization = vi.fn(() => true); const consumeGatewayRestartIntentSync = vi.fn(() => false); const isGatewaySigusr1RestartExternallyAllowed = vi.fn(() => false); @@ -117,19 +132,17 @@ const reloadTaskRuntimeStateFromStore = vi.fn(); const clearRuntimeConfigSnapshot = vi.fn(); const restartGatewayProcessWithFreshPid = vi.fn< (_opts?: { env?: NodeJS.ProcessEnv }) => { - mode: "spawned" | "supervised" | "disabled" | "failed"; - pid?: number; + mode: "supervised" | "disabled" | "failed"; detail?: string; handoffSpawned?: Promise; } >(() => ({ mode: "disabled" })); const respawnGatewayProcessForUpdate = vi.fn< (_opts?: { env?: NodeJS.ProcessEnv }) => { - mode: "spawned" | "supervised" | "disabled" | "failed"; + mode: "spawned" | "disabled" | "failed"; pid?: number; detail?: string; child?: { kill: () => void }; - handoffSpawned?: Promise; } >(() => ({ mode: "disabled", detail: "OPENCLAW_NO_RESPAWN" })); const markUpdateRestartSentinelFailure = vi.fn<(reason: string) => Promise>( @@ -188,6 +201,19 @@ vi.mock("../../infra/restart-intent.js", () => ({ consumeGatewayRestartIntentSync: () => consumeGatewayRestartIntentSync(), })); +vi.mock("../../infra/update-managed-service-handoff.js", () => ({ + cancelManagedServiceUpdateHandoff: (identity: ManagedUpdateOwner) => + cancelManagedServiceUpdateHandoff(identity), + claimManagedServiceUpdateHandoff: (identity: ManagedUpdateOwner) => + claimManagedServiceUpdateHandoff(identity), + requestManagedServiceUpdateHandoffPark: (identity: ManagedUpdateOwner) => + requestManagedServiceUpdateHandoffPark(identity), + commitManagedServiceUpdateHandoff: ( + identity: ManagedUpdateOwner, + outcome?: "update" | "restore", + ) => commitManagedServiceUpdateHandoff(identity, outcome), +})); + vi.mock("../../infra/gateway-suspend-coordinator.js", () => ({ resetGatewaySuspendCoordinatorForLifecycleRestart: () => resetGatewaySuspendCoordinatorForLifecycleRestart(), @@ -473,8 +499,29 @@ function expectRestartHandoffCall(expected: { } let gatewayWorkAdmissionActual: typeof import("../../process/gateway-work-admission.js"); +let supervisorEnvSnapshot: ReturnType | undefined; beforeEach(async () => { + vi.useRealTimers(); + supervisorEnvSnapshot = captureEnv([...SUPERVISOR_HINT_ENV_VARS]); + for (const key of SUPERVISOR_HINT_ENV_VARS) { + deleteTestEnvValue(key); + } + + // clearAllMocks preserves queued one-shot results. A skipped lifecycle branch + // must not shift a stale supervisor or respawn decision into the next case. + consumeGatewaySigusr1RestartIntent.mockReset(); + consumeGatewaySigusr1RestartIntent.mockReturnValue(null); + peekGatewaySigusr1RestartReason.mockReset(); + peekGatewaySigusr1RestartReason.mockReturnValue(undefined); + restartGatewayProcessWithFreshPid.mockReset(); + restartGatewayProcessWithFreshPid.mockReturnValue({ mode: "disabled" }); + respawnGatewayProcessForUpdate.mockReset(); + respawnGatewayProcessForUpdate.mockReturnValue({ + mode: "disabled", + detail: "OPENCLAW_NO_RESPAWN", + }); + gatewayWorkAdmissionActual = await vi.importActual("../../process/gateway-work-admission.js"); gatewayWorkAdmissionActual.resetGatewayWorkAdmission(); createGatewayActiveWorkSnapshot.mockReset(); @@ -485,6 +532,23 @@ beforeEach(async () => { options?.onSnapshot?.(snapshot); return { drained: snapshot.idle, snapshot }; }); + cancelManagedServiceUpdateHandoff.mockReset(); + cancelManagedServiceUpdateHandoff.mockResolvedValue("restored-in-process"); + claimManagedServiceUpdateHandoff.mockReset(); + claimManagedServiceUpdateHandoff.mockReturnValue(true); + requestManagedServiceUpdateHandoffPark.mockReset(); + requestManagedServiceUpdateHandoffPark.mockResolvedValue(true); + commitManagedServiceUpdateHandoff.mockReset(); + commitManagedServiceUpdateHandoff.mockResolvedValue(true); +}); + +afterEach(() => { + supervisorEnvSnapshot?.restore(); + supervisorEnvSnapshot = undefined; + vi.useRealTimers(); + if (originalPlatformDescriptor) { + Object.defineProperty(process, "platform", originalPlatformDescriptor); + } }); describe("runGatewayLoop", () => { @@ -1501,9 +1565,14 @@ describe("runGatewayLoop", () => { await withIsolatedSignals(async ({ captureSignal }) => { const close = vi.fn(async () => {}); const startupNeverReturns = new Promise(() => {}); + let markStartupEntered: () => void = () => {}; + const startupEntered = new Promise((resolve) => { + markStartupEntered = resolve; + }); const { runtime, exited } = createRuntimeWithExitSignal(); const completeBoot = vi.fn(); const start = vi.fn(async () => { + markStartupEntered(); await startupNeverReturns; return createGatewayServer(close); }); @@ -1515,6 +1584,7 @@ describe("runGatewayLoop", () => { completeBoot, }); await vi.advanceTimersByTimeAsync(0); + await startupEntered; const sigusr1 = captureSignal("SIGUSR1"); sigusr1(); @@ -2033,11 +2103,12 @@ describe("runGatewayLoop", () => { }); }); - it("releases the lock before exiting on spawned restart", async () => { + it("releases the lock before exiting on supervised restart", async () => { vi.clearAllMocks(); peekGatewaySigusr1RestartReason.mockReturnValue(undefined); const originalTraceEnv = process.env.OPENCLAW_GATEWAY_RESTART_TRACE; process.env.OPENCLAW_GATEWAY_RESTART_TRACE = "1"; + process.env.OPENCLAW_SUPERVISOR_MODE = "external"; try { await withIsolatedSignals(async ({ captureSignal }) => { @@ -2046,11 +2117,7 @@ describe("runGatewayLoop", () => { release: lockRelease, }); - // Override process-respawn to return "spawned" mode - restartGatewayProcessWithFreshPid.mockReturnValueOnce({ - mode: "spawned", - pid: 9999, - }); + restartGatewayProcessWithFreshPid.mockReturnValueOnce({ mode: "supervised" }); const exitCallOrder: string[] = []; const { runtime, exited } = await createSignaledLoopHarness(exitCallOrder); @@ -2068,9 +2135,10 @@ describe("runGatewayLoop", () => { const [respawnOpts] = restartGatewayProcessWithFreshPid.mock.calls[0] ?? []; expect(respawnOpts?.env?.OPENCLAW_GATEWAY_RESTART_TRACE_STARTED_AT_MS).toMatch(/^\d/u); expect(respawnOpts?.env?.OPENCLAW_GATEWAY_RESTART_TRACE_LAST_AT_MS).toMatch(/^\d/u); - expect(writeGatewayRestartHandoffSync).not.toHaveBeenCalled(); + expect(writeGatewayRestartHandoffSync).toHaveBeenCalledOnce(); }); } finally { + delete process.env.OPENCLAW_SUPERVISOR_MODE; if (originalTraceEnv === undefined) { delete process.env.OPENCLAW_GATEWAY_RESTART_TRACE; } else { @@ -2363,11 +2431,12 @@ describe("runGatewayLoop", () => { async (reason) => { vi.clearAllMocks(); peekGatewaySigusr1RestartReason.mockReturnValue(reason); - respawnGatewayProcessForUpdate.mockReturnValueOnce({ + restartGatewayProcessWithFreshPid.mockReturnValueOnce({ mode: "supervised", }); try { setPlatform("freebsd"); + process.env.OPENCLAW_SUPERVISOR_MODE = "external"; await withIsolatedSignals(async ({ captureSignal }) => { const { runtime, exited } = await createSignaledLoopHarness(); const sigusr1 = captureSignal("SIGUSR1"); @@ -2381,8 +2450,10 @@ describe("runGatewayLoop", () => { reason, supervisorMode: "external", }); + expect(respawnGatewayProcessForUpdate).not.toHaveBeenCalled(); }); } finally { + delete process.env.OPENCLAW_SUPERVISOR_MODE; if (originalPlatformDescriptor) { Object.defineProperty(process, "platform", originalPlatformDescriptor); } @@ -2393,7 +2464,7 @@ describe("runGatewayLoop", () => { it("falls back in-process when a launchd update handoff fails to spawn", async () => { vi.clearAllMocks(); peekGatewaySigusr1RestartReason.mockReturnValue("update.run"); - respawnGatewayProcessForUpdate.mockReturnValueOnce({ + restartGatewayProcessWithFreshPid.mockReturnValueOnce({ mode: "supervised", handoffSpawned: Promise.resolve(false), }); @@ -2431,7 +2502,7 @@ describe("runGatewayLoop", () => { vi.clearAllMocks(); peekGatewaySigusr1RestartReason.mockReturnValue("update.run"); process.env.OPENCLAW_SUPERVISOR_MODE = "external"; - respawnGatewayProcessForUpdate.mockReturnValueOnce({ + restartGatewayProcessWithFreshPid.mockReturnValueOnce({ mode: "supervised", }); writeGatewayRestartHandoffSync.mockReturnValueOnce(null); @@ -2461,19 +2532,12 @@ describe("runGatewayLoop", () => { } }); - it("upgrades an accepted restart when a managed update arrives during shutdown", async () => { + it("upgrades an accepted restart when an update arrives during shutdown", async () => { vi.clearAllMocks(); - consumeGatewayRestartIntentPayloadSync.mockReset(); - consumeGatewayRestartIntentPayloadSync.mockReturnValue(null); - consumeGatewaySigusr1RestartIntent.mockReset(); - consumeGatewaySigusr1RestartIntent.mockReturnValue(null); - consumeGatewaySigusr1RestartAuthorization.mockReset(); - consumeGatewaySigusr1RestartAuthorization.mockReturnValue(true); - peekGatewaySigusr1RestartReason.mockReset(); peekGatewaySigusr1RestartReason .mockReturnValueOnce("config.patch") .mockReturnValueOnce("update.auto"); - respawnGatewayProcessForUpdate.mockReturnValueOnce({ mode: "supervised" }); + restartGatewayProcessWithFreshPid.mockReturnValueOnce({ mode: "supervised" }); let releaseClose: () => void = () => {}; const close = vi.fn( @@ -2482,9 +2546,9 @@ describe("runGatewayLoop", () => { releaseClose = resolve; }), ); - + setPlatform("freebsd"); + process.env.OPENCLAW_SUPERVISOR_MODE = "external"; try { - setPlatform("freebsd"); await withIsolatedSignals(async ({ captureSignal }) => { const { start, started } = createSignaledStart(close); const { runtime, exited } = createRuntimeWithExitSignal(); @@ -2492,58 +2556,41 @@ describe("runGatewayLoop", () => { await waitForStart(started); const sigusr1 = captureSignal("SIGUSR1"); - try { - sigusr1(); - await waitForLoopCondition( - () => close.mock.calls.length === 1, - "restart close did not start", - ); - expect(armShutdownHardExitWatchdog).toHaveBeenCalledTimes(1); - sigusr1(); - await waitForLoopCondition( - () => - gatewayLog.info.mock.calls.some(([message]) => - String(message).includes("upgrading to update.auto"), - ), - "accepted restart was not upgraded", - ); - expect(cancelShutdownHardExitWatchdog).toHaveBeenCalledTimes(1); - expect(armShutdownHardExitWatchdog).toHaveBeenCalledTimes(2); + sigusr1(); + await waitForLoopCondition( + () => close.mock.calls.length === 1, + "restart close did not start", + ); + sigusr1(); + await waitForLoopCondition( + () => + gatewayLog.info.mock.calls.some(([message]) => + String(message).includes("upgrading to update.auto"), + ), + "accepted restart was not upgraded", + ); - releaseClose(); - await expect(exited).resolves.toBe(0); - expect(cancelShutdownHardExitWatchdog).toHaveBeenCalledTimes(2); - expect(respawnGatewayProcessForUpdate).toHaveBeenCalledTimes(1); - expectRestartHandoffCall({ - restartKind: "update-process", - reason: "update.auto", - supervisorMode: "external", - }); - } finally { - releaseClose(); - await exited; - } + releaseClose(); + await expect(exited).resolves.toBe(0); + expect(restartGatewayProcessWithFreshPid).toHaveBeenCalledOnce(); + expectRestartHandoffCall({ + restartKind: "update-process", + reason: "update.auto", + supervisorMode: "external", + }); }); } finally { - if (originalPlatformDescriptor) { - Object.defineProperty(process, "platform", originalPlatformDescriptor); - } + releaseClose(); + delete process.env.OPENCLAW_SUPERVISOR_MODE; } }); - it("reads a managed update upgrade after asynchronous lock release", async () => { + it("reads an update upgrade after asynchronous lock release", async () => { vi.clearAllMocks(); - consumeGatewayRestartIntentPayloadSync.mockReset(); - consumeGatewayRestartIntentPayloadSync.mockReturnValue(null); - consumeGatewaySigusr1RestartIntent.mockReset(); - consumeGatewaySigusr1RestartIntent.mockReturnValue(null); - consumeGatewaySigusr1RestartAuthorization.mockReset(); - consumeGatewaySigusr1RestartAuthorization.mockReturnValue(true); - peekGatewaySigusr1RestartReason.mockReset(); peekGatewaySigusr1RestartReason .mockReturnValueOnce("config.patch") .mockReturnValueOnce("update.auto"); - respawnGatewayProcessForUpdate.mockReturnValueOnce({ mode: "supervised" }); + restartGatewayProcessWithFreshPid.mockReturnValueOnce({ mode: "supervised" }); let releaseLock: () => void = () => {}; const lockReleaseBlocked = new Promise((resolve) => { @@ -2553,31 +2600,157 @@ describe("runGatewayLoop", () => { await lockReleaseBlocked; }); acquireGatewayLock.mockResolvedValueOnce({ release: lockRelease }); + process.env.OPENCLAW_SUPERVISOR_MODE = "external"; + try { + await withIsolatedSignals(async ({ captureSignal }) => { + const { runtime, exited } = await createSignaledLoopHarness(); + const sigusr1 = captureSignal("SIGUSR1"); + sigusr1(); + await waitForLoopCondition( + () => lockRelease.mock.calls.length === 1, + "restart did not reach lock release", + ); + sigusr1(); + await waitForLoopCondition( + () => + gatewayLog.info.mock.calls.some(([message]) => + String(message).includes("upgrading to update.auto"), + ), + "lock-release restart was not upgraded", + ); - await withIsolatedSignals(async ({ captureSignal }) => { - const { runtime, exited } = await createSignaledLoopHarness(); - const sigusr1 = captureSignal("SIGUSR1"); - - sigusr1(); - await waitForLoopCondition( - () => lockRelease.mock.calls.length === 1, - "restart did not reach lock release", - ); - sigusr1(); - await waitForLoopCondition( - () => - gatewayLog.info.mock.calls.some(([message]) => - String(message).includes("upgrading to update.auto"), - ), - "lock-release restart was not upgraded", - ); - + releaseLock(); + await expect(exited).resolves.toBe(0); + expect(restartGatewayProcessWithFreshPid).toHaveBeenCalledOnce(); + expect(runtime.exit).toHaveBeenCalledWith(0); + }); + } finally { releaseLock(); - await expect(exited).resolves.toBe(0); - expect(respawnGatewayProcessForUpdate).toHaveBeenCalledTimes(1); - expect(restartGatewayProcessWithFreshPid).not.toHaveBeenCalled(); - expect(runtime.exit).toHaveBeenCalledWith(0); + delete process.env.OPENCLAW_SUPERVISOR_MODE; + } + }); + + it("recovers in process after exactly cancelling a replacement managed owner before exit", async () => { + vi.clearAllMocks(); + const replacementOwner = { ...managedUpdateSuccessorOwner, handoffId: "replacement-handoff" }; + consumeGatewaySigusr1RestartIntent + .mockReturnValueOnce({ reason: "update.run", successorOwner: managedUpdateSuccessorOwner }) + .mockReturnValueOnce({ reason: "update.auto", successorOwner: replacementOwner }); + cancelManagedServiceUpdateHandoff + .mockResolvedValueOnce("restored-in-process") + .mockResolvedValueOnce("restored-in-process"); + + let releaseCommit: () => void = () => {}; + const commitBlocked = new Promise((resolve) => { + releaseCommit = resolve; }); + commitManagedServiceUpdateHandoff.mockImplementationOnce(async () => { + await commitBlocked; + return true; + }); + setPlatform("linux"); + process.env.OPENCLAW_SERVICE_MARKER = "openclaw"; + process.env.OPENCLAW_SERVICE_KIND = "gateway"; + try { + await withIsolatedSignals(async ({ captureSignal }) => { + const { start, runtime, exited } = await createSignaledLoopHarness(); + const sigusr1 = captureSignal("SIGUSR1"); + const sigint = captureSignal("SIGINT"); + sigusr1(); + await waitForLoopCondition( + () => commitManagedServiceUpdateHandoff.mock.calls.length === 1, + "managed owner did not reach its final helper commit", + ); + sigusr1(); + await waitForLoopCondition( + () => consumeGatewaySigusr1RestartIntent.mock.calls.length === 2, + "replacement owner was not admitted before exit", + ); + releaseCommit(); + await waitForLoopCondition( + () => start.mock.calls.length === 2, + "replacement managed owner cancellation did not reopen gateway admission", + ); + + expect(requestManagedServiceUpdateHandoffPark).toHaveBeenCalledExactlyOnceWith( + managedUpdateSuccessorOwner, + ); + expect(cancelManagedServiceUpdateHandoff).toHaveBeenNthCalledWith( + 1, + managedUpdateSuccessorOwner, + ); + expect(cancelManagedServiceUpdateHandoff).toHaveBeenNthCalledWith(2, replacementOwner); + expect(commitManagedServiceUpdateHandoff).toHaveBeenCalledExactlyOnceWith( + managedUpdateSuccessorOwner, + "update", + ); + expect(runtime.exit).not.toHaveBeenCalled(); + expect(start).toHaveBeenCalledTimes(2); + expect(gatewayWorkAdmissionActual.isGatewayWorkAdmissionClosed()).toBe(false); + + sigint(); + await expect(exited).resolves.toBe(0); + }); + } finally { + releaseCommit(); + delete process.env.OPENCLAW_SERVICE_MARKER; + delete process.env.OPENCLAW_SERVICE_KIND; + } + }); + + it("reopens admission after a broken control pipe waits for the exact helper to exit", async () => { + vi.clearAllMocks(); + consumeGatewaySigusr1RestartIntent.mockReturnValueOnce({ + reason: "update.run", + successorOwner: managedUpdateSuccessorOwner, + }); + requestManagedServiceUpdateHandoffPark.mockResolvedValueOnce(false); + let releaseHelperExit: () => void = () => {}; + const helperExit = new Promise((resolve) => { + releaseHelperExit = resolve; + }); + cancelManagedServiceUpdateHandoff.mockImplementationOnce(async () => { + await helperExit; + return "restored-in-process"; + }); + setPlatform("linux"); + process.env.OPENCLAW_SERVICE_MARKER = "openclaw"; + process.env.OPENCLAW_SERVICE_KIND = "gateway"; + + try { + await withIsolatedSignals(async ({ captureSignal }) => { + const { start, runtime, exited } = await createSignaledLoopHarness(); + const sigusr1 = captureSignal("SIGUSR1"); + const sigint = captureSignal("SIGINT"); + + sigusr1(); + await waitForLoopCondition( + () => cancelManagedServiceUpdateHandoff.mock.calls.length === 1, + "broken helper control pipe did not begin exact-owner cancellation", + ); + expect(start).toHaveBeenCalledOnce(); + expect(gatewayWorkAdmissionActual.isGatewayWorkAdmissionClosed()).toBe(true); + releaseHelperExit(); + await waitForLoopCondition( + () => start.mock.calls.length === 2, + "broken helper control pipe left the gateway permanently draining", + ); + + expect(cancelManagedServiceUpdateHandoff).toHaveBeenCalledExactlyOnceWith( + managedUpdateSuccessorOwner, + ); + expect(commitManagedServiceUpdateHandoff).not.toHaveBeenCalled(); + expect(runtime.exit).not.toHaveBeenCalled(); + expect(gatewayWorkAdmissionActual.isGatewayWorkAdmissionClosed()).toBe(false); + + sigint(); + await expect(exited).resolves.toBe(0); + }); + } finally { + releaseHelperExit(); + delete process.env.OPENCLAW_SERVICE_MARKER; + delete process.env.OPENCLAW_SERVICE_KIND; + } }); it("probes the configured gateway host for update respawn health", async () => { diff --git a/src/cli/gateway-cli/run-loop.ts b/src/cli/gateway-cli/run-loop.ts index e90a6089328c..e8a790ee19d2 100644 --- a/src/cli/gateway-cli/run-loop.ts +++ b/src/cli/gateway-cli/run-loop.ts @@ -1,4 +1,5 @@ // In-process gateway run loop, restart signaling, drain, and update respawn handling. +import type { ChildProcess } from "node:child_process"; import { randomUUID } from "node:crypto"; import net from "node:net"; import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; @@ -17,6 +18,7 @@ import { type GatewayBootLifecycleCompletion, } from "../../infra/gateway-boot-lifecycle.js"; import { acquireGatewayLock } from "../../infra/gateway-lock.js"; +import type { GatewayRestartIntent } from "../../infra/restart-intent.js"; import type { GatewayRestartEmitter } from "../../infra/restart.js"; import { flushLogger } from "../../logging/logger.js"; import { createSubsystemLogger } from "../../logging/subsystem.js"; @@ -42,17 +44,11 @@ const LOG_FLUSH_EXIT_TIMEOUT_MS = 4_000; const HARD_EXIT_WATCHDOG_GRACE_MS = 2_000; type GatewayRunSignalAction = "stop" | "restart"; -type RestartDrainTimeoutMs = number | undefined; -type RestartIntentOptions = { - reason?: string; - force?: boolean; - waitMs?: number; -}; type GatewayRunSignalRequest = { action: GatewayRunSignalAction; signal: string; restartReason?: string; - restartIntent?: RestartIntentOptions; + restartIntent?: GatewayRestartIntent; }; type GatewayLifecycleRuntimeModule = typeof import("./lifecycle.runtime.js"); @@ -70,20 +66,11 @@ const loadGatewayLifecycleRuntimeModule = () => gatewayLifecycleRuntimeLoader.lo async function waitForGatewayPortReady(host: string, port: number): Promise { return await new Promise((resolve) => { const socket = net.createConnection({ host, port }); - let settled = false; const finish = (value: boolean) => { - if (settled) { - return; - } - settled = true; - clearTimeout(timer); - socket.removeAllListeners(); socket.destroy(); resolve(value); }; - const timer = setTimeout(() => { - finish(false); - }, UPDATE_RESPAWN_HEALTH_POLL_MS); + socket.setTimeout(UPDATE_RESPAWN_HEALTH_POLL_MS, () => finish(false)); socket.once("connect", () => finish(true)); socket.once("error", () => finish(false)); }); @@ -141,6 +128,11 @@ export async function runGatewayLoop(params: { // here pulls the lifecycle re-export graph into memory, immune to later disk // rotation. const eagerLifecycleRuntime = await loadGatewayLifecycleRuntimeModule(); + const supervisorMode = eagerLifecycleRuntime.detectGatewayRespawnSupervisor( + process.env, + process.platform, + { includeLinuxOpenClawGatewayServiceMarker: true }, + ); let lock = await acquireGatewayLock({ port: params.lockPort }); let server: Awaited> | null = null; let shuttingDown = false; @@ -149,12 +141,22 @@ export async function runGatewayLoop(params: { // Defer lifecycle signals from that window until the loop can close and advance. let pendingStartupRequest: GatewayRunSignalRequest | null = null; let activeRestartRequest: GatewayRunSignalRequest | null = null; + let committedGenericSuccessor: ChildProcess | true | null = null; let forceActiveRestartExit: (() => void) | null = null; let pendingStartupForceExitTimer: ReturnType | null = null; let restartDrainingMarked = false; let startupFailedWithoutServerHandle = false; const processInstanceId = randomUUID(); const waitForHealthyChild = params.waitForHealthyChild ?? waitForHealthyGatewayChild; + const getManagedUpdateOwner = () => + (pendingStartupRequest ?? activeRestartRequest)?.restartIntent?.successorOwner; + const sameManagedUpdateOwner = ( + left: GatewayRestartIntent["successorOwner"], + right: GatewayRestartIntent["successorOwner"], + ) => + Boolean( + left && right && left.handoffId === right.handoffId && left.installRoot === right.installRoot, + ); const cleanupSignals = () => { process.removeListener("SIGTERM", onSigterm); @@ -165,7 +167,13 @@ export async function runGatewayLoop(params: { cleanupSignals(); params.runtime.exit(code); }; - const exitProcessAfterLogFlush = async (code: number) => { + const exitProcessAfterLogFlush = async ( + code: number, + initialOwner?: GatewayRestartIntent["successorOwner"], + initialOutcome: "update" | "restore" = "update", + ): Promise => { + let ownerToCommit = initialOwner; + let commitOutcome = initialOutcome; // Graceful signal/restart paths call process.exit(), which skips beforeExit. let flushTimer: ReturnType | undefined; const flushed = await Promise.race([ @@ -174,69 +182,179 @@ export async function runGatewayLoop(params: { flushTimer = setTimeout(() => resolve(false), LOG_FLUSH_EXIT_TIMEOUT_MS); }), ]); - if (flushTimer) { - clearTimeout(flushTimer); - } + clearTimeout(flushTimer); if (!flushed) { gatewayLog.warn( `log flush did not settle within ${LOG_FLUSH_EXIT_TIMEOUT_MS}ms; continuing shutdown`, ); } - exitProcess(code); + for (;;) { + const owner = getManagedUpdateOwner(); + if (!owner) { + if (!ownerToCommit) { + exitProcess(code); + } + return; + } + if ( + sameManagedUpdateOwner(owner, ownerToCommit) && + eagerLifecycleRuntime.claimManagedServiceUpdateHandoff(owner) && + (await eagerLifecycleRuntime.commitManagedServiceUpdateHandoff(owner, commitOutcome)) && + sameManagedUpdateOwner(getManagedUpdateOwner(), owner) && + eagerLifecycleRuntime.claimManagedServiceUpdateHandoff(owner) + ) { + // Keep exact request ownership live through the synchronous exit call. + exitProcess(code); + return; + } + await markRestartHandoffUnavailable(); + const ownerToCancel = ownerToCommit ?? owner; + const restoration = await cancelManagedUpdateHandoffBeforeRecovery(ownerToCancel); + if (!restoration) { + const child = committedGenericSuccessor === true ? null : committedGenericSuccessor; + if (child && child.exitCode === null && child.signalCode === null) { + const exited = new Promise((resolve) => { + child.once("exit", () => { + resolve(); + }); + }); + try { + child.kill("SIGKILL"); + await exited; + } catch {} + } + return; + } + if (restoration === "restart-after-exit") { + ownerToCommit = ownerToCancel; + commitOutcome = "restore"; + const currentRequest = pendingStartupRequest ?? activeRestartRequest; + if ( + currentRequest && + !sameManagedUpdateOwner(currentRequest.restartIntent?.successorOwner, ownerToCancel) + ) { + currentRequest.restartIntent = { + ...currentRequest.restartIntent, + successorOwner: ownerToCancel, + }; + } + continue; + } + if (!committedGenericSuccessor && initialOwner) { + return reacquireAndResumeInProcessRestart(getManagedUpdateOwner() ?? owner); + } + exitProcess(code); + return; + } }; - const completeForcedStop = (reason: string) => { - params.completeBoot?.({ outcome: "forced_stop", reason }); - }; - const writeStabilityBundle = async (reason: string, error?: unknown) => { - const { writeDiagnosticStabilityBundleForFailureSync } = - await loadGatewayLifecycleRuntimeModule(); - const result = writeDiagnosticStabilityBundleForFailureSync(reason, error); + const writeStabilityBundle = (reason: string, error?: unknown) => { + const result = eagerLifecycleRuntime.writeDiagnosticStabilityBundleForFailureSync( + reason, + error, + ); if ("message" in result) { gatewayLog.warn(result.message); } }; - const releaseLockIfHeld = async (): Promise => { - if (!lock) { - return false; - } - await lock.release(); + const releaseLockIfHeld = async (): Promise => { + await lock?.release(); lock = null; - return true; }; - const reacquireLockForInProcessRestart = async (): Promise => { + const cancelManagedUpdateHandoffBeforeRecovery = async ( + initialOwner = getManagedUpdateOwner(), + ): Promise => { + let owner = initialOwner; + let requiresParentExit = false; try { - lock = await acquireGatewayLock({ port: params.lockPort }); - return true; + for (;;) { + if (!owner) { + return requiresParentExit ? "restart-after-exit" : "restored-in-process"; + } + const restoration = await eagerLifecycleRuntime.cancelManagedServiceUpdateHandoff(owner); + if (!restoration) { + gatewayLog.error("managed update handoff cancellation unconfirmed; remaining draining"); + return false; + } + requiresParentExit ||= restoration === "restart-after-exit"; + const replacement = getManagedUpdateOwner(); + if (!replacement || sameManagedUpdateOwner(owner, replacement)) { + return requiresParentExit ? "restart-after-exit" : "restored-in-process"; + } + owner = replacement; + } } catch (err) { - gatewayLog.error(`failed to reacquire gateway lock for in-process restart: ${String(err)}`); - exitProcess(1); + gatewayLog.error(`managed update handoff cancellation failed: ${formatErrorMessage(err)}`); return false; } }; - const confirmLaunchdHandoff = async (respawn: { - handoffSpawned?: Promise; - }): Promise => { - const delay = new Promise((resolve) => { - setTimeout(resolve, LAUNCHD_SUPERVISED_RESTART_EXIT_DELAY_MS); - }); - const spawned = respawn.handoffSpawned - ? await Promise.race([respawn.handoffSpawned, delay.then(() => true)]) - : false; - // Preserve the crash-loop throttle window even when spawn settles early. - await delay; - return spawned; + const forceExitAfterStabilityBundle = async (reason: string) => { + try { + writeStabilityBundle(reason); + } finally { + const owner = getManagedUpdateOwner(); + if (owner) { + forceActiveRestartExit?.(); + } + const restoration = await cancelManagedUpdateHandoffBeforeRecovery(owner); + if (restoration) { + params.completeBoot?.({ outcome: "forced_stop", reason }); + if (restoration === "restart-after-exit") { + await exitProcessAfterLogFlush(1, owner, "restore"); + } else { + exitProcess(1); + } + } + } }; - const handleRestartAfterServerClose = async () => { + const reacquireAndResumeInProcessRestart = async ( + alreadyCancelledOwner?: GatewayRestartIntent["successorOwner"], + ): Promise => { + for (;;) { + const restartRequest = activeRestartRequest; + const restartOwner = restartRequest?.restartIntent?.successorOwner; + const restoration = sameManagedUpdateOwner(restartOwner, alreadyCancelledOwner) + ? "restored-in-process" + : await cancelManagedUpdateHandoffBeforeRecovery(restartOwner); + if (!restoration) { + return; + } + if (restoration === "restart-after-exit") { + await releaseLockIfHeld(); + return exitProcessAfterLogFlush(0, restartOwner, "restore"); + } + if (activeRestartRequest !== restartRequest) { + continue; + } + try { + lock = await acquireGatewayLock({ port: params.lockPort }); + } catch (err) { + if (activeRestartRequest !== restartRequest) { + continue; + } + gatewayLog.error(`failed to reacquire gateway lock for in-process restart: ${String(err)}`); + exitProcess(1); + return; + } + if (activeRestartRequest === restartRequest) { + activeRestartRequest = null; + shuttingDown = false; + restartResolver?.(); + return; + } + await releaseLockIfHeld(); + } + }; + const markRestartHandoffUnavailable = async (reason = "restart-handoff-unavailable") => { + await eagerLifecycleRuntime.markUpdateRestartSentinelFailure(reason).catch((err: unknown) => { + gatewayLog.warn(`failed to mark update restart ${reason}: ${String(err)}`); + }); + }; + const handleRestartAfterServerClose = async ( + expectedOwner?: GatewayRestartIntent["successorOwner"], + cancelled = false, + ): Promise => { await releaseLockIfHeld(); - const { - detectGatewayRespawnSupervisor, - markUpdateRestartSentinelFailure, - respawnGatewayProcessForUpdate, - restartGatewayProcessWithFreshPid, - writeGatewayRestartHandoffSync, - } = await loadGatewayLifecycleRuntimeModule(); - // Lock release and lazy lifecycle loading may yield while a managed update - // upgrades this restart. Keep the request live until a restart path commits. + // Lock release may yield while a managed update upgrades this restart. const restartReason = activeRestartRequest?.restartReason; params.completeBoot?.({ outcome: "planned_restart", @@ -244,214 +362,122 @@ export async function runGatewayLoop(params: { }); const isUpdateRestart = isUpdateProcessRestartReason(restartReason); - if (isUpdateRestart) { - const restartTraceHandoff = captureGatewayRestartTraceHandoff(); - const respawn = respawnGatewayProcessForUpdate({ - env: createGatewayRestartTraceHandoffEnv(restartTraceHandoff), - }); - if (respawn.mode === "spawned") { - const port = params.lockPort; - const healthy = - typeof port === "number" - ? await waitForHealthyChild(port, respawn.pid, params.healthHost ?? "127.0.0.1") - : false; - if (healthy) { - activeRestartRequest = null; - gatewayLog.info( - `restart mode: update process respawn (spawned pid ${respawn.pid ?? "unknown"})`, - ); - await exitProcessAfterLogFlush(0); - return; - } - gatewayLog.warn( - `update respawn child did not become healthy (${respawn.pid ?? "unknown"}); falling back to in-process restart`, - ); - try { - respawn.child?.kill(); - } catch { - // Best-effort; parent fallback keeps the gateway reachable for recovery. - } - await markUpdateRestartSentinelFailure("restart-unhealthy").catch((err: unknown) => { - gatewayLog.warn(`failed to mark update restart sentinel unhealthy: ${String(err)}`); - }); - if (!(await reacquireLockForInProcessRestart())) { - return; - } - shuttingDown = false; - restartResolver?.(); - return; + if (cancelled) { + return reacquireAndResumeInProcessRestart(expectedOwner); + } + if (activeRestartRequest?.restartIntent?.successorOwner) { + if (!expectedOwner) { + gatewayLog.error("managed update handoff arrived after successor parking closed"); + await markRestartHandoffUnavailable(); + return reacquireAndResumeInProcessRestart(); } - if (respawn.mode === "supervised") { - const supervisorMode = detectGatewayRespawnSupervisor(process.env, process.platform); - markGatewayRestartTrace("restart.full-process-handoff", [ - ["kind", "update-process"], - ["mode", respawn.mode], - ["supervisorMode", supervisorMode ?? "external"], - ]); - const handoff = writeGatewayRestartHandoffSync({ - restartKind: "update-process", - reason: restartReason, - processInstanceId, - supervisorMode: supervisorMode ?? "external", - restartTrace: captureGatewayRestartTraceHandoff(), - }); - if (supervisorMode === "external" && !handoff) { - gatewayLog.warn( - "external supervisor restart handoff could not be persisted; falling back to in-process restart", - ); - await markUpdateRestartSentinelFailure("restart-handoff-unavailable").catch( - (err: unknown) => { - gatewayLog.warn(`failed to mark update restart handoff unavailable: ${String(err)}`); - }, - ); - if (!(await reacquireLockForInProcessRestart())) { - return; - } - activeRestartRequest = null; - shuttingDown = false; - restartResolver?.(); - return; + gatewayLog.info("restart mode: managed update handoff owns successor"); + return exitProcessAfterLogFlush(0, expectedOwner); + } + + const respawnOptions = { + env: createGatewayRestartTraceHandoffEnv(captureGatewayRestartTraceHandoff()), + }; + const isStandaloneUpdate = isUpdateRestart && !supervisorMode; + const respawn = isStandaloneUpdate + ? eagerLifecycleRuntime.respawnGatewayProcessForUpdate(respawnOptions) + : eagerLifecycleRuntime.restartGatewayProcessWithFreshPid(respawnOptions); + if (respawn.mode === "spawned") { + const port = params.lockPort; + const healthy = + typeof port === "number" + ? await waitForHealthyChild(port, respawn.pid, params.healthHost ?? "127.0.0.1") + : false; + if (healthy) { + committedGenericSuccessor = respawn.child ?? true; + gatewayLog.info( + `restart mode: update process respawn (spawned pid ${respawn.pid ?? "unknown"})`, + ); + return exitProcessAfterLogFlush(0); + } + gatewayLog.warn( + `update respawn child did not become healthy (${respawn.pid ?? "unknown"}); falling back to in-process restart`, + ); + try { + respawn.child?.kill(); + } catch { + // Best-effort; parent fallback keeps the gateway reachable for recovery. + } + await markRestartHandoffUnavailable("restart-unhealthy"); + return reacquireAndResumeInProcessRestart(); + } + if (respawn.mode === "supervised") { + const restartKind = isUpdateRestart ? "update-process" : "full-process"; + markGatewayRestartTrace("restart.full-process-handoff", [ + ["kind", restartKind], + ["mode", respawn.mode], + ["pid", "none"], + ["supervisorMode", supervisorMode ?? "none"], + ]); + const handoff = eagerLifecycleRuntime.writeGatewayRestartHandoffSync({ + restartKind, + reason: restartReason, + processInstanceId, + supervisorMode: supervisorMode ?? "external", + restartTrace: captureGatewayRestartTraceHandoff(), + }); + if (supervisorMode === "external" && !handoff) { + gatewayLog.warn( + "external supervisor restart handoff could not be persisted; falling back to in-process restart", + ); + if (isUpdateRestart) { + await markRestartHandoffUnavailable(); } - gatewayLog.info("restart mode: update process respawn (supervisor restart)"); - if (supervisorMode === "launchd" && !(await confirmLaunchdHandoff(respawn))) { + return reacquireAndResumeInProcessRestart(); + } + gatewayLog.info("restart mode: full process restart (supervisor restart)"); + if (supervisorMode === "launchd") { + const delay = new Promise((resolve) => { + setTimeout(resolve, LAUNCHD_SUPERVISED_RESTART_EXIT_DELAY_MS); + }); + const spawned = respawn.handoffSpawned + ? await Promise.race([respawn.handoffSpawned, delay.then(() => true)]) + : false; + // Preserve the crash-loop throttle window even when spawn settles early. + await delay; + if (!spawned) { + writeStabilityBundle("gateway.restart_handoff_spawn_failed"); gatewayLog.warn( "launchd restart handoff failed to spawn; falling back to in-process restart", ); - await markUpdateRestartSentinelFailure("restart-handoff-unavailable").catch( - (err: unknown) => { - gatewayLog.warn(`failed to mark update restart handoff unavailable: ${String(err)}`); - }, - ); - if (!(await reacquireLockForInProcessRestart())) { - return; + if (isUpdateRestart) { + await markRestartHandoffUnavailable(); } - activeRestartRequest = null; - shuttingDown = false; - restartResolver?.(); - return; - } - activeRestartRequest = null; - await exitProcessAfterLogFlush(0); - return; - } - if (respawn.mode === "failed") { - gatewayLog.warn( - `update respawn failed (${respawn.detail ?? "unknown error"}); falling back to in-process restart`, - ); - await markUpdateRestartSentinelFailure("restart-unhealthy").catch((err: unknown) => { - gatewayLog.warn(`failed to mark update restart sentinel unhealthy: ${String(err)}`); - }); - } else { - gatewayLog.info( - `restart mode: in-process restart (${respawn.detail ?? "OPENCLAW_NO_RESPAWN"})`, - ); - } - if (!(await reacquireLockForInProcessRestart())) { - return; - } - activeRestartRequest = null; - shuttingDown = false; - restartResolver?.(); - return; - } - - // Release the lock BEFORE spawning so the child can acquire it immediately. - const restartTraceHandoff = captureGatewayRestartTraceHandoff(); - const respawn = restartGatewayProcessWithFreshPid({ - env: createGatewayRestartTraceHandoffEnv(restartTraceHandoff), - }); - if (respawn.mode === "spawned" || respawn.mode === "supervised") { - const supervisorMode = - respawn.mode === "supervised" - ? detectGatewayRespawnSupervisor(process.env, process.platform) - : null; - const modeLabel = - respawn.mode === "spawned" - ? `spawned pid ${respawn.pid ?? "unknown"}` - : "supervisor restart"; - markGatewayRestartTrace("restart.full-process-handoff", [ - ["kind", "full-process"], - ["mode", respawn.mode], - ["pid", respawn.mode === "spawned" ? (respawn.pid ?? "unknown") : "none"], - ["supervisorMode", supervisorMode ?? "none"], - ]); - if (respawn.mode === "supervised") { - const handoff = writeGatewayRestartHandoffSync({ - restartKind: "full-process", - reason: restartReason, - processInstanceId, - supervisorMode: supervisorMode ?? "external", - restartTrace: captureGatewayRestartTraceHandoff(), - }); - if (supervisorMode === "external" && !handoff) { - gatewayLog.warn( - "external supervisor restart handoff could not be persisted; falling back to in-process restart", - ); - if (!(await reacquireLockForInProcessRestart())) { - return; - } - activeRestartRequest = null; - shuttingDown = false; - restartResolver?.(); - return; + return reacquireAndResumeInProcessRestart(); } } - gatewayLog.info(`restart mode: full process restart (${modeLabel})`); - if (supervisorMode === "launchd" && !(await confirmLaunchdHandoff(respawn))) { - await writeStabilityBundle("gateway.restart_handoff_spawn_failed"); - gatewayLog.warn( - "launchd restart handoff failed to spawn; falling back to in-process restart", - ); - if (!(await reacquireLockForInProcessRestart())) { - return; - } - activeRestartRequest = null; - shuttingDown = false; - restartResolver?.(); - return; - } - activeRestartRequest = null; - await exitProcessAfterLogFlush(0); - return; + committedGenericSuccessor = true; + return exitProcessAfterLogFlush(0); } if (respawn.mode === "failed") { - await writeStabilityBundle("gateway.restart_respawn_failed"); + if (!isStandaloneUpdate) { + writeStabilityBundle("gateway.restart_respawn_failed"); + } gatewayLog.warn( - `full process restart failed (${respawn.detail ?? "unknown error"}); falling back to in-process restart`, + `${isStandaloneUpdate ? "update respawn" : "full process restart"} failed (${respawn.detail ?? "unknown error"}); falling back to in-process restart`, ); + if (isUpdateRestart) { + await markRestartHandoffUnavailable("restart-unhealthy"); + } } else { gatewayLog.info( `restart mode: in-process restart (${respawn.detail ?? "OPENCLAW_NO_RESPAWN"})`, ); } - if (isUpdateProcessRestartReason(activeRestartRequest?.restartReason)) { - await handleRestartAfterServerClose(); - return; + if (!isUpdateRestart && isUpdateProcessRestartReason(activeRestartRequest?.restartReason)) { + return handleRestartAfterServerClose(); } - if (!(await reacquireLockForInProcessRestart())) { - return; - } - if (isUpdateProcessRestartReason(activeRestartRequest?.restartReason)) { - await handleRestartAfterServerClose(); - return; - } - activeRestartRequest = null; - shuttingDown = false; - restartResolver?.(); + return reacquireAndResumeInProcessRestart(); }; - const handleStopAfterServerClose = async () => { - params.completeBoot?.({ outcome: "clean_stop", reason: "gateway.stop" }); - await releaseLockIfHeld(); - await exitProcessAfterLogFlush(0); - }; - const SUPERVISOR_STOP_TIMEOUT_MS = 30_000; const SHUTDOWN_TIMEOUT_MS = SUPERVISOR_STOP_TIMEOUT_MS - 5_000; const clearPendingStartupForceExitTimer = () => { - if (!pendingStartupForceExitTimer) { - return; - } - clearTimeout(pendingStartupForceExitTimer); + clearTimeout(pendingStartupForceExitTimer ?? undefined); pendingStartupForceExitTimer = null; }; const armPendingStartupForceExitTimer = () => { @@ -463,20 +489,13 @@ export async function runGatewayLoop(params: { gatewayLog.error( "startup restart request timed out before gateway returned a close handle; exiting for supervisor recovery", ); - void (async () => { - try { - await writeStabilityBundle("gateway.restart_startup_request_timeout"); - } finally { - completeForcedStop("gateway.restart_startup_request_timeout"); - exitProcess(1); - } - })(); + void forceExitAfterStabilityBundle("gateway.restart_startup_request_timeout"); }, SHUTDOWN_TIMEOUT_MS); pendingStartupForceExitTimer.unref?.(); }; - const resolveRestartDrainTimeoutMs = async ( - restartIntent?: RestartIntentOptions, - ): Promise => { + const resolveRestartDrainTimeoutMs = ( + restartIntent?: GatewayRestartIntent, + ): number | undefined => { if (restartIntent?.force) { return 0; } @@ -484,8 +503,7 @@ export async function runGatewayLoop(params: { return restartIntent.waitMs > 0 ? Math.floor(restartIntent.waitMs) : undefined; } try { - const { resolveGatewayRestartDeferralTimeoutMs } = await loadGatewayLifecycleRuntimeModule(); - return resolveGatewayRestartDeferralTimeoutMs(); + return eagerLifecycleRuntime.resolveGatewayRestartDeferralTimeoutMs(); } catch { return DEFAULT_RESTART_DRAIN_TIMEOUT_MS; } @@ -515,21 +533,9 @@ export async function runGatewayLoop(params: { } forceExitTimer = setTimeout(() => { gatewayLog.error("shutdown timed out; exiting without full cleanup"); - void (async () => { - try { - await writeStabilityBundle( - isRestart ? "gateway.restart_shutdown_timeout" : "gateway.stop_shutdown_timeout", - ); - } finally { - // Keep the in-process watchdog below the supervisor stop budget so this - // path wins before launchd/systemd escalates to a hard kill. Exit - // non-zero on any timeout so supervised installs restart cleanly. - completeForcedStop( - isRestart ? "gateway.restart_shutdown_timeout" : "gateway.stop_shutdown_timeout", - ); - exitProcess(1); - } - })(); + void forceExitAfterStabilityBundle( + isRestart ? "gateway.restart_shutdown_timeout" : "gateway.stop_shutdown_timeout", + ); }, forceExitMs); if (params.ownsProcessLifecycle === true) { hardExitWatchdog = armShutdownHardExitWatchdog({ @@ -543,31 +549,36 @@ export async function runGatewayLoop(params: { } }; const clearForceExitTimer = () => { - if (forceExitTimer) { - clearTimeout(forceExitTimer); - forceExitTimer = null; - } + clearTimeout(forceExitTimer ?? undefined); + forceExitTimer = null; hardExitWatchdog?.cancel(); hardExitWatchdog = null; }; if (isRestart) { forceActiveRestartExit = () => { clearForceExitTimer(); - armForceExitTimer(SHUTDOWN_TIMEOUT_MS); + if (!getManagedUpdateOwner()) { + armForceExitTimer(SHUTDOWN_TIMEOUT_MS); + } }; } void (async () => { - const restartDrainTimeoutMs = isRestart - ? await resolveRestartDrainTimeoutMs(restartIntent) - : 0; + let managedUpdateOwner: GatewayRestartIntent["successorOwner"]; + let managedUpdateCancellation: + | false + | "restored-in-process" + | "restart-after-exit" + | undefined; + const restartDrainTimeoutMs = isRestart ? resolveRestartDrainTimeoutMs(restartIntent) : 0; const restartDrainDeadlineAt = isRestart && restartDrainTimeoutMs !== undefined ? Date.now() + restartDrainTimeoutMs : undefined; + // Managed helpers must reach native parking before either exit watchdog can arm. if (!isRestart) { armForceExitTimer(SHUTDOWN_TIMEOUT_MS); - } else if (restartDrainTimeoutMs !== undefined) { + } else if (restartDrainTimeoutMs !== undefined && !getManagedUpdateOwner()) { // Allow extra time for draining active turns on explicitly capped restarts. armForceExitTimer(restartDrainTimeoutMs + SHUTDOWN_TIMEOUT_MS); } @@ -576,21 +587,6 @@ export async function runGatewayLoop(params: { restartDrainTimeoutMs === undefined ? "without a timeout" : `with timeout ${restartDrainTimeoutMs}ms`; - const armCloseForceExitTimerForIndefiniteRestart = () => { - if (isRestart && restartDrainTimeoutMs === undefined) { - armForceExitTimer(SHUTDOWN_TIMEOUT_MS); - } - }; - const resolveRestartCloseDrainTimeoutMs = () => { - if (!isRestart) { - return null; - } - if (restartDrainTimeoutMs === undefined) { - return Math.max(0, SHUTDOWN_TIMEOUT_MS - RESTART_CLOSE_REPLY_DRAIN_SHUTDOWN_RESERVE_MS); - } - return Math.max(0, (restartDrainDeadlineAt ?? Date.now()) - Date.now()); - }; - try { // On restart, wait for the canonical process activity inventory before // tearing down the server so active work can settle. @@ -610,12 +606,6 @@ export async function runGatewayLoop(params: { markRestartAbortedMainSessions, waitForGatewayActiveWork, } = await loadGatewayLifecycleRuntimeModule(); - const collectActiveRestartSessionKeys = () => { - return new Set(listActiveEmbeddedRunSessionKeys()); - }; - const collectActiveRestartSessionIds = () => { - return new Set(listActiveEmbeddedRunSessionIds()); - }; let activeRestartSessionKeysAtDrainStart = new Set(); let activeRestartSessionIdsAtDrainStart = new Set(); let hasMarkedActiveMainSessionsForRestart = false; @@ -627,11 +617,11 @@ export async function runGatewayLoop(params: { } const sessionKeys = new Set([ ...activeRestartSessionKeysAtDrainStart, - ...collectActiveRestartSessionKeys(), + ...listActiveEmbeddedRunSessionKeys(), ]); const sessionIds = new Set([ ...activeRestartSessionIdsAtDrainStart, - ...collectActiveRestartSessionIds(), + ...listActiveEmbeddedRunSessionIds(), ]); if (sessionKeys.size === 0 && sessionIds.size === 0) { return; @@ -643,9 +633,7 @@ export async function runGatewayLoop(params: { sessionIds, reason, }); - if (result.marked > 0) { - hasMarkedActiveMainSessionsForRestart = true; - } + hasMarkedActiveMainSessionsForRestart = result.marked > 0; } catch (err) { gatewayLog.warn( `failed to mark interrupted main sessions for restart recovery: ${String(err)}`, @@ -662,8 +650,8 @@ export async function runGatewayLoop(params: { const initialSnapshot = createGatewayActiveWorkSnapshot(); activeWorkAtDrainStart = initialSnapshot.counts.totalActive; activeRunsAtDrainStart = initialSnapshot.counts.embeddedRuns; - activeRestartSessionKeysAtDrainStart = collectActiveRestartSessionKeys(); - activeRestartSessionIdsAtDrainStart = collectActiveRestartSessionIds(); + activeRestartSessionKeysAtDrainStart = new Set(listActiveEmbeddedRunSessionKeys()); + activeRestartSessionIdsAtDrainStart = new Set(listActiveEmbeddedRunSessionIds()); // Best-effort abort for compacting runs so transcript settlement does // not remain pending across restart boundaries. @@ -725,9 +713,7 @@ export async function runGatewayLoop(params: { ["force", restartIntent?.force === true], ], ); - } - - if (!isRestart) { + } else { // Keep all process-owned work alive without spending the shutdown reserve // that server teardown and the supervisor watchdog need. try { @@ -746,8 +732,48 @@ export async function runGatewayLoop(params: { } } - armCloseForceExitTimerForIndefiniteRestart(); - const closeDrainTimeoutMs = resolveRestartCloseDrainTimeoutMs(); + if (isRestart && activeRestartRequest?.restartIntent?.successorOwner) { + const owner = activeRestartRequest.restartIntent.successorOwner; + managedUpdateOwner = owner; + try { + if ( + !sameManagedUpdateOwner(getManagedUpdateOwner(), owner) || + !(await eagerLifecycleRuntime.requestManagedServiceUpdateHandoffPark(owner)) || + !sameManagedUpdateOwner(getManagedUpdateOwner(), owner) || + !eagerLifecycleRuntime.claimManagedServiceUpdateHandoff(owner) + ) { + throw new Error("managed update helper lost exact ownership during service parking"); + } + } catch (err) { + clearForceExitTimer(); + gatewayLog.error( + `managed update handoff could not park ${supervisorMode}: ${String(err)}`, + ); + await markRestartHandoffUnavailable(); + managedUpdateCancellation = await cancelManagedUpdateHandoffBeforeRecovery(owner); + if (!managedUpdateCancellation) { + return; + } + if (managedUpdateCancellation === "restart-after-exit") { + await releaseLockIfHeld(); + await exitProcessAfterLogFlush(0, owner, "restore"); + return; + } + } + } + + if ( + isRestart && + !forceExitTimer && + (!managedUpdateOwner || managedUpdateCancellation === "restored-in-process") + ) { + armForceExitTimer(SHUTDOWN_TIMEOUT_MS); + } + const closeDrainTimeoutMs = !isRestart + ? null + : restartDrainTimeoutMs === undefined + ? SHUTDOWN_TIMEOUT_MS - RESTART_CLOSE_REPLY_DRAIN_SHUTDOWN_RESERVE_MS + : Math.max(0, (restartDrainDeadlineAt ?? Date.now()) - Date.now()); await server?.close({ reason: isRestart ? "gateway restarting" : "gateway stopping", restartExpectedMs: isRestart ? 1500 : null, @@ -756,17 +782,28 @@ export async function runGatewayLoop(params: { } catch (err) { gatewayLog.error(`shutdown step failed (gateway server close): ${formatErrorMessage(err)}`); } finally { - server = null; + const handoffClosed = + managedUpdateCancellation !== false && managedUpdateCancellation !== "restart-after-exit"; + if (handoffClosed) { + server = null; + } if (isRestart) { try { - await handleRestartAfterServerClose(); + if (handoffClosed) { + await handleRestartAfterServerClose( + managedUpdateOwner, + managedUpdateCancellation === "restored-in-process", + ); + } } finally { clearForceExitTimer(); forceActiveRestartExit = null; } } else { clearForceExitTimer(); - await handleStopAfterServerClose(); + params.completeBoot?.({ outcome: "clean_stop", reason: "gateway.stop" }); + await releaseLockIfHeld(); + await exitProcessAfterLogFlush(0); } } })(); @@ -788,7 +825,7 @@ export async function runGatewayLoop(params: { action: GatewayRunSignalAction, signal: string, restartReason?: string, - restartIntent?: RestartIntentOptions, + restartIntent?: GatewayRestartIntent, ) => { const acceptedRequest = { action, signal, restartReason, restartIntent }; if (shuttingDown) { @@ -797,7 +834,12 @@ export async function runGatewayLoop(params: { action === "restart" && isUpdateProcessRestartReason(restartReason) && currentRestartRequest?.action === "restart" && - !isUpdateProcessRestartReason(currentRestartRequest.restartReason) + (!isUpdateProcessRestartReason(currentRestartRequest.restartReason) || + (restartIntent?.successorOwner && + !sameManagedUpdateOwner( + restartIntent.successorOwner, + currentRestartRequest.restartIntent?.successorOwner, + ))) ) { const upgradedRequest = { ...currentRestartRequest, @@ -900,6 +942,10 @@ export async function runGatewayLoop(params: { if (restartIntent) { abortPendingChannelReloads(); const authorized = consumeGatewaySigusr1RestartAuthorization(); + const processLocalIntent = authorized ? consumeGatewaySigusr1RestartIntent() : null; + if (processLocalIntent?.successorOwner) { + Object.assign(restartIntent, processLocalIntent); + } markRestartDraining(); if (authorized) { markGatewaySigusr1RestartHandled(); @@ -1088,7 +1134,7 @@ export async function runGatewayLoop(params: { } const errMsg = formatErrorMessage(err); const errStack = err instanceof Error && err.stack ? `\n${err.stack}` : ""; - await writeStabilityBundle("gateway.restart_startup_failed", err); + writeStabilityBundle("gateway.restart_startup_failed", err); gatewayLog.error( `gateway startup failed: ${errMsg}. ` + `Process will stay alive; fix the issue and restart.${errStack}`, diff --git a/src/gateway/server-methods/update-run-campaign.test.ts b/src/gateway/server-methods/update-run-campaign.test.ts index 89c771b2dc81..0f55f154b280 100644 --- a/src/gateway/server-methods/update-run-campaign.test.ts +++ b/src/gateway/server-methods/update-run-campaign.test.ts @@ -48,6 +48,7 @@ const startManagedServiceUpdateHandoffMock = vi.fn< command: "openclaw update --yes --timeout 1800", logPath: "/tmp/openclaw-update-run-handoff/handoff.log", handoffId: "handoff-1", + installRoot: "/tmp/openclaw", })); const scheduleGatewaySigusr1RestartMock = vi.fn(() => ({ scheduled: true })); const logGatewayInfoMock = vi.fn(); @@ -87,8 +88,8 @@ vi.mock("../../infra/restart-sentinel.js", async () => { }; }); -vi.mock("../../infra/restart.js", () => ({ - resolveGatewayRestartDeferralTimeoutMs: () => 300_000, +vi.mock("../../infra/restart.js", async () => ({ + ...(await vi.importActual("../../infra/restart.js")), scheduleGatewaySigusr1Restart: scheduleGatewaySigusr1RestartMock, })); diff --git a/src/gateway/server-methods/update.test.ts b/src/gateway/server-methods/update.test.ts index 2bafc27df1cb..e095ccac0667 100644 --- a/src/gateway/server-methods/update.test.ts +++ b/src/gateway/server-methods/update.test.ts @@ -55,13 +55,14 @@ type ManagedServiceUpdateHandoffResult = Awaited< > >; const startManagedServiceUpdateHandoffMock = vi.fn< - (params?: { handoffId?: string }) => Promise + (params?: { handoffId?: string; root?: string }) => Promise >(async (params) => ({ status: "started", pid: 12345, command: "openclaw update --yes --timeout 1800", logPath: "/tmp/openclaw-update-run-handoff/handoff.log", - handoffId: params?.handoffId, + handoffId: params?.handoffId ?? "handoff-default", + installRoot: params?.root ?? "/tmp/openclaw", })); const scheduleGatewaySigusr1RestartMock = vi.fn(() => ({ scheduled: true })); @@ -88,9 +89,7 @@ vi.mock("../../config/config.js", () => ({ readConfigFileSnapshot: readConfigFileSnapshotMock, })); -vi.mock("../../config/commands.flags.js", () => ({ - isRestartEnabled: isRestartEnabledMock, -})); +vi.mock("../../config/commands.flags.js", () => ({ isRestartEnabled: isRestartEnabledMock })); vi.mock("../../config/sessions.js", () => ({ extractDeliveryInfo: (sessionKey: string | undefined) => { @@ -124,19 +123,12 @@ vi.mock("../../infra/restart-sentinel.js", async () => { }; }); -vi.mock("../../infra/restart.js", () => ({ - resolveGatewayRestartDeferralTimeoutMs: (timeoutMs: unknown) => { - if (typeof timeoutMs !== "number" || !Number.isFinite(timeoutMs)) { - return 300_000; - } - return timeoutMs <= 0 ? undefined : Math.floor(timeoutMs); - }, +vi.mock("../../infra/restart.js", async () => ({ + ...(await vi.importActual("../../infra/restart.js")), scheduleGatewaySigusr1Restart: scheduleGatewaySigusr1RestartMock, })); -vi.mock("../../infra/package-json.js", () => ({ - readPackageVersion: readPackageVersionMock, -})); +vi.mock("../../infra/package-json.js", () => ({ readPackageVersion: readPackageVersionMock })); vi.mock("../../version.js", () => ({ get VERSION() { @@ -291,12 +283,13 @@ beforeEach(() => { recordLatestUpdateRestartSentinelMock.mockClear(); startManagedServiceUpdateHandoffMock.mockClear(); startManagedServiceUpdateHandoffMock.mockImplementation( - async (params?: { handoffId?: string }) => ({ + async (params?: { handoffId?: string; root?: string }) => ({ status: "started" as const, pid: 12345, command: "openclaw update --yes --timeout 1800", logPath: "/tmp/openclaw-update-run-handoff/handoff.log", - handoffId: params?.handoffId, + handoffId: params?.handoffId ?? "handoff-default", + installRoot: params?.root ?? "/tmp/openclaw", }), ); scheduleGatewaySigusr1RestartMock.mockClear(); @@ -550,8 +543,21 @@ describe("update.run restart scheduling", () => { const [restartParams] = firstMockCall( scheduleGatewaySigusr1RestartMock, "gateway restart schedule", - ) as [{ delayMs?: number; reason?: string; skipCooldown?: boolean; skipDeferral?: boolean }]; + ) as [ + { + delayMs?: number; + reason?: string; + successorOwner?: unknown; + skipCooldown?: boolean; + skipDeferral?: boolean; + }, + ]; expect(restartParams?.reason).toBe("update.run"); + expect(restartParams?.successorOwner).toEqual({ + kind: "managed-update-handoff", + handoffId: handoffParams.handoffId, + installRoot: "/tmp/openclaw-global", + }); expect(restartParams?.skipCooldown).toBe(true); expect(restartParams?.skipDeferral).toBe(true); expect(payload?.ok).toBe(true); @@ -673,6 +679,7 @@ describe("update.run restart scheduling", () => { expect(startManagedServiceUpdateHandoffMock).toHaveBeenCalledWith( expect.objectContaining({ supervisor: "systemd", + restartDrainTimeoutMs: 300_000, restartDelayMs: 2000, }), ); @@ -680,12 +687,34 @@ describe("update.run restart scheduling", () => { expect.objectContaining({ delayMs: 2000, reason: "update.run", + successorOwner: { + kind: "managed-update-handoff", + handoffId: expect.any(String), + installRoot: "/tmp/openclaw-global", + }, skipCooldown: true, skipDeferral: true, }), ); }); + it.each(["launchd", "systemd"] as const)( + "normalizes overflow-sized %s handoff and restart delays together", + async (supervisor) => { + detectRespawnSupervisorMock.mockReturnValueOnce(supervisor); + mockGlobalInstallSurface(); + + await invokeUpdateRun({ restartDelayMs: 2_147_153_648 }); + + expect(startManagedServiceUpdateHandoffMock).toHaveBeenCalledWith( + expect.objectContaining({ supervisor, restartDelayMs: 60_000 }), + ); + expect(scheduleGatewaySigusr1RestartMock).toHaveBeenCalledWith( + expect.objectContaining({ delayMs: 60_000 }), + ); + }, + ); + it("starts managed package handoff when the gateway cwd is unavailable", async () => { detectRespawnSupervisorMock.mockReturnValueOnce("launchd"); mockGlobalInstallSurface(); @@ -902,27 +931,29 @@ describe("update.run restart scheduling", () => { expect(payload?.handoff?.status).toBe("started"); }); - it("does not hand off systemd-supervised git/dev updates from generic systemd markers alone", async () => { + it("hands marker-only systemd git/dev updates to the helper for exact ownership verification", async () => { detectRespawnSupervisorMock.mockReturnValueOnce("systemd"); mockGitInstallSurface("/tmp/openclaw-git"); const payload = await withProcessEnv( { OPENCLAW_SYSTEMD_UNIT: undefined, - INVOCATION_ID: "8a77e69a8f604bf0b7984879b9f17a7c", + OPENCLAW_SERVICE_MARKER: "openclaw", + OPENCLAW_SERVICE_KIND: "gateway", }, () => captureUpdateRunPayload(), ); expect(runGatewayUpdateMock).not.toHaveBeenCalled(); - expect(startManagedServiceUpdateHandoffMock).not.toHaveBeenCalled(); - expect(scheduleGatewaySigusr1RestartMock).not.toHaveBeenCalled(); - expect(payload?.ok).toBe(false); - expect(payload?.restart).toBeNull(); + expect(startManagedServiceUpdateHandoffMock).toHaveBeenCalledWith( + expect.objectContaining({ root: "/tmp/openclaw-git", supervisor: "systemd" }), + ); + expect(scheduleGatewaySigusr1RestartMock).toHaveBeenCalledOnce(); + expect(payload?.ok).toBe(true); expect(payload?.result?.status).toBe("skipped"); - expect(payload?.result?.reason).toBe("managed-service-handoff-unavailable"); + expect(payload?.result?.reason).toBe("managed-service-handoff-started"); expect(payload?.result?.mode).toBe("git"); - expect(payload?.handoff?.status).toBe("unavailable"); + expect(payload?.handoff?.status).toBe("started"); }); it("returns a safe command when package updates cannot be handed off", async () => { @@ -961,11 +992,16 @@ describe("update.run restart scheduling", () => { expect(payload?.result?.mode).toBe("npm"); }); - it("delegates update.run without mutating or restarting under external supervision", async () => { + it("keeps external update supervision authoritative even with native systemd markers", async () => { mockGlobalInstallSurface(); + detectRespawnSupervisorMock.mockReturnValue("systemd"); - const payload = await withProcessEnv({ OPENCLAW_SUPERVISOR_MODE: "external" }, () => - captureUpdateRunPayload(), + const payload = await withProcessEnv( + { + OPENCLAW_SUPERVISOR_MODE: "external", + OPENCLAW_SYSTEMD_UNIT: "openclaw-gateway.service", + }, + () => captureUpdateRunPayload(), ); expect(runGatewayUpdateMock).not.toHaveBeenCalled(); diff --git a/src/gateway/server-methods/update.ts b/src/gateway/server-methods/update.ts index 39816b162f19..ed10dc5253d5 100644 --- a/src/gateway/server-methods/update.ts +++ b/src/gateway/server-methods/update.ts @@ -13,7 +13,6 @@ import { isRestartEnabled } from "../../config/commands.flags.js"; import { readConfigFileSnapshot } from "../../config/config.js"; import { extractDeliveryInfo } from "../../config/sessions.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; -import { GATEWAY_SERVICE_KIND, GATEWAY_SERVICE_MARKER } from "../../daemon/constants.js"; import { EXTERNAL_SUPERVISOR_UPDATE_REQUIRED_REASON, isGatewayExternallySupervised, @@ -21,6 +20,7 @@ import { import { readPackageVersion } from "../../infra/package-json.js"; import { type RestartSentinelPayload, writeRestartSentinel } from "../../infra/restart-sentinel.js"; import { + normalizeGatewayRestartDelayMs, resolveGatewayRestartDeferralTimeoutMs, scheduleGatewaySigusr1Restart, } from "../../infra/restart.js"; @@ -114,47 +114,6 @@ async function readPreUpdateConfigForPostCoreFinalize(): Promise< }; } -function resolveManagedServiceHandoffRestartDelayMs( - restartDelayMs: number | undefined, - supervisor: ReturnType, -): number { - const resolvedDelayMs = restartDelayMs ?? MANAGED_HANDOFF_RESTART_DELAY_MS; - if (supervisor !== "systemd") { - return resolvedDelayMs; - } - // systemd needs a short grace period after the handoff process starts before - // the gateway exits, otherwise the service can restart before handoff state is durable. - return Math.max(resolvedDelayMs, MANAGED_HANDOFF_RESTART_DELAY_MS); -} - -function hasManagedServiceHandoffContext( - env: NodeJS.ProcessEnv, - supervisor: ReturnType, -): boolean { - if (supervisor === "launchd") { - return Boolean( - env.OPENCLAW_LAUNCHD_LABEL?.trim() || - env.LAUNCH_JOB_LABEL?.trim() || - env.LAUNCH_JOB_NAME?.trim() || - env.XPC_SERVICE_NAME?.trim(), - ); - } - if (supervisor === "systemd") { - // Ambient systemd markers only prove that a service manager started this - // process. The detached CLI needs the durable unit name to stop the same - // gateway before mutating the install root. - return Boolean(env.OPENCLAW_SYSTEMD_UNIT?.trim()); - } - if (supervisor === "schtasks") { - return Boolean( - env.OPENCLAW_WINDOWS_TASK_NAME?.trim() || - (env.OPENCLAW_SERVICE_MARKER?.trim() === GATEWAY_SERVICE_MARKER && - env.OPENCLAW_SERVICE_KIND?.trim() === GATEWAY_SERVICE_KIND), - ); - } - return false; -} - export const updateHandlers: GatewayRequestHandlers = { "update.status": async ({ params, respond, context }) => { if (!assertValidParams(params, validateUpdateStatusParams, "update.status", respond)) { @@ -269,8 +228,9 @@ export const updateHandlers: GatewayRequestHandlers = { threadId: requestedThreadId, note, continuationMessage, - restartDelayMs, + restartDelayMs: requestedRestartDelayMs, } = parseRestartRequestParams(params); + const restartDelayMs = normalizeGatewayRestartDelayMs(requestedRestartDelayMs); const { deliveryContext: sessionDeliveryContext, threadId: sessionThreadId } = extractDeliveryInfo(sessionKey); const deliveryContext = requestedDeliveryContext ?? sessionDeliveryContext; @@ -312,16 +272,15 @@ export const updateHandlers: GatewayRequestHandlers = { installKind: status.installKind, git: status.git, }).channel; - const supervisor = detectRespawnSupervisor(process.env, process.platform); - const hasHandoffContext = supervisor - ? hasManagedServiceHandoffContext(process.env, supervisor) - : false; + const supervisor = detectRespawnSupervisor(process.env, process.platform, { + includeLinuxOpenClawGatewayServiceMarker: true, + }); const requiresManagedServiceHandoff = installSurface.kind === "global" || (installSurface.kind === "git" && supervisor !== null); const managedGitPreflightFailure = installSurface.kind === "git" && effectiveChannel === "dev" && - hasHandoffContext && + supervisor && !isGatewayExternallySupervised() ? await runGatewayUpdatePreflight(installRoot, timeoutMs, adoptedDevTarget) : undefined; @@ -386,15 +345,16 @@ export const updateHandlers: GatewayRequestHandlers = { ...(handoffChannel ? { channel: handoffChannel } : {}), ...(adoptedPackageTargetVersion ? { tag: adoptedPackageTargetVersion } : {}), }); - if (supervisor && hasHandoffContext) { + if (supervisor) { try { const beforeVersion = await readPackageVersion(installRoot); const startedAt = Date.now(); const handoffId = randomUUID(); - const managedRestartDelayMs = resolveManagedServiceHandoffRestartDelayMs( - restartDelayMs, - supervisor, - ); + // systemd needs startup grace before the Gateway exits and its state becomes durable. + const managedRestartDelayMs = + supervisor === "systemd" + ? Math.max(restartDelayMs, MANAGED_HANDOFF_RESTART_DELAY_MS) + : restartDelayMs; sentinelMeta.handoffId = handoffId; sentinelMeta.root = resolveUpdateInstallRoot(installRoot); // Managed services update from a detached helper so the running @@ -416,7 +376,7 @@ export const updateHandlers: GatewayRequestHandlers = { sentinelMeta.handoffId = started.handoffId ?? handoffId; // The owner pairs helper creation with parent exit before any // persistence can fail. Joiners leave both to the active owner. - if (ownsManagedServiceHandoff) { + if (started.status === "started") { handoff = { status: "started", ...(started.pid ? { pid: started.pid } : {}), @@ -425,6 +385,11 @@ export const updateHandlers: GatewayRequestHandlers = { managedHandoffRestart = scheduleGatewaySigusr1Restart({ delayMs: managedRestartDelayMs, reason: "update.run", + successorOwner: { + kind: "managed-update-handoff", + handoffId: started.handoffId, + installRoot: started.installRoot, + }, skipDeferral: true, skipCooldown: true, audit: { diff --git a/src/infra/infra-runtime.test.ts b/src/infra/infra-runtime.test.ts index 8dc12d35c110..2a98d6e5c857 100644 --- a/src/infra/infra-runtime.test.ts +++ b/src/infra/infra-runtime.test.ts @@ -9,8 +9,14 @@ import { tryBeginGatewayRootWorkAdmission, } from "../process/gateway-work-admission.js"; type RestartModule = typeof import("./restart.js"); +const managedSuccessorOwner = { + kind: "managed-update-handoff", + handoffId: "managed-handoff", + installRoot: "/canonical/install", +} as const; let consumeGatewaySigusr1RestartAuthorization: RestartModule["consumeGatewaySigusr1RestartAuthorization"]; +let consumeGatewaySigusr1RestartIntent: RestartModule["consumeGatewaySigusr1RestartIntent"]; let deferGatewayRestartUntilIdle: RestartModule["deferGatewayRestartUntilIdle"]; let isGatewaySigusr1RestartExternallyAllowed: RestartModule["isGatewaySigusr1RestartExternallyAllowed"]; let markGatewaySigusr1RestartHandled: RestartModule["markGatewaySigusr1RestartHandled"]; @@ -114,6 +120,7 @@ describe("infra runtime", () => { ); ({ consumeGatewaySigusr1RestartAuthorization, + consumeGatewaySigusr1RestartIntent, deferGatewayRestartUntilIdle, isGatewaySigusr1RestartExternallyAllowed, markGatewaySigusr1RestartHandled, @@ -445,17 +452,16 @@ describe("infra runtime", () => { const beforeEmit = vi.fn(async () => { await preparationBlocked; }); - let resolveSignal: () => void = () => {}; - const signalEmitted = new Promise((resolve) => { - resolveSignal = resolve; - }); - const handler = () => resolveSignal(); + const staleEmitRestart = vi.fn(() => ({ status: "failed" as const })); + const emitSpy = vi.spyOn(process, "emit"); + const handler = () => {}; process.on("SIGUSR1", handler); try { scheduleGatewaySigusr1Restart({ delayMs: 0, reason: "config.patch", - emitHooks: { beforeEmit }, + sessionKey: "agent:main:session-A", + emitHooks: { beforeEmit, emitRestart: staleEmitRestart }, }); await vi.advanceTimersByTimeAsync(0); await Promise.resolve(); @@ -464,14 +470,71 @@ describe("infra runtime", () => { const update = scheduleGatewaySigusr1Restart({ delayMs: 0, reason: "update.auto", + successorOwner: managedSuccessorOwner, skipDeferral: true, }); expect(update.coalesced).toBe(true); releasePreparation(); - await signalEmitted; + await vi.advanceTimersByTimeAsync(0); + + expect(staleEmitRestart).not.toHaveBeenCalled(); + expect(emitSpy).toHaveBeenCalledWith("SIGUSR1"); + expect(peekGatewaySigusr1RestartReason()).toBe("update.auto"); + expect(consumeGatewaySigusr1RestartIntent()).toEqual({ + reason: "update.auto", + successorOwner: managedSuccessorOwner, + }); + } finally { + process.removeListener("SIGUSR1", handler); + } + }); + + it("retains managed successor ownership when an ordinary restart pulls the timer earlier", async () => { + const handler = () => {}; + process.on("SIGUSR1", handler); + try { + scheduleGatewaySigusr1Restart({ + delayMs: 1_000, + reason: "update.auto", + successorOwner: managedSuccessorOwner, + }); + scheduleGatewaySigusr1Restart({ delayMs: 0, reason: "config.patch" }); + + await vi.advanceTimersByTimeAsync(0); expect(peekGatewaySigusr1RestartReason()).toBe("update.auto"); + expect(consumeGatewaySigusr1RestartIntent()).toEqual({ + reason: "update.auto", + successorOwner: managedSuccessorOwner, + }); + } finally { + process.removeListener("SIGUSR1", handler); + } + }); + + it("replaces stale managed successor ownership when its replacement coalesces", async () => { + const replacementOwner = { ...managedSuccessorOwner, handoffId: "replacement-handoff" }; + const handler = () => {}; + process.on("SIGUSR1", handler); + try { + scheduleGatewaySigusr1Restart({ + delayMs: 1_000, + reason: "update.auto", + successorOwner: managedSuccessorOwner, + }); + const replacement = scheduleGatewaySigusr1Restart({ + delayMs: 1_000, + reason: "update.auto", + successorOwner: replacementOwner, + }); + + expect(replacement.coalesced).toBe(true); + await vi.advanceTimersByTimeAsync(1_000); + expect(consumeGatewaySigusr1RestartIntent()).toEqual({ + reason: "update.auto", + successorOwner: replacementOwner, + }); } finally { process.removeListener("SIGUSR1", handler); } @@ -651,7 +714,7 @@ describe("infra runtime", () => { it("rejects coalesced emit hooks from a different session while preparation is in flight (#86742)", async () => { // Pins the CWE-200 in-flight preparation race: pendingRestartSessionKey // must stay alive through await beforeEmit(), otherwise a coalesced - // different-session caller slips past updatePendingRestartEmitHooks + // different-session caller slips past canReplacePendingRestartEmitHooks // and chains its own hooks while preparation runs. let releaseSessionAPrep: () => void = () => {}; const sessionAPrepBlocked = new Promise((resolve) => { @@ -1219,6 +1282,7 @@ describe("infra runtime", () => { const forced = scheduleGatewaySigusr1Restart({ delayMs: 0, reason: "update.run", + successorOwner: managedSuccessorOwner, skipDeferral: true, }); @@ -1226,6 +1290,10 @@ describe("infra runtime", () => { expect(emitSpy).toHaveBeenCalledWith("SIGUSR1"); expect(staleBeforeEmit).not.toHaveBeenCalled(); expect(peekGatewaySigusr1RestartReason()).toBe("update.run"); + expect(consumeGatewaySigusr1RestartIntent()).toEqual({ + reason: "update.run", + successorOwner: managedSuccessorOwner, + }); } finally { process.removeListener("SIGUSR1", handler); } diff --git a/src/infra/process-respawn.test.ts b/src/infra/process-respawn.test.ts index acd70e5f6551..f18c57c4b8a5 100644 --- a/src/infra/process-respawn.test.ts +++ b/src/infra/process-respawn.test.ts @@ -354,22 +354,6 @@ describe("respawnGatewayProcessForUpdate", () => { expect(spawnMock).not.toHaveBeenCalled(); }); - it("actively schedules launchd update relaunch before exiting", () => { - clearSupervisorHints(); - setPlatform("darwin"); - process.env.OPENCLAW_LAUNCHD_LABEL = "ai.openclaw.gateway"; - process.env.OPENCLAW_NO_RESPAWN = "1"; - - const result = respawnGatewayProcessForUpdate(); - - expect(result.mode).toBe("supervised"); - expect(result.handoffSpawned).toBeInstanceOf(Promise); - expect(scheduleLaunchdHandoffMock).toHaveBeenCalledWith({ - mode: "start-after-exit", - waitForPid: process.pid, - }); - }); - it("allows detached respawn on unmanaged Windows during updates", () => { clearSupervisorHints(); setPlatform("win32"); @@ -397,18 +381,6 @@ describe("respawnGatewayProcessForUpdate", () => { ); }); - it("delegates update restarts to external supervision without spawning", () => { - clearSupervisorHints(); - setPlatform("linux"); - process.env.OPENCLAW_SUPERVISOR_MODE = "external"; - - const result = respawnGatewayProcessForUpdate(); - - expect(result).toEqual({ mode: "supervised" }); - expect(spawnMock).not.toHaveBeenCalled(); - expect(triggerOpenClawRestartMock).not.toHaveBeenCalled(); - }); - it("rewrites a pnpm-versioned OpenClaw entry before detached update respawn", () => { clearSupervisorHints(); setPlatform("linux"); @@ -497,19 +469,6 @@ describe("respawnGatewayProcessForUpdate", () => { expect(onCallOrder).toBeLessThan(unrefCallOrder); }); - it("exits to a managed supervisor for updates even when respawn is disabled", () => { - clearSupervisorHints(); - setPlatform("linux"); - process.env.OPENCLAW_NO_RESPAWN = "1"; - process.env.OPENCLAW_SERVICE_MARKER = "openclaw"; - process.env.OPENCLAW_SERVICE_KIND = "gateway"; - - const result = respawnGatewayProcessForUpdate(); - - expect(result).toEqual({ mode: "supervised" }); - expect(spawnMock).not.toHaveBeenCalled(); - }); - it("returns failed when update detached respawn throws", () => { delete process.env.OPENCLAW_NO_RESPAWN; clearSupervisorHints(); diff --git a/src/infra/process-respawn.ts b/src/infra/process-respawn.ts index 2a54e905f4bb..07af462d4c2b 100644 --- a/src/infra/process-respawn.ts +++ b/src/infra/process-respawn.ts @@ -7,16 +7,16 @@ import { formatErrorMessage } from "./errors.js"; import { triggerOpenClawRestart } from "./restart.js"; import { detectGatewayRespawnSupervisor } from "./supervisor-markers.js"; -type RespawnMode = "spawned" | "supervised" | "disabled" | "failed"; - type GatewayRespawnResult = { - mode: RespawnMode; - pid?: number; + mode: "supervised" | "disabled" | "failed"; detail?: string; handoffSpawned?: Promise; }; -type GatewayUpdateRespawnResult = GatewayRespawnResult & { +type GatewayUpdateRespawnResult = { + mode: "spawned" | "disabled" | "failed"; + pid?: number; + detail?: string; child?: ChildProcess; }; type GatewayRespawnOptions = { @@ -35,39 +35,6 @@ function rewritePnpmVersionedOpenClawEntryPath(entryPath: string): string { ); } -function spawnDetachedGatewayProcess(opts: GatewayRespawnOptions = {}): { - child: ChildProcess; - pid?: number; -} { - const [entryArg, ...entryArgs] = process.argv.slice(1); - const args = [ - ...process.execArgv, - ...(entryArg ? [rewritePnpmVersionedOpenClawEntryPath(entryArg)] : []), - ...entryArgs, - ]; - const child = spawn(process.execPath, args, { - env: opts.env ? { ...process.env, ...opts.env } : process.env, - detached: true, - stdio: "inherit", - }); - // Detached spawn failures can arrive asynchronously after spawn() returns. - // Keep this listener before unref() so the parent does not crash during handoff. - child.on("error", () => {}); - child.unref(); - return { child, pid: child.pid ?? undefined }; -} - -function scheduleLaunchdRestartAfterExit(): GatewayRespawnResult { - const handoff = scheduleDetachedLaunchdRestartHandoff({ - mode: "start-after-exit", - waitForPid: process.pid, - }); - if (!handoff.ok) { - return { mode: "failed", detail: handoff.error }; - } - return { mode: "supervised", handoffSpawned: handoff.value }; -} - /** * Attempt to restart this process with a fresh PID. * - supervised environments (launchd/systemd/schtasks): caller should exit and let supervisor restart @@ -84,7 +51,13 @@ export function restartGatewayProcessWithFreshPid( const supervisor = detectGatewayRespawnSupervisor(process.env); if (supervisor) { if (supervisor === "launchd") { - return scheduleLaunchdRestartAfterExit(); + const handoff = scheduleDetachedLaunchdRestartHandoff({ + mode: "start-after-exit", + waitForPid: process.pid, + }); + return handoff.ok + ? { mode: "supervised", handoffSpawned: handoff.value } + : { mode: "failed", detail: handoff.error }; } if (supervisor === "schtasks") { const restart = triggerOpenClawRestart(); @@ -97,68 +70,46 @@ export function restartGatewayProcessWithFreshPid( } return { mode: "supervised" }; } - if (process.platform === "win32") { - // Detached respawn is unsafe on Windows without an identified Scheduled Task: - // the child becomes orphaned if the original process exits. - return { - mode: "disabled", - detail: "win32: detached respawn unsupported without Scheduled Task markers", - }; - } - if (isContainerEnvironment()) { - return { - mode: "disabled", - detail: "container: use in-process restart to keep PID 1 alive", - }; - } - - return { - mode: "disabled", - detail: "unmanaged: use in-process restart to keep custom supervisor PID tracking stable", - }; + // Unmanaged Windows or containers cannot safely surrender their tracked process. + const detail = + process.platform === "win32" + ? "win32: detached respawn unsupported without Scheduled Task markers" + : isContainerEnvironment() + ? "container: use in-process restart to keep PID 1 alive" + : "unmanaged: use in-process restart to keep custom supervisor PID tracking stable"; + return { mode: "disabled", detail }; } /** * Update restarts must replace the OS process so the new code runs from a * fresh module graph after package files have changed on disk. * - * Unlike the generic restart path, update mode allows detached respawn on - * unmanaged Windows installs because there is no safe in-process fallback once - * the installed package contents have been replaced. + * The caller resolves supervisor ownership first; this path is only for an + * unmanaged process whose installed package contents have been replaced. */ export function respawnGatewayProcessForUpdate( opts: GatewayRespawnOptions = {}, ): GatewayUpdateRespawnResult { - const supervisor = detectGatewayRespawnSupervisor(process.env, process.platform, { - includeLinuxOpenClawGatewayServiceMarker: true, - }); - if (supervisor) { - // Managed update handoffs require the original PID to exit before the - // detached helper can mutate the install, even when respawn is disabled. - if (supervisor === "launchd") { - return scheduleLaunchdRestartAfterExit(); - } - if (supervisor === "schtasks") { - const restart = triggerOpenClawRestart(); - if (!restart.ok) { - return { - mode: "failed", - detail: restart.detail ?? `${restart.method} restart failed`, - }; - } - } - return { mode: "supervised" }; - } if (isTruthyEnvValue(process.env.OPENCLAW_NO_RESPAWN)) { return { mode: "disabled", detail: "OPENCLAW_NO_RESPAWN" }; } try { - const { child, pid } = spawnDetachedGatewayProcess(opts); - return { mode: "spawned", pid, child }; + const [entryArg, ...entryArgs] = process.argv.slice(1); + const args = [ + ...process.execArgv, + ...(entryArg ? [rewritePnpmVersionedOpenClawEntryPath(entryArg)] : []), + ...entryArgs, + ]; + const child = spawn(process.execPath, args, { + env: opts.env ? { ...process.env, ...opts.env } : process.env, + detached: true, + stdio: "inherit", + }); + // Register before unref: late detached-spawn failures must not crash the parent. + child.on("error", () => {}); + child.unref(); + return { mode: "spawned", pid: child.pid ?? undefined, child }; } catch (err) { - return { - mode: "failed", - detail: formatErrorMessage(err), - }; + return { mode: "failed", detail: formatErrorMessage(err) }; } } diff --git a/src/infra/restart-intent.test.ts b/src/infra/restart-intent.test.ts index 777b0d1b47c3..781f0859f98b 100644 --- a/src/infra/restart-intent.test.ts +++ b/src/infra/restart-intent.test.ts @@ -120,7 +120,7 @@ describe("gateway restart intent", () => { expect(readIntentRow(env)).toBeUndefined(); }); - it("round-trips restart reason, force, and wait options", () => { + it("round-trips restart options without persisting process-local successor identity", () => { const env = createIntentEnv(); expect( @@ -128,7 +128,15 @@ describe("gateway restart intent", () => { env, targetPid: process.pid, reason: "gateway.restart", - intent: { force: true, waitMs: 12_345 }, + intent: { + force: true, + waitMs: 12_345, + successorOwner: { + kind: "managed-update-handoff", + handoffId: "private-handoff", + installRoot: "/private/install", + }, + }, }), ).toBe(true); diff --git a/src/infra/restart-intent.ts b/src/infra/restart-intent.ts index d48da1b683c9..54a1d7719873 100644 --- a/src/infra/restart-intent.ts +++ b/src/infra/restart-intent.ts @@ -32,12 +32,14 @@ export type GatewayRestartIntent = { reason?: string; force?: boolean; waitMs?: number; + // Process-local only: persisted restart requests cannot delegate successor ownership. + successorOwner?: { + kind: "managed-update-handoff"; + handoffId: string; + installRoot: string; + }; }; -function normalizeRestartIntentPid(pid: number | undefined): number | null { - return asPositiveSafeInteger(pid) ?? null; -} - export function normalizeRestartIntentReason(reason: string | undefined): string | undefined { const normalized = reason?.trim(); return normalized ? truncateUtf16Safe(normalized, 200) : undefined; @@ -49,7 +51,7 @@ export function writeGatewayRestartIntentSync(opts: { intent?: GatewayRestartIntent; reason?: string; }): boolean { - const targetPid = normalizeRestartIntentPid(opts.targetPid); + const targetPid = asPositiveSafeInteger(opts.targetPid) ?? null; if (targetPid === null) { return false; } diff --git a/src/infra/restart.test.ts b/src/infra/restart.test.ts index ede6768341ef..27a015bf36a8 100644 --- a/src/infra/restart.test.ts +++ b/src/infra/restart.test.ts @@ -36,7 +36,12 @@ vi.mock("../config/paths.js", () => ({ const { cleanStaleGatewayProcessesSync, findGatewayPidsOnPortSync } = await import("./restart-stale-pids.js"); -const { triggerOpenClawRestart } = await import("./restart.js"); +const { + normalizeGatewayRestartDelayMs, + resetGatewayRestartStateForInProcessRestart, + scheduleGatewaySigusr1Restart, + triggerOpenClawRestart, +} = await import("./restart.js"); const envSnapshot = captureFullEnv(); @@ -262,3 +267,34 @@ describe("triggerOpenClawRestart", () => { ); }); }); + +describe("gateway restart delay normalization", () => { + it.each([ + { requested: undefined, effective: 2000 }, + { requested: Number.NaN, effective: 2000 }, + { requested: Number.POSITIVE_INFINITY, effective: 2000 }, + { requested: -1, effective: 0 }, + { requested: 1500.8, effective: 1500 }, + { requested: 2_147_153_648, effective: 60_000 }, + ])("normalizes $requested to $effective ms", ({ requested, effective }) => { + expect(normalizeGatewayRestartDelayMs(requested)).toBe(effective); + }); + + it("does not emit an overflow-sized restart before its effective 60-second delay", async () => { + vi.useFakeTimers(); + const killSpy = vi.spyOn(process, "kill").mockImplementation(() => true); + try { + const restart = scheduleGatewaySigusr1Restart({ + delayMs: 2_147_153_648, + skipCooldown: true, + }); + + expect(restart.delayMs).toBe(60_000); + await vi.advanceTimersByTimeAsync(59_999); + expect(killSpy).not.toHaveBeenCalled(); + } finally { + resetGatewayRestartStateForInProcessRestart(); + vi.useRealTimers(); + } + }); +}); diff --git a/src/infra/restart.ts b/src/infra/restart.ts index f5409c7c6154..e00456a4f950 100644 --- a/src/infra/restart.ts +++ b/src/infra/restart.ts @@ -45,6 +45,7 @@ let lastRestartEmittedAt = 0; let pendingRestartTimer: ReturnType | null = null; let pendingRestartDueAt = 0; let pendingRestartReason: string | undefined; +let pendingRestartSuccessorOwner: GatewayRestartIntent["successorOwner"]; let pendingRestartEmitHooks: RestartEmitHooks | undefined; let pendingRestartSessionKey: string | undefined; let pendingRestartSkipDeferral = false; @@ -63,12 +64,11 @@ function hasUnconsumedRestartSignal(): boolean { } function clearPendingScheduledRestart(): void { - if (pendingRestartTimer) { - clearTimeout(pendingRestartTimer); - } + clearTimeout(pendingRestartTimer ?? undefined); pendingRestartTimer = null; pendingRestartDueAt = 0; pendingRestartReason = undefined; + pendingRestartSuccessorOwner = undefined; pendingRestartEmitHooks = undefined; pendingRestartSessionKey = undefined; pendingRestartSkipDeferral = false; @@ -179,19 +179,12 @@ function formatRestartAudit(audit: RestartAuditInfo | undefined): string { const clientIp = typeof audit?.clientIp === "string" && audit.clientIp.trim() ? audit.clientIp.trim() : null; const changed = summarizeChangedPaths(audit?.changedPaths); - const fields = []; - if (actor) { - fields.push(`actor=${actor}`); - } - if (deviceId) { - fields.push(`device=${deviceId}`); - } - if (clientIp) { - fields.push(`ip=${clientIp}`); - } - if (changed) { - fields.push(`changedPaths=${changed}`); - } + const fields = [ + actor && `actor=${actor}`, + deviceId && `device=${deviceId}`, + clientIp && `ip=${clientIp}`, + changed && `changedPaths=${changed}`, + ].filter(Boolean); return fields.length > 0 ? fields.join(" ") : "actor="; } @@ -297,10 +290,7 @@ export function requestGatewayRestartWithSignalAdmission( } function resetSigusr1AuthorizationIfExpired(now = Date.now()) { - if (sigusr1AuthorizedCount <= 0) { - return; - } - if (now <= sigusr1AuthorizedUntil) { + if (sigusr1AuthorizedCount <= 0 || now <= sigusr1AuthorizedUntil) { return; } sigusr1AuthorizedCount = 0; @@ -315,9 +305,8 @@ export function isGatewaySigusr1RestartExternallyAllowed() { return sigusr1ExternalAllowed; } -function authorizeGatewaySigusr1Restart(delayMs = 0) { - const delay = Math.max(0, Math.floor(delayMs)); - const expiresAt = Date.now() + delay + SIGUSR1_AUTH_GRACE_MS; +function authorizeGatewaySigusr1Restart() { + const expiresAt = Date.now() + SIGUSR1_AUTH_GRACE_MS; sigusr1AuthorizedCount += 1; if (expiresAt > sigusr1AuthorizedUntil) { sigusr1AuthorizedUntil = expiresAt; @@ -406,44 +395,22 @@ type GatewayRestartEmitResult = | { status: "coalesced" } | { status: "failed" }; +export function resolveGatewayRestartDeferralTimeoutMs(): number; +export function resolveGatewayRestartDeferralTimeoutMs(timeoutMs: unknown): number | undefined; export function resolveGatewayRestartDeferralTimeoutMs(timeoutMs?: unknown): number | undefined { if (typeof timeoutMs !== "number" || !Number.isFinite(timeoutMs)) { return DEFAULT_RESTART_DEFERRAL_TIMEOUT_MS; } - if (timeoutMs <= 0) { - return undefined; - } - return Math.floor(timeoutMs); + return timeoutMs > 0 ? Math.floor(timeoutMs) : undefined; } function canReplacePendingRestartEmitHooks( hooks: RestartEmitHooks | undefined, sessionKey: string | undefined, ): boolean { - if (!hooks) { - return true; - } - return pendingRestartSessionKey === undefined || pendingRestartSessionKey === sessionKey; -} - -// Returns true when the new hooks took ownership of the pending restart slot. -// Coalesced callers from a different sessionKey are rejected to prevent the -// cross-session continuation overwrite documented in #86742 (CWE-200). -function updatePendingRestartEmitHooks( - hooks: RestartEmitHooks | undefined, - sessionKey: string | undefined, -): boolean { - if (!canReplacePendingRestartEmitHooks(hooks, sessionKey)) { - return false; - } - if (!hooks) { - return false; - } - pendingRestartEmitHooks = hooks; - if (sessionKey !== undefined) { - pendingRestartSessionKey = sessionKey; - } - return true; + return ( + !hooks || pendingRestartSessionKey === undefined || pendingRestartSessionKey === sessionKey + ); } async function rejectPreparedRestartHook(hooks: RestartEmitHooks | undefined): Promise { @@ -556,24 +523,26 @@ async function emitPreparedGatewayRestartUnderAdmission( pendingRestartEmitHooks = undefined; } - // Slot settled and no awaits remain before emission — release ownership for - // every emission attempt, not only hookless ones, so a later session can - // claim continuation hooks for the next restart cycle. - pendingRestartSessionKey = undefined; - // Track every successfully prepared hook set (parked + caller) so non-emitted // outcomes can roll back both the gateway-tool sentinel and reload preflight. - const preparedHooksList: RestartEmitHooks[] = []; - if (preparedParked) { - preparedHooksList.push(preparedParked); - } + const preparedHooksList: RestartEmitHooks[] = preparedParked ? [preparedParked] : []; if (hooks && callerPrepared) { preparedHooksList.push(hooks); } // With caller hooks, emission stays the caller's (or falls back to the core // signal path if its preparation failed); parked hooks never own emission // when a caller is present. - const emitOwner = hooks ? (callerPrepared ? hooks : undefined) : preparedParked; + const emitOwner = + hooks && callerPrepared + ? hooks + : hooks || (pendingRestartSuccessorOwner && pendingRestartSessionKey === undefined) + ? undefined + : preparedParked; + + // Slot settled and no awaits remain before emission — release ownership for + // every emission attempt, not only hookless ones, so a later session can + // claim continuation hooks for the next restart cycle. + pendingRestartSessionKey = undefined; if (!isCurrent()) { await rejectPreparedRestartHooks(preparedHooksList); @@ -586,8 +555,15 @@ async function emitPreparedGatewayRestartUnderAdmission( ? pendingRestartReason : undefined; const resolvedReason = preferredReason ?? reasonOverride; + const successorOwner = pendingRestartSuccessorOwner ?? intent?.successorOwner; const resolvedIntent = - preferredReason && intent ? { ...intent, reason: preferredReason } : intent; + preferredReason || successorOwner + ? { + ...intent, + ...(resolvedReason ? { reason: resolvedReason } : {}), + ...(successorOwner ? { successorOwner } : {}), + } + : intent; const emitResult = emitOwner?.emitRestart ? emitOwner.emitRestart(resolvedReason, resolvedIntent) : requestGatewayRestartWithSignalAdmission(resolvedReason, resolvedIntent); @@ -862,18 +838,11 @@ function formatSpawnDetail(result: { return "unknown error"; } } - const stderr = clean(result.stderr); - if (stderr) { - return stderr; - } - const stdout = clean(result.stdout); - if (stdout) { - return stdout; - } - if (typeof result.status === "number") { - return `exit ${result.status}`; - } - return "unknown error"; + return ( + clean(result.stderr) || + clean(result.stdout) || + (typeof result.status === "number" ? `exit ${result.status}` : "unknown error") + ); } function normalizeSystemdUnit(raw?: string, profile?: string): string { @@ -1009,6 +978,12 @@ export type ScheduledRestart = { emitHooksQueued: boolean; }; +export function normalizeGatewayRestartDelayMs(delayMs?: number): number { + return typeof delayMs === "number" && Number.isFinite(delayMs) + ? Math.min(Math.max(Math.floor(delayMs), 0), 60_000) + : 2000; +} + export function scheduleGatewaySigusr1Restart(opts?: { delayMs?: number; reason?: string; @@ -1018,24 +993,35 @@ export function scheduleGatewaySigusr1Restart(opts?: { sessionKey?: string; skipDeferral?: boolean; skipCooldown?: boolean; + successorOwner?: GatewayRestartIntent["successorOwner"]; }): ScheduledRestart { - const delayMsRaw = - typeof opts?.delayMs === "number" && Number.isFinite(opts.delayMs) - ? Math.floor(opts.delayMs) - : 2000; - const delayMs = Math.min(Math.max(delayMsRaw, 0), 60_000); + const delayMs = normalizeGatewayRestartDelayMs(opts?.delayMs); const reason = normalizeRestartIntentReason(opts?.reason); - const hasSigusr1Listener = process.listenerCount("SIGUSR1") > 0; - const mode = hasSigusr1Listener ? "emit" : process.platform === "win32" ? "supervisor" : "signal"; + const mode: ScheduledRestart["mode"] = + process.listenerCount("SIGUSR1") > 0 + ? "emit" + : process.platform === "win32" + ? "supervisor" + : "signal"; const nowMs = Date.now(); - const skipCooldown = opts?.skipCooldown === true; - const cooldownMsApplied = skipCooldown - ? 0 - : Math.max(0, lastRestartEmittedAt + RESTART_COOLDOWN_MS - nowMs); + const cooldownMsApplied = + opts?.skipCooldown === true + ? 0 + : Math.max(0, lastRestartEmittedAt + RESTART_COOLDOWN_MS - nowMs); + const restartResultBase = { + ok: true, + pid: process.pid, + signal: "SIGUSR1" as const, + reason, + mode, + cooldownMsApplied, + }; const requestedDueAt = nowMs + delayMs + cooldownMsApplied; const skipDeferral = opts?.skipDeferral === true; let nextPendingEmitHooks = opts?.emitHooks; let nextPendingSessionKey = opts?.sessionKey; + let nextPendingReason = reason; + let nextPendingSuccessorOwner = opts?.successorOwner; if (hasUnconsumedRestartSignal()) { if (shouldPreferRestartReason(reason, emittedRestartReason)) { @@ -1045,18 +1031,20 @@ export function scheduleGatewaySigusr1Restart(opts?: { emittedRestartIntent = { ...emittedRestartIntent, reason }; } } + if (opts?.successorOwner) { + emittedRestartIntent = { + ...emittedRestartIntent, + ...(emittedRestartReason ? { reason: emittedRestartReason } : {}), + successorOwner: opts.successorOwner, + }; + } restartLog.warn( `restart request coalesced (already in-flight) reason=${reason ?? "unspecified"} ${formatRestartAudit(opts?.audit)}`, ); return { - ok: true, - pid: process.pid, - signal: "SIGUSR1", + ...restartResultBase, delayMs: 0, - reason, - mode, coalesced: true, - cooldownMsApplied, // SIGUSR1 already emitted; the new caller's hooks cannot run for this cycle. emitHooksQueued: false, }; @@ -1064,44 +1052,39 @@ export function scheduleGatewaySigusr1Restart(opts?: { if (pendingRestartTimer || pendingRestartPreparing) { const remainingMs = pendingRestartPreparing ? 0 : Math.max(0, pendingRestartDueAt - nowMs); + // Hookless forced restarts that own no sentinel may preserve an accepted + // pending hook; update/handoff callers rely on the default clear path. + const preservePendingHooks = + opts?.preservePendingEmitHooksOnDeferralBypass === true && + opts?.emitHooks === undefined && + pendingRestartSessionKey !== undefined; if (pendingRestartPreparing && skipDeferral && activeDeferralPolls.size > 0) { restartLog.warn( `restart request bypassed active deferral reason=${reason ?? "unspecified"} pendingReason=${pendingRestartReason ?? "unspecified"} ${formatRestartAudit(opts?.audit)}`, ); clearActiveDeferralPolls(); pendingRestartReason = reason; - // Hookless forced restarts that own no sentinel may preserve an accepted - // pending hook; update/handoff callers rely on the default clear path. - const preservePendingHooks = - opts?.preservePendingEmitHooksOnDeferralBypass === true && - opts?.emitHooks === undefined && - pendingRestartSessionKey !== undefined; + pendingRestartSuccessorOwner = opts?.successorOwner ?? pendingRestartSuccessorOwner; if (!preservePendingHooks) { pendingRestartEmitHooks = opts?.emitHooks; pendingRestartSessionKey = opts?.sessionKey; } void emitPreparedGatewayRestart(undefined, reason); return { - ok: true, - pid: process.pid, - signal: "SIGUSR1", + ...restartResultBase, delayMs: 0, - reason, - mode, coalesced: false, - cooldownMsApplied, emitHooksQueued: opts?.emitHooks !== undefined, }; } - const shouldUpgradeToSkipDeferral = skipDeferral && !pendingRestartSkipDeferral; const shouldPullEarlier = !pendingRestartPreparing && - (requestedDueAt < pendingRestartDueAt || shouldUpgradeToSkipDeferral); + (requestedDueAt < pendingRestartDueAt || (skipDeferral && !pendingRestartSkipDeferral)); if (shouldPullEarlier) { - const preservePendingHooks = - opts?.preservePendingEmitHooksOnDeferralBypass === true && - opts?.emitHooks === undefined && - pendingRestartSessionKey !== undefined; + if (shouldPreferRestartReason(pendingRestartReason, reason)) { + nextPendingReason = pendingRestartReason; + } + nextPendingSuccessorOwner ??= pendingRestartSuccessorOwner; if ( !preservePendingHooks && !canReplacePendingRestartEmitHooks(opts?.emitHooks, opts?.sessionKey) @@ -1109,79 +1092,75 @@ export function scheduleGatewaySigusr1Restart(opts?: { restartLog.warn( `restart continuation dropped: another session owns the pending restart (callerSessionKey=${opts?.sessionKey ?? "unspecified"} pendingSessionKey=${pendingRestartSessionKey ?? "unspecified"})`, ); - if (pendingRestartTimer) { - clearTimeout(pendingRestartTimer); - } + clearTimeout(pendingRestartTimer ?? undefined); pendingRestartTimer = null; pendingRestartDueAt = requestedDueAt; - pendingRestartReason = reason; + pendingRestartReason = nextPendingReason; + pendingRestartSuccessorOwner = nextPendingSuccessorOwner; pendingRestartSkipDeferral = pendingRestartSkipDeferral || skipDeferral; armPendingRestartTimer(requestedDueAt, nowMs); return { - ok: true, - pid: process.pid, - signal: "SIGUSR1", + ...restartResultBase, delayMs: Math.max(0, requestedDueAt - nowMs), - reason, - mode, coalesced: true, - cooldownMsApplied, emitHooksQueued: false, }; } - const preservedEmitHooks = preservePendingHooks ? pendingRestartEmitHooks : undefined; - const preservedSessionKey = preservePendingHooks ? pendingRestartSessionKey : undefined; + if (preservePendingHooks) { + nextPendingEmitHooks = pendingRestartEmitHooks; + nextPendingSessionKey = pendingRestartSessionKey; + } restartLog.warn( `restart request rescheduled earlier reason=${reason ?? "unspecified"} pendingReason=${pendingRestartReason ?? "unspecified"} oldDelayMs=${remainingMs} newDelayMs=${Math.max(0, requestedDueAt - nowMs)} ${formatRestartAudit(opts?.audit)}`, ); clearPendingScheduledRestart(); - if (preservePendingHooks) { - nextPendingEmitHooks = preservedEmitHooks; - nextPendingSessionKey = preservedSessionKey; - } } else { - if (shouldPreferRestartReason(reason, pendingRestartReason)) { + const restartReasonPromoted = shouldPreferRestartReason(reason, pendingRestartReason); + if (restartReasonPromoted) { pendingRestartReason = reason; } + pendingRestartSuccessorOwner = opts?.successorOwner ?? pendingRestartSuccessorOwner; pendingRestartSkipDeferral = pendingRestartSkipDeferral || skipDeferral; restartLog.warn( `restart request coalesced (already scheduled) reason=${reason ?? "unspecified"} pendingReason=${pendingRestartReason ?? "unspecified"} delayMs=${remainingMs} ${formatRestartAudit(opts?.audit)}`, ); - const emitHooksQueued = updatePendingRestartEmitHooks(opts?.emitHooks, opts?.sessionKey); + const emitHooksQueued = + opts?.emitHooks !== undefined && + canReplacePendingRestartEmitHooks(opts.emitHooks, opts.sessionKey); + if ( + emitHooksQueued || + (!preservePendingHooks && + opts?.emitHooks === undefined && + (restartReasonPromoted || opts?.successorOwner !== undefined)) + ) { + pendingRestartEmitHooks = opts?.emitHooks; + pendingRestartSessionKey = opts?.emitHooks ? opts.sessionKey : undefined; + } if (opts?.emitHooks && !emitHooksQueued) { restartLog.warn( `restart continuation dropped: another session owns the pending restart (callerSessionKey=${opts.sessionKey ?? "unspecified"} pendingSessionKey=${pendingRestartSessionKey ?? "unspecified"})`, ); } return { - ok: true, - pid: process.pid, - signal: "SIGUSR1", + ...restartResultBase, delayMs: remainingMs, - reason, - mode, coalesced: true, - cooldownMsApplied, emitHooksQueued, }; } } pendingRestartDueAt = requestedDueAt; - pendingRestartReason = reason; + pendingRestartReason = nextPendingReason; + pendingRestartSuccessorOwner = nextPendingSuccessorOwner; pendingRestartEmitHooks = nextPendingEmitHooks; pendingRestartSessionKey = nextPendingSessionKey; pendingRestartSkipDeferral = skipDeferral; armPendingRestartTimer(requestedDueAt, nowMs); return { - ok: true, - pid: process.pid, - signal: "SIGUSR1", + ...restartResultBase, delayMs: Math.max(0, requestedDueAt - nowMs), - reason, - mode, coalesced: false, - cooldownMsApplied, emitHooksQueued: opts?.emitHooks !== undefined, }; } diff --git a/src/infra/supervisor-markers.test.ts b/src/infra/supervisor-markers.test.ts index 6c94341f393d..37677ce3204e 100644 --- a/src/infra/supervisor-markers.test.ts +++ b/src/infra/supervisor-markers.test.ts @@ -106,6 +106,15 @@ describe("detectRespawnSupervisor", () => { "win32", ), ).toBeNull(); + expect( + detectRespawnSupervisor( + { + OPENCLAW_SERVICE_MARKER: "other", + OPENCLAW_SERVICE_KIND: "gateway", + }, + "win32", + ), + ).toBeNull(); }); it("ignores service markers on non-Windows platforms and unknown platforms", () => { diff --git a/src/infra/supervisor-markers.ts b/src/infra/supervisor-markers.ts index c62aba76953d..4ba64f6db4b6 100644 --- a/src/infra/supervisor-markers.ts +++ b/src/infra/supervisor-markers.ts @@ -75,9 +75,7 @@ export function detectRespawnSupervisor( if (hasAnyHint(env, SUPERVISOR_HINTS.schtasks)) { return "schtasks"; } - const marker = env.OPENCLAW_SERVICE_MARKER?.trim(); - const serviceKind = env.OPENCLAW_SERVICE_KIND?.trim(); - return marker && serviceKind === "gateway" ? "schtasks" : null; + return hasOpenClawGatewayServiceMarker(env) ? "schtasks" : null; } return null; } diff --git a/src/infra/update-managed-service-handoff-command.test.ts b/src/infra/update-managed-service-handoff-command.test.ts index 9821389b8280..1b76a326a342 100644 --- a/src/infra/update-managed-service-handoff-command.test.ts +++ b/src/infra/update-managed-service-handoff-command.test.ts @@ -1,24 +1,33 @@ // Managed-service handoff command tests cover immutable update target serialization. import { EventEmitter } from "node:events"; import fs from "node:fs/promises"; +import os from "node:os"; import path from "node:path"; import { PassThrough } from "node:stream"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { parseDevUpdateTargetEnv, type DevUpdateTarget } from "./update-dev-target.js"; +import { signalMockManagedUpdateHandoffReady } from "./update-managed-service-handoff.test-support.js"; const spawnMock = vi.hoisted(() => vi.fn()); const tempDirs = new Set(); +const mockedHandoffLeaseCleanups = new Set<() => void>(); +const MOCK_INSTALL_ROOT = path.join(os.tmpdir(), `openclaw-handoff-command-${process.pid}`); -function createReadyChild() { +function createReadyChild(_command: string, args: string[]) { const child = Object.assign(new EventEmitter(), { - pid: 24680, + pid: process.pid, exitCode: null, signalCode: null, + stdin: new PassThrough(), stdout: new PassThrough(), unref: vi.fn(), }); process.nextTick(() => { - child.stdout.write("OPENCLAW_UPDATE_HANDOFF_READY\n"); + signalMockManagedUpdateHandoffReady({ + child, + paramsPath: args.at(-1) ?? "", + cleanups: mockedHandoffLeaseCleanups, + }); }); return child; } @@ -37,6 +46,9 @@ beforeEach(() => { }); afterEach(async () => { + for (const cleanup of mockedHandoffLeaseCleanups) { + cleanup(); + } await Promise.all([...tempDirs].map((dir) => fs.rm(dir, { recursive: true, force: true }))); tempDirs.clear(); vi.resetModules(); @@ -47,18 +59,23 @@ async function startHandoffAndReadCommand(params: { tag?: string; devTarget?: DevUpdateTarget; env?: NodeJS.ProcessEnv; + restartDelayMs?: number; + restartDrainTimeoutMs?: number; }): Promise<{ command: string; commandArgv: string[] | undefined; + parentExitTimeoutMs: number; + parentExitDeadlineAt: number; spawnEnv: NodeJS.ProcessEnv | undefined; }> { const { startManagedServiceUpdateHandoff } = await import("./update-managed-service-handoff.js"); const result = await startManagedServiceUpdateHandoff({ - root: "/tmp/openclaw", - restartDrainTimeoutMs: 300_000, + root: MOCK_INSTALL_ROOT, + restartDrainTimeoutMs: params.restartDrainTimeoutMs ?? 300_000, + ...(params.restartDelayMs === undefined ? {} : { restartDelayMs: params.restartDelayMs }), channel: params.channel, ...(params.tag ? { tag: params.tag } : {}), - parentPid: 12345, + parentPid: process.pid, execPath: "/usr/local/bin/node", argv1: "/opt/openclaw/openclaw.mjs", meta: {}, @@ -75,22 +92,45 @@ async function startHandoffAndReadCommand(params: { tempDirs.add(path.dirname(paramsPath)); const helperParams = JSON.parse(await fs.readFile(paramsPath, "utf-8")) as { commandArgv?: string[]; + parentExitTimeoutMs: number; + parentExitDeadlineAt: number; }; const metaPath = path.join(path.dirname(paramsPath), "sentinel-meta.json"); const metaFile = JSON.parse(await fs.readFile(metaPath, "utf-8")) as { meta?: { root?: string }; }; expect(metaFile.meta?.root).toBe( - await fs.realpath("/tmp/openclaw").catch(() => path.resolve("/tmp/openclaw")), + await fs.realpath(MOCK_INSTALL_ROOT).catch(() => path.resolve(MOCK_INSTALL_ROOT)), ); return { command: result.command, commandArgv: helperParams.commandArgv, + parentExitTimeoutMs: helperParams.parentExitTimeoutMs, + parentExitDeadlineAt: helperParams.parentExitDeadlineAt, spawnEnv: spawnCall?.[2]?.env, }; } describe("managed service update handoff command", () => { + it.each([ + { drain: 300_000, expected: 390_000 }, + { drain: Number.MAX_SAFE_INTEGER, expected: 2_147_483_647 }, + ])( + "serializes a bounded timer-safe restart deadline for drain $drain", + async ({ drain, expected }) => { + const startedAt = Date.now(); + const result = await startHandoffAndReadCommand({ + channel: "beta", + restartDelayMs: 60_000, + restartDrainTimeoutMs: drain, + }); + + expect(result.parentExitTimeoutMs).toBe(expected); + expect(result.parentExitDeadlineAt).toBeGreaterThanOrEqual(startedAt + expected); + expect(result.parentExitDeadlineAt).toBeLessThanOrEqual(Date.now() + expected); + }, + ); + it("serializes extended-stable into the detached CLI command", async () => { const result = await startHandoffAndReadCommand({ channel: "extended-stable" }); diff --git a/src/infra/update-managed-service-handoff-cross-process.test.ts b/src/infra/update-managed-service-handoff-cross-process.test.ts index 029453ae44af..e0c12c7b2b47 100644 --- a/src/infra/update-managed-service-handoff-cross-process.test.ts +++ b/src/infra/update-managed-service-handoff-cross-process.test.ts @@ -5,20 +5,27 @@ import os from "node:os"; import path from "node:path"; import { PassThrough } from "node:stream"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { isPidAlive } from "../shared/pid-alive.js"; +import { signalMockManagedUpdateHandoffReady } from "./update-managed-service-handoff.test-support.js"; const spawnMock = vi.hoisted(() => vi.fn()); const tempDirs = new Set(); +const handoffParents = new Map(); +const mockedHandoffLeaseCleanups = new Set<() => void>(); -function createReadyChild() { +function createReadyChild(_command: string, args: string[]) { const child = Object.assign(new EventEmitter(), { - pid: 24680, + pid: process.pid, exitCode: null, signalCode: null, + stdin: new PassThrough(), stdout: new PassThrough(), unref: vi.fn(), }); - process.nextTick(() => { - child.stdout.write("OPENCLAW_UPDATE_HANDOFF_READY\n"); + process.nextTick(signalMockManagedUpdateHandoffReady, { + child, + paramsPath: args.at(-1) ?? "", + cleanups: mockedHandoffLeaseCleanups, }); return child; } @@ -31,12 +38,24 @@ vi.mock("node:child_process", async () => { }); }); +vi.mock("../daemon/systemd-scope.js", async (importOriginal) => ({ + ...(await importOriginal()), + findInstalledSystemdGatewayScope: vi.fn(async () => null), +})); + beforeEach(() => { spawnMock.mockReset(); spawnMock.mockImplementation(createReadyChild); }); afterEach(async () => { + for (const parent of handoffParents.values()) { + parent.stdin?.end(); + } + handoffParents.clear(); + for (const cleanup of mockedHandoffLeaseCleanups) { + cleanup(); + } await Promise.all([...tempDirs].map((dir) => fs.rm(dir, { recursive: true, force: true }))); tempDirs.clear(); vi.resetModules(); @@ -51,20 +70,12 @@ async function pathExists(filePath: string): Promise { } } -function processIsAlive(pid: number): boolean { - try { - process.kill(pid, 0); - return true; - } catch { - return false; - } -} - async function prepareConcurrentHandoffHelper(): Promise<{ tmpDir: string; helperScriptPath: string; baseParams: Record; }> { + const { DatabaseSync } = await import("node:sqlite"); const { startManagedServiceUpdateHandoff } = await import("./update-managed-service-handoff.js"); const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-handoff-concurrent-test-")); tempDirs.add(tmpDir); @@ -84,12 +95,20 @@ async function prepareConcurrentHandoffHelper(): Promise<{ const [, args] = spawnMock.mock.calls.at(-1) as unknown as [string, string[]]; const helperScriptPath = args[0] ?? ""; + const baseParams = JSON.parse(await fs.readFile(args[1] ?? "", "utf-8")) as Record< + string, + unknown + >; + const database = new DatabaseSync(String(baseParams.updateLeaseDatabasePath)); + try { + database + .prepare("DELETE FROM managed_update_handoffs WHERE install_root = ? AND owner = ?") + .run(String(baseParams.updateLeaseKey), String(baseParams.updateLeaseOwner)); + } finally { + database.close(); + } tempDirs.add(path.dirname(helperScriptPath)); - return { - tmpDir, - helperScriptPath, - baseParams: JSON.parse(await fs.readFile(args[1] ?? "", "utf-8")) as Record, - }; + return { tmpDir, helperScriptPath, baseParams }; } async function writeConcurrentHandoffParams(params: { @@ -101,13 +120,27 @@ async function writeConcurrentHandoffParams(params: { stateDatabasePath?: string; leaseDatabasePath?: string; }): Promise { + const { spawn } = + await vi.importActual("node:child_process"); + const { getFileLockProcessStartTime } = await import("../shared/pid-alive.js"); + const parent = spawn(process.execPath, ["-e", "process.stdin.resume()"], { + stdio: ["pipe", "ignore", "ignore"], + }); + const parentPid = parent.pid; + const startIdentity = parentPid ? getFileLockProcessStartTime(parentPid) : null; + if (!parentPid || startIdentity === null) { + parent.kill("SIGKILL"); + throw new Error("expected a parent process with a stable start identity"); + } const paramsPath = path.join(params.tmpDir, `${params.name}.json`); + handoffParents.set(paramsPath, parent); await fs.writeFile( paramsPath, `${JSON.stringify( { ...params.baseParams, - parentPid: 0, + parentPid, + parentStartIdentity: String(startIdentity), parentExitTimeoutMs: 5_000, handoffId: params.owner, updateLeaseOwner: params.owner, @@ -128,18 +161,42 @@ async function writeConcurrentHandoffParams(params: { return paramsPath; } +function driveHandoffProtocol( + child: import("node:child_process").ChildProcess, + paramsPath: string, +): void { + let buffered = ""; + child.stdout?.on("data", (chunk: Buffer | string) => { + buffered += chunk.toString(); + let newline: number; + while ((newline = buffered.indexOf("\n")) >= 0) { + const line = buffered.slice(0, newline); + buffered = buffered.slice(newline + 1); + if (line === "OPENCLAW_UPDATE_HANDOFF_READY") { + child.stdin?.write("park\n"); + } else if (line === "parked") { + child.stdin?.write("commit\n"); + } else if (line === "committed") { + handoffParents.get(paramsPath)?.stdin?.end(); + } + } + }); +} + async function runHelper(params: { execFile: typeof import("node:child_process").execFile; helperScriptPath: string; paramsPath: string; cwd: string; + env?: NodeJS.ProcessEnv; }): Promise<{ code: number | null; stdout: string; stderr: string }> { return await new Promise((resolve) => { - params.execFile( + const child = params.execFile( process.execPath, [params.helperScriptPath, params.paramsPath], - { cwd: params.cwd, encoding: "utf8" }, + { cwd: params.cwd, encoding: "utf8", ...(params.env ? { env: params.env } : {}) }, (error, stdout, stderr) => { + handoffParents.get(params.paramsPath)?.stdin?.end(); const childError = error as NodeJS.ErrnoException | null; resolve({ code: typeof childError?.code === "number" ? childError.code : 0, @@ -148,15 +205,17 @@ async function runHelper(params: { }); }, ); + driveHandoffProtocol(child, params.paramsPath); }); } describe("managed service update handoff cross-process lease", () => { it("joins the durable owner reported by a replacement helper", async () => { const child = Object.assign(new EventEmitter(), { - pid: 24681, + pid: process.pid, exitCode: null, signalCode: null, + stdin: new PassThrough(), stdout: new PassThrough(), unref: vi.fn(), }); @@ -171,7 +230,7 @@ describe("managed service update handoff cross-process lease", () => { const result = await startManagedServiceUpdateHandoff({ root: "/tmp/openclaw", restartDrainTimeoutMs: 300_000, - parentPid: 12345, + parentPid: process.pid, execPath: "/usr/local/bin/node", argv1: "/opt/openclaw/openclaw.mjs", handoffId: "replacement-handoff", @@ -187,6 +246,488 @@ describe("managed service update handoff cross-process lease", () => { expect(result).not.toHaveProperty("pid"); }); + it.runIf(process.platform !== "win32").each([ + { label: "its exact dead helper", replacement: "exact", reclaimed: true }, + { label: "a mismatched owner", replacement: "owner", reclaimed: false }, + { label: "a mismatched helper identity", replacement: "identity", reclaimed: false }, + { label: "another live process", replacement: "live", reclaimed: false }, + { label: "an unknown malformed process identity", replacement: "unknown", reclaimed: false }, + ] as const)( + "claims a separate systemd scope helper and fences cancellation against $label", + async ({ replacement, reclaimed }) => { + const { DatabaseSync } = await import("node:sqlite"); + const { getFileLockProcessStartTime } = await import("../shared/pid-alive.js"); + const { spawn } = + await vi.importActual("node:child_process"); + const { + cancelManagedServiceUpdateHandoff, + claimManagedServiceUpdateHandoff, + startManagedServiceUpdateHandoff, + } = await import("./update-managed-service-handoff.js"); + const tmpDir = await fs.realpath( + await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-handoff-scope-wrapper-")), + ); + tempDirs.add(tmpDir); + const helperExitPath = path.join(tmpDir, "nested-helper-exited"); + const launcherPath = path.join(tmpDir, "systemd-run"); + await fs.writeFile( + launcherPath, + `#!${process.execPath} +const fs = require("node:fs"); +const { spawn } = require("node:child_process"); +const [command, scriptPath, paramsPath] = process.argv.slice(-3); +const helper = spawn(command, [scriptPath, paramsPath], { stdio: ["pipe", "pipe", "ignore"] }); +let helperAlive = true; +helper.stdin.on("error", () => {}); +helper.stdout.pipe(process.stdout, { end: false }); +helper.once("exit", () => { + helperAlive = false; + fs.writeFileSync(${JSON.stringify(helperExitPath)}, String(helper.pid)); +}); +process.stdin.on("data", (chunk) => { + if (helperAlive) { + helper.stdin.write(chunk); + } else if (chunk.toString().includes("cancel\\n")) { + process.stdout.write("cancelled\\n", () => process.exit(0)); + } +}); +`, + { mode: 0o700 }, + ); + spawnMock.mockImplementationOnce(spawn); + + const handoffId = `scope-wrapper-${replacement}-${path.basename(tmpDir)}`; + let launcher: import("node:child_process").ChildProcess | undefined; + let helperPid = 0; + let leaseDatabasePath: string | undefined; + let leaseOwner = handoffId; + try { + const started = await startManagedServiceUpdateHandoff({ + root: tmpDir, + restartDrainTimeoutMs: 5_000, + restartDelayMs: 0, + parentPid: process.pid, + execPath: process.execPath, + argv1: "/opt/openclaw/openclaw.mjs", + supervisor: "systemd", + handoffId, + meta: { handoffId }, + env: { + OPENCLAW_STATE_DIR: tmpDir, + PATH: `${tmpDir}${path.delimiter}${process.env.PATH ?? ""}`, + }, + }); + if (started.status !== "started" || !started.pid) { + throw new Error("expected the systemd scope wrapper to start a real handoff"); + } + launcher = spawnMock.mock.results.at(-1)?.value as + | import("node:child_process").ChildProcess + | undefined; + tempDirs.add(path.dirname(started.logPath)); + const [, args] = spawnMock.mock.calls.at(-1) as unknown as [string, string[]]; + const helperParams = JSON.parse(await fs.readFile(args.at(-1) ?? "", "utf8")) as { + updateLeaseDatabasePath: string; + updateLeaseKey: string; + }; + leaseDatabasePath = helperParams.updateLeaseDatabasePath; + const identity = { + kind: "managed-update-handoff" as const, + handoffId, + installRoot: started.installRoot, + }; + const leaseDatabase = new DatabaseSync(leaseDatabasePath); + let originalRow: { owner: string; payload_json: string }; + try { + const current = leaseDatabase + .prepare( + "SELECT owner, payload_json FROM managed_update_handoffs WHERE install_root = ?", + ) + .get(helperParams.updateLeaseKey) as + | { owner: string; payload_json: string } + | undefined; + if (!current) { + throw new Error("expected the nested handoff helper to own its durable lease"); + } + originalRow = current; + const helperIdentity = JSON.parse(current.payload_json) as { + pid: number; + startIdentity: string; + }; + helperPid = helperIdentity.pid; + + expect(helperPid).not.toBe(started.pid); + expect(launcher?.pid).toBe(started.pid); + expect(isPidAlive(helperPid)).toBe(true); + expect(claimManagedServiceUpdateHandoff(identity)).toBe(true); + + process.kill(helperPid, "SIGKILL"); + await vi.waitFor( + async () => { + await expect(fs.readFile(helperExitPath, "utf8")).resolves.toBe(String(helperPid)); + expect(isPidAlive(helperPid)).toBe(false); + expect(isPidAlive(started.pid!)).toBe(true); + }, + { interval: 10, timeout: 5_000 }, + ); + expect(claimManagedServiceUpdateHandoff(identity)).toBe(false); + + let payload = current.payload_json; + if (replacement === "owner") { + leaseOwner = `${handoffId}-foreign`; + } else if (replacement === "identity") { + payload = JSON.stringify({ + version: 1, + pid: helperPid, + startIdentity: `${helperIdentity.startIdentity}-mismatched`, + }); + } else if (replacement === "live") { + const liveStartIdentity = getFileLockProcessStartTime(process.pid); + if (liveStartIdentity === null) { + throw new Error("expected the live replacement to have a stable process identity"); + } + payload = JSON.stringify({ + version: 1, + pid: process.pid, + startIdentity: String(liveStartIdentity), + }); + } else if (replacement === "unknown") { + payload = JSON.stringify({ version: 1, pid: helperPid, startIdentity: null }); + } + if (replacement !== "exact") { + leaseDatabase + .prepare( + "UPDATE managed_update_handoffs SET owner = ?, payload_json = ? WHERE install_root = ? AND owner = ?", + ) + .run(leaseOwner, payload, helperParams.updateLeaseKey, handoffId); + originalRow = { owner: leaseOwner, payload_json: payload }; + } + } finally { + leaseDatabase.close(); + } + + await expect(cancelManagedServiceUpdateHandoff(identity)).resolves.toBe( + reclaimed ? "restored-in-process" : false, + ); + const retained = new DatabaseSync(leaseDatabasePath, { readOnly: true }); + try { + expect( + retained + .prepare( + "SELECT owner, payload_json FROM managed_update_handoffs WHERE install_root = ?", + ) + .get(helperParams.updateLeaseKey), + ).toEqual(reclaimed ? undefined : originalRow!); + } finally { + retained.close(); + } + } finally { + if (helperPid > 0 && isPidAlive(helperPid)) { + process.kill(helperPid, "SIGKILL"); + } + if (launcher?.pid && isPidAlive(launcher.pid)) { + launcher.kill("SIGKILL"); + } + if (leaseDatabasePath) { + const cleanup = new DatabaseSync(leaseDatabasePath); + try { + cleanup + .prepare("DELETE FROM managed_update_handoffs WHERE install_root = ? AND owner = ?") + .run(tmpDir, leaseOwner); + } finally { + cleanup.close(); + } + } + } + }, + ); + + it.each([ + ["an unavailable parent start identity", null, false], + ["a reused parent start identity", "0", false], + ["an existing live owner with a null start identity", "current", true], + ])("rejects %s before announcing ownership", async (_label, identity, invalidOwner) => { + const { execFile } = + await vi.importActual("node:child_process"); + const { getFileLockProcessStartTime } = await import("../shared/pid-alive.js"); + const { DatabaseSync } = await import("node:sqlite"); + const { tmpDir, helperScriptPath, baseParams } = await prepareConcurrentHandoffHelper(); + const markerPath = path.join(tmpDir, "invalid-parent-update-ran"); + const paramsPath = await writeConcurrentHandoffParams({ + tmpDir, + baseParams, + name: "invalid-parent", + owner: "invalid-parent-owner", + commandArgv: [ + process.execPath, + "-e", + `require("node:fs").writeFileSync(${JSON.stringify(markerPath)}, "ran")`, + ], + }); + const params = JSON.parse(await fs.readFile(paramsPath, "utf8")) as Record; + const currentStartIdentity = getFileLockProcessStartTime(process.pid); + if (currentStartIdentity === null) { + throw new Error("expected the live parent to have a stable start identity"); + } + await fs.writeFile( + paramsPath, + JSON.stringify({ + ...params, + parentPid: process.pid, + parentStartIdentity: identity === "current" ? String(currentStartIdentity) : identity, + }), + ); + const leaseDatabasePath = String(params.updateLeaseDatabasePath); + const leaseKey = String(params.updateLeaseKey); + const invalidPayload = JSON.stringify({ + version: 1, + pid: process.pid, + startIdentity: null, + }); + const invalidUpdatedAt = Date.now(); + if (invalidOwner) { + await fs.mkdir(path.dirname(leaseDatabasePath), { recursive: true, mode: 0o700 }); + const db = new DatabaseSync(leaseDatabasePath); + try { + db.exec( + "CREATE TABLE IF NOT EXISTS managed_update_handoffs (install_root TEXT NOT NULL PRIMARY KEY, owner TEXT NOT NULL, payload_json TEXT NOT NULL, updated_at INTEGER NOT NULL) STRICT;", + ); + db.prepare( + "INSERT INTO managed_update_handoffs (install_root, owner, payload_json, updated_at) VALUES (?, ?, ?, ?)", + ).run(leaseKey, "unverifiable-owner", invalidPayload, invalidUpdatedAt); + } finally { + db.close(); + } + if (process.platform !== "win32") { + await fs.chmod(leaseDatabasePath, 0o600); + } + } + try { + const result = await runHelper({ execFile, helperScriptPath, paramsPath, cwd: tmpDir }); + + expect(result.code).toBe(1); + expect(result.stdout).not.toContain("OPENCLAW_UPDATE_HANDOFF_READY"); + await expect(pathExists(markerPath)).resolves.toBe(false); + if (await pathExists(leaseDatabasePath)) { + const db = new DatabaseSync(leaseDatabasePath, { readOnly: true }); + try { + const row = db + .prepare( + "SELECT owner, payload_json, updated_at FROM managed_update_handoffs WHERE install_root = ?", + ) + .get(leaseKey); + expect(row).toEqual( + invalidOwner + ? { + owner: "unverifiable-owner", + payload_json: invalidPayload, + updated_at: invalidUpdatedAt, + } + : undefined, + ); + } finally { + db.close(); + } + } + } finally { + if (invalidOwner) { + const db = new DatabaseSync(leaseDatabasePath); + db.prepare("DELETE FROM managed_update_handoffs WHERE install_root = ?").run(leaseKey); + db.close(); + } + } + }); + + it("preserves a live durable owner when its current start identity cannot be observed", async () => { + const { execFile } = + await vi.importActual("node:child_process"); + const { DatabaseSync } = await import("node:sqlite"); + const { getFileLockProcessStartTime } = await import("../shared/pid-alive.js"); + const { tmpDir, helperScriptPath, baseParams } = await prepareConcurrentHandoffHelper(); + const ownerStartIdentity = getFileLockProcessStartTime(process.pid); + if (ownerStartIdentity === null) { + throw new Error("expected the live lease owner to have a stable process identity"); + } + const leaseDatabasePath = String(baseParams.updateLeaseDatabasePath); + const leaseKey = String(baseParams.updateLeaseKey); + const owner = "identity-probe-unavailable-owner"; + const payload = JSON.stringify({ + version: 1, + pid: process.pid, + startIdentity: String(ownerStartIdentity), + }); + await fs.mkdir(path.dirname(leaseDatabasePath), { recursive: true, mode: 0o700 }); + const database = new DatabaseSync(leaseDatabasePath); + try { + database.exec( + "CREATE TABLE IF NOT EXISTS managed_update_handoffs (install_root TEXT NOT NULL PRIMARY KEY, owner TEXT NOT NULL, payload_json TEXT NOT NULL, updated_at INTEGER NOT NULL) STRICT;", + ); + database + .prepare( + "INSERT INTO managed_update_handoffs (install_root, owner, payload_json, updated_at) VALUES (?, ?, ?, ?)", + ) + .run(leaseKey, owner, payload, Date.now()); + } finally { + database.close(); + } + if (process.platform !== "win32") { + await fs.chmod(leaseDatabasePath, 0o600); + } + const preloadPath = path.join(tmpDir, "unavailable-owner-identity.cjs"); + await fs.writeFile( + preloadPath, + `const fs = require("node:fs"); +const childProcess = require("node:child_process"); +const ownerPid = ${process.pid}; +const readFileSync = fs.readFileSync; +fs.readFileSync = function(filePath, ...args) { + if (filePath === "/proc/" + ownerPid + "/stat") { + throw Object.assign(new Error("identity probe unavailable"), { code: "EIO" }); + } + return readFileSync.call(this, filePath, ...args); +}; +const spawnSync = childProcess.spawnSync; +childProcess.spawnSync = function(command, args, options) { + if (args.some((argument) => String(argument).includes(String(ownerPid)))) { + return { status: null, stdout: "", error: new Error("identity probe unavailable") }; + } + return spawnSync.call(this, command, args, options); +}; +`, + ); + const markerPath = path.join(tmpDir, "identity-probe-unavailable-updater-ran"); + const paramsPath = await writeConcurrentHandoffParams({ + tmpDir, + baseParams, + name: "identity-probe-unavailable", + owner: "competing-owner", + commandArgv: [ + process.execPath, + "-e", + `require("node:fs").writeFileSync(${JSON.stringify(markerPath)}, "ran")`, + ], + }); + try { + const result = await runHelper({ + execFile, + helperScriptPath, + paramsPath, + cwd: tmpDir, + env: { ...process.env, NODE_OPTIONS: `--require ${preloadPath}` }, + }); + + expect(result, result.stderr).toMatchObject({ + code: 0, + stdout: expect.stringContaining(`HANDOFF_BUSY ${owner}`), + }); + const retained = new DatabaseSync(leaseDatabasePath, { readOnly: true }); + try { + expect( + retained + .prepare( + "SELECT owner, payload_json FROM managed_update_handoffs WHERE install_root = ?", + ) + .get(leaseKey), + ).toEqual({ owner, payload_json: payload }); + } finally { + retained.close(); + } + await expect(pathExists(markerPath)).resolves.toBe(false); + } finally { + const cleanup = new DatabaseSync(leaseDatabasePath); + cleanup + .prepare("DELETE FROM managed_update_handoffs WHERE install_root = ? AND owner = ?") + .run(leaseKey, owner); + cleanup.close(); + } + }); + + it("releases exact helper ownership when cancellation cannot record its terminal sentinel", async () => { + const { spawn } = + await vi.importActual("node:child_process"); + const { DatabaseSync } = await import("node:sqlite"); + const { tmpDir, helperScriptPath, baseParams } = await prepareConcurrentHandoffHelper(); + const markerPath = path.join(tmpDir, "sentinel-failure-updater-ran"); + const owner = "sentinel-failure-owner"; + const paramsPath = await writeConcurrentHandoffParams({ + tmpDir, + baseParams, + name: "sentinel-failure", + owner, + commandArgv: [ + process.execPath, + "-e", + `require("node:fs").writeFileSync(${JSON.stringify(markerPath)}, "ran")`, + ], + }); + const stateDatabasePath = String(baseParams.stateDatabasePath); + await fs.mkdir(path.dirname(stateDatabasePath), { recursive: true }); + const stateDb = new DatabaseSync(stateDatabasePath); + try { + stateDb.exec( + [ + "CREATE TABLE gateway_restart_sentinel (", + "sentinel_key TEXT NOT NULL PRIMARY KEY, version INTEGER NOT NULL,", + "kind TEXT NOT NULL, status TEXT NOT NULL, ts INTEGER NOT NULL,", + "session_key TEXT, thread_id TEXT, delivery_channel TEXT, delivery_to TEXT,", + "delivery_account_id TEXT, message TEXT, continuation_json TEXT, doctor_hint TEXT,", + "stats_json TEXT, payload_json TEXT NOT NULL, updated_at_ms INTEGER NOT NULL", + ") STRICT;", + "CREATE TRIGGER reject_update_sentinel_write", + "BEFORE INSERT ON gateway_restart_sentinel", + "WHEN NEW.sentinel_key = 'current'", + "BEGIN SELECT RAISE(ABORT, 'sentinel write rejected'); END;", + ].join(" "), + ); + } finally { + stateDb.close(); + } + const leaseDatabasePath = String(baseParams.updateLeaseDatabasePath); + const leaseKey = String(baseParams.updateLeaseKey); + const helper = spawn(process.execPath, [helperScriptPath, paramsPath], { + cwd: tmpDir, + stdio: ["pipe", "pipe", "pipe"], + }); + let output = ""; + helper.stdout.on("data", (chunk: Buffer | string) => { + output += chunk.toString(); + if (output.includes("OPENCLAW_UPDATE_HANDOFF_READY") && !output.includes("cancelled")) { + helper.stdin.write("cancel\n"); + } + }); + const exited = new Promise((resolve) => { + helper.once("close", resolve); + }); + + try { + await expect(exited).resolves.toBe(0); + expect(output).toContain("cancelled"); + const db = new DatabaseSync(leaseDatabasePath, { readOnly: true }); + try { + expect( + db + .prepare("SELECT owner FROM managed_update_handoffs WHERE install_root = ?") + .get(leaseKey), + ).toBeUndefined(); + } finally { + db.close(); + } + const log = await fs.readFile(path.join(tmpDir, "sentinel-failure.log"), "utf8"); + expect(log).toContain("failed to write update sentinel failure"); + expect(log).toContain("sentinel write rejected"); + await expect(pathExists(markerPath)).resolves.toBe(false); + } finally { + if (helper.exitCode === null && helper.signalCode === null) { + helper.kill("SIGKILL"); + } + const db = new DatabaseSync(leaseDatabasePath); + db.prepare("DELETE FROM managed_update_handoffs WHERE install_root = ? AND owner = ?").run( + leaseKey, + owner, + ); + db.close(); + } + }); + it.runIf(process.platform === "win32")( "reclaims a reused Windows PID with a different creation identity", async () => { @@ -330,8 +871,9 @@ describe("managed service update handoff cross-process lease", () => { const first = spawn(process.execPath, [helperScriptPath, firstParamsPath], { cwd: tmpDir, - stdio: ["ignore", "pipe", "pipe"], + stdio: ["pipe", "pipe", "pipe"], }); + driveHandoffProtocol(first, firstParamsPath); let firstStdout = ""; first.stdout.on("data", (chunk) => (firstStdout += chunk)); let orphanPid = 0; @@ -344,7 +886,7 @@ describe("managed service update handoff cross-process lease", () => { { interval: 10, timeout: 5_000 }, ); orphanPid = Number(await fs.readFile(orphanPidPath, "utf8")); - expect(processIsAlive(orphanPid)).toBe(true); + expect(isPidAlive(orphanPid)).toBe(true); first.kill("SIGKILL"); await new Promise((resolve) => { first.once("close", () => resolve()); @@ -363,7 +905,7 @@ describe("managed service update handoff cross-process lease", () => { await expect(pathExists(secondStartedPath)).resolves.toBe(false); await fs.writeFile(releaseOrphanPath, "release"); - await vi.waitFor(() => expect(processIsAlive(orphanPid)).toBe(false), { + await vi.waitFor(() => expect(isPidAlive(orphanPid)).toBe(false), { interval: 20, timeout: 5_000, }); @@ -384,7 +926,7 @@ describe("managed service update handoff cross-process lease", () => { if (first.exitCode === null) { first.kill("SIGKILL"); } - if (orphanPid > 0 && processIsAlive(orphanPid)) { + if (orphanPid > 0 && isPidAlive(orphanPid)) { process.kill(orphanPid, "SIGKILL"); } } @@ -437,8 +979,9 @@ describe("managed service update handoff cross-process lease", () => { const first = spawn(process.execPath, [helperScriptPath, firstParamsPath], { cwd: tmpDir, - stdio: ["ignore", "pipe", "pipe"], + stdio: ["pipe", "pipe", "pipe"], }); + driveHandoffProtocol(first, firstParamsPath); let firstStdout = ""; let firstStderr = ""; first.stdout.on("data", (chunk) => (firstStdout += chunk)); diff --git a/src/infra/update-managed-service-handoff-lifecycle.test-support.ts b/src/infra/update-managed-service-handoff-lifecycle.test-support.ts new file mode 100644 index 000000000000..f8e25060bcca --- /dev/null +++ b/src/infra/update-managed-service-handoff-lifecycle.test-support.ts @@ -0,0 +1,145 @@ +export type ManagedServiceManagerBoundaryOptions = { + cancelAfterPark?: boolean; + parentExitTimeoutMs?: number; + launchdFault?: "wrong-parent" | "missing-restored-pid" | "dead-restored-pid"; + launchdTeardown?: { + bootoutDelayMs?: number; + clockEachCommandMs?: number; + loadedPrints?: number; + pendingBootstrapFailures?: number; + pendingOperationInProgress?: number; + }; + lateCommand?: "park" | "commit"; + overdueCommit?: boolean; + systemdFault?: "start-failed" | "dead-restored-pid"; +}; + +export type ManagedServiceCommandTiming = { + action: string; + startedAtMs: number; + timeoutMs: number; +}; + +export function createManagedServiceManagerFixtureScript(params: { + kind: "systemd" | "launchd"; + parentPid: number; + statePath: string; + commandsPath: string; + options?: ManagedServiceManagerBoundaryOptions; +}): string { + const { commandsPath, kind, options, parentPid, statePath } = params; + return `#!${process.execPath} +const fs = require("node:fs"); +const args = process.argv.slice(2); +const statePath = ${JSON.stringify(statePath)}; +const state = fs.existsSync(statePath) ? JSON.parse(fs.readFileSync(statePath, "utf8")) : {}; +fs.appendFileSync(${JSON.stringify(commandsPath)}, args.join(" ") + "\\n"); +const action = args.find((arg) => ["show", "stop", "reset-failed", "start", "print", "disable", "bootout", "enable", "bootstrap", "kickstart"].includes(arg)); +if (${JSON.stringify(kind)} === "systemd") { + if (action === "stop") state.parked = true; + if (action === "stop" && ${Boolean(options?.lateCommand)}) setTimeout(() => {}, 200); + if (action === "reset-failed") state.reset = true; + if (action === "start" && ${JSON.stringify(options?.systemdFault)} === "start-failed") { + state.startFailed = true; + process.stderr.write("start limit hit\\n"); + process.exitCode = 1; + } else if (action === "start") state.restored = true; + if (action === "show") { + const active = !state.parked || state.restored; + const restoredPid = ${JSON.stringify(options?.systemdFault)} === "dead-restored-pid" ? 2147483647 : ${process.pid}; + process.stdout.write([ + "Id=openclaw-gateway.service", + "LoadState=loaded", + "ActiveState=" + (active ? "active" : "inactive"), + "MainPID=" + (state.restored ? restoredPid : active ? ${parentPid} : 0), + "ExecMainStartTimestampMonotonic=" + (state.restored ? "222" : "111"), + ].join("\\n") + "\\n"); + } + } else { + if (action === "disable") state.disabled = true; + if (action === "bootout") { + state.parked = true; + state.loadedPrintsRemaining = ${options?.launchdTeardown?.loadedPrints ?? 0}; + state.pendingBootstrapFailures = ${options?.launchdTeardown?.pendingBootstrapFailures ?? 0}; + state.pendingOperationInProgress = ${options?.launchdTeardown?.pendingOperationInProgress ?? 0}; + const delay = ${options?.launchdTeardown?.bootoutDelayMs ?? 0}; + if (delay) setTimeout(() => { + state.bootoutCompleted = true; + fs.writeFileSync(statePath, JSON.stringify(state)); + }, delay); + } + if (action === "enable") state.disabled = false; + if (action === "bootstrap" || action === "kickstart") { + state.bootstrapAttempts = (state.bootstrapAttempts || 0) + 1; + if (state.pendingOperationInProgress > 0) { + state.pendingOperationInProgress -= 1; + state.operationInProgressObserved = (state.operationInProgressObserved || 0) + 1; + process.stderr.write("Bootstrap failed: 37: Operation already in progress\\n"); + process.exitCode = 37; + } else if (!state.unloaded) { + process.stderr.write("Bootstrap failed: 37: Operation already in progress\\n"); + process.exitCode = 37; + } else if (action === "bootstrap" && state.pendingBootstrapFailures > 0) { + state.pendingBootstrapFailures -= 1; + process.stderr.write("Bootstrap failed: 5: Input/output error\\n"); + process.exitCode = 5; + } else state.restored = true; + } + if (action === "print") { + let parentAlive = false; + try { process.kill(${parentPid}, 0); parentAlive = true; } catch {} + if (state.parked && !state.restored && !parentAlive) { + if (state.loadedPrintsRemaining > 0) { + state.loadedPrintsRemaining -= 1; + state.loadedPrintsObserved = (state.loadedPrintsObserved || 0) + 1; + } else { + state.unloaded = true; + process.stderr.write("Could not find service\\n"); + fs.writeFileSync(statePath, JSON.stringify(state)); + process.exit(113); + } + } + const fault = ${JSON.stringify(options?.launchdFault)}; + if (state.restored && fault === "missing-restored-pid") { + process.stdout.write("state = running\\n"); + } else { + const restoredPid = fault === "dead-restored-pid" ? 2147483647 : ${process.pid}; + const currentPid = fault === "wrong-parent" ? ${process.pid} : ${parentPid}; + process.stdout.write("state = running\\npid = " + (state.restored ? restoredPid : currentPid) + "\\n"); + } + } +} +fs.writeFileSync(statePath, JSON.stringify(state)); +`; +} + +export function createManagedServiceLaunchdClockPreload(params: { + commandTimingsPath: string; + clockEachCommandMs: number; +}): string { + return [ + 'const fs = require("node:fs");', + 'const children = require("node:child_process");', + "const actualSpawn = children.spawn;", + "const actualSetTimeout = global.setTimeout;", + "const startedAt = Date.now();", + "let elapsed = 0;", + "Date.now = () => startedAt + elapsed;", + "global.setTimeout = (callback, delay, ...args) => {", + " if (delay === 500) {", + " elapsed += delay;", + " return actualSetTimeout(callback, 0, ...args);", + " }", + " return actualSetTimeout(callback, delay, ...args);", + "};", + "children.spawn = (command, args, options) => {", + ' if (command === "launchctl") {', + " const timeoutMs = options.timeout;", + " const startedAtMs = Date.now();", + ` fs.appendFileSync(${JSON.stringify(params.commandTimingsPath)}, JSON.stringify({ action: args[0], startedAtMs, timeoutMs }) + "\\n");`, + ` elapsed += Math.min(${params.clockEachCommandMs}, timeoutMs);`, + " }", + " return actualSpawn(command, args, options);", + "};", + ].join("\n"); +} diff --git a/src/infra/update-managed-service-handoff-lifecycle.test.ts b/src/infra/update-managed-service-handoff-lifecycle.test.ts index 34539d0f2137..b7ee95225dcc 100644 --- a/src/infra/update-managed-service-handoff-lifecycle.test.ts +++ b/src/infra/update-managed-service-handoff-lifecycle.test.ts @@ -5,59 +5,73 @@ import { EventEmitter } from "node:events"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; +import { DatabaseSync } from "node:sqlite"; import { PassThrough, type Readable } from "node:stream"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { getFileLockProcessStartTime } from "../shared/pid-alive.js"; import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; import { closeOpenClawStateDatabaseForTest, openOpenClawStateDatabase, } from "../state/openclaw-state-db.js"; -import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js"; -import { - executeSqliteQuerySync, - executeSqliteQueryTakeFirstSync, - getNodeSqliteKysely, -} from "./kysely-sync.js"; +import { executeSqliteQueryTakeFirstSync, getNodeSqliteKysely } from "./kysely-sync.js"; import { SUPERVISOR_HINT_ENV_VARS } from "./supervisor-markers.js"; import { CONTROL_PLANE_UPDATE_SENTINEL_META_ENV } from "./update-control-plane-sentinel.js"; import { cleanupStaleManagedServiceUpdateHandoffs, MANAGED_SERVICE_UPDATE_HANDOFF_TEMP_PREFIX, } from "./update-managed-service-handoff-cleanup.js"; +import { + createManagedServiceLaunchdClockPreload, + createManagedServiceManagerFixtureScript, + type ManagedServiceCommandTiming, + type ManagedServiceManagerBoundaryOptions, +} from "./update-managed-service-handoff-lifecycle.test-support.js"; +import { signalMockManagedUpdateHandoffReady } from "./update-managed-service-handoff.test-support.js"; const { forceKillChildProcessTreeMock, spawnMock } = vi.hoisted(() => ({ forceKillChildProcessTreeMock: vi.fn(), spawnMock: vi.fn(), })); const FAST_WAIT_OPTS = { interval: 1 } as const; +const MOCK_INSTALL_ROOT = path.join(os.tmpdir(), `openclaw-handoff-lifecycle-${process.pid}`); function createSpawnMock(params?: { pid?: number }) { const child = Object.assign(new EventEmitter(), { - pid: params?.pid ?? 24680, + pid: params?.pid ?? process.pid, exitCode: null, signalCode: null, + stdin: new PassThrough(), stdout: new PassThrough(), unref: vi.fn(), }); return child; } -function signalHandoffReady(child: ReturnType): void { - child.stdout.write("OPENCLAW_UPDATE_HANDOFF_READY\n"); -} +const mockedHandoffLeaseCleanups = new Set<() => void>(); async function waitForHandoffReady(output: Readable | null): Promise { + return waitForHandoffResponse(output, "OPENCLAW_UPDATE_HANDOFF_READY"); +} + +async function waitForHandoffResponse(output: Readable | null, expected: string): Promise { if (!output) { throw new Error("expected managed handoff helper stdout"); } - let buffered = ""; - for await (const chunk of output) { - buffered = `${buffered}${chunk.toString()}`.slice(-1024); - if (buffered.includes("OPENCLAW_UPDATE_HANDOFF_READY\n")) { - return; - } - } - throw new Error("managed handoff helper exited before readiness"); + await new Promise((resolve, reject) => { + let buffered = ""; + const onData = (chunk: Buffer | string) => { + buffered = `${buffered}${chunk.toString()}`.slice(-1024); + if (buffered.includes(`${expected}\n`)) { + output.removeListener("data", onData); + output.removeListener("end", onEnd); + resolve(); + } + }; + const onEnd = () => reject(new Error(`managed handoff helper exited before ${expected}`)); + output.on("data", onData); + output.once("end", onEnd); + }); } vi.mock("node:child_process", async () => { @@ -81,10 +95,14 @@ type GatewayRestartSentinelDatabase = Pick { forceKillChildProcessTreeMock.mockReset(); spawnMock.mockReset(); - spawnMock.mockImplementation(() => { + spawnMock.mockImplementation((_command: string, args: string[]) => { const child = createSpawnMock(); process.nextTick(() => { - signalHandoffReady(child); + signalMockManagedUpdateHandoffReady({ + child, + paramsPath: args.at(-1) ?? "", + cleanups: mockedHandoffLeaseCleanups, + }); }); return child; }); @@ -92,6 +110,9 @@ beforeEach(() => { afterEach(async () => { vi.useRealTimers(); + for (const cleanup of mockedHandoffLeaseCleanups) { + cleanup(); + } closeOpenClawStateDatabaseForTest(); await Promise.all([...tempDirs].map((dir) => fs.rm(dir, { recursive: true, force: true }))); tempDirs.clear(); @@ -107,70 +128,6 @@ async function pathExists(filePath: string): Promise { } } -function writeRestartSentinelRow(env: NodeJS.ProcessEnv, sentinel: unknown): void { - const { db } = openOpenClawStateDatabase({ env }); - const stateDb = getNodeSqliteKysely(db); - const payload = - sentinel && typeof sentinel === "object" && (sentinel as { version?: unknown }).version === 1 - ? (sentinel as { payload?: unknown }).payload - : null; - if (!payload || typeof payload !== "object") { - throw new Error("expected versioned restart sentinel payload"); - } - const record = payload as { - kind?: unknown; - status?: unknown; - ts?: unknown; - sessionKey?: unknown; - threadId?: unknown; - deliveryContext?: { channel?: unknown; to?: unknown; accountId?: unknown }; - message?: unknown; - continuation?: unknown; - doctorHint?: unknown; - stats?: unknown; - }; - const revision = - typeof (sentinel as { revision?: unknown }).revision === "number" - ? (sentinel as { revision: number }).revision - : Date.now(); - executeSqliteQuerySync( - db, - stateDb.insertInto("gateway_restart_sentinel").values({ - sentinel_key: record.kind === "revision-floor" ? "revision-floor" : "current", - version: 1, - kind: typeof record.kind === "string" ? record.kind : "update", - status: typeof record.status === "string" ? record.status : "skipped", - ts: typeof record.ts === "number" ? record.ts : Date.now(), - session_key: typeof record.sessionKey === "string" ? record.sessionKey : null, - thread_id: typeof record.threadId === "string" ? record.threadId : null, - delivery_channel: - typeof record.deliveryContext?.channel === "string" ? record.deliveryContext.channel : null, - delivery_to: - typeof record.deliveryContext?.to === "string" ? record.deliveryContext.to : null, - delivery_account_id: - typeof record.deliveryContext?.accountId === "string" - ? record.deliveryContext.accountId - : null, - message: typeof record.message === "string" ? record.message : null, - continuation_json: record.continuation ? JSON.stringify(record.continuation) : null, - doctor_hint: typeof record.doctorHint === "string" ? record.doctorHint : null, - stats_json: record.stats ? JSON.stringify(record.stats) : null, - payload_json: JSON.stringify(payload), - updated_at_ms: revision, - }), - ); -} - -function replaceRestartSentinelRow(env: NodeJS.ProcessEnv, sentinel: unknown): void { - const { db } = openOpenClawStateDatabase({ env }); - const stateDb = getNodeSqliteKysely(db); - executeSqliteQuerySync( - db, - stateDb.deleteFrom("gateway_restart_sentinel").where("sentinel_key", "=", "current"), - ); - writeRestartSentinelRow(env, sentinel); -} - function readRestartSentinelPayload(env: NodeJS.ProcessEnv, key = "current"): unknown { const { db } = openOpenClawStateDatabase({ env }); const stateDb = getNodeSqliteKysely(db); @@ -186,248 +143,274 @@ function readRestartSentinelPayload(env: NodeJS.ProcessEnv, key = "current"): un : null; } -async function runHelperWithExistingSentinel(params: { - handoffId?: string; - metaHandoffId?: string; - prepareStateDatabase?: (env: NodeJS.ProcessEnv) => Promise | void; - sentinel?: unknown; - deepStatePath?: boolean; - commandDelayMs?: number; - whileHelperRunning?: (env: NodeJS.ProcessEnv) => Promise | void; -}) { - const { execFile } = - await vi.importActual("node:child_process"); - const { startManagedServiceUpdateHandoff } = await import("./update-managed-service-handoff.js"); - const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-handoff-helper-test-")); - tempDirs.add(tmpDir); - let stateDir = tmpDir; - while ( - params.deepStatePath && - resolveOpenClawStateSqlitePath({ OPENCLAW_STATE_DIR: stateDir }).length <= 260 - ) { - stateDir = path.join(stateDir, `segment-${"x".repeat(24)}`); - } - const env = { OPENCLAW_STATE_DIR: stateDir } as NodeJS.ProcessEnv; - - await startManagedServiceUpdateHandoff({ - root: tmpDir, - timeoutMs: 1_800_000, - restartDrainTimeoutMs: 300_000, - restartDelayMs: 500, - parentPid: process.pid, - execPath: "/usr/local/bin/node", - argv1: "/opt/openclaw/openclaw.mjs", - ...(params.handoffId ? { handoffId: params.handoffId } : {}), - env, - meta: { - ...(params.metaHandoffId ? { handoffId: params.metaHandoffId } : {}), - sessionKey: "agent:test:webchat:dm:user-123", - continuationMessage: "continue after restart", - }, - }); - - const [, args] = spawnMock.mock.calls.at(-1) as unknown as [ - string, - string[], - { env: NodeJS.ProcessEnv; detached?: boolean; cwd?: string }, - ]; - const helperScriptPath = args[0] ?? ""; - tempDirs.add(path.dirname(helperScriptPath)); - const helperParams = JSON.parse(await fs.readFile(args[1] ?? "", "utf-8")) as Record< - string, - unknown - >; - await params.prepareStateDatabase?.(env); - if (params.sentinel !== undefined) { - writeRestartSentinelRow(env, params.sentinel); - } - const helperParamsPath = path.join(tmpDir, "helper-params.json"); - const exitedParentPid = await spawnExitedPid(); - const failureScript = `setTimeout(() => process.exit(1), ${params.commandDelayMs ?? 0})`; - await fs.writeFile( - helperParamsPath, - `${JSON.stringify( - { - ...helperParams, - parentPid: exitedParentPid, - parentExitTimeoutMs: 5_000, - commandArgv: [process.execPath, "-e", failureScript], - logPath: path.join(tmpDir, "handoff.log"), - sensitivePaths: [], - }, - null, - 2, - )}\n`, - ); - - const resultPromise = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>( - (resolve) => { - execFile(process.execPath, [helperScriptPath, helperParamsPath], { cwd: tmpDir }, (err) => { - const childError = err as (NodeJS.ErrnoException & { signal?: NodeJS.Signals }) | null; - resolve({ - code: typeof childError?.code === "number" ? childError.code : 0, - signal: childError?.signal ?? null, - }); - }); - }, - ); - await params.whileHelperRunning?.(env); - const result = await resultPromise; - - return { result, env }; -} - -async function createLegacyRestartSentinelTable(env: NodeJS.ProcessEnv): Promise { - const sqlite = await import("node:sqlite"); - const stateDatabasePath = resolveOpenClawStateSqlitePath(env); - await fs.mkdir(path.dirname(stateDatabasePath), { recursive: true }); - const db = new sqlite.DatabaseSync(stateDatabasePath); - try { - db.exec(` - CREATE TABLE gateway_restart_sentinel ( - sentinel_key TEXT NOT NULL PRIMARY KEY, - version INTEGER NOT NULL, - kind TEXT NOT NULL, - status TEXT NOT NULL, - ts INTEGER NOT NULL, - session_key TEXT, - thread_id TEXT, - payload_json TEXT NOT NULL, - updated_at_ms INTEGER NOT NULL - ); - `); - } finally { - db.close(); - } -} - -async function spawnExitedPid(): Promise { +async function runManagedServiceManagerBoundary( + kind: "systemd" | "launchd", + options?: ManagedServiceManagerBoundaryOptions, +): Promise<{ + commands: string[]; + parentSignal: NodeJS.Signals | null; + state: Record; + sentinel: unknown; + commandTimings: ManagedServiceCommandTiming[]; +}> { const { spawn } = await vi.importActual("node:child_process"); - return await new Promise((resolve) => { - const child = spawn(process.execPath, ["-e", ""], { stdio: "ignore" }); - const pid = child.pid ?? 0; - child.once("exit", () => resolve(pid)); - }); -} - -async function runHelperWithCommand(params: { - commandArgv: string[]; - parentPid?: number; - parentExitTimeoutMs?: number | null; - serviceRecovery?: Record; - pathPrepend?: string; -}): Promise<{ - ready: Promise; - completion: Promise<{ code: number }>; - logPath: string; -}> { - const { execFile } = - await vi.importActual("node:child_process"); const { startManagedServiceUpdateHandoff } = await import("./update-managed-service-handoff.js"); - const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-handoff-recovery-test-")); - tempDirs.add(tmpDir); - - await startManagedServiceUpdateHandoff({ - root: tmpDir, - timeoutMs: 1_800_000, - restartDrainTimeoutMs: 300_000, - restartDelayMs: 0, - parentPid: process.pid, - execPath: "/usr/local/bin/node", - argv1: "/opt/openclaw/openclaw.mjs", - env: { OPENCLAW_STATE_DIR: tmpDir }, - meta: { sessionKey: "agent:test:webchat:dm:user-123" }, - }); - - const [, args] = spawnMock.mock.calls.at(-1) as unknown as [string, string[]]; - const helperScriptPath = args[0] ?? ""; - tempDirs.add(path.dirname(helperScriptPath)); - const baseParams = JSON.parse(await fs.readFile(args[1] ?? "", "utf-8")) as Record< - string, - unknown - >; - - const helperParamsPath = path.join(tmpDir, "helper-params.json"); - const logPath = path.join(tmpDir, "handoff.log"); - await fs.writeFile( - helperParamsPath, - `${JSON.stringify( - { - ...baseParams, - parentPid: params.parentPid ?? (await spawnExitedPid()), - parentExitTimeoutMs: - params.parentExitTimeoutMs === undefined ? 5000 : params.parentExitTimeoutMs, - cwd: tmpDir, - commandArgv: params.commandArgv, - logPath, - sensitivePaths: [], - ...(params.serviceRecovery ? { serviceRecovery: params.serviceRecovery } : {}), - }, - null, - 2, - )}\n`, + const root = await fs.realpath( + await fs.mkdtemp(path.join(os.tmpdir(), `openclaw-${kind}-manager-boundary-`)), ); - - const childEnv = { + tempDirs.add(root); + const commandsPath = path.join(root, "manager-commands.log"); + const statePath = path.join(root, "manager-state.json"); + const updaterPath = path.join(root, "updater-ran"); + const commandTimingsPath = path.join(root, "manager-command-timings.jsonl"); + const parent = spawn(process.execPath, ["-e", "process.stdin.resume()"], { + stdio: ["pipe", "ignore", "ignore"], + }); + const parentPid = parent.pid; + const parentStartIdentity = parentPid ? getFileLockProcessStartTime(parentPid) : null; + if (!parentPid || parentStartIdentity === null) { + parent.kill("SIGKILL"); + throw new Error("expected the managed Gateway parent to have a stable process identity"); + } + const recovery = + kind === "systemd" + ? { kind, unit: "openclaw-gateway.service" } + : { + kind, + uid: 501, + label: "ai.openclaw.gateway", + plistPath: path.join(root, "ai.openclaw.gateway.plist"), + }; + await fs.writeFile( + path.join(root, kind === "systemd" ? "systemctl" : "launchctl"), + createManagedServiceManagerFixtureScript({ + kind, + parentPid, + statePath, + commandsPath, + options, + }), + { + mode: 0o755, + }, + ); + const env = { ...process.env, - ...(params.pathPrepend - ? { PATH: `${params.pathPrepend}${path.delimiter}${process.env.PATH ?? ""}` } - : {}), + OPENCLAW_STATE_DIR: root, + PATH: `${root}${path.delimiter}${process.env.PATH ?? ""}`, }; - let child!: ReturnType; - const completion = new Promise<{ code: number }>((resolve) => { - child = execFile( - process.execPath, - [helperScriptPath, helperParamsPath], - { env: childEnv }, - (err) => { - const childError = err as NodeJS.ErrnoException | null; - resolve({ code: typeof childError?.code === "number" ? childError.code : 0 }); - }, + let helper: import("node:child_process").ChildProcess | undefined; + try { + await startManagedServiceUpdateHandoff({ + root, + restartDrainTimeoutMs: 300_000, + parentPid, + execPath: process.execPath, + argv1: process.argv[1], + handoffId: `${kind}-boundary`, + env, + meta: { handoffId: `${kind}-boundary` }, + }); + const [, generatedArgs] = spawnMock.mock.calls.at(-1) as [string, string[]]; + const scriptPath = generatedArgs[0]; + const generatedParamsPath = generatedArgs[1]; + if (!scriptPath || !generatedParamsPath) { + throw new Error("expected generated managed handoff script and parameters"); + } + const generated = JSON.parse(await fs.readFile(generatedParamsPath, "utf8")) as Record< + string, + unknown + >; + const mockedChild = spawnMock.mock.results.at(-1)?.value as ReturnType; + mockedChild.emit("exit", 0, null); + tempDirs.add(path.dirname(scriptPath)); + const paramsPath = path.join(root, "manager-helper.json"); + await fs.writeFile( + paramsPath, + JSON.stringify({ + ...generated, + parentPid, + parentStartIdentity: String(parentStartIdentity), + ...(options?.parentExitTimeoutMs === undefined + ? {} + : { + parentExitDeadlineAt: Date.now() + options.parentExitTimeoutMs, + parentExitTimeoutMs: options.parentExitTimeoutMs, + }), + ...(options?.overdueCommit ? { parentExitDeadlineAt: Date.now() - 1 } : {}), + serviceRecovery: recovery, + commandArgv: [ + process.execPath, + "-e", + [ + `const fs = require("node:fs");`, + ...(kind === "launchd" + ? [ + `const state = JSON.parse(fs.readFileSync(${JSON.stringify(statePath)}, "utf8"));`, + `if (!state.unloaded) process.exit(19);`, + `state.updaterObservedUnloaded = true;`, + `fs.writeFileSync(${JSON.stringify(statePath)}, JSON.stringify(state));`, + ] + : []), + `fs.writeFileSync(${JSON.stringify(updaterPath)}, "ran");process.exit(7);`, + ].join(""), + ], + sensitivePaths: [], + }), ); - }); - return { ready: waitForHandoffReady(child.stdout), completion, logPath }; -} + let helperEnv: NodeJS.ProcessEnv = env; + if (options?.launchdTeardown?.clockEachCommandMs) { + const preloadPath = path.join(root, "launchd-clock-preload.cjs"); + await fs.writeFile( + preloadPath, + createManagedServiceLaunchdClockPreload({ + commandTimingsPath, + clockEachCommandMs: options.launchdTeardown.clockEachCommandMs, + }), + ); + helperEnv = { ...env, NODE_OPTIONS: `--require ${preloadPath}` }; + } + const runningHelper = spawn(process.execPath, [scriptPath, paramsPath], { + env: helperEnv, + stdio: ["pipe", "pipe", "pipe"], + }); + helper = runningHelper; + let stdout = ""; + runningHelper.stdout?.on("data", (chunk) => { + stdout += chunk.toString(); + }); + let stderr = ""; + runningHelper.stderr?.on("data", (chunk) => { + stderr += chunk.toString(); + }); + const completion = new Promise((resolve, reject) => { + runningHelper.once("error", reject); + runningHelper.once("close", resolve); + }); + await waitForHandoffReady(runningHelper.stdout); -async function writeFakeSystemctl(): Promise<{ binDir: string; recordPath: string }> { - const binDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-recovery-bin-")); - tempDirs.add(binDir); - const recordPath = path.join(binDir, "systemctl-calls.log"); - await fs.writeFile( - path.join(binDir, "systemctl"), - `#!/bin/sh\necho "$@" >> '${recordPath}'\nexit 0\n`, - { mode: 0o755 }, - ); - return { binDir, recordPath }; -} - -async function writeFakeLaunchctl(): Promise<{ binDir: string; recordPath: string }> { - const binDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-launchctl-bin-")); - tempDirs.add(binDir); - const recordPath = path.join(binDir, "launchctl-calls.log"); - const countPath = path.join(binDir, "launchctl-kickstart-count"); - await fs.writeFile( - path.join(binDir, "launchctl"), - `#!/bin/sh -echo "$@" >> '${recordPath}' -if [ "$1" = "kickstart" ]; then - count=0 - if [ -f '${countPath}' ]; then - count=$(cat '${countPath}') - fi - count=$((count + 1)) - echo "$count" > '${countPath}' - [ "$count" -gt 1 ] - exit $? -fi -[ "$1" = "enable" ] && exit 0 -[ "$1" = "bootstrap" ] && exit 1 -exit 1 -`, - { mode: 0o755 }, - ); - return { binDir, recordPath }; + const databasePath = String(generated.updateLeaseDatabasePath); + const owner = String(generated.updateLeaseOwner); + const readLease = (): Record | null => { + const db = new DatabaseSync(databasePath, { readOnly: true }); + try { + const row = db + .prepare( + "SELECT payload_json FROM managed_update_handoffs WHERE install_root = ? AND owner = ?", + ) + .get(root, owner) as { payload_json: string } | undefined; + return row ? (JSON.parse(row.payload_json) as Record) : null; + } finally { + db.close(); + } + }; + expect(readLease()).toEqual({ + version: 1, + pid: runningHelper.pid, + startIdentity: expect.any(String), + }); + await expect(pathExists(commandsPath)).resolves.toBe(false); + if (options?.parentExitTimeoutMs !== undefined) { + if (options.lateCommand) { + await vi.waitFor( + async () => { + await expect(fs.readFile(commandsPath, "utf8")).resolves.toContain( + "--no-block stop openclaw-gateway.service", + ); + }, + { interval: 5, timeout: options.parentExitTimeoutMs + 3_000 }, + ); + const response = waitForHandoffResponse( + runningHelper.stdout, + options.lateCommand === "park" ? "parked" : "restore-after-exit", + ); + runningHelper.stdin?.write(`${options.lateCommand}\n`); + await response; + } + const timeout = options.parentExitTimeoutMs + (options.launchdTeardown ? 8_000 : 3_000); + let timer: ReturnType | undefined; + try { + expect( + await Promise.race([ + completion, + new Promise((_resolve, reject) => { + timer = setTimeout( + () => reject(new Error("managed helper did not restore the stalled parent")), + timeout, + ); + }), + ]), + stderr, + ).toBe(0); + } finally { + clearTimeout(timer); + } + expect(parent.signalCode).toBe("SIGKILL"); + expect(stdout).not.toContain("committed\n"); + await expect(pathExists(updaterPath)).resolves.toBe(false); + } else if (options?.launchdFault === "wrong-parent") { + const cancelled = waitForHandoffResponse(runningHelper.stdout, "cancelled"); + runningHelper.stdin?.write("park\n"); + await cancelled; + expect(await completion, stderr).toBe(0); + expect(parent.exitCode).toBeNull(); + expect(parent.signalCode).toBeNull(); + await expect(pathExists(updaterPath)).resolves.toBe(false); + } else { + const parked = waitForHandoffResponse(runningHelper.stdout, "parked"); + runningHelper.stdin?.write("park\n"); + await parked; + expect(parent.exitCode).toBeNull(); + await expect(pathExists(updaterPath)).resolves.toBe(false); + if (options?.cancelAfterPark) { + const restoring = waitForHandoffResponse(runningHelper.stdout, "restore-after-exit"); + runningHelper.stdin?.write("cancel\n"); + await restoring; + expect(stdout).not.toContain("committed\n"); + parent.stdin?.end(); + expect(await completion, stderr).toBe(0); + await expect(pathExists(updaterPath)).resolves.toBe(false); + } else if (options?.overdueCommit) { + runningHelper.stdin?.write("commit\n"); + await vi.waitFor( + () => expect(stdout).toMatch(/(?:committed|restore-after-exit)\n/u), + FAST_WAIT_OPTS, + ); + expect(stdout).not.toContain("committed\n"); + expect(stdout).toContain("restore-after-exit\n"); + parent.stdin?.end(); + expect(await completion, stderr).toBe(0); + await expect(pathExists(updaterPath)).resolves.toBe(false); + } else { + const committed = waitForHandoffResponse(runningHelper.stdout, "committed"); + runningHelper.stdin?.write("commit\n"); + await committed; + parent.stdin?.end(); + const code = await completion; + const helperLog = await fs.readFile(String(generated.logPath), "utf8").catch(() => ""); + expect(code, `${stderr}\n${helperLog}`).toBe(7); + await expect(fs.readFile(updaterPath, "utf8")).resolves.toBe("ran"); + } + } + expect(readLease()).toBeNull(); + return { + commands: (await fs.readFile(commandsPath, "utf8")).trim().split("\n"), + parentSignal: parent.signalCode, + state: JSON.parse(await fs.readFile(statePath, "utf8")) as Record, + sentinel: readRestartSentinelPayload({ OPENCLAW_STATE_DIR: root }), + commandTimings: (await fs.readFile(commandTimingsPath, "utf8").catch(() => "")) + .trim() + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line) as ManagedServiceCommandTiming), + }; + } finally { + parent.stdin?.end(); + if (helper && helper.exitCode === null && helper.signalCode === null) { + helper.kill("SIGKILL"); + } + } } describe("managed service update handoff", () => { @@ -440,9 +423,9 @@ describe("managed service update handoff", () => { await import("./update-managed-service-handoff.js"); const resultPromise = startManagedServiceUpdateHandoff({ - root: "/tmp/openclaw", + root: MOCK_INSTALL_ROOT, restartDrainTimeoutMs: 300_000, - parentPid: 12345, + parentPid: process.pid, execPath: "/definitely/missing/openclaw-node", argv1: "/opt/openclaw/openclaw.mjs", meta: { sessionKey: "agent:test:webchat:dm:user-123" }, @@ -472,9 +455,9 @@ describe("managed service update handoff", () => { await import("./update-managed-service-handoff.js"); const resultPromise = startManagedServiceUpdateHandoff({ - root: "/tmp/openclaw", + root: MOCK_INSTALL_ROOT, restartDrainTimeoutMs: 300_000, - parentPid: 12345, + parentPid: process.pid, execPath: "/usr/local/bin/node", argv1: "/opt/openclaw/openclaw.mjs", supervisor: "systemd", @@ -506,9 +489,9 @@ describe("managed service update handoff", () => { await import("./update-managed-service-handoff.js"); const resultPromise = startManagedServiceUpdateHandoff({ - root: "/tmp/openclaw", - restartDrainTimeoutMs: undefined, - parentPid: 12345, + root: MOCK_INSTALL_ROOT, + restartDrainTimeoutMs: 300_000, + parentPid: process.pid, execPath: "/usr/local/bin/node", argv1: "/opt/openclaw/openclaw.mjs", meta: {}, @@ -545,11 +528,11 @@ describe("managed service update handoff", () => { ) as NodeJS.ProcessEnv; const result = await startManagedServiceUpdateHandoff({ - root: "/tmp/openclaw", + root: MOCK_INSTALL_ROOT, timeoutMs: 1_800_000, restartDrainTimeoutMs: 300_000, restartDelayMs: 500, - parentPid: 12345, + parentPid: process.pid, execPath: "/usr/local/bin/node", argv1: "/opt/openclaw/openclaw.mjs", env: { @@ -596,11 +579,11 @@ describe("managed service update handoff", () => { await fs.writeFile(systemdRunPath, "#!/bin/sh\nexit 0\n", { mode: 0o755 }); const result = await startManagedServiceUpdateHandoff({ - root: "/tmp/openclaw", + root: MOCK_INSTALL_ROOT, timeoutMs: 1_800_000, restartDrainTimeoutMs: 300_000, restartDelayMs: 500, - parentPid: 12345, + parentPid: process.pid, execPath: "/usr/local/bin/node", argv1: "/opt/openclaw/openclaw.mjs", handoffId: "handoff-123", @@ -667,61 +650,322 @@ describe("managed service update handoff", () => { expect(options.env.OPENCLAW_UPDATE_RUN_HANDOFF).toBe("1"); }); - itUnix( - "starts the managed gateway service when the update command fails after handoff", - async () => { - const { binDir, recordPath } = await writeFakeSystemctl(); - const { completion } = await runHelperWithCommand({ - commandArgv: [process.execPath, "-e", "process.exit(7)"], - serviceRecovery: { kind: "systemd", unit: "openclaw-gateway.service" }, - pathPrepend: binDir, - }); - const result = await completion; + itUnix("parks and restores the exact user-systemd service from its detached helper", async () => { + const { commands, sentinel, state } = await runManagedServiceManagerBoundary("systemd"); + const verbs = commands.map((command) => + command.split(" ").find((part) => ["show", "stop", "reset-failed", "start"].includes(part)), + ); - expect(result.code).toBe(7); - await expect(fs.readFile(recordPath, "utf-8")).resolves.toBe( - "--user start openclaw-gateway.service\n", + expect(verbs).toEqual(["show", "stop", "show", "reset-failed", "start", "show"]); + expect(commands.every((command) => command.startsWith("--user "))).toBe(true); + expect(commands[0]).toContain( + "--property=Id,LoadState,ActiveState,MainPID,ExecMainStartTimestampMonotonic", + ); + expect(commands[1]).toContain("--no-block stop openclaw-gateway.service"); + expect(state).toMatchObject({ parked: true, reset: true, restored: true }); + expect(sentinel).toMatchObject({ + payload: { + status: "error", + stats: { + reason: "managed-service-handoff-failed", + steps: expect.arrayContaining([ + expect.objectContaining({ name: "service-restore", log: { exitCode: 0 } }), + ]), + }, + }, + }); + }); + + itUnix.each([ + ["without a late control command", undefined], + ["with an idempotent late park", "park"], + ["with a fenced late commit", "commit"], + ] as const)( + "restores a stalled managed parent when the deadline expires before park %s", + async (_label, lateCommand) => { + const { commands, parentSignal, sentinel, state } = await runManagedServiceManagerBoundary( + "systemd", + { parentExitTimeoutMs: 500, ...(lateCommand ? { lateCommand } : {}) }, ); + + expect(parentSignal).toBe("SIGKILL"); + expect(commands).toEqual( + expect.arrayContaining([ + expect.stringContaining("--no-block stop openclaw-gateway.service"), + expect.stringContaining("start openclaw-gateway.service"), + ]), + ); + expect(state).toMatchObject({ parked: true, reset: true, restored: true }); + expect(sentinel).toMatchObject({ + payload: { + status: "error", + stats: { + reason: "managed-service-handoff-cancelled", + steps: expect.arrayContaining([ + expect.objectContaining({ name: "service-restore", log: { exitCode: 0 } }), + ]), + }, + }, + }); }, ); - it("leaves the gateway service alone when the update command succeeds", async () => { - const { binDir, recordPath } = await writeFakeSystemctl(); - const { completion } = await runHelperWithCommand({ - commandArgv: [process.execPath, "-e", "process.exit(0)"], - serviceRecovery: { kind: "systemd", unit: "openclaw-gateway.service" }, - pathPrepend: binDir, - }); - const result = await completion; + itUnix("rejects an overdue commit before its delayed deadline callback executes", async () => { + const { commands, parentSignal, sentinel, state } = await runManagedServiceManagerBoundary( + "systemd", + { overdueCommit: true }, + ); - expect(result.code).toBe(0); - await expect(pathExists(recordPath)).resolves.toBe(false); + expect(parentSignal).toBeNull(); + expect(commands.filter((command) => command.includes("reset-failed"))).toHaveLength(1); + expect( + commands.filter((command) => command.includes("start openclaw-gateway.service")), + ).toHaveLength(1); + expect(state).toMatchObject({ parked: true, reset: true, restored: true }); + expect(sentinel).toMatchObject({ + payload: { + status: "error", + stats: { + reason: "managed-service-handoff-cancelled", + steps: expect.arrayContaining([ + expect.objectContaining({ name: "service-restore", log: { exitCode: 0 } }), + ]), + }, + }, + }); }); - itUnix("retries launchd start when bootstrap reports an already-loaded label", async () => { - const { binDir, recordPath } = await writeFakeLaunchctl(); - const { completion } = await runHelperWithCommand({ - commandArgv: [process.execPath, "-e", "process.exit(7)"], - serviceRecovery: { - kind: "launchd", - uid: 501, - label: "com.example.openclaw", - plistPath: "/Users/test/Library/LaunchAgents/com.example.openclaw.plist", - }, - pathPrepend: binDir, - }); - const result = await completion; + itUnix.each([ + ["cannot restart", "start-failed", { startFailed: true }], + ["reports a dead replacement PID", "dead-restored-pid", { restored: true }], + ] as const)( + "records one durable failure when the canonical systemd service %s", + async (_label, systemdFault, expectedState) => { + const { commands, parentSignal, sentinel, state } = await runManagedServiceManagerBoundary( + "systemd", + { parentExitTimeoutMs: 500, systemdFault }, + ); - expect(result.code).toBe(7); - await expect(fs.readFile(recordPath, "utf-8")).resolves.toBe( - [ - "kickstart gui/501/com.example.openclaw", - "enable gui/501/com.example.openclaw", - "bootstrap gui/501 /Users/test/Library/LaunchAgents/com.example.openclaw.plist", - "kickstart gui/501/com.example.openclaw", - "", - ].join("\n"), + expect(parentSignal).toBe("SIGKILL"); + expect(commands.filter((command) => command.includes("reset-failed"))).toHaveLength(1); + expect( + commands.filter((command) => command.includes("start openclaw-gateway.service")), + ).toHaveLength(1); + expect(state).toMatchObject({ parked: true, reset: true, ...expectedState }); + expect(sentinel).toMatchObject({ + payload: { + status: "error", + stats: { + reason: "managed-service-handoff-restore-failed", + steps: expect.arrayContaining([ + expect.objectContaining({ name: "service-restore", log: { exitCode: 1 } }), + ]), + }, + }, + }); + }, + ); + + itUnix("parks and restores the exact launchd service from its detached helper", async () => { + const { commands, sentinel, state } = await runManagedServiceManagerBoundary("launchd"); + const verbs = commands.map((command) => command.split(" ")[0]); + const disable = verbs.indexOf("disable"); + const bootout = verbs.indexOf("bootout"); + const enable = verbs.indexOf("enable"); + const restart = verbs.findIndex((verb) => verb === "bootstrap" || verb === "kickstart"); + + expect(disable).toBeGreaterThan(0); + expect(commands[0]).toBe("print gui/501/ai.openclaw.gateway"); + expect(bootout).toBeGreaterThan(disable); + expect(enable).toBeGreaterThan(bootout); + expect(verbs.slice(bootout + 1, enable)).toContain("print"); + expect(restart).toBeGreaterThan(enable); + expect(verbs.lastIndexOf("print")).toBeGreaterThan(restart); + expect(commands[disable]).toBe("disable gui/501/ai.openclaw.gateway"); + expect(commands[bootout]).toBe("bootout gui/501/ai.openclaw.gateway"); + expect(commands.every((command) => !command.includes("kickstart -k"))).toBe(true); + expect(state).toMatchObject({ disabled: false, parked: true, restored: true }); + expect(sentinel).toMatchObject({ + payload: { + status: "error", + stats: { + reason: "managed-service-handoff-failed", + steps: expect.arrayContaining([ + expect.objectContaining({ name: "service-restore", log: { exitCode: 0 } }), + ]), + }, + }, + }); + }); + + itUnix.each([ + { + label: "keeps bootout alive beyond the short command timeout before authorizing the updater", + options: { launchdTeardown: { bootoutDelayMs: 5_250, loadedPrints: 2 } }, + updaterRan: true, + }, + { + label: "restores a cancelled handoff after loaded teardown and transient bootstrap EIO", + options: { + cancelAfterPark: true, + launchdTeardown: { loadedPrints: 2, pendingBootstrapFailures: 2 }, + }, + updaterRan: false, + }, + { + label: "restores an expired handoff after loaded teardown and transient bootstrap EIO", + options: { + parentExitTimeoutMs: 500, + launchdTeardown: { loadedPrints: 2, pendingBootstrapFailures: 2 }, + }, + updaterRan: false, + }, + { + label: + "retries canonical bootstrap when an operation-in-progress service disappears during restoration", + options: { + cancelAfterPark: true, + launchdTeardown: { loadedPrints: 2, pendingOperationInProgress: 1 }, + }, + updaterRan: false, + }, + ])( + "$label", + async ({ options, updaterRan }) => { + const { commands, parentSignal, sentinel, state } = await runManagedServiceManagerBoundary( + "launchd", + options, + ); + const verbs = commands.map((command) => command.split(" ")[0]); + + expect(state).toMatchObject({ + disabled: false, + parked: true, + unloaded: true, + restored: true, + loadedPrintsObserved: 2, + ...(updaterRan + ? { bootoutCompleted: true, updaterObservedUnloaded: true } + : { + pendingBootstrapFailures: 0, + bootstrapAttempts: "pendingOperationInProgress" in options.launchdTeardown ? 2 : 3, + ...("pendingOperationInProgress" in options.launchdTeardown + ? { operationInProgressObserved: 1, pendingOperationInProgress: 0 } + : {}), + }), + }); + expect(verbs.filter((verb) => verb === "print").length).toBeGreaterThanOrEqual(4); + expect(parentSignal).toBe("parentExitTimeoutMs" in options ? "SIGKILL" : null); + expect(sentinel).toMatchObject({ + payload: { + status: "error", + stats: { + reason: updaterRan + ? "managed-service-handoff-failed" + : "managed-service-handoff-cancelled", + steps: expect.arrayContaining([ + expect.objectContaining({ name: "service-restore", log: { exitCode: 0 } }), + ]), + }, + }, + }); + }, + 20_000, + ); + + itUnix( + "never starts launchd bootstrap after its absolute restoration deadline or grants a command excess time", + async () => { + const { commandTimings, commands, sentinel, state } = await runManagedServiceManagerBoundary( + "launchd", + { + cancelAfterPark: true, + launchdTeardown: { clockEachCommandMs: 5_000, loadedPrints: 4 }, + }, + ); + const restoreIndex = commandTimings.findIndex(({ action }) => action === "enable"); + expect(restoreIndex).toBeGreaterThan(0); + const restoration = commandTimings.slice(restoreIndex); + const restoreStartedAtMs = restoration[0]?.startedAtMs ?? 0; + + expect(restoration.map(({ action }) => action)).toEqual([ + "enable", + "print", + "print", + "print", + "print", + "print", + ]); + expect(commands.some((command) => command.startsWith("bootstrap "))).toBe(false); + for (const { startedAtMs, timeoutMs } of restoration) { + const elapsedMs = startedAtMs - restoreStartedAtMs; + expect(elapsedMs).toBeLessThan(30_000); + expect(timeoutMs).toBeLessThanOrEqual(5_000); + expect(elapsedMs + timeoutMs).toBeLessThanOrEqual(30_000); + } + expect(restoration.at(-1)?.timeoutMs).toBeLessThan(5_000); + expect(state).toMatchObject({ disabled: false, parked: true, unloaded: true }); + expect(state.restored).toBeUndefined(); + expect(sentinel).toMatchObject({ + payload: { + status: "error", + stats: { + reason: "managed-service-handoff-restore-failed", + steps: expect.arrayContaining([ + expect.objectContaining({ name: "service-restore", log: { exitCode: 1 } }), + ]), + }, + }, + }); + }, + 15_000, + ); + + itUnix( + "rejects a launchd target owned by a different parent without native mutation", + async () => { + const { commands, sentinel, state } = await runManagedServiceManagerBoundary("launchd", { + launchdFault: "wrong-parent", + }); + + expect(commands).toEqual(["print gui/501/ai.openclaw.gateway"]); + expect(state).toEqual({}); + expect(sentinel).toMatchObject({ + payload: { + status: "error", + stats: { reason: "managed-service-handoff-cancelled" }, + }, + }); + }, + ); + + itUnix.each([ + ["a missing PID", "missing-restored-pid"], + ["a dead PID", "dead-restored-pid"], + ] as const)("rejects launchd restoration reporting running with %s", async (_label, fault) => { + const { commands, sentinel, state } = await runManagedServiceManagerBoundary("launchd", { + launchdFault: fault, + }); + + expect(commands).toEqual( + expect.arrayContaining([ + "disable gui/501/ai.openclaw.gateway", + "bootout gui/501/ai.openclaw.gateway", + "enable gui/501/ai.openclaw.gateway", + ]), ); + expect(state).toMatchObject({ disabled: false, parked: true, restored: true }); + expect(sentinel).toMatchObject({ + payload: { + status: "error", + stats: { + reason: "managed-service-handoff-restore-failed", + steps: expect.arrayContaining([ + expect.objectContaining({ name: "service-restore", log: { exitCode: 1 } }), + ]), + }, + }, + }); }); it("passes a gateway service recovery descriptor for each supervisor", async () => { @@ -735,7 +979,12 @@ describe("managed service update handoff", () => { kind: "launchd", uid: typeof process.getuid === "function" ? process.getuid() : 501, label: "test.gateway", - plistPath: path.normalize("/Users/test/Library/LaunchAgents/test.gateway.plist"), + plistPath: path.posix.join( + "/Users/test", + "Library", + "LaunchAgents", + "test.gateway.plist", + ), }, }, { @@ -747,11 +996,11 @@ describe("managed service update handoff", () => { for (const testCase of cases) { const result = await startManagedServiceUpdateHandoff({ - root: "/tmp/openclaw", + root: MOCK_INSTALL_ROOT, timeoutMs: 1_800_000, restartDrainTimeoutMs: 300_000, restartDelayMs: 500, - parentPid: 12345, + parentPid: process.pid, execPath: "/usr/local/bin/node", argv1: "/opt/openclaw/openclaw.mjs", supervisor: testCase.supervisor, @@ -772,215 +1021,6 @@ describe("managed service update handoff", () => { } }); - it("writes a fallback update failure when no restart sentinel row exists", async () => { - const { result, env } = await runHelperWithExistingSentinel({ - handoffId: "handoff-123", - metaHandoffId: "handoff-123", - }); - - expect(result).toEqual({ code: 1, signal: null }); - expect(readRestartSentinelPayload(env)).toMatchObject({ - version: 1, - payload: { - kind: "update", - status: "error", - sessionKey: "agent:test:webchat:dm:user-123", - stats: { - handoffId: "handoff-123", - reason: "managed-service-handoff-failed", - }, - }, - }); - if (process.platform !== "win32") { - const mode = (await fs.stat(resolveOpenClawStateSqlitePath(env))).mode & 0o777; - expect(mode).toBe(0o600); - } - }); - - it.runIf(process.platform === "win32")( - "writes fallback state through the detached helper beyond MAX_PATH", - async () => { - const { result, env } = await runHelperWithExistingSentinel({ - deepStatePath: true, - handoffId: "handoff-windows-long-path", - metaHandoffId: "handoff-windows-long-path", - }); - const statePath = resolveOpenClawStateSqlitePath(env); - expect(statePath.startsWith("\\\\?\\")).toBe(false); - expect(statePath.length).toBeGreaterThan(260); - expect(result).toEqual({ code: 1, signal: null }); - expect(readRestartSentinelPayload(env)).toMatchObject({ - payload: { status: "error" }, - }); - }, - ); - - it("waits for a concurrent state writer before persisting the fallback failure", async () => { - let lockReleased: Promise | undefined; - const { result, env } = await runHelperWithExistingSentinel({ - handoffId: "handoff-locked", - metaHandoffId: "handoff-locked", - prepareStateDatabase: async (stateEnv) => { - openOpenClawStateDatabase({ env: stateEnv }); - closeOpenClawStateDatabaseForTest(); - const sqlite = await import("node:sqlite"); - const lock = new sqlite.DatabaseSync(resolveOpenClawStateSqlitePath(stateEnv)); - lock.exec("BEGIN IMMEDIATE;"); - lockReleased = new Promise((resolve) => { - setTimeout(() => { - lock.exec("COMMIT;"); - lock.close(); - resolve(); - }, 200); - }); - }, - }); - await lockReleased; - - expect(result).toEqual({ code: 1, signal: null }); - expect(readRestartSentinelPayload(env)).toMatchObject({ - version: 1, - payload: { - status: "error", - stats: { - handoffId: "handoff-locked", - reason: "managed-service-handoff-failed", - }, - }, - }); - }); - - it("repairs legacy restart sentinel columns before writing fallback failures", async () => { - const { result, env } = await runHelperWithExistingSentinel({ - handoffId: "handoff-123", - metaHandoffId: "handoff-123", - prepareStateDatabase: createLegacyRestartSentinelTable, - }); - - expect(result).toEqual({ code: 1, signal: null }); - expect(readRestartSentinelPayload(env)).toMatchObject({ - version: 1, - payload: { - kind: "update", - status: "error", - stats: { - reason: "managed-service-handoff-failed", - }, - }, - }); - }); - - it("does not overwrite a restart sentinel owned by another startup task", async () => { - const unrelatedSentinel = { - version: 1, - payload: { - kind: "config", - status: "skipped", - message: "preserve this restart task", - stats: { reason: "config-restart-pending" }, - }, - }; - const { result, env } = await runHelperWithExistingSentinel({ - sentinel: unrelatedSentinel, - }); - - expect(result).toEqual({ code: 1, signal: null }); - expect(readRestartSentinelPayload(env)).toMatchObject(unrelatedSentinel); - }); - - it("does not overwrite a newer pending update handoff sentinel", async () => { - const newerSentinel = { - version: 1, - payload: { - kind: "update", - status: "skipped", - message: "new handoff still pending", - stats: { - mode: "npm", - handoffId: "newer-handoff", - reason: "managed-service-handoff-started", - steps: [], - durationMs: 0, - }, - }, - }; - const { result, env } = await runHelperWithExistingSentinel({ - handoffId: "old-handoff", - metaHandoffId: "old-handoff", - sentinel: newerSentinel, - }); - - expect(result).toEqual({ code: 1, signal: null }); - expect(readRestartSentinelPayload(env)).toMatchObject(newerSentinel); - }); - - it("preserves a newer sentinel written while the detached helper is active", async () => { - const oldSentinel = { - version: 1, - revision: 100, - payload: { - kind: "update", - status: "skipped", - ts: 100, - stats: { - handoffId: "old-handoff", - reason: "managed-service-handoff-started", - }, - }, - }; - const newerSentinel = { - version: 1, - revision: 200, - payload: { - kind: "restart", - status: "ok", - ts: 200, - }, - }; - const { env } = await runHelperWithExistingSentinel({ - handoffId: "old-handoff", - metaHandoffId: "old-handoff", - sentinel: oldSentinel, - commandDelayMs: 200, - whileHelperRunning: async (stateEnv) => { - await new Promise((resolve) => { - setTimeout(resolve, 50); - }); - replaceRestartSentinelRow(stateEnv, newerSentinel); - }, - }); - - expect(readRestartSentinelPayload(env)).toMatchObject({ - payload: newerSentinel.payload, - revision: 200, - }); - }); - - it("advances the durable revision floor even when it is ahead of the clock", async () => { - const futureRevision = Date.now() + 60_000; - const { env } = await runHelperWithExistingSentinel({ - handoffId: "handoff-future-revision", - metaHandoffId: "handoff-future-revision", - sentinel: { - version: 1, - revision: futureRevision, - payload: { - kind: "revision-floor", - status: "skipped", - ts: 123, - stats: { - handoffId: "handoff-future-revision", - reason: "managed-service-handoff-started", - }, - }, - }, - }); - - expect(readRestartSentinelPayload(env, "revision-floor")).toMatchObject({ - revision: futureRevision + 1, - }); - }); - it("sweeps stale handoff temp directories while keeping fresh handoff logs", async () => { const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-handoff-cleanup-test-")); tempDirs.add(tmpDir); @@ -1006,76 +1046,4 @@ describe("managed service update handoff", () => { await expect(pathExists(freshDir)).resolves.toBe(true); await expect(pathExists(unrelatedDir)).resolves.toBe(true); }); - - it.each([ - ["the configured restart drain and shutdown reserve (#99666)", 60_000, 2_000, 92_000], - ["indefinitely when restart draining has no deadline", undefined, 0, null], - ])("waits %s", async (_name, restartDrainTimeoutMs, restartDelayMs, expectedTimeoutMs) => { - const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-handoff-timeout-test-")); - tempDirs.add(tmpDir); - const { startManagedServiceUpdateHandoff } = - await import("./update-managed-service-handoff.js"); - - await startManagedServiceUpdateHandoff({ - root: tmpDir, - restartDrainTimeoutMs, - restartDelayMs, - parentPid: process.pid, - execPath: "/usr/local/bin/node", - argv1: "/opt/openclaw/openclaw.mjs", - env: {}, - meta: { sessionKey: "agent:test:webchat:dm:user-123" }, - }); - - const [, args] = spawnMock.mock.calls.at(-1) as unknown as [string, string[]]; - const helperParams = JSON.parse(await fs.readFile(args[1] ?? "", "utf-8")) as { - parentExitTimeoutMs?: unknown; - }; - expect(helperParams.parentExitTimeoutMs).toBe(expectedTimeoutMs); - }); - - it.each([ - ["past the expected parent-exit deadline", 50, true], - ["through an indefinite parent wait", null, false], - ])("runs the update only after the parent exits %s", async (_name, timeoutMs, expectLateLog) => { - const { spawn } = - await vi.importActual("node:child_process"); - const markerDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-handoff-marker-test-")); - tempDirs.add(markerDir); - const markerPath = path.join(markerDir, "update-ran"); - const markerScript = `require("node:fs").writeFileSync(${JSON.stringify(markerPath)}, "ran")`; - const parent = spawn(process.execPath, ["-e", "process.stdin.resume()"], { - stdio: ["pipe", "ignore", "ignore"], - }); - let completion: Promise<{ code: number }> | undefined; - - try { - const helper = await runHelperWithCommand({ - commandArgv: [process.execPath, "-e", markerScript], - parentPid: parent.pid, - parentExitTimeoutMs: timeoutMs, - }); - completion = helper.completion; - await helper.ready; - - if (expectLateLog) { - await vi.waitFor( - async () => { - const log = await fs.readFile(helper.logPath, "utf-8"); - const expected = `gateway parent pid ${parent.pid} exceeded expected handoff timeout; continuing to wait`; - expect(log.split(expected)).toHaveLength(2); - }, - { interval: 10, timeout: 2_000 }, - ); - } - expect(parent.exitCode).toBeNull(); - await expect(pathExists(markerPath)).resolves.toBe(false); - parent.stdin.end(); - await expect(completion).resolves.toEqual({ code: 0 }); - await expect(fs.readFile(markerPath, "utf-8")).resolves.toBe("ran"); - } finally { - parent.stdin.end(); - await completion?.catch(() => undefined); - } - }); }); diff --git a/src/infra/update-managed-service-handoff-ownership.test.ts b/src/infra/update-managed-service-handoff-ownership.test.ts index b4a29826bfe0..e281acd160e1 100644 --- a/src/infra/update-managed-service-handoff-ownership.test.ts +++ b/src/infra/update-managed-service-handoff-ownership.test.ts @@ -5,7 +5,7 @@ import { EventEmitter } from "node:events"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { PassThrough } from "node:stream"; +import { PassThrough, type Readable } from "node:stream"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; import { @@ -14,20 +14,40 @@ import { } from "../state/openclaw-state-db.js"; import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js"; import { claimOpenClawStateOwnership } from "../state/openclaw-state-ownership-operations.js"; -import { executeSqliteQuerySync, getNodeSqliteKysely } from "./kysely-sync.js"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "./kysely-sync.js"; +import { signalMockManagedUpdateHandoffReady } from "./update-managed-service-handoff.test-support.js"; const { spawnMock } = vi.hoisted(() => ({ spawnMock: vi.fn() })); function createSpawnMock() { return Object.assign(new EventEmitter(), { - pid: 24680, + pid: process.pid, exitCode: null, signalCode: null, + stdin: new PassThrough(), stdout: new PassThrough(), unref: vi.fn(), }); } +async function waitForHandoffLine(output: Readable | null, expected: string): Promise { + await new Promise((resolve, reject) => { + const onData = (chunk: Buffer | string) => { + if (!chunk.toString().includes(`${expected}\n`)) { + return; + } + output?.removeListener("data", onData); + resolve(); + }; + output?.on("data", onData); + output?.once("end", () => reject(new Error(`helper exited before ${expected}`))); + }); +} + vi.mock("node:child_process", async () => { const { mockNodeChildProcessModule } = await import("../gateway/server-methods/node-child-process.test-support.js"); @@ -37,82 +57,170 @@ vi.mock("node:child_process", async () => { }); const tempDirs = new Set(); +const mockedHandoffLeaseCleanups = new Set<() => void>(); type GatewayRestartSentinelDatabase = Pick; beforeEach(() => { spawnMock.mockReset(); - spawnMock.mockImplementation(() => { + spawnMock.mockImplementation((_command: string, args: string[]) => { const child = createSpawnMock(); process.nextTick(() => { - child.stdout.write("OPENCLAW_UPDATE_HANDOFF_READY\n"); + signalMockManagedUpdateHandoffReady({ + child, + paramsPath: args.at(-1) ?? "", + cleanups: mockedHandoffLeaseCleanups, + }); }); return child; }); }); afterEach(async () => { + for (const cleanup of mockedHandoffLeaseCleanups) { + cleanup(); + } closeOpenClawStateDatabaseForTest(); await Promise.all([...tempDirs].map((dir) => fs.rm(dir, { recursive: true, force: true }))); tempDirs.clear(); vi.resetModules(); }); -function writeRestartSentinelRow( - env: NodeJS.ProcessEnv, - sentinel: { - version: 1; - revision: number; - payload: { - kind: string; - status: string; - ts: number; - stats: Record; - }; - }, -): void { +function writeRestartSentinelRow(env: NodeJS.ProcessEnv, sentinel: unknown): void { + const { db } = openOpenClawStateDatabase({ env }); + const stateDb = getNodeSqliteKysely(db); + const payload = + sentinel && typeof sentinel === "object" && (sentinel as { version?: unknown }).version === 1 + ? (sentinel as { payload?: unknown }).payload + : null; + if (!payload || typeof payload !== "object") { + throw new Error("expected versioned restart sentinel payload"); + } + const record = payload as { + kind?: unknown; + status?: unknown; + ts?: unknown; + sessionKey?: unknown; + threadId?: unknown; + deliveryContext?: { channel?: unknown; to?: unknown; accountId?: unknown }; + message?: unknown; + continuation?: unknown; + doctorHint?: unknown; + stats?: unknown; + }; + const revision = + typeof (sentinel as { revision?: unknown }).revision === "number" + ? (sentinel as { revision: number }).revision + : Date.now(); + executeSqliteQuerySync( + db, + stateDb.insertInto("gateway_restart_sentinel").values({ + sentinel_key: record.kind === "revision-floor" ? "revision-floor" : "current", + version: 1, + kind: typeof record.kind === "string" ? record.kind : "update", + status: typeof record.status === "string" ? record.status : "skipped", + ts: typeof record.ts === "number" ? record.ts : Date.now(), + session_key: typeof record.sessionKey === "string" ? record.sessionKey : null, + thread_id: typeof record.threadId === "string" ? record.threadId : null, + delivery_channel: + typeof record.deliveryContext?.channel === "string" ? record.deliveryContext.channel : null, + delivery_to: + typeof record.deliveryContext?.to === "string" ? record.deliveryContext.to : null, + delivery_account_id: + typeof record.deliveryContext?.accountId === "string" + ? record.deliveryContext.accountId + : null, + message: typeof record.message === "string" ? record.message : null, + continuation_json: record.continuation ? JSON.stringify(record.continuation) : null, + doctor_hint: typeof record.doctorHint === "string" ? record.doctorHint : null, + stats_json: record.stats ? JSON.stringify(record.stats) : null, + payload_json: JSON.stringify(payload), + updated_at_ms: revision, + }), + ); +} + +function replaceRestartSentinelRow(env: NodeJS.ProcessEnv, sentinel: unknown): void { const { db } = openOpenClawStateDatabase({ env }); const stateDb = getNodeSqliteKysely(db); executeSqliteQuerySync( db, - stateDb.insertInto("gateway_restart_sentinel").values({ - sentinel_key: "current", - version: sentinel.version, - kind: sentinel.payload.kind, - status: sentinel.payload.status, - ts: sentinel.payload.ts, - session_key: null, - thread_id: null, - delivery_channel: null, - delivery_to: null, - delivery_account_id: null, - message: null, - continuation_json: null, - doctor_hint: null, - stats_json: JSON.stringify(sentinel.payload.stats), - payload_json: JSON.stringify(sentinel.payload), - updated_at_ms: sentinel.revision, - }), + stateDb.deleteFrom("gateway_restart_sentinel").where("sentinel_key", "=", "current"), ); + writeRestartSentinelRow(env, sentinel); +} + +function readRestartSentinelPayload(env: NodeJS.ProcessEnv, key = "current"): unknown { + const { db } = openOpenClawStateDatabase({ env }); + const stateDb = getNodeSqliteKysely(db); + const row = executeSqliteQueryTakeFirstSync( + db, + stateDb + .selectFrom("gateway_restart_sentinel") + .select(["version", "payload_json", "updated_at_ms"]) + .where("sentinel_key", "=", key), + ); + return row + ? { version: row.version, payload: JSON.parse(row.payload_json), revision: row.updated_at_ms } + : null; +} + +async function createLegacyRestartSentinelTable(env: NodeJS.ProcessEnv): Promise { + const sqlite = await import("node:sqlite"); + const stateDatabasePath = resolveOpenClawStateSqlitePath(env); + await fs.mkdir(path.dirname(stateDatabasePath), { recursive: true }); + const db = new sqlite.DatabaseSync(stateDatabasePath); + try { + db.exec(` + CREATE TABLE gateway_restart_sentinel ( + sentinel_key TEXT NOT NULL PRIMARY KEY, + version INTEGER NOT NULL, + kind TEXT NOT NULL, + status TEXT NOT NULL, + ts INTEGER NOT NULL, + session_key TEXT, + thread_id TEXT, + payload_json TEXT NOT NULL, + updated_at_ms INTEGER NOT NULL + ); + `); + } finally { + db.close(); + } } async function runOwnershipHelper(params: { handoffId?: string; metaHandoffId?: string; - prepareStateDatabase: (env: NodeJS.ProcessEnv) => Promise | void; - whileHelperRunning?: (context: { logPath: string }) => Promise | void; + prepareStateDatabase?: (env: NodeJS.ProcessEnv) => Promise | void; + sentinel?: unknown; + deepStatePath?: boolean; + commandDelayMs?: number; + commandExitCode?: number; + runnerFault?: "closed-stdin"; + whileHelperRunning?: (context: { + env: NodeJS.ProcessEnv; + logPath: string; + }) => Promise | void; }) { - const { execFile } = + const { spawn } = await vi.importActual("node:child_process"); + const { getFileLockProcessStartTime } = await import("../shared/pid-alive.js"); const { startManagedServiceUpdateHandoff } = await import("./update-managed-service-handoff.js"); const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-handoff-ownership-test-")); tempDirs.add(tmpDir); - const env = { OPENCLAW_STATE_DIR: tmpDir } as NodeJS.ProcessEnv; + let stateDir = tmpDir; + while ( + params.deepStatePath && + resolveOpenClawStateSqlitePath({ OPENCLAW_STATE_DIR: stateDir }).length <= 260 + ) { + stateDir = path.join(stateDir, `segment-${"x".repeat(24)}`); + } + const env = { OPENCLAW_STATE_DIR: stateDir } as NodeJS.ProcessEnv; await startManagedServiceUpdateHandoff({ root: tmpDir, timeoutMs: 1_800_000, restartDrainTimeoutMs: 300_000, - restartDelayMs: 500, parentPid: process.pid, execPath: "/usr/local/bin/node", argv1: "/opt/openclaw/openclaw.mjs", @@ -124,6 +232,8 @@ async function runOwnershipHelper(params: { continuationMessage: "continue after restart", }, }); + const mockedLauncher = spawnMock.mock.results.at(-1)?.value as ReturnType; + mockedLauncher.emit("exit", 0, null); const [, args, spawnOptions] = spawnMock.mock.calls.at(-1) as unknown as [ string, @@ -136,17 +246,63 @@ async function runOwnershipHelper(params: { string, unknown >; - await params.prepareStateDatabase(env); + await params.prepareStateDatabase?.(env); + if (params.sentinel !== undefined) { + writeRestartSentinelRow(env, params.sentinel); + } + const parent = spawn(process.execPath, ["-e", "process.stdin.resume()"], { + stdio: ["pipe", "ignore", "ignore"], + }); + const parentPid = parent.pid; + const startIdentity = parentPid ? getFileLockProcessStartTime(parentPid) : null; + if (!parentPid || startIdentity === null) { + parent.kill("SIGKILL"); + throw new Error("expected a parent process with a stable start identity"); + } const helperParamsPath = path.join(tmpDir, "helper-params.json"); const logPath = path.join(tmpDir, "handoff.log"); + const updaterPath = path.join(tmpDir, "updater-ran"); + const runnerClosedPath = path.join(tmpDir, "runner-closed-stdin"); + const preloadPath = path.join(tmpDir, "runner-fault-preload.cjs"); + if (params.runnerFault === "closed-stdin") { + await fs.writeFile( + preloadPath, + `const fs = require("node:fs"); +const childProcess = require("node:child_process"); +const originalSpawn = childProcess.spawn; +const closedPath = ${JSON.stringify(runnerClosedPath)}; +childProcess.spawn = function(command, args, options) { + if (command !== process.execPath || args[0] !== "-e" || + !args[1].includes('process.stdin.once("data"')) { + return originalSpawn.apply(this, arguments); + } + const runnerScript = 'const fs=require("node:fs");fs.closeSync(0);' + + 'fs.writeFileSync(' + JSON.stringify(closedPath) + ',String(process.pid));' + + 'setTimeout(() => process.exit(0),5000)'; + const child = originalSpawn.call(this, command, ["-e", runnerScript, args[2]], options); + const deadline = Date.now() + 5000; + const waitWord = new Int32Array(new SharedArrayBuffer(4)); + while (!fs.existsSync(closedPath) && Date.now() < deadline) { + Atomics.wait(waitWord, 0, 0, 10); + } + if (!fs.existsSync(closedPath)) throw new Error("runner did not close stdin"); + return child; +}; +`, + ); + } await fs.writeFile( helperParamsPath, `${JSON.stringify( { ...helperParams, - parentPid: 0, - parentExitTimeoutMs: 5_000, - commandArgv: [process.execPath, "-e", "process.exit(7)"], + parentPid, + parentStartIdentity: String(startIdentity), + commandArgv: [ + process.execPath, + "-e", + `require("node:fs").writeFileSync(${JSON.stringify(updaterPath)},"ran");setTimeout(() => process.exit(${params.commandExitCode ?? 1}), ${params.commandDelayMs ?? 0})`, + ], logPath, sensitivePaths: [], }, @@ -155,27 +311,55 @@ async function runOwnershipHelper(params: { )}\n`, ); + const helper = spawn(process.execPath, [helperScriptPath, helperParamsPath], { + cwd: tmpDir, + env: { + ...spawnOptions.env, + ...(params.runnerFault ? { NODE_OPTIONS: `--require ${preloadPath}` } : {}), + }, + stdio: ["pipe", "pipe", params.runnerFault ? "pipe" : "ignore"], + }); + const helperInput = helper.stdin; + if (!helperInput) { + throw new Error("expected the managed update helper to expose its control pipe"); + } + let stderr = ""; + helper.stderr?.on("data", (chunk: Buffer | string) => { + stderr += chunk.toString(); + }); const resultPromise = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>( (resolve) => { - execFile( - process.execPath, - [helperScriptPath, helperParamsPath], - { cwd: tmpDir, env: spawnOptions.env }, - (err) => { - const childError = err as (NodeJS.ErrnoException & { signal?: NodeJS.Signals }) | null; - resolve({ - code: typeof childError?.code === "number" ? childError.code : 0, - signal: childError?.signal ?? null, - }); - }, - ); + helper.once("close", (code, signal) => resolve({ code, signal })); }, ); - await params.whileHelperRunning?.({ logPath }); - return { result: await resultPromise, env, logPath }; + await waitForHandoffLine(helper.stdout, "OPENCLAW_UPDATE_HANDOFF_READY"); + const parked = waitForHandoffLine(helper.stdout, "parked"); + helperInput.write("park\n"); + await parked; + const committed = waitForHandoffLine(helper.stdout, "committed"); + helperInput.write("commit\n"); + await committed; + parent.stdin.end(); + await params.whileHelperRunning?.({ env, logPath }); + const result = await resultPromise; + if (params.runnerFault) { + const runnerPid = Number(await fs.readFile(runnerClosedPath, "utf8")); + try { + process.kill(runnerPid, "SIGKILL"); + } catch {} + } + return { + result, + env, + logPath, + stderr, + updaterPath, + leaseDatabasePath: String(helperParams.updateLeaseDatabasePath), + leaseKey: String(helperParams.updateLeaseKey), + }; } -describe("managed service update handoff external ownership", () => { +describe("managed service update handoff state ownership and sentinel persistence", () => { it("refuses fallback writes to externally owned state without the supervisor marker", async () => { let before: | { @@ -188,6 +372,7 @@ describe("managed service update handoff external ownership", () => { } | undefined; const { result, env, logPath } = await runOwnershipHelper({ + commandExitCode: 7, prepareStateDatabase: async (stateEnv) => { const externalEnv = { ...stateEnv, OPENCLAW_SUPERVISOR_MODE: "external" }; claimOpenClawStateOwnership("gateway-supervisor", { env: externalEnv }); @@ -247,6 +432,7 @@ describe("managed service update handoff external ownership", () => { let helperResult: Awaited> | undefined; try { helperResult = await runOwnershipHelper({ + commandExitCode: 7, handoffId: "handoff-ownership-race", metaHandoffId: "handoff-ownership-race", prepareStateDatabase: async (stateEnv) => { @@ -321,4 +507,249 @@ describe("managed service update handoff external ownership", () => { /race-supervisor.*OPENCLAW_SUPERVISOR_MODE=external/u, ); }); + + it("writes a fallback update failure when no restart sentinel row exists", async () => { + const { result, env } = await runOwnershipHelper({ + handoffId: "handoff-123", + metaHandoffId: "handoff-123", + }); + + expect(result).toEqual({ code: 1, signal: null }); + expect(readRestartSentinelPayload(env)).toMatchObject({ + version: 1, + payload: { + kind: "update", + status: "error", + sessionKey: "agent:test:webchat:dm:user-123", + stats: { + handoffId: "handoff-123", + reason: "managed-service-handoff-failed", + }, + }, + }); + if (process.platform !== "win32") { + const mode = (await fs.stat(resolveOpenClawStateSqlitePath(env))).mode & 0o777; + expect(mode).toBe(0o600); + } + }); + + it("reclaims the exact runner lease and records failure when its stdin closes before the update gate", async () => { + const { DatabaseSync } = await import("node:sqlite"); + const { result, env, stderr, updaterPath, leaseDatabasePath, leaseKey } = + await runOwnershipHelper({ + handoffId: "handoff-runner-closed-stdin", + metaHandoffId: "handoff-runner-closed-stdin", + runnerFault: "closed-stdin", + }); + + expect(result).toEqual({ code: 1, signal: null }); + expect(stderr).not.toMatch(/Unhandled 'error' event|Error: write EPIPE/u); + await expect(fs.access(updaterPath)).rejects.toMatchObject({ code: "ENOENT" }); + + const leaseDatabase = new DatabaseSync(leaseDatabasePath, { readOnly: true }); + try { + expect( + leaseDatabase + .prepare("SELECT owner FROM managed_update_handoffs WHERE install_root = ?") + .get(leaseKey), + ).toBeUndefined(); + } finally { + leaseDatabase.close(); + } + + expect(readRestartSentinelPayload(env)).toMatchObject({ + payload: { + kind: "update", + status: "error", + stats: { + handoffId: "handoff-runner-closed-stdin", + reason: expect.stringMatching(/^managed-service-handoff-(?:helper|spawn)-failed$/u), + }, + }, + }); + }); + + it.runIf(process.platform === "win32")( + "writes fallback state through the detached helper beyond MAX_PATH", + async () => { + const { result, env } = await runOwnershipHelper({ + deepStatePath: true, + handoffId: "handoff-windows-long-path", + metaHandoffId: "handoff-windows-long-path", + }); + const statePath = resolveOpenClawStateSqlitePath(env); + expect(statePath.startsWith("\\\\?\\")).toBe(false); + expect(statePath.length).toBeGreaterThan(260); + expect(result).toEqual({ code: 1, signal: null }); + expect(readRestartSentinelPayload(env)).toMatchObject({ + payload: { status: "error" }, + }); + }, + ); + + it("waits for a concurrent state writer before persisting the fallback failure", async () => { + let lockReleased: Promise | undefined; + const { result, env } = await runOwnershipHelper({ + handoffId: "handoff-locked", + metaHandoffId: "handoff-locked", + prepareStateDatabase: async (stateEnv) => { + openOpenClawStateDatabase({ env: stateEnv }); + closeOpenClawStateDatabaseForTest(); + const sqlite = await import("node:sqlite"); + const lock = new sqlite.DatabaseSync(resolveOpenClawStateSqlitePath(stateEnv)); + lock.exec("BEGIN IMMEDIATE;"); + lockReleased = new Promise((resolve) => { + setTimeout(() => { + lock.exec("COMMIT;"); + lock.close(); + resolve(); + }, 200); + }); + }, + }); + await lockReleased; + + expect(result).toEqual({ code: 1, signal: null }); + expect(readRestartSentinelPayload(env)).toMatchObject({ + version: 1, + payload: { + status: "error", + stats: { + handoffId: "handoff-locked", + reason: "managed-service-handoff-failed", + }, + }, + }); + }); + + it("repairs legacy restart sentinel columns before writing fallback failures", async () => { + const { result, env } = await runOwnershipHelper({ + handoffId: "handoff-123", + metaHandoffId: "handoff-123", + prepareStateDatabase: createLegacyRestartSentinelTable, + }); + + expect(result).toEqual({ code: 1, signal: null }); + expect(readRestartSentinelPayload(env)).toMatchObject({ + version: 1, + payload: { + kind: "update", + status: "error", + stats: { + reason: "managed-service-handoff-failed", + }, + }, + }); + }); + + it("does not overwrite a restart sentinel owned by another startup task", async () => { + const unrelatedSentinel = { + version: 1, + payload: { + kind: "config", + status: "skipped", + message: "preserve this restart task", + stats: { reason: "config-restart-pending" }, + }, + }; + const { result, env } = await runOwnershipHelper({ + sentinel: unrelatedSentinel, + }); + + expect(result).toEqual({ code: 1, signal: null }); + expect(readRestartSentinelPayload(env)).toMatchObject(unrelatedSentinel); + }); + + it("does not overwrite a newer pending update handoff sentinel", async () => { + const newerSentinel = { + version: 1, + payload: { + kind: "update", + status: "skipped", + message: "new handoff still pending", + stats: { + mode: "npm", + handoffId: "newer-handoff", + reason: "managed-service-handoff-started", + steps: [], + durationMs: 0, + }, + }, + }; + const { result, env } = await runOwnershipHelper({ + handoffId: "old-handoff", + metaHandoffId: "old-handoff", + sentinel: newerSentinel, + }); + + expect(result).toEqual({ code: 1, signal: null }); + expect(readRestartSentinelPayload(env)).toMatchObject(newerSentinel); + }); + + it("preserves a newer sentinel written while the detached helper is active", async () => { + const oldSentinel = { + version: 1, + revision: 100, + payload: { + kind: "update", + status: "skipped", + ts: 100, + stats: { + handoffId: "old-handoff", + reason: "managed-service-handoff-started", + }, + }, + }; + const newerSentinel = { + version: 1, + revision: 200, + payload: { + kind: "restart", + status: "ok", + ts: 200, + }, + }; + const { env } = await runOwnershipHelper({ + handoffId: "old-handoff", + metaHandoffId: "old-handoff", + sentinel: oldSentinel, + commandDelayMs: 200, + whileHelperRunning: async ({ env: stateEnv }) => { + await new Promise((resolve) => { + setTimeout(resolve, 50); + }); + replaceRestartSentinelRow(stateEnv, newerSentinel); + }, + }); + + expect(readRestartSentinelPayload(env)).toMatchObject({ + payload: newerSentinel.payload, + revision: 200, + }); + }); + + it("advances the durable revision floor even when it is ahead of the clock", async () => { + const futureRevision = Date.now() + 60_000; + const { env } = await runOwnershipHelper({ + handoffId: "handoff-future-revision", + metaHandoffId: "handoff-future-revision", + sentinel: { + version: 1, + revision: futureRevision, + payload: { + kind: "revision-floor", + status: "skipped", + ts: 123, + stats: { + handoffId: "handoff-future-revision", + reason: "managed-service-handoff-started", + }, + }, + }, + }); + + expect(readRestartSentinelPayload(env, "revision-floor")).toMatchObject({ + revision: futureRevision + 1, + }); + }); }); diff --git a/src/infra/update-managed-service-handoff-single-flight.test.ts b/src/infra/update-managed-service-handoff-single-flight.test.ts new file mode 100644 index 000000000000..05a8af6bdc06 --- /dev/null +++ b/src/infra/update-managed-service-handoff-single-flight.test.ts @@ -0,0 +1,775 @@ +// Process-local handoff sharing complements the durable cross-process lease. +import { EventEmitter } from "node:events"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { PassThrough } from "node:stream"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import { + signalMockManagedUpdateHandoffReady, + type MockManagedUpdateHandoffLeaseFailure, +} from "./update-managed-service-handoff.test-support.js"; + +const spawnMock = vi.hoisted(() => vi.fn()); +const forceKillChildProcessTreeMock = vi.hoisted(() => vi.fn()); +const findInstalledSystemdGatewayScopeMock = vi.hoisted(() => + vi.fn( + async (_env: NodeJS.ProcessEnv) => + null as { + scope: "user" | "system"; + unitName: string; + unitPath: string; + } | null, + ), +); +const tempRoots = useAutoCleanupTempDirTracker(afterEach); +const mockedHandoffLeaseCleanups = new Set<() => void>(); +const MOCK_INSTALL_ROOT = path.join(os.tmpdir(), `openclaw-handoff-single-flight-${process.pid}`); + +function createReadyChild( + pid: number, + paramsPath: string, + failure?: MockManagedUpdateHandoffLeaseFailure, +) { + const child = Object.assign(new EventEmitter(), { + pid, + exitCode: null, + signalCode: null, + stdin: new PassThrough(), + stdout: new PassThrough(), + unref: vi.fn(), + }); + process.nextTick(() => { + signalMockManagedUpdateHandoffReady({ + child, + paramsPath, + cleanups: mockedHandoffLeaseCleanups, + ...(child.pid === process.pid ? {} : { startIdentity: 17 }), + failure, + }); + }); + return child; +} + +vi.mock("node:child_process", async () => { + const { mockNodeChildProcessModule } = + await import("../gateway/server-methods/node-child-process.test-support.js"); + return mockNodeChildProcessModule({ + spawn: spawnMock as unknown as typeof import("node:child_process").spawn, + }); +}); + +vi.mock("../daemon/systemd-scope.js", async (importOriginal) => ({ + ...(await importOriginal()), + findInstalledSystemdGatewayScope: findInstalledSystemdGatewayScopeMock, +})); + +vi.mock("../process/child-process-tree.js", async (importOriginal) => ({ + ...(await importOriginal()), + forceKillChildProcessTree: forceKillChildProcessTreeMock, +})); + +beforeEach(async () => { + let pid = 24680; + const liveChildren = new Set(); + const processIdentity = await import("../shared/pid-alive.js"); + const parentStartIdentity = processIdentity.getFileLockProcessStartTime(process.pid); + vi.spyOn(processIdentity, "getFileLockProcessStartTime").mockImplementation((targetPid) => + targetPid === process.pid ? parentStartIdentity : liveChildren.has(targetPid) ? 17 : null, + ); + vi.spyOn(processIdentity, "isPidAlive").mockImplementation( + (targetPid) => targetPid === process.pid || liveChildren.has(targetPid), + ); + forceKillChildProcessTreeMock.mockReset(); + forceKillChildProcessTreeMock.mockImplementation((child: ReturnType) => { + child.stdout.destroy(); + }); + findInstalledSystemdGatewayScopeMock.mockReset(); + findInstalledSystemdGatewayScopeMock.mockResolvedValue(null); + spawnMock.mockReset(); + spawnMock.mockImplementation((_command: string, args: string[]) => { + const child = createReadyChild(pid++, args.at(-1) ?? ""); + liveChildren.add(child.pid); + child.once("exit", () => liveChildren.delete(child.pid)); + return child; + }); +}); + +afterEach(async () => { + for (const cleanup of mockedHandoffLeaseCleanups) { + cleanup(); + } + const handoffDirs = spawnMock.mock.calls.flatMap((call) => { + const args = call[1] as string[] | undefined; + const scriptPath = args?.[0]; + return scriptPath ? [path.dirname(scriptPath)] : []; + }); + await Promise.all(handoffDirs.map((dir) => fs.rm(dir, { recursive: true, force: true }))); + vi.restoreAllMocks(); + vi.resetModules(); +}); + +const baseParams = { + restartDrainTimeoutMs: 300_000, + parentPid: process.pid, + execPath: "/usr/local/bin/node", + argv1: "/opt/openclaw/openclaw.mjs", +}; + +describe("managed service update handoff single-flight", () => { + it.each([ + ["does not exist", "absent"], + ["has malformed helper identity", "malformed"], + ["belongs to a different owner", "wrong-owner"], + ["identifies a dead helper", "dead-helper"], + ] as const)("rejects READY when the durable helper lease %s", async (_label, failure) => { + spawnMock.mockImplementationOnce((_command: string, args: string[]) => + createReadyChild(process.pid, args.at(-1) ?? "", failure), + ); + const { claimManagedServiceUpdateHandoff, startManagedServiceUpdateHandoff } = + await import("./update-managed-service-handoff.js"); + const identity = { + kind: "managed-update-handoff" as const, + handoffId: `invalid-ready-${failure}`, + installRoot: `${MOCK_INSTALL_ROOT}-${failure}`, + }; + + await expect( + startManagedServiceUpdateHandoff({ + ...baseParams, + root: identity.installRoot, + handoffId: identity.handoffId, + meta: {}, + }), + ).rejects.toThrow(/lease|helper|identity/u); + + const child = spawnMock.mock.results[0]?.value as ReturnType; + expect(forceKillChildProcessTreeMock).toHaveBeenCalledExactlyOnceWith(child); + expect(child.unref).not.toHaveBeenCalled(); + expect(claimManagedServiceUpdateHandoff(identity)).toBe(false); + }); + + it("rejects system-scope systemd before spawning or reserving handoff ownership", async () => { + findInstalledSystemdGatewayScopeMock.mockResolvedValueOnce({ + scope: "system", + unitName: "openclaw-gateway.service", + unitPath: "/etc/systemd/system/openclaw-gateway.service", + }); + const { claimManagedServiceUpdateHandoff, startManagedServiceUpdateHandoff } = + await import("./update-managed-service-handoff.js"); + const root = `${MOCK_INSTALL_ROOT}-system-scope`; + + await expect( + startManagedServiceUpdateHandoff({ + ...baseParams, + root, + handoffId: "system-handoff", + supervisor: "systemd", + env: { OPENCLAW_SYSTEMD_UNIT: "openclaw-gateway.service" }, + meta: {}, + }), + ).rejects.toThrow(/user-scope systemd unit.*manual system-service update/); + expect(spawnMock).not.toHaveBeenCalled(); + expect( + claimManagedServiceUpdateHandoff({ + kind: "managed-update-handoff", + handoffId: "system-handoff", + installRoot: root, + }), + ).toBe(false); + + await expect( + startManagedServiceUpdateHandoff({ ...baseParams, root, meta: {} }), + ).resolves.toMatchObject({ status: "started" }); + expect(spawnMock).toHaveBeenCalledOnce(); + const owner = spawnMock.mock.results[0]?.value as ReturnType; + owner.emit("exit", 0, null); + }); + + it("shares one same-root helper until its lifecycle ends", async () => { + const { startManagedServiceUpdateHandoff } = + await import("./update-managed-service-handoff.js"); + const first = startManagedServiceUpdateHandoff({ + ...baseParams, + root: MOCK_INSTALL_ROOT, + handoffId: "handoff-first", + meta: { handoffId: "handoff-first" }, + }); + const second = startManagedServiceUpdateHandoff({ + ...baseParams, + root: MOCK_INSTALL_ROOT, + handoffId: "handoff-second", + meta: { handoffId: "handoff-second" }, + }); + + const outcomes = await Promise.all([first, second]); + expect(outcomes).toEqual([ + expect.objectContaining({ status: "started", handoffId: "handoff-first" }), + expect.objectContaining({ status: "joined", handoffId: "handoff-first" }), + ]); + expect(outcomes[1]).not.toHaveProperty("installRoot"); + expect(spawnMock).toHaveBeenCalledOnce(); + + const owner = spawnMock.mock.results[0]?.value as ReturnType; + owner.emit("exit", 0, null); + const next = startManagedServiceUpdateHandoff({ + ...baseParams, + root: MOCK_INSTALL_ROOT, + handoffId: "handoff-next", + meta: { handoffId: "handoff-next" }, + }); + + await expect(next).resolves.toMatchObject({ + status: "started", + handoffId: "handoff-next", + }); + expect(spawnMock).toHaveBeenCalledTimes(2); + const nextOwner = spawnMock.mock.results[1]?.value as ReturnType; + nextOwner.emit("exit", 0, null); + }); + + it.each([ + ["has exited before its ChildProcess notification", "dead"], + ["reuses its PID for another process", "reused"], + ["no longer exposes its process start identity", "unknown"], + ])("rejects a helper claim when its operating-system process %s", async (_label, failure) => { + vi.restoreAllMocks(); + const { spawn } = + await vi.importActual("node:child_process"); + spawnMock.mockImplementation(spawn); + const processIdentity = await import("../shared/pid-alive.js"); + const root = await fs.realpath(tempRoots.make("openclaw-helper-process-identity-")); + const parent = spawn(process.execPath, ["-e", "process.stdin.resume()"], { + stdio: ["pipe", "ignore", "ignore"], + }); + try { + const { + cancelManagedServiceUpdateHandoff, + claimManagedServiceUpdateHandoff, + startManagedServiceUpdateHandoff, + } = await import("./update-managed-service-handoff.js"); + const started = await startManagedServiceUpdateHandoff({ + root, + restartDrainTimeoutMs: 300_000, + parentPid: parent.pid, + execPath: process.execPath, + argv1: process.argv[1], + env: { ...process.env, OPENCLAW_STATE_DIR: root }, + meta: {}, + }); + if (started.status !== "started" || !started.pid) { + throw new Error("expected real detached handoff ownership"); + } + const identity = { kind: "managed-update-handoff" as const, ...started }; + expect(claimManagedServiceUpdateHandoff(identity)).toBe(true); + if (failure === "dead") { + vi.spyOn(processIdentity, "isPidAlive").mockReturnValue(false); + } else { + const helperStartIdentity = processIdentity.getFileLockProcessStartTime(started.pid); + if (helperStartIdentity === null) { + throw new Error("expected the real detached helper to have a process identity"); + } + vi.spyOn(processIdentity, "getFileLockProcessStartTime").mockReturnValue( + failure === "reused" ? helperStartIdentity + 1 : null, + ); + } + expect(claimManagedServiceUpdateHandoff(identity)).toBe(false); + vi.restoreAllMocks(); + await expect(cancelManagedServiceUpdateHandoff(identity)).resolves.toBe( + "restored-in-process", + ); + } finally { + vi.restoreAllMocks(); + parent.stdin?.end(); + } + }); + + it("terminates the exact helper when its initial start identity is unavailable", async () => { + const processIdentity = await import("../shared/pid-alive.js"); + vi.mocked(processIdentity.getFileLockProcessStartTime) + .mockReturnValueOnce(17) + .mockReturnValueOnce(null); + const { claimManagedServiceUpdateHandoff, startManagedServiceUpdateHandoff } = + await import("./update-managed-service-handoff.js"); + const identity = { + kind: "managed-update-handoff" as const, + handoffId: "identity-unavailable", + installRoot: `${MOCK_INSTALL_ROOT}-identity-unavailable`, + }; + + await expect( + startManagedServiceUpdateHandoff({ + ...baseParams, + root: identity.installRoot, + handoffId: identity.handoffId, + meta: {}, + }), + ).rejects.toThrow("process start identity is unavailable"); + + const child = spawnMock.mock.results[0]?.value as ReturnType; + expect(forceKillChildProcessTreeMock).toHaveBeenCalledExactlyOnceWith(child); + expect(child.unref).not.toHaveBeenCalled(); + expect(claimManagedServiceUpdateHandoff(identity)).toBe(false); + }); + + it("reclaims only its exact dead detached helper before reopening the install root", async () => { + vi.restoreAllMocks(); + const { spawn } = + await vi.importActual("node:child_process"); + const { DatabaseSync } = await import("node:sqlite"); + const processIdentity = await import("../shared/pid-alive.js"); + const { getFileLockProcessStartTime } = processIdentity; + spawnMock.mockImplementation(spawn); + const root = await fs.realpath(tempRoots.make("openclaw-dead-handoff-owner-")); + const markerPath = path.join(root, "updater-ran"); + const updaterPath = path.join(root, "updater.cjs"); + await fs.writeFile( + updaterPath, + `require("node:fs").writeFileSync(${JSON.stringify(markerPath)}, "ran")`, + ); + const parent = spawn(process.execPath, ["-e", "process.stdin.resume()"], { + stdio: ["pipe", "ignore", "ignore"], + }); + const { cancelManagedServiceUpdateHandoff, startManagedServiceUpdateHandoff } = + await import("./update-managed-service-handoff.js"); + let leaseDatabasePath: string | undefined; + let deadOwner: string | undefined; + let replacement: Awaited> | undefined; + try { + const start = () => + startManagedServiceUpdateHandoff({ + root, + restartDrainTimeoutMs: 300_000, + parentPid: parent.pid, + execPath: process.execPath, + argv1: updaterPath, + env: { ...process.env, OPENCLAW_STATE_DIR: root }, + meta: {}, + }); + const started = await start(); + if (started.status !== "started" || !started.pid) { + throw new Error("expected real detached handoff ownership"); + } + deadOwner = started.handoffId; + const helperStartIdentity = getFileLockProcessStartTime(started.pid); + if (helperStartIdentity === null) { + throw new Error("expected the detached helper to have a stable process identity"); + } + const helper = spawnMock.mock.results[0]?.value as import("node:child_process").ChildProcess; + const [, args] = spawnMock.mock.calls[0] as [string, string[]]; + leaseDatabasePath = ( + JSON.parse(await fs.readFile(args[1] ?? "", "utf8")) as { + updateLeaseDatabasePath: string; + } + ).updateLeaseDatabasePath; + const readLease = () => { + const db = new DatabaseSync(leaseDatabasePath!, { readOnly: true }); + try { + return db + .prepare( + "SELECT owner, payload_json FROM managed_update_handoffs WHERE install_root = ?", + ) + .get(root) as { owner: string; payload_json: string } | undefined; + } finally { + db.close(); + } + }; + const initialLease = readLease(); + expect(initialLease?.owner).toBe(started.handoffId); + expect(JSON.parse(initialLease?.payload_json ?? "null")).toEqual({ + version: 1, + pid: started.pid, + startIdentity: String(helperStartIdentity), + }); + + const helperExited = new Promise((resolve) => { + helper.once("exit", () => resolve()); + }); + helper.kill("SIGKILL"); + await helperExited; + expect(readLease()).toEqual(initialLease); + + const ownStartIdentity = getFileLockProcessStartTime(process.pid); + if (ownStartIdentity === null || !initialLease) { + throw new Error("expected complete live-parent and dead-helper lease identities"); + } + const originalPayload = { + version: 1, + pid: started.pid, + startIdentity: String(helperStartIdentity), + }; + const rejectedOwners = [ + { + label: "replacement owner", + owner: "replacement-owner", + payload: originalPayload, + }, + { + label: "different live process", + owner: started.handoffId, + payload: { + ...originalPayload, + pid: process.pid, + startIdentity: String(ownStartIdentity), + }, + }, + { + label: "different recorded start identity", + owner: started.handoffId, + payload: { ...originalPayload, startIdentity: `${helperStartIdentity}-reused` }, + }, + { + label: "malformed process identity", + owner: started.handoffId, + payload: { ...originalPayload, startIdentity: null }, + }, + { + label: "noncanonical process identity", + owner: started.handoffId, + payload: { ...originalPayload, unexpected: true }, + }, + ]; + const writeLease = (owner: string, payload: unknown) => { + if (!leaseDatabasePath) { + throw new Error("expected the detached helper lease database path"); + } + const db = new DatabaseSync(leaseDatabasePath); + try { + db.prepare( + "UPDATE managed_update_handoffs SET owner = ?, payload_json = ? WHERE install_root = ?", + ).run(owner, JSON.stringify(payload), root); + } finally { + db.close(); + } + }; + const identity = { kind: "managed-update-handoff" as const, ...started }; + for (const rejected of rejectedOwners) { + writeLease(rejected.owner, rejected.payload); + await expect(cancelManagedServiceUpdateHandoff(identity), rejected.label).resolves.toBe( + false, + ); + expect(readLease(), rejected.label).toEqual({ + owner: rejected.owner, + payload_json: JSON.stringify(rejected.payload), + }); + } + writeLease(started.handoffId, originalPayload); + const unknownDeath = vi.spyOn(processIdentity, "isPidDefinitelyDead").mockReturnValue(false); + await expect(cancelManagedServiceUpdateHandoff(identity)).resolves.toBe(false); + expect(readLease()).toEqual(initialLease); + unknownDeath.mockRestore(); + + await expect(cancelManagedServiceUpdateHandoff(identity)).resolves.toBe( + "restored-in-process", + ); + expect(readLease()).toBeUndefined(); + await expect(fs.access(markerPath)).rejects.toThrow(); + + replacement = await start(); + if (replacement.status !== "started") { + throw new Error("expected cancellation to reopen the install root"); + } + expect(replacement.installRoot).toBe(root); + expect(replacement.handoffId).not.toBe(started.handoffId); + await expect( + cancelManagedServiceUpdateHandoff({ kind: "managed-update-handoff", ...replacement }), + ).resolves.toBe("restored-in-process"); + replacement = undefined; + expect(readLease()).toBeUndefined(); + await expect(fs.access(markerPath)).rejects.toThrow(); + } finally { + if (replacement?.status === "started") { + await cancelManagedServiceUpdateHandoff({ + kind: "managed-update-handoff", + ...replacement, + }); + } + if (leaseDatabasePath && deadOwner) { + const db = new DatabaseSync(leaseDatabasePath); + db.prepare("DELETE FROM managed_update_handoffs WHERE install_root = ? AND owner = ?").run( + root, + deadOwner, + ); + db.close(); + } + parent.stdin?.end(); + } + }); + + it("waits for the exact helper to release its lease after an immediate control-pipe EPIPE", async () => { + vi.restoreAllMocks(); + const { spawn } = + await vi.importActual("node:child_process"); + const { DatabaseSync } = await import("node:sqlite"); + spawnMock.mockImplementation(spawn); + const root = await fs.realpath(tempRoots.make("openclaw-handoff-control-epipe-")); + const parent = spawn(process.execPath, ["-e", "process.stdin.resume()"], { + stdio: ["pipe", "ignore", "ignore"], + }); + const { + cancelManagedServiceUpdateHandoff, + requestManagedServiceUpdateHandoffPark, + startManagedServiceUpdateHandoff, + } = await import("./update-managed-service-handoff.js"); + const started = await startManagedServiceUpdateHandoff({ + root, + restartDrainTimeoutMs: 300_000, + parentPid: parent.pid, + execPath: process.execPath, + argv1: process.argv[1], + env: { ...process.env, OPENCLAW_STATE_DIR: root }, + meta: {}, + }); + if (started.status !== "started") { + throw new Error("expected real detached helper ownership"); + } + const identity = { kind: "managed-update-handoff" as const, ...started }; + const helper = spawnMock.mock.results[0]?.value as import("node:child_process").ChildProcess; + const input = helper.stdin; + if (!input) { + throw new Error("expected the detached helper control pipe"); + } + const [, args] = spawnMock.mock.calls[0] as [string, string[]]; + const { updateLeaseDatabasePath } = JSON.parse(await fs.readFile(args[1] ?? "", "utf8")) as { + updateLeaseDatabasePath: string; + }; + let controlDestroyed = false; + const destroyed = vi + .spyOn(input, "destroyed", "get") + .mockImplementation(() => controlDestroyed); + const write = vi.spyOn(input, "write").mockImplementation((( + _chunk: unknown, + callback: unknown, + ) => { + controlDestroyed = true; + const error = Object.assign(new Error("write EPIPE"), { code: "EPIPE" }); + if (typeof callback === "function") { + callback(error); + } + input.emit("error", error); + return false; + }) as typeof input.write); + + try { + await expect(requestManagedServiceUpdateHandoffPark(identity)).resolves.toBe(false); + expect(helper.exitCode).toBeNull(); + let cancellationSettled = false; + const cancellation = cancelManagedServiceUpdateHandoff(identity).then((result) => { + cancellationSettled = true; + return result; + }); + await new Promise((resolve) => { + setImmediate(resolve); + }); + + expect(cancellationSettled).toBe(false); + expect(helper.exitCode).toBeNull(); + write.mockRestore(); + destroyed.mockRestore(); + input.end(); + + await expect(cancellation).resolves.toBe("restored-in-process"); + expect(helper.exitCode !== null || helper.signalCode !== null).toBe(true); + const database = new DatabaseSync(updateLeaseDatabasePath, { readOnly: true }); + try { + expect( + database + .prepare("SELECT owner FROM managed_update_handoffs WHERE install_root = ?") + .get(root), + ).toBeUndefined(); + } finally { + database.close(); + } + } finally { + write.mockRestore(); + destroyed.mockRestore(); + input.end(); + parent.stdin?.end(); + } + }); + + it("joins canonical aliases while distinct install roots remain independent", async () => { + const tempDir = tempRoots.make("openclaw-handoff-root-"); + const root = path.join(tempDir, "install"); + const alias = path.join(tempDir, "install-alias"); + const otherRoot = path.join(tempDir, "other"); + await fs.mkdir(root); + await fs.mkdir(otherRoot); + await fs.symlink(root, alias, "dir"); + const { startManagedServiceUpdateHandoff } = + await import("./update-managed-service-handoff.js"); + + const owner = await startManagedServiceUpdateHandoff({ + ...baseParams, + root, + handoffId: "handoff-root", + meta: {}, + }); + await expect( + startManagedServiceUpdateHandoff({ ...baseParams, root: alias, meta: {} }), + ).resolves.toMatchObject({ status: "joined", handoffId: "handoff-root" }); + const other = await startManagedServiceUpdateHandoff({ + ...baseParams, + root: otherRoot, + handoffId: "handoff-other", + meta: {}, + }); + + expect(owner).toMatchObject({ + status: "started", + handoffId: "handoff-root", + installRoot: await fs.realpath(root), + }); + expect(other).toMatchObject({ status: "started", handoffId: "handoff-other" }); + expect(spawnMock).toHaveBeenCalledTimes(2); + for (const result of spawnMock.mock.results) { + (result.value as ReturnType).emit("exit", 0, null); + } + }); + + it.each([ + ["releases the root for another owner", false, false, false, false], + ["retains a claimed exited owner until release is confirmed", false, false, true, false], + ["treats control-pipe disconnect as cancellation", false, false, true, true], + ["wins a concurrent parent exit without running the updater", true, false, false, false], + ["refuses recovery when another owner replaces the completed helper", false, true, true, false], + ])("cancellation %s", async (_label, exitParent, replaceOwner, claimBeforeExit, disconnect) => { + vi.restoreAllMocks(); + const { spawn } = + await vi.importActual("node:child_process"); + const { DatabaseSync } = await import("node:sqlite"); + const { getFileLockProcessStartTime } = await import("../shared/pid-alive.js"); + const replacementStartIdentity = getFileLockProcessStartTime(process.pid); + if (replacementStartIdentity === null) { + throw new Error("expected the replacement owner to have a stable process identity"); + } + spawnMock.mockImplementation(spawn); + const root = await fs.realpath(tempRoots.make("openclaw-cancel-owner-")); + const markerPath = path.join(root, "updater-ran"); + const updaterPath = path.join(root, "updater.cjs"); + await fs.writeFile( + updaterPath, + `require("node:fs").writeFileSync(${JSON.stringify(markerPath)}, "ran")`, + ); + const parent = spawn(process.execPath, ["-e", "process.stdin.resume()"], { + stdio: ["pipe", "ignore", "ignore"], + }); + const { + cancelManagedServiceUpdateHandoff, + claimManagedServiceUpdateHandoff, + startManagedServiceUpdateHandoff, + } = await import("./update-managed-service-handoff.js"); + const start = () => + startManagedServiceUpdateHandoff({ + root, + restartDrainTimeoutMs: 300_000, + parentPid: parent.pid, + execPath: process.execPath, + argv1: updaterPath, + env: { ...process.env, OPENCLAW_STATE_DIR: root }, + meta: {}, + }); + const started = await start(); + if (started.status !== "started") { + throw new Error("expected handoff ownership"); + } + const identity = { kind: "managed-update-handoff" as const, ...started }; + if (claimBeforeExit) { + expect(claimManagedServiceUpdateHandoff(identity)).toBe(true); + expect(claimManagedServiceUpdateHandoff(identity)).toBe(true); + } + await expect( + cancelManagedServiceUpdateHandoff({ ...identity, handoffId: "joined" }), + ).resolves.toBe(false); + const child = spawnMock.mock.results[0]?.value as import("node:child_process").ChildProcess; + const [, args] = spawnMock.mock.calls[0] as [string, string[]]; + const helper = JSON.parse(await fs.readFile(args[1] ?? "", "utf8")) as { + updateLeaseDatabasePath: string; + stateDatabasePath: string; + }; + let joinedAfterExit: ReturnType | undefined; + child.once("exit", () => { + if (!exitParent) { + joinedAfterExit = start(); + } + if (replaceOwner) { + const replacement = new DatabaseSync(helper.updateLeaseDatabasePath); + replacement + .prepare( + "INSERT INTO managed_update_handoffs (install_root, owner, payload_json, updated_at) VALUES (?, ?, ?, ?)", + ) + .run( + root, + "replacement", + JSON.stringify({ + version: 1, + pid: process.pid, + startIdentity: String(replacementStartIdentity), + }), + Date.now(), + ); + replacement.close(); + } + }); + let cancellation: ReturnType; + if (claimBeforeExit) { + const exited = new Promise((resolve) => { + child.once("exit", () => resolve()); + }); + if (disconnect) { + child.stdin?.end(); + } else { + child.stdin?.write("cancel\n"); + } + await exited; + expect(claimManagedServiceUpdateHandoff(identity)).toBe(false); + expect(joinedAfterExit).toBeDefined(); + cancellation = cancelManagedServiceUpdateHandoff(identity); + } else { + cancellation = cancelManagedServiceUpdateHandoff(identity); + if (exitParent) { + parent.stdin?.end(); + } + } + await expect(cancellation).resolves.toBe(replaceOwner ? false : "restored-in-process"); + expect(child.exitCode !== null || child.signalCode !== null).toBe(true); + const sentinel = new DatabaseSync(helper.stateDatabasePath, { readOnly: true }); + const terminal = sentinel + .prepare("SELECT payload_json FROM gateway_restart_sentinel WHERE sentinel_key = 'current'") + .get() as { payload_json: string }; + sentinel.close(); + expect(JSON.parse(terminal.payload_json)).toMatchObject({ + status: "error", + stats: { reason: "managed-service-handoff-cancelled" }, + }); + if (joinedAfterExit) { + await expect(joinedAfterExit).resolves.toMatchObject({ + status: "joined", + handoffId: started.handoffId, + }); + expect(spawnMock).toHaveBeenCalledOnce(); + } + await expect(fs.access(markerPath)).rejects.toThrow(); + if (!replaceOwner && !exitParent) { + const priorSentinel = new DatabaseSync(helper.stateDatabasePath); + priorSentinel + .prepare("DELETE FROM gateway_restart_sentinel WHERE sentinel_key = 'current'") + .run(); + priorSentinel.close(); + const next = await start(); + if (next.status !== "started") { + throw new Error("expected replacement ownership"); + } + await expect( + cancelManagedServiceUpdateHandoff({ kind: "managed-update-handoff", ...next }), + ).resolves.toBe("restored-in-process"); + } + if (replaceOwner) { + const replacement = new DatabaseSync(helper.updateLeaseDatabasePath); + replacement.prepare("DELETE FROM managed_update_handoffs WHERE install_root = ?").run(root); + replacement.close(); + } + parent.stdin?.end(); + }); +}); diff --git a/src/infra/update-managed-service-handoff.test-support.ts b/src/infra/update-managed-service-handoff.test-support.ts new file mode 100644 index 000000000000..524107063cb1 --- /dev/null +++ b/src/infra/update-managed-service-handoff.test-support.ts @@ -0,0 +1,84 @@ +import type { EventEmitter } from "node:events"; +import fs from "node:fs"; +import path from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import type { Writable } from "node:stream"; +import { getFileLockProcessStartTime } from "../shared/pid-alive.js"; + +export type MockManagedUpdateHandoffLeaseFailure = + | "absent" + | "malformed" + | "wrong-owner" + | "dead-helper"; + +export function signalMockManagedUpdateHandoffReady(params: { + child: EventEmitter & { pid: number; stdout: Pick }; + paramsPath: string; + cleanups: Set<() => void>; + startIdentity?: number; + failure?: MockManagedUpdateHandoffLeaseFailure; +}): void { + const { child, cleanups, failure } = params; + if (child.stdout.destroyed) { + return; + } + const lease = JSON.parse(fs.readFileSync(params.paramsPath, "utf8")) as { + updateLeaseDatabasePath: string; + updateLeaseKey: string; + updateLeaseOwner: string; + }; + const startIdentity = params.startIdentity ?? getFileLockProcessStartTime(child.pid); + if (startIdentity === null) { + throw new Error("expected the mocked handoff child to have a live process identity"); + } + fs.mkdirSync(path.dirname(lease.updateLeaseDatabasePath), { recursive: true, mode: 0o700 }); + const owner = + failure === "wrong-owner" ? `${lease.updateLeaseOwner}-replacement` : lease.updateLeaseOwner; + const payload = JSON.stringify({ + version: 1, + pid: failure === "dead-helper" ? child.pid + 1_000_000 : child.pid, + startIdentity: failure === "malformed" ? null : String(startIdentity), + }); + const db = new DatabaseSync(lease.updateLeaseDatabasePath); + try { + if (process.platform !== "win32") { + fs.chmodSync(lease.updateLeaseDatabasePath, 0o600); + } + db.exec("PRAGMA busy_timeout = 5000;"); + db.exec( + "CREATE TABLE IF NOT EXISTS managed_update_handoffs " + + "(install_root TEXT NOT NULL PRIMARY KEY, owner TEXT NOT NULL, " + + "payload_json TEXT NOT NULL, updated_at INTEGER NOT NULL) STRICT", + ); + if (failure !== "absent") { + db.prepare( + "INSERT INTO managed_update_handoffs " + + "(install_root, owner, payload_json, updated_at) VALUES (?, ?, ?, ?) " + + "ON CONFLICT(install_root) DO UPDATE SET updated_at = excluded.updated_at " + + "WHERE owner = excluded.owner AND payload_json = excluded.payload_json", + ).run(lease.updateLeaseKey, owner, payload, Date.now()); + } + } finally { + db.close(); + } + if (failure !== "absent") { + const cleanup = () => { + cleanups.delete(cleanup); + const cleanupDb = new DatabaseSync(lease.updateLeaseDatabasePath); + try { + cleanupDb.exec("PRAGMA busy_timeout = 5000;"); + cleanupDb + .prepare( + "DELETE FROM managed_update_handoffs " + + "WHERE install_root = ? AND owner = ? AND payload_json = ?", + ) + .run(lease.updateLeaseKey, owner, payload); + } finally { + cleanupDb.close(); + } + }; + cleanups.add(cleanup); + child.once("exit", cleanup); + } + child.stdout.write("OPENCLAW_UPDATE_HANDOFF_READY\n"); +} diff --git a/src/infra/update-managed-service-handoff.ts b/src/infra/update-managed-service-handoff.ts index 2c388c2208ca..bb5434735815 100644 --- a/src/infra/update-managed-service-handoff.ts +++ b/src/infra/update-managed-service-handoff.ts @@ -5,14 +5,24 @@ import { randomUUID } from "node:crypto"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { - resolveGatewayLaunchAgentLabel, - resolveGatewaySystemdServiceName, - resolveGatewayWindowsTaskName, -} from "../daemon/constants.js"; +import type { DatabaseSync } from "node:sqlite"; +import { isRecord } from "@openclaw/normalization-core/record-coerce"; +import { resolveGatewayWindowsTaskName } from "../daemon/constants.js"; +import { resolveLaunchAgentLabel } from "../daemon/launchd-label.js"; +import { resolveLaunchAgentPlistPath } from "../daemon/launchd-service-files.js"; +import { findInstalledSystemdGatewayScope } from "../daemon/systemd-scope.js"; +import { resolveSystemdServiceName } from "../daemon/systemd-service-files.js"; import { forceKillChildProcessTree } from "../process/child-process-tree.js"; +import { + getFileLockProcessStartTime, + isPidAlive, + isPidDefinitelyDead, +} from "../shared/pid-alive.js"; import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js"; -import { resolveNodeSqliteLocation } from "./node-sqlite.js"; +import { resolveExecutableFromPathEnv } from "./executable-path.js"; +import { executeSqliteQueryTakeFirstSync, getNodeSqliteKysely } from "./kysely-sync.js"; +import { openNodeSqliteDatabase, resolveNodeSqliteLocation } from "./node-sqlite.js"; +import type { GatewayRestartIntent } from "./restart-intent.js"; import { SUPERVISOR_HINT_ENV_VARS, type RespawnSupervisor } from "./supervisor-markers.js"; import { resolvePreferredOpenClawTmpDir } from "./tmp-openclaw-dir.js"; import type { UpdateChannel } from "./update-channels.js"; @@ -25,55 +35,41 @@ import { resolveUpdateInstallRoot } from "./update-install-root.js"; import { MANAGED_SERVICE_UPDATE_HANDOFF_TEMP_PREFIX } from "./update-managed-service-handoff-cleanup.js"; import type { UpdateRestartSentinelMeta } from "./update-restart-sentinel-payload.js"; -// The Gateway may spend its full restart-drain budget before entering the -// bounded shutdown phase. This estimate only controls the late-parent -// diagnostic; exact parent exit is the update mutation boundary. (#99666) +// The helper deadline covers scheduled restart delay, the full Gateway drain +// budget, and its bounded parent-exit shutdown reserve. (#99666) const PARENT_EXIT_SHUTDOWN_RESERVE_MS = 30_000; const HANDOFF_READY_TIMEOUT_MS = 30_000; const HANDOFF_READY_MARKER = "OPENCLAW_UPDATE_HANDOFF_READY\n"; const HANDOFF_BUSY_MARKER = "HANDOFF_BUSY "; const HANDOFF_STATE_DATABASE_BUSY_TIMEOUT_MS = 5_000; -const SYSTEMD_RUN_CANDIDATE_PATHS = ["/usr/bin/systemd-run", "/bin/systemd-run"] as const; const SERVICE_IDENTITY_ENV_VARS = new Set([ "OPENCLAW_LAUNCHD_LABEL", "OPENCLAW_SYSTEMD_UNIT", "OPENCLAW_WINDOWS_TASK_NAME", ] as const); -type HandoffChild = ChildProcess & { stdout: NonNullable }; -type HandoffReadiness = { status: "ready" } | { status: "joined"; handoffId?: string }; - +type HandoffChild = ChildProcess & { + stdin: NonNullable; + stdout: NonNullable; +}; const HANDOFF_COMMAND_RUNNER_SCRIPT = String.raw` const { spawn } = require("node:child_process"); -const fs = require("node:fs"); -const params = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); -const gateDeadline = Date.now() + 30000; -const waitBuffer = new Int32Array(new SharedArrayBuffer(4)); -while (!fs.existsSync(params.runnerGatePath)) { - if (Date.now() >= gateDeadline) { - process.exit(1); +process.stdin.once("data", (decision) => { + if (decision.toString() !== "go") return; + const argv = JSON.parse(process.argv[1]); + if (process.platform !== "win32" && typeof process.execve === "function") { + process.execve(argv[0], argv, process.env); } - Atomics.wait(waitBuffer, 0, 0, 25); -} -if (process.platform !== "win32" && typeof process.execve === "function") { - process.execve(params.commandArgv[0], params.commandArgv, process.env); -} -const child = spawn(params.commandArgv[0], params.commandArgv.slice(1), { - cwd: params.commandCwd, - env: process.env, - stdio: "inherit", -}); -child.once("error", () => { - process.exitCode = 1; -}); -child.once("exit", (code, signal) => { - process.exitCode = typeof code === "number" ? code : signal ? 1 : 0; + const child = spawn(argv[0], argv.slice(1), { env: process.env, stdio: "inherit" }); + child.once("error", () => { process.exitCode = 1; }); + child.once("exit", (code, signal) => { + process.exitCode = typeof code === "number" ? code : signal ? 1 : 0; + }); }); `; const HANDOFF_SCRIPT = String.raw` const { spawn, spawnSync } = require("node:child_process"); const fs = require("node:fs"); -const os = require("node:os"); const path = require("node:path"); const { pathToFileURL } = require("node:url"); @@ -97,40 +93,19 @@ function isPidAlive(pid) { try { process.kill(pid, 0); } catch (err) { - return Boolean(err && err.code === "EPERM"); + return Boolean(err && err.code !== "ESRCH"); } if (process.platform === "linux") { try { const status = fs.readFileSync("/proc/" + pid + "/status", "utf8"); return !/^State:\s+Z/m.test(status); } catch { - return false; + return true; } } return true; } -function parseWindowsProcessStartTime(raw) { - const value = String(raw || "").trim().replace(/^CreationDate=/i, ""); - const parsedIso = Date.parse(value); - if (Number.isFinite(parsedIso)) { - return parsedIso; - } - const dmtf = value.match(/^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})\.(\d{6})([+-])(\d{3})$/); - if (!dmtf) return null; - const localTimeMs = Date.UTC( - Number(dmtf[1]), - Number(dmtf[2]) - 1, - Number(dmtf[3]), - Number(dmtf[4]), - Number(dmtf[5]), - Number(dmtf[6]), - Math.floor(Number(dmtf[7]) / 1000), - ); - const offsetMs = Number(dmtf[9]) * 60000 * (dmtf[8] === "+" ? 1 : -1); - return localTimeMs - offsetMs; -} - function readProcessStartIdentity(pid) { if (!isPidAlive(pid)) { return null; @@ -147,81 +122,30 @@ function readProcessStartIdentity(pid) { return null; } } - if (process.platform === "darwin") { - try { - const result = spawnSync("/bin/ps", ["-o", "lstart=", "-p", String(pid)], { - encoding: "utf8", - env: { ...process.env, LC_ALL: "C", TZ: "UTC" }, - stdio: ["ignore", "pipe", "ignore"], - timeout: 1000, - }); - const value = typeof result.stdout === "string" ? result.stdout.trim() : ""; - return result.status === 0 && value ? value : null; - } catch { - return null; - } - } - if (process.platform === "win32") { - const powershell = spawnSync( - "powershell.exe", - [ - "-NoProfile", - "-NonInteractive", - "-Command", - '$process = Get-CimInstance Win32_Process -Filter "ProcessId = ' + - pid + - '" -ErrorAction Stop; [Console]::Out.Write($process.CreationDate.ToUniversalTime().ToString("o"))', - ], - { encoding: "utf8", timeout: 1500, windowsHide: true }, - ); - if (!powershell.error && powershell.status === 0) { - const startedAt = parseWindowsProcessStartTime(powershell.stdout); - if (startedAt !== null) return String(startedAt); - } - const wmic = spawnSync( - "wmic.exe", - ["process", "where", "ProcessId=" + pid, "get", "CreationDate", "/value"], - { encoding: "utf8", timeout: 1500, windowsHide: true }, - ); - if (!wmic.error && wmic.status === 0) { - const line = String(wmic.stdout || "") - .split(/\r?\n/) - .find((entry) => /^CreationDate=/i.test(entry.trim())); - const startedAt = parseWindowsProcessStartTime(line); - if (startedAt !== null) return String(startedAt); - } - } - return null; + const windows = process.platform === "win32"; + if (!windows && process.platform !== "darwin") return null; + const args = windows + ? ["-NoProfile", "-NonInteractive", "-Command", "(Get-Process -Id " + pid + ").StartTime.ToString('o')"] + : ["-o", "lstart=", "-p", String(pid)]; + const result = spawnSync(windows ? "powershell.exe" : "/bin/ps", args, { + encoding: "utf8", env: { ...process.env, LC_ALL: "C", TZ: "UTC" }, + stdio: ["ignore", "pipe", "ignore"], timeout: 1000, killSignal: "SIGKILL", windowsHide: windows, + }); + const startedAt = Date.parse(String(result.stdout || "").trim() + (windows ? "" : " UTC")); + return !result.error && result.status === 0 && Number.isFinite(startedAt) + ? String(Math.floor(startedAt / (windows ? 1 : 1000))) + : null; } function parseLeaseCommandIdentity(value) { - if (typeof value !== "string" || !value) return null; - try { - const parsed = JSON.parse(value); - if ( - !parsed || - parsed.version !== 1 || - !Number.isInteger(parsed.pid) || - parsed.pid <= 0 || - (parsed.startIdentity !== null && typeof parsed.startIdentity !== "string") - ) { - return null; - } - return parsed; - } catch { - return null; - } -} - -function leaseCommandIsAlive(payloadJson) { - const identity = parseLeaseCommandIdentity(payloadJson); - if (!identity || !isPidAlive(identity.pid)) { - return false; - } - if (identity.startIdentity === null) { - return true; - } - return readProcessStartIdentity(identity.pid) === identity.startIdentity; + const parsed = parseJsonColumn(value); + return parsed && + parsed.version === 1 && + Number.isInteger(parsed.pid) && + parsed.pid > 0 && + typeof parsed.startIdentity === "string" && parsed.startIdentity.length > 0 + ? parsed + : null; } function sleep(ms) { @@ -238,41 +162,6 @@ function cleanupSensitiveFiles() { } } -function resolveExistingDirectory(candidates) { - for (const candidate of candidates) { - if (!candidate || typeof candidate !== "string") { - continue; - } - try { - const stat = fs.statSync(candidate); - if (stat.isDirectory()) { - return candidate; - } - } catch { - // Try the next candidate. - } - } - return undefined; -} - -function readJsonFile(filePath) { - try { - return JSON.parse(fs.readFileSync(filePath, "utf-8")); - } catch { - return null; - } -} - -function isPendingUpdatePayload(payload) { - const reason = payload && payload.stats && payload.stats.reason; - return ( - payload && - payload.kind === "update" && - payload.status === "skipped" && - (reason === "managed-service-handoff-started" || reason === "restart-health-pending") - ); -} - // Keep this self-contained helper aligned with resolveImmutableSqliteFileUri; // the detached script cannot import the TypeScript runtime after replacement. function resolveImmutableStateDatabaseUri(databasePath) { @@ -311,14 +200,7 @@ function assertStateDatabaseWriteAllowed(database) { .prepare("SELECT value_json FROM config_machine_state WHERE state_key = 'gateway.supervision' LIMIT 1") .get(); if (!row) return; - let value = null; - if (typeof row.value_json === "string") { - try { - value = JSON.parse(row.value_json); - } catch { - // The shared owner contract below rejects invalid JSON and shape together. - } - } + const value = parseJsonColumn(row.value_json); const keys = value && typeof value === "object" && !Array.isArray(value) ? Object.keys(value).sort() : []; @@ -353,49 +235,46 @@ function openStateDatabase() { return null; } let db = null; - let transactionOpen = false; try { assertStateDatabaseWriteAllowed(); const sqlite = require("node:sqlite"); fs.mkdirSync(path.dirname(params.stateDatabasePath), { recursive: true, mode: 0o700 }); db = new sqlite.DatabaseSync(params.nodeSqliteLocation); db.exec("PRAGMA busy_timeout = ${HANDOFF_STATE_DATABASE_BUSY_TIMEOUT_MS};"); - db.exec("BEGIN IMMEDIATE;"); - transactionOpen = true; - assertStateDatabaseWriteAllowed(db); - db.exec([ - "CREATE TABLE IF NOT EXISTS gateway_restart_sentinel (", - "sentinel_key TEXT NOT NULL PRIMARY KEY,", - "version INTEGER NOT NULL,", - "kind TEXT NOT NULL,", - "status TEXT NOT NULL,", - "ts INTEGER NOT NULL,", - "session_key TEXT,", - "thread_id TEXT,", - "delivery_channel TEXT,", - "delivery_to TEXT,", - "delivery_account_id TEXT,", - "message TEXT,", - "continuation_json TEXT,", - "doctor_hint TEXT,", - "stats_json TEXT,", - "payload_json TEXT NOT NULL,", - "updated_at_ms INTEGER NOT NULL", - ") STRICT;", - "CREATE INDEX IF NOT EXISTS idx_gateway_restart_sentinel_ts", - "ON gateway_restart_sentinel(ts DESC, sentinel_key);", - ].join(" ")); - ensureGatewayRestartSentinelColumns(db); - hardenStateDatabaseFiles(); - db.exec("COMMIT;"); - transactionOpen = false; + runManagedUpdateLeaseTransaction(db, () => { + assertStateDatabaseWriteAllowed(db); + db.exec([ + "CREATE TABLE IF NOT EXISTS gateway_restart_sentinel (", + "sentinel_key TEXT NOT NULL PRIMARY KEY,", + "version INTEGER NOT NULL,", + "kind TEXT NOT NULL,", + "status TEXT NOT NULL,", + "ts INTEGER NOT NULL,", + "session_key TEXT,", + "thread_id TEXT,", + "delivery_channel TEXT,", + "delivery_to TEXT,", + "delivery_account_id TEXT,", + "message TEXT,", + "continuation_json TEXT,", + "doctor_hint TEXT,", + "stats_json TEXT,", + "payload_json TEXT NOT NULL,", + "updated_at_ms INTEGER NOT NULL", + ") STRICT;", + "CREATE INDEX IF NOT EXISTS idx_gateway_restart_sentinel_ts", + "ON gateway_restart_sentinel(ts DESC, sentinel_key);", + ].join(" ")); + const columns = new Set(db.prepare("PRAGMA table_info(gateway_restart_sentinel)").all().map((row) => row.name)); + for (const column of ["delivery_channel", "delivery_to", "delivery_account_id", "message", "continuation_json", "doctor_hint", "stats_json"]) { + if (!columns.has(column)) db.exec("ALTER TABLE gateway_restart_sentinel ADD COLUMN " + column + " TEXT;"); + } + for (const suffix of ["", "-wal", "-shm"]) { + try { fs.chmodSync(params.stateDatabasePath + suffix, 0o600); } catch {} + } + }); return db; } catch (err) { - if (transactionOpen) { - try { - db.exec("ROLLBACK;"); - } catch {} - } try { db?.close(); } catch {} @@ -404,51 +283,6 @@ function openStateDatabase() { } } -function tableHasColumn(db, tableName, columnName) { - try { - return db.prepare("PRAGMA table_info(" + tableName + ")").all().some((row) => row && row.name === columnName); - } catch { - return false; - } -} - -function ensureColumn(db, tableName, columnSql) { - const columnName = columnSql.trim().split(/\s+/, 1)[0]; - if (!columnName || tableHasColumn(db, tableName, columnName)) { - return; - } - db.exec("ALTER TABLE " + tableName + " ADD COLUMN " + columnSql + ";"); -} - -function ensureGatewayRestartSentinelColumns(db) { - ensureColumn(db, "gateway_restart_sentinel", "delivery_channel TEXT"); - ensureColumn(db, "gateway_restart_sentinel", "delivery_to TEXT"); - ensureColumn(db, "gateway_restart_sentinel", "delivery_account_id TEXT"); - ensureColumn(db, "gateway_restart_sentinel", "message TEXT"); - ensureColumn(db, "gateway_restart_sentinel", "continuation_json TEXT"); - ensureColumn(db, "gateway_restart_sentinel", "doctor_hint TEXT"); - ensureColumn(db, "gateway_restart_sentinel", "stats_json TEXT"); -} - -function hardenStateDatabaseFiles() { - if (!params.stateDatabasePath || typeof params.stateDatabasePath !== "string") { - return; - } - for (const filePath of [ - params.stateDatabasePath, - params.stateDatabasePath + "-wal", - params.stateDatabasePath + "-shm", - ]) { - try { - if (fs.existsSync(filePath)) { - fs.chmodSync(filePath, 0o600); - } - } catch { - // Best effort only. - } - } -} - // This profile-independent SQLite coordinator owns one updater per canonical // install root. Process identity, not time, controls stale takeover. let managedUpdateLease = null; @@ -508,29 +342,26 @@ function openManagedUpdateLeaseDatabase() { } function runManagedUpdateLeaseTransaction(db, operation) { - let transactionOpen = false; + db.exec("BEGIN IMMEDIATE;"); try { - db.exec("BEGIN IMMEDIATE;"); - transactionOpen = true; const result = operation(); db.exec("COMMIT;"); - transactionOpen = false; return result; } catch (err) { - if (transactionOpen) { - try { - db.exec("ROLLBACK;"); - } catch {} - } + try { + db.exec("ROLLBACK;"); + } catch {} throw err; } } function buildLeaseProcessPayload(pid) { + const startIdentity = readProcessStartIdentity(pid); + if (!startIdentity) throw new Error("managed update process start identity is unavailable"); return JSON.stringify({ version: 1, pid, - startIdentity: readProcessStartIdentity(pid), + startIdentity, }); } @@ -548,7 +379,12 @@ function acquireManagedUpdateLease() { "SELECT owner, payload_json FROM managed_update_handoffs WHERE install_root = ?", ) .get(key); - if (current && leaseCommandIsAlive(current.payload_json)) { + const currentIdentity = current && parseLeaseCommandIdentity(current.payload_json); + if (current && !currentIdentity) { + throw new Error("existing managed update lease process identity is invalid"); + } + if (currentIdentity && isPidAlive(currentIdentity.pid) && + (readProcessStartIdentity(currentIdentity.pid) || currentIdentity.startIdentity) === currentIdentity.startIdentity) { return { acquired: false, owner: typeof current.owner === "string" ? current.owner : undefined, @@ -583,25 +419,18 @@ function acquireManagedUpdateLease() { } } -function bindManagedUpdateLeaseToProcess(pid) { +function bindManagedUpdateLeaseToProcess(pid, expectedIdentity, nextIdentity = buildLeaseProcessPayload(pid)) { const lease = managedUpdateLease; if (!lease || !managedUpdateLeaseOwned || !Number.isInteger(pid) || pid <= 0) { return false; } try { - runManagedUpdateLeaseTransaction(lease.db, () => { - const updated = lease.db - .prepare( - [ - "UPDATE managed_update_handoffs SET payload_json = ?, updated_at = ?", - "WHERE install_root = ? AND owner = ?", - ].join(" "), - ) - .run(buildLeaseProcessPayload(pid), Date.now(), lease.key, lease.owner); - if (updated.changes !== 1) { - throw new Error("managed update lease process binding was lost"); - } - }); + const updated = lease.db.prepare( + "UPDATE managed_update_handoffs SET payload_json = ?, updated_at = ? " + + "WHERE install_root = ? AND owner = ?" + (expectedIdentity ? " AND payload_json = ?" : ""), + ).run(nextIdentity, Date.now(), lease.key, lease.owner, + ...(expectedIdentity ? [expectedIdentity] : [])); + if (updated.changes !== 1) throw new Error("managed update lease process binding was lost"); return true; } catch (err) { managedUpdateLeaseOwned = false; @@ -613,6 +442,17 @@ function bindManagedUpdateLeaseToProcess(pid) { } } +function ownsManagedUpdateLease() { + const lease = managedUpdateLease; + if (!lease || !managedUpdateLeaseOwned) return false; + const row = lease.db + .prepare("SELECT owner, payload_json FROM managed_update_handoffs WHERE install_root = ?") + .get(lease.key); + const identity = row && row.owner === lease.owner ? parseLeaseCommandIdentity(row.payload_json) : null; + return Boolean(identity && identity.pid === process.pid && + identity.startIdentity === readProcessStartIdentity(process.pid)); +} + function releaseManagedUpdateLease() { const lease = managedUpdateLease; managedUpdateLease = null; @@ -621,13 +461,9 @@ function releaseManagedUpdateLease() { } try { if (managedUpdateLeaseOwned) { - runManagedUpdateLeaseTransaction(lease.db, () => { - lease.db - .prepare( - "DELETE FROM managed_update_handoffs WHERE install_root = ? AND owner = ?", - ) - .run(lease.key, lease.owner); - }); + lease.db.prepare( + "DELETE FROM managed_update_handoffs WHERE install_root = ? AND owner = ?", + ).run(lease.key, lease.owner); } } catch (err) { appendLog( @@ -643,11 +479,8 @@ function releaseManagedUpdateLease() { } function parseJsonColumn(value) { - if (typeof value !== "string" || !value) { - return null; - } try { - return JSON.parse(value); + return typeof value === "string" && value ? JSON.parse(value) : null; } catch { return null; } @@ -695,35 +528,14 @@ function readRestartSentinelRecord(db) { return { revision: row.updated_at_ms, payload }; } -function readRestartSentinelRevisionFloor(db) { - const row = db - .prepare("SELECT updated_at_ms FROM gateway_restart_sentinel WHERE sentinel_key = ?") - .get("revision-floor"); - if (!row) return null; - if (!Number.isSafeInteger(row.updated_at_ms)) { +function writeRestartSentinelPayload(db, payload, currentRevision) { + const floor = db.prepare( + "SELECT updated_at_ms FROM gateway_restart_sentinel WHERE sentinel_key = 'revision-floor'", + ).get(); + if (floor && !Number.isSafeInteger(floor.updated_at_ms)) { throw new Error("restart sentinel revision floor is outside the safe integer range"); } - return row.updated_at_ms; -} - -function advanceRestartSentinelRevisionFloor(db, revision) { - const payloadJson = JSON.stringify({ kind: "restart", status: "skipped", ts: revision }); - db.prepare( - [ - "INSERT INTO gateway_restart_sentinel (", - "sentinel_key, version, kind, status, ts, session_key, thread_id,", - "delivery_channel, delivery_to, delivery_account_id, message, continuation_json,", - "doctor_hint, stats_json, payload_json, updated_at_ms", - ") VALUES ('revision-floor', 1, 'restart', 'skipped', ?, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, ?, ?)", - "ON CONFLICT(sentinel_key) DO UPDATE SET", - "ts = excluded.ts, payload_json = excluded.payload_json, updated_at_ms = excluded.updated_at_ms", - ].join(" "), - ).run(revision, payloadJson, revision); -} - -function writeRestartSentinelPayload(db, payload, currentRevision) { - const revisionFloor = readRestartSentinelRevisionFloor(db); - const updatedAtMs = Math.max(Date.now(), Math.max(currentRevision || 0, revisionFloor || 0) + 1); + const updatedAtMs = Math.max(Date.now(), Math.max(currentRevision || 0, floor?.updated_at_ms || 0) + 1); if (!Number.isSafeInteger(updatedAtMs)) { throw new Error("restart sentinel revision exhausted the safe integer range"); } @@ -773,15 +585,29 @@ function writeRestartSentinelPayload(db, payload, currentRevision) { } if (changed) { // This runs inside the same BEGIN IMMEDIATE section as the guarded current-row write. - advanceRestartSentinelRevisionFloor(db, updatedAtMs); + const floorPayload = JSON.stringify({ kind: "restart", status: "skipped", ts: updatedAtMs }); + db.prepare( + [ + "INSERT INTO gateway_restart_sentinel (", + "sentinel_key, version, kind, status, ts, session_key, thread_id,", + "delivery_channel, delivery_to, delivery_account_id, message, continuation_json,", + "doctor_hint, stats_json, payload_json, updated_at_ms", + ") VALUES ('revision-floor', 1, 'restart', 'skipped', ?, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, ?, ?)", + "ON CONFLICT(sentinel_key) DO UPDATE SET", + "ts = excluded.ts, payload_json = excluded.payload_json, updated_at_ms = excluded.updated_at_ms", + ].join(" "), + ).run(updatedAtMs, floorPayload, updatedAtMs); } return changed; } -function buildFallbackFailurePayload(reason) { - const metaFile = params.metaPath ? readJsonFile(params.metaPath) : null; +function markUpdateSentinelFailureIfPending(reason, restored) { + let metaFile; + try { + metaFile = JSON.parse(fs.readFileSync(params.metaPath, "utf-8")); + } catch {} const meta = metaFile && metaFile.version === 1 && metaFile.meta ? metaFile.meta : {}; - const payload = { + const fallbackPayload = { kind: "update", status: "error", ts: Date.now(), @@ -797,147 +623,216 @@ function buildFallbackFailurePayload(reason) { durationMs: 0, }, }; - if (typeof meta.sessionKey === "string" && meta.sessionKey.trim()) { - payload.sessionKey = meta.sessionKey; + for (const key of ["sessionKey", "threadId"]) { + if (typeof meta[key] === "string" && meta[key].trim()) fallbackPayload[key] = meta[key]; } if (meta.deliveryContext && typeof meta.deliveryContext === "object") { - payload.deliveryContext = meta.deliveryContext; + fallbackPayload.deliveryContext = meta.deliveryContext; } - if (typeof meta.threadId === "string" && meta.threadId.trim()) { - payload.threadId = meta.threadId; - } - return payload; -} - -function markUpdateSentinelFailureIfPending(reason) { - const snapshotDb = openStateDatabase(); - if (!snapshotDb) return; - let snapshot; - try { - snapshot = readRestartSentinelRecord(snapshotDb); - } catch { - return; - } finally { - try { - snapshotDb.close(); - } catch {} - } - const fallbackPayload = snapshot === null ? buildFallbackFailurePayload(reason) : null; - const db = openStateDatabase(); - if (!db) return; - let transactionOpen = false; + if (!db) return false; + let recorded = false; try { - db.exec("BEGIN IMMEDIATE;"); - transactionOpen = true; - assertStateDatabaseWriteAllowed(db); - const current = readRestartSentinelRecord(db); - if ( - (snapshot === null && current !== null) || - (snapshot !== null && - (current === null || current.revision !== snapshot.revision)) - ) { - db.exec("COMMIT;"); - transactionOpen = false; - return; - } - - let payload = current && current.payload; - if (payload && (payload.kind !== "update" || !isPendingUpdatePayload(payload))) { - db.exec("COMMIT;"); - transactionOpen = false; - return; - } - const handoffId = typeof params.handoffId === "string" ? params.handoffId.trim() : ""; - if (payload && handoffId && (!payload.stats || payload.stats.handoffId !== handoffId)) { - db.exec("COMMIT;"); - transactionOpen = false; - return; - } - if (payload) { - payload = { ...payload, status: "error" }; - delete payload.continuation; - payload.stats = { ...(payload.stats || {}), reason }; - } else { - payload = fallbackPayload; - } - if (!payload) { - throw new Error("restart sentinel disappeared before guarded failure write"); - } - if (!writeRestartSentinelPayload(db, payload, current ? current.revision : null)) { - throw new Error("restart sentinel changed before guarded failure write"); - } - db.exec("COMMIT;"); - transactionOpen = false; + runManagedUpdateLeaseTransaction(db, () => { + assertStateDatabaseWriteAllowed(db); + const current = readRestartSentinelRecord(db); + let payload = current && current.payload; + const handoffId = typeof params.handoffId === "string" ? params.handoffId.trim() : ""; + if ( + (payload && (payload.kind !== "update" || payload.status !== "skipped" || + !["managed-service-handoff-started", "restart-health-pending"].includes(payload.stats?.reason)) && + !(typeof restored === "boolean" && payload.kind === "update" && payload.status === "error")) || + (payload && handoffId && (!payload.stats || payload.stats.handoffId !== handoffId)) + ) { + return; + } + if (payload) { + payload = { ...payload, status: "error" }; + delete payload.continuation; + payload.stats = { ...(payload.stats || {}), reason }; + } else { + payload = fallbackPayload; + } + if (typeof restored === "boolean") { + payload.stats.steps = [ + ...(payload.stats.steps || []), + { name: "service-restore", command: params.serviceRecovery.kind, log: { exitCode: restored ? 0 : 1 } }, + ]; + } + if (!writeRestartSentinelPayload(db, payload, current ? current.revision : null)) { + throw new Error("restart sentinel changed before guarded failure write"); + } + recorded = true; + }); } catch (err) { - if (transactionOpen) { - try { - db.exec("ROLLBACK;"); - } catch {} - } appendLog("failed to write update sentinel failure: " + (err && err.stack ? err.stack : String(err))); } finally { try { db.close(); } catch {} } + return recorded; } -function runCommandSync(command, args) { - try { - const result = spawnSync(command, args, { stdio: "ignore", timeout: 30000 }); - return typeof result.status === "number" ? result.status : 1; - } catch { - return 1; - } +function runServiceCommand(command, args, onSpawn, deadline) { + if (!ownsManagedUpdateLease()) return Promise.resolve({ code: 1, stdout: "", stderr: "" }); + return new Promise((resolve) => { + const cap = args[0] === "bootout" ? ${PARENT_EXIT_SHUTDOWN_RESERVE_MS} : 5000; + const remaining = deadline === undefined ? cap : deadline - Date.now(); + if (remaining <= 0) return resolve({ code: 1, stdout: "", stderr: "" }); + let stdout = "", stderr = ""; + const child = spawn(command, args, { + stdio: ["ignore", "pipe", "pipe"], killSignal: "SIGKILL", + timeout: Math.min(cap, remaining), + }); + child.stdout?.on("data", (chunk) => { stdout = (stdout + chunk).slice(-8192); }); + child.stderr?.on("data", (chunk) => { stderr = (stderr + chunk).slice(-8192); }); + child.once("spawn", () => onSpawn?.()); + child.once("error", (error) => { stderr = String(error); }); + child.once("close", (code) => resolve({ code: typeof code === "number" ? code : 1, stdout, stderr })); + }); } -function runServiceCommand(command, args) { - return managedUpdateLeaseOwned ? runCommandSync(command, args) : 1; +async function inspectSystemdService(unit) { + const result = await runServiceCommand("systemctl", ["--user", "show", unit, + "--property=Id,LoadState,ActiveState,MainPID,ExecMainStartTimestampMonotonic"]); + if (result.code !== 0) return null; + return Object.fromEntries(result.stdout.trim().split(/\r?\n/).map((line) => { + const index = line.indexOf("="); + return [line.slice(0, index), line.slice(index + 1)]; + })); } -function startGatewayServiceBestEffort() { +function isLaunchdNotLoaded(result) { + return /no such process|could not find service|not found/i.test(result.stderr || result.stdout); +} + +let parkedServiceGeneration = null; +let restorationArmed = false; +let pendingServiceStop; + +async function parkGatewayService() { const recovery = params.serviceRecovery; - if (!recovery || typeof recovery !== "object" || !recovery.kind) { - return; + if (!recovery || recovery.kind === "schtasks") return; + if (readProcessStartIdentity(params.parentPid) !== params.parentStartIdentity) { + throw new Error("managed update parent identity changed before parking"); } - let target = ""; - let status = 1; if (recovery.kind === "systemd") { - target = recovery.unit; - status = runServiceCommand("systemctl", ["--user", "start", recovery.unit]); - } else if (recovery.kind === "launchd") { - target = recovery.label; - const serviceTarget = "gui/" + recovery.uid + "/" + recovery.label; - status = runServiceCommand("launchctl", ["kickstart", serviceTarget]); - if (status !== 0) { - runServiceCommand("launchctl", ["enable", serviceTarget]); - status = runServiceCommand("launchctl", [ - "bootstrap", - "gui/" + recovery.uid, - recovery.plistPath, - ]); - if (status !== 0) { - // Bootstrap can fail when the label is already loaded. Retry start-only - // so recovery does not bounce a gateway that is already running. - status = runServiceCommand("launchctl", ["kickstart", serviceTarget]); - } + const current = await inspectSystemdService(recovery.unit); + if (!current || current.Id !== recovery.unit || current.LoadState !== "loaded" || + current.ActiveState !== "active" || current.MainPID !== String(params.parentPid) || + !/^[1-9]\d*$/.test(current.ExecMainStartTimestampMonotonic || "") || + !ownsManagedUpdateLease() || + readProcessStartIdentity(params.parentPid) !== params.parentStartIdentity) { + throw new Error("systemd service does not match the exact active gateway parent"); } - } else if (recovery.kind === "schtasks") { - target = recovery.taskName; - status = runServiceCommand("schtasks.exe", ["/Run", "/TN", recovery.taskName]); - } else { + parkedServiceGeneration = current.ExecMainStartTimestampMonotonic; + // A submitted stop can kill its parent even when the transport later times out. + restorationArmed = true; + const stopped = await runServiceCommand("systemctl", ["--user", "--no-block", "stop", recovery.unit]); + if (stopped.code !== 0) throw new Error("systemd stop submission failed: " + stopped.stderr); return; } - appendLog( - "gateway service recovery " + - (status === 0 ? "succeeded" : "failed status=" + status) + - " target=" + - target, + if (recovery.kind !== "launchd") throw new Error("unsupported managed update supervisor"); + const target = "gui/" + recovery.uid + "/" + recovery.label; + const inspection = await runServiceCommand("launchctl", ["print", target]); + const parentMatch = /^\s*pid\s*=\s*([1-9]\d*)\s*$/im.exec(inspection.stdout); + if (inspection.code !== 0 || Number(parentMatch?.[1]) !== params.parentPid || + !ownsManagedUpdateLease() || + readProcessStartIdentity(params.parentPid) !== params.parentStartIdentity) { + throw new Error("launchd service does not match the exact active gateway parent"); + } + restorationArmed = true; + const disabled = await runServiceCommand("launchctl", ["disable", target]); + if (disabled.code !== 0) throw new Error("launchctl disable failed: " + disabled.stderr); + if (!ownsManagedUpdateLease() || + readProcessStartIdentity(params.parentPid) !== params.parentStartIdentity) { + throw new Error("managed update owner changed before launchd bootout"); + } + // bootout gets launchd's full teardown budget; its accepted spawn acknowledges parking. + await new Promise((resolve, reject) => { + pendingServiceStop = runServiceCommand("launchctl", ["bootout", target], resolve); + pendingServiceStop.then((result) => { + if (result.code !== 0 && !isLaunchdNotLoaded(result)) { + reject(new Error("launchctl bootout failed: " + result.stderr)); + } + }); + }); +} + +async function restoreGatewayService(reason) { + const recovery = params.serviceRecovery; + let restored = false; + if (recovery?.kind === "systemd") { + const run = (args) => runServiceCommand("systemctl", ["--user", ...args]); + await run(["reset-failed", recovery.unit]); + const started = await run(["start", recovery.unit]); + const current = started.code === 0 && await inspectSystemdService(recovery.unit); + restored = Boolean(current && current.Id === recovery.unit && + current.LoadState === "loaded" && current.ActiveState === "active" && + /^[1-9]\d*$/.test(current.MainPID || "") && current.MainPID !== String(params.parentPid) && + isPidAlive(Number(current.MainPID)) && + /^[1-9]\d*$/.test(current.ExecMainStartTimestampMonotonic || "") && + current.ExecMainStartTimestampMonotonic !== parkedServiceGeneration); + } else if (recovery?.kind === "launchd") { + const target = "gui/" + recovery.uid + "/" + recovery.label; + const deadline = Date.now() + ${PARENT_EXIT_SHUTDOWN_RESERVE_MS}; + const run = (args) => runServiceCommand("launchctl", args, undefined, deadline); + const enabled = await run(["enable", target]); + let kickstarted = false; + for (let inspection = enabled; enabled.code === 0 && Date.now() < deadline;) { + inspection = await run(["print", target]); + if (inspection.code === 0) { + const pid = Number(/^\s*pid\s*=\s*([1-9]\d*)\s*$/im.exec(inspection.stdout)?.[1]); + if (pid !== params.parentPid && isPidAlive(pid)) { + restored = true; + break; + } + // launchd retains the old label until its ExitTimeOut-bounded teardown completes. + if (pid === params.parentPid) { + await sleep(Math.min(500, Math.max(0, deadline - Date.now()))); + continue; + } + if (kickstarted) break; + kickstarted = true; + inspection = await run(["kickstart", target]); + } else if (isLaunchdNotLoaded(inspection)) { + inspection = await run(["bootstrap", "gui/" + recovery.uid, recovery.plistPath]); + } else break; + if (inspection.code === 0) continue; + const detail = inspection.stderr || inspection.stdout; + if (inspection.code === 130 || + /already exists in domain|operation already in progress|bootstrap failed: 37/i.test(detail)) continue; + if (kickstarted && isLaunchdNotLoaded(inspection)) continue; + if (!/bootstrap failed: 5|input\/output error/i.test(detail)) break; + await sleep(Math.min(500, Math.max(0, deadline - Date.now()))); + } + } else if (recovery?.kind === "schtasks") { + restored = (await runServiceCommand("schtasks.exe", ["/Run", "/TN", recovery.taskName])).code === 0; + } + appendLog("gateway service recovery " + (restored ? "succeeded" : "failed")); + const recorded = markUpdateSentinelFailureIfPending( + restored ? reason : "managed-service-handoff-restore-failed", restored, ); + if (!recorded) { + appendLog("managed update restoration result could not be durably recorded"); + } + return restored && recorded; } (async () => { + if (!Number.isInteger(params.parentPid) || params.parentPid <= 0 || + typeof params.parentStartIdentity !== "string" || !params.parentStartIdentity) { + throw new Error("managed update parent process identity is unavailable"); + } + if (isPidAlive(params.parentPid) && + readProcessStartIdentity(params.parentPid) !== params.parentStartIdentity) { + throw new Error("managed update parent process identity changed"); + } + if (!Number.isFinite(params.parentExitTimeoutMs) || params.parentExitTimeoutMs < 0 || + !Number.isFinite(params.parentExitDeadlineAt)) { + throw new Error("managed update parent exit deadline is unavailable"); + } const lease = acquireManagedUpdateLease(); if (!lease.acquired) { appendLog( @@ -948,74 +843,171 @@ function startGatewayServiceBestEffort() { await sleep(25); return; } - fs.writeSync(1, ${JSON.stringify(HANDOFF_READY_MARKER)}); - + let outcome; + let wake; + let deadlineExpired = false; + const parentExitDeadline = setTimeout(() => { + deadlineExpired = true; + if (outcome !== "update") outcome = "restore"; + wake?.(); + }, params.parentExitTimeoutMs); try { - let deadline = - typeof params.parentExitTimeoutMs === "number" - ? Date.now() + params.parentExitTimeoutMs - : null; - while (isPidAlive(params.parentPid)) { - if (deadline !== null && Date.now() >= deadline) { - appendLog( - "gateway parent pid " + - params.parentPid + - " exceeded expected handoff timeout; continuing to wait", - ); - deadline = null; + fs.writeSync(1, ${JSON.stringify(HANDOFF_READY_MARKER)}); + const commands = []; + let input = ""; + let disconnected = false; + process.stdin.setEncoding("utf8"); + process.stdin.on("data", (chunk) => { + input += chunk; + if (input.length > 64) return process.stdin.destroy(); + let newline; + while ((newline = input.indexOf("\n")) >= 0) { + if (commands.length >= 4) return process.stdin.destroy(); + commands.push(input.slice(0, newline)); + input = input.slice(newline + 1); + } + wake?.(); + }); + process.stdin.once("close", () => { + disconnected = true; + wake?.(); + }); + const reply = (line) => fs.writeSync(1, line + "\n"); + let parked = false; + while (isPidAlive(params.parentPid)) { + if (!ownsManagedUpdateLease()) throw new Error("managed update lease no longer owns the helper"); + if (readProcessStartIdentity(params.parentPid) !== params.parentStartIdentity) { + if (isPidAlive(params.parentPid)) throw new Error("managed update parent process identity changed"); + await new Promise((resolve) => setImmediate(resolve)); + if (!commands.length) break; + } + if (deadlineExpired) { + deadlineExpired = false; + if (!parked) { + await parkGatewayService(); + parked = true; + } + if (ownsManagedUpdateLease() && + readProcessStartIdentity(params.parentPid) === params.parentStartIdentity) { + try { process.kill(params.parentPid, "SIGKILL"); } catch {} + } + } + const command = commands.shift(); + if (command === "park") { + try { + if (!parked) await parkGatewayService(); + parked = true; + reply("parked"); + } catch (error) { + appendLog("managed service parking failed: " + String(error)); + if (restorationArmed) { + outcome = "restore"; + reply("restore-after-exit"); + } else { + markUpdateSentinelFailureIfPending("managed-service-handoff-cancelled"); + reply("cancelled"); + return; + } + } + } else if (command === "commit" && parked) { + const restoring = outcome === "restore" || Date.now() >= params.parentExitDeadlineAt; + outcome = restoring ? "restore" : "update"; + reply(restoring ? "restore-after-exit" : "committed"); + } else if (command === "cancel" || (disconnected && outcome !== "update")) { + if (!restorationArmed) { + markUpdateSentinelFailureIfPending("managed-service-handoff-cancelled"); + if (command) reply("cancelled"); + return; + } + outcome = "restore"; + if (command) reply("restore-after-exit"); + } else if (command === "restore-commit" && outcome === "restore") { + reply("committed"); + } else if (command) { + throw new Error("invalid managed update control command"); + } + await Promise.race([sleep(25), new Promise((resolve) => { wake = resolve; })]); + } + clearTimeout(parentExitDeadline); + const stopped = pendingServiceStop ? await pendingServiceStop : null; + if (stopped && stopped.code !== 0 && !isLaunchdNotLoaded(stopped)) { + throw new Error("launchctl bootout failed: " + stopped.stderr); + } + if (outcome !== "update") { + if (restorationArmed) await restoreGatewayService("managed-service-handoff-cancelled"); + else markUpdateSentinelFailureIfPending("managed-service-handoff-cancelled"); + return; + } + if (params.serviceRecovery?.kind === "systemd") { + const unit = params.serviceRecovery.unit; + const current = await inspectSystemdService(unit); + if (!current || current.Id !== unit || current.LoadState !== "loaded" || + current.ActiveState !== "inactive" || current.MainPID !== "0" || + current.ExecMainStartTimestampMonotonic !== parkedServiceGeneration) { + throw new Error("systemd service remained active or changed execution generation"); + } + } + if (params.serviceRecovery?.kind === "launchd") { + const target = "gui/" + params.serviceRecovery.uid + "/" + params.serviceRecovery.label; + const deadline = Date.now() + ${PARENT_EXIT_SHUTDOWN_RESERVE_MS}; + for (;;) { + const result = await runServiceCommand("launchctl", ["print", target], undefined, deadline); + if (result.code !== 0) { + if (!isLaunchdNotLoaded(result)) throw new Error("launchctl print failed: " + result.stderr); + break; + } + if (Date.now() >= deadline) throw new Error("launchd service remained loaded after parent exit"); + await sleep(Math.min(500, Math.max(0, deadline - Date.now()))); } - await sleep(250); } appendLog("starting managed update command: " + params.commandLabel); let outputFd; try { outputFd = fs.openSync(params.logPath, "a", 0o600); - const commandCwd = - resolveExistingDirectory([ - params.cwd, - os.homedir(), - os.tmpdir(), - path.parse(process.execPath).root, - ]) || params.cwd; - if (commandCwd !== params.cwd) { - appendLog("managed update command cwd fallback: " + params.cwd + " -> " + commandCwd); - } - fs.writeFileSync( - params.runnerParamsPath, - JSON.stringify({ - commandArgv: params.commandArgv, - commandCwd, - runnerGatePath: params.runnerGatePath, - }), - { mode: 0o600 }, - ); - const child = spawn(process.execPath, [params.runnerScriptPath, params.runnerParamsPath], { - cwd: commandCwd, + const child = spawn(process.execPath, ["-e", ${JSON.stringify(HANDOFF_COMMAND_RUNNER_SCRIPT)}, JSON.stringify(params.commandArgv)], { + cwd: params.cwd, env: process.env, detached: true, - stdio: ["ignore", outputFd, outputFd], + stdio: ["pipe", outputFd, outputFd], }); - if (!bindManagedUpdateLeaseToProcess(child.pid)) { - try { - child.kill("SIGKILL"); - } catch {} - throw new Error("managed update runner lease binding failed"); - } - fs.writeFileSync(params.runnerGatePath, "go", { mode: 0o600 }); - appendLog("managed update command pid=" + (child.pid || "unknown")); - const exit = await new Promise((resolve) => { + const exited = new Promise((resolve) => { child.once("error", (err) => resolve({ error: err })); child.once("exit", (code, signal) => resolve({ code, signal })); }); - if (!bindManagedUpdateLeaseToProcess(process.pid)) { + child.stdin.on("error", () => {}); + const runnerIdentity = buildLeaseProcessPayload(child.pid); + if (!bindManagedUpdateLeaseToProcess(child.pid, undefined, runnerIdentity)) { + try { + child.kill("SIGKILL"); + } catch {} + await exited; + throw new Error("managed update runner lease binding failed"); + } + try { + await new Promise((resolve, reject) => { + child.stdin.once("error", reject); + child.stdin.once("close", () => reject(new Error("managed update runner stdin closed"))); + child.once("exit", () => reject(new Error("managed update runner exited before its gate"))); + child.stdin.write("go", (error) => error ? reject(error) : resolve()); + }); + child.stdin.end(); + } catch (error) { + try { child.kill("SIGKILL"); } catch {} + await exited; + bindManagedUpdateLeaseToProcess(process.pid, runnerIdentity); + throw error; + } + appendLog("managed update command pid=" + (child.pid || "unknown")); + const exit = await exited; + if (!bindManagedUpdateLeaseToProcess(process.pid, runnerIdentity)) { process.exitCode = 1; return; } if (exit && exit.error) { appendLog("managed update command failed to start: " + (exit.error && exit.error.stack ? exit.error.stack : String(exit.error))); - markUpdateSentinelFailureIfPending("managed-service-handoff-spawn-failed"); - startGatewayServiceBestEffort(); + if (params.serviceRecovery) await restoreGatewayService("managed-service-handoff-spawn-failed"); + else markUpdateSentinelFailureIfPending("managed-service-handoff-spawn-failed"); process.exitCode = 1; return; } @@ -1025,14 +1017,10 @@ function startGatewayServiceBestEffort() { " signal=" + (exit && exit.signal ? exit.signal : "null"), ); - if (exit && typeof exit.code === "number" && exit.code !== 0) { - markUpdateSentinelFailureIfPending("managed-service-handoff-failed"); - startGatewayServiceBestEffort(); - process.exitCode = exit.code; - } else if (exit && exit.signal) { - markUpdateSentinelFailureIfPending("managed-service-handoff-failed"); - startGatewayServiceBestEffort(); - process.exitCode = 1; + if (exit && (exit.signal || (typeof exit.code === "number" && exit.code !== 0))) { + if (params.serviceRecovery) await restoreGatewayService("managed-service-handoff-failed"); + else markUpdateSentinelFailureIfPending("managed-service-handoff-failed"); + process.exitCode = exit.code || 1; } } finally { if (outputFd !== undefined) { @@ -1045,14 +1033,16 @@ function startGatewayServiceBestEffort() { } } catch (err) { appendLog("handoff failed: " + (err && err.stack ? err.stack : String(err))); - markUpdateSentinelFailureIfPending("managed-service-handoff-helper-failed"); if (managedUpdateLeaseOwned) { bindManagedUpdateLeaseToProcess(process.pid); - startGatewayServiceBestEffort(); + if (restorationArmed) await restoreGatewayService("managed-service-handoff-helper-failed"); + else markUpdateSentinelFailureIfPending("managed-service-handoff-helper-failed"); } process.exitCode = 1; } finally { + clearTimeout(parentExitDeadline); releaseManagedUpdateLease(); + process.stdin.destroy(); cleanupSensitiveFiles(); } })().catch((err) => { @@ -1065,10 +1055,10 @@ function startGatewayServiceBestEffort() { type ManagedServiceUpdateHandoffParams = { root: string; timeoutMs?: number; - restartDrainTimeoutMs: number | undefined; + restartDrainTimeoutMs: number; + restartDelayMs?: number; channel?: UpdateChannel; tag?: string; - restartDelayMs?: number; meta: UpdateRestartSentinelMeta; handoffId?: string; supervisor?: RespawnSupervisor | null; @@ -1079,25 +1069,26 @@ type ManagedServiceUpdateHandoffParams = { parentPid?: number; }; -type StartedManagedServiceUpdateHandoff = { - status: "started"; +type ManagedServiceUpdateHandoffResult = { pid?: number; command: string; logPath: string; - handoffId?: string; -}; +} & ( + | { status: "started"; handoffId: string; installRoot: string } + | { status: "joined"; handoffId?: string } +); -type ManagedServiceUpdateHandoffResult = Omit & { - status: "started" | "joined"; +type ActiveManagedServiceUpdateHandoff = { + handoffId: string; + flight?: Promise; + launcher?: HandoffChild; + launcherStartIdentity?: number | null; + helper?: { owner: string; pid: number; startIdentity: string }; + claimed?: boolean; + cancelling?: boolean; + exited?: boolean; }; - -function isNodeLikeRuntime(execPath: string | undefined): boolean { - if (!execPath?.trim()) { - return false; - } - const base = path.basename(execPath).toLowerCase(); - return base === "node" || base === "node.exe" || base === "bun" || base === "bun.exe"; -} +const activeManagedServiceUpdateHandoffs = new Map(); function resolveUpdateCliArgv(params: { timeoutMs?: number; @@ -1122,7 +1113,7 @@ function resolveUpdateCliArgv(params: { if (execPath && argv1) { return [execPath, argv1, ...updateArgs]; } - if (execPath && !isNodeLikeRuntime(execPath)) { + if (execPath && !/^(?:node|bun)(?:\.exe)?$/iu.test(path.basename(execPath))) { return [execPath, ...updateArgs]; } return ["openclaw", ...updateArgs]; @@ -1133,17 +1124,9 @@ export function formatManagedServiceUpdateCommand(params?: { channel?: UpdateChannel; tag?: string; }): string { - const args = ["openclaw", "update", "--yes"]; - if (params?.channel) { - args.push("--channel", params.channel); - } - if (params?.tag) { - args.push("--tag", params.tag); - } - if (typeof params?.timeoutMs === "number" && Number.isFinite(params.timeoutMs)) { - args.push("--timeout", String(Math.max(1, Math.ceil(params.timeoutMs / 1000)))); - } - return args.join(" "); + return resolveUpdateCliArgv(params ?? {}) + .toSpliced(3, 1) + .join(" "); } type GatewayServiceRecovery = @@ -1156,25 +1139,12 @@ function resolveGatewayServiceRecovery( env: NodeJS.ProcessEnv, ): GatewayServiceRecovery | undefined { if (supervisor === "systemd") { - const override = env.OPENCLAW_SYSTEMD_UNIT?.trim(); - const unit = override - ? override.endsWith(".service") - ? override - : `${override}.service` - : `${resolveGatewaySystemdServiceName(env.OPENCLAW_PROFILE)}.service`; - return { kind: "systemd", unit }; + return { kind: "systemd", unit: `${resolveSystemdServiceName(env)}.service` }; } if (supervisor === "launchd") { - const label = - env.OPENCLAW_LAUNCHD_LABEL?.trim() || resolveGatewayLaunchAgentLabel(env.OPENCLAW_PROFILE); + const label = resolveLaunchAgentLabel(env); const uid = typeof process.getuid === "function" ? process.getuid() : 501; - const home = env.HOME?.trim() || os.homedir(); - return { - kind: "launchd", - uid, - label, - plistPath: path.join(home, "Library", "LaunchAgents", `${label}.plist`), - }; + return { kind: "launchd", uid, label, plistPath: resolveLaunchAgentPlistPath(env) }; } if (supervisor === "schtasks") { const taskName = @@ -1184,219 +1154,89 @@ function resolveGatewayServiceRecovery( return undefined; } -function stripSupervisorHintEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { - const next = { ...env }; - for (const key of SUPERVISOR_HINT_ENV_VARS) { - if (SERVICE_IDENTITY_ENV_VARS.has(key)) { - continue; - } - delete next[key]; - } - return next; -} - -async function resolveManagedServiceHandoffCwd(root: string): Promise { - const candidates = [os.homedir(), os.tmpdir(), path.dirname(process.execPath), root]; - for (const candidate of candidates) { - if (!candidate.trim()) { - continue; - } - try { - const stat = await fs.stat(candidate); - if (stat.isDirectory()) { - return candidate; - } - } catch { - // Try the next candidate. - } - } - return root; -} - function resolveManagedUpdateLeaseDatabasePath(): string { return path.join(resolvePreferredOpenClawTmpDir(), "managed-update-handoffs.sqlite"); } -async function resolveExecutableOnPath( - name: string, - env: NodeJS.ProcessEnv, - fallbackPaths: readonly string[], -): Promise { - const candidates = new Set(); - const pathValue = env.PATH?.trim(); - if (pathValue) { - for (const dir of pathValue.split(path.delimiter)) { - if (dir.trim()) { - candidates.add(path.join(dir, name)); - } - } - } - for (const candidate of fallbackPaths) { - candidates.add(candidate); - } - - for (const candidate of candidates) { - try { - await fs.access(candidate, fs.constants.X_OK); - return candidate; - } catch { - // Try the next candidate. - } - } - return null; -} - -function sanitizeSystemdUnitFragment(value: string | undefined): string { - const normalized = value?.trim().replace(/[^A-Za-z0-9_.:@-]+/gu, "-") ?? ""; - return normalized.replace(/^-+|-+$/gu, "").slice(0, 80); -} - -function buildSystemdHandoffUnitName(handoffId: string | undefined): string { - const suffix = - sanitizeSystemdUnitFragment(handoffId) || - sanitizeSystemdUnitFragment(`${process.pid}-${Date.now()}`) || - "handoff"; - return `openclaw-update-${suffix}.scope`; -} - -async function waitForHandoffReady(child: HandoffChild): Promise { - const output = child.stdout; - - return await new Promise((resolve, reject) => { +function waitForHandoffResponse(child: HandoffChild, command?: string): Promise { + return new Promise((resolve, reject) => { + const output = child.stdout; let settled = false; let buffered = ""; - const parseReadiness = (): HandoffReadiness | null => { - if (buffered.includes(HANDOFF_READY_MARKER)) { - return { status: "ready" }; - } - const busyIndex = buffered.indexOf(HANDOFF_BUSY_MARKER); - if (busyIndex < 0) { - return null; - } - const valueStart = busyIndex + HANDOFF_BUSY_MARKER.length; - const valueEnd = buffered.indexOf("\n", valueStart); - if (valueEnd < 0) { - return null; - } - const handoffId = buffered.slice(valueStart, valueEnd).trim(); - return { status: "joined", ...(handoffId ? { handoffId } : {}) }; - }; - const cleanup = () => { - clearTimeout(timeout); - child.removeListener("error", onError); - child.removeListener("exit", onExit); - output.removeListener("data", onData); - output.removeListener("error", onOutputError); - output.destroy(); - }; - const finish = (result: HandoffReadiness | null, err?: Error) => { + const finish = (result: string | Error) => { if (settled) { return; } settled = true; - cleanup(); - if (err) { - reject(err); - } else if (result) { - resolve(result); + clearTimeout(timeout); + child.removeListener("error", finish); + child.removeListener("exit", onExit); + output.removeListener("data", onData); + output.removeListener("error", onOutputError); + child.stdin.removeListener("error", finish).removeListener("close", onInputClose); + if (result instanceof Error) { + if (!command) { + output.destroy(); + } + reject(result); } else { - reject(new Error("managed update handoff readiness result is unavailable")); + resolve(result); } }; - const onError = (err: Error) => finish(null, err); const onExit = (code: number | null, signal: NodeJS.Signals | null) => { - const readiness = parseReadiness(); finish( - readiness, - readiness - ? undefined - : new Error( - `managed update handoff exited before signaling readiness (code=${code ?? "null"}, signal=${signal ?? "null"})`, - ), + new Error( + `managed update handoff exited before ${command ? "responding" : "signaling readiness"} (code=${code ?? "null"}, signal=${signal ?? "null"})`, + ), ); }; - const terminateBeforeFailure = () => { - if (typeof child.pid !== "number" || child.pid <= 0) { - return; - } - // A helper that loaded its parameters is armed even if its readiness - // marker is lost. Stop the detached tree before reporting failure. - forceKillChildProcessTree(child); - }; const onOutputError = (err: Error) => { - terminateBeforeFailure(); - finish(null, err); + if (!command && child.pid) { + // A loaded helper is armed even when its readiness marker was lost. + forceKillChildProcessTree(child); + } + finish(err); }; + const onInputClose = () => finish(new Error("managed update handoff control input closed")); const onData = (chunk: Buffer | string) => { buffered = `${buffered}${chunk.toString()}`.slice(-1024); - const readiness = parseReadiness(); - if (readiness) { - finish(readiness); + const newline = buffered.indexOf("\n"); + if (newline >= 0) { + finish(buffered.slice(0, newline)); } }; const timeout = setTimeout(() => { - terminateBeforeFailure(); - finish(null, new Error("managed update handoff did not signal readiness within 30 seconds")); + const phase = command ? "respond" : "signal readiness"; + onOutputError(new Error(`managed update handoff did not ${phase} within 30 seconds`)); }, HANDOFF_READY_TIMEOUT_MS); - child.once("error", onError); - child.once("exit", onExit); - output.once("error", onOutputError); - output.on("data", onData); + child.once("error", finish).once("exit", onExit); + output.once("error", onOutputError).on("data", onData); + child.stdin.once("error", finish).once("close", onInputClose); + if (command) { + child.stdin.write(`${command}\n`, (error) => { + if (error) { + finish(error); + } + }); + } }); } -async function resolveHandoffSpawn(params: { - supervisor?: RespawnSupervisor | null; - env: NodeJS.ProcessEnv; - execPath: string; - scriptPath: string; - paramsPath: string; - handoffId?: string; -}): Promise<{ command: string; args: string[] }> { - if (params.supervisor !== "systemd") { - return { - command: params.execPath, - args: [params.scriptPath, params.paramsPath], - }; - } - - const systemdRunPath = await resolveExecutableOnPath( - "systemd-run", - params.env, - SYSTEMD_RUN_CANDIDATE_PATHS, - ); - if (!systemdRunPath) { - throw new Error( - "systemd-run is required to start the managed update handoff outside openclaw-gateway.service", - ); - } - - return { - command: systemdRunPath, - args: [ - "--user", - "--scope", - "--collect", - `--unit=${buildSystemdHandoffUnitName(params.handoffId)}`, - params.execPath, - params.scriptPath, - params.paramsPath, - ], - }; -} - async function spawnManagedServiceUpdateHandoff( - params: ManagedServiceUpdateHandoffParams, + params: ManagedServiceUpdateHandoffParams & { handoffId: string }, rootIdentity: string, + owner: ActiveManagedServiceUpdateHandoff, ): Promise { + const parentPid = params.parentPid ?? process.pid; + const parentStartIdentity = getFileLockProcessStartTime(parentPid); + if (parentStartIdentity === null) { + throw new Error("managed update parent process start identity is unavailable"); + } const dir = await fs.mkdtemp(path.join(os.tmpdir(), MANAGED_SERVICE_UPDATE_HANDOFF_TEMP_PREFIX)); const scriptPath = path.join(dir, "handoff.cjs"); const paramsPath = path.join(dir, "handoff.json"); const metaPath = path.join(dir, "sentinel-meta.json"); - const runnerScriptPath = path.join(dir, "update-runner.cjs"); - const runnerParamsPath = path.join(dir, "update-runner.json"); - const runnerGatePath = path.join(dir, "update-runner.go"); const logPath = path.join(dir, "handoff.log"); const commandArgv = resolveUpdateCliArgv({ timeoutMs: params.timeoutMs, @@ -1410,22 +1250,47 @@ async function spawnManagedServiceUpdateHandoff( channel: params.channel, tag: params.tag, }); - const handoffCwd = await resolveManagedServiceHandoffCwd(params.root); const metaFile: ControlPlaneUpdateSentinelMetaFile = { version: 1, meta: { ...params.meta, root: rootIdentity }, }; - const stateDatabasePath = resolveOpenClawStateSqlitePath(params.env ?? process.env); + const serviceEnv = params.env ?? process.env; + let spawnCommand = params.execPath ?? process.execPath; + const spawnArgs = [scriptPath, paramsPath]; + if (params.supervisor === "systemd") { + const systemdRun = resolveExecutableFromPathEnv( + "systemd-run", + [serviceEnv.PATH ?? "", "/usr/bin", "/bin"].join(path.delimiter), + serviceEnv, + ); + if (!systemdRun) { + throw new Error("systemd-run is required to launch a transient user scope"); + } + const normalized = params.handoffId.trim().replace(/[^A-Za-z0-9_.:@-]+/gu, "-"); + const suffix = + normalized.replace(/^-+|-+$/gu, "").slice(0, 80) || `${process.pid}-${Date.now()}`; + spawnArgs.unshift( + "--user", + "--scope", + "--collect", + `--unit=openclaw-update-${suffix}.scope`, + spawnCommand, + ); + spawnCommand = systemdRun; + } + const stateDatabasePath = resolveOpenClawStateSqlitePath(serviceEnv); + const parentExitTimeoutMs = Math.min( + 2_147_483_647, + Math.max(0, params.restartDelayMs ?? 0) + + Math.max(0, params.restartDrainTimeoutMs) + + PARENT_EXIT_SHUTDOWN_RESERVE_MS, + ); const helperParams = { - parentPid: params.parentPid ?? process.pid, - // An undefined drain timeout is the configured indefinite-wait contract. - parentExitTimeoutMs: - params.restartDrainTimeoutMs === undefined - ? null - : Math.max(0, params.restartDelayMs ?? 0) + - Math.max(0, params.restartDrainTimeoutMs) + - PARENT_EXIT_SHUTDOWN_RESERVE_MS, - cwd: handoffCwd, + parentPid, + parentStartIdentity: String(parentStartIdentity), + parentExitTimeoutMs, + parentExitDeadlineAt: Date.now() + parentExitTimeoutMs, + cwd: dir, commandArgv, commandLabel, handoffId: params.handoffId, @@ -1436,89 +1301,315 @@ async function spawnManagedServiceUpdateHandoff( updateLeaseDatabasePath: resolveManagedUpdateLeaseDatabasePath(), updateLeaseKey: rootIdentity, updateLeaseOwner: params.handoffId, - runnerScriptPath, - runnerParamsPath, - runnerGatePath, - sensitivePaths: [ - scriptPath, - paramsPath, - metaPath, - runnerScriptPath, - runnerParamsPath, - runnerGatePath, - ], - serviceRecovery: resolveGatewayServiceRecovery(params.supervisor, params.env ?? process.env), + sensitivePaths: [scriptPath, paramsPath, metaPath], + serviceRecovery: resolveGatewayServiceRecovery(params.supervisor, serviceEnv), }; let child!: HandoffChild; - let readiness!: HandoffReadiness; + let readiness!: string; + const onExit = () => { + // Keep exact ownership until cancellation proves the durable lease was released. + owner.exited = true; + }; try { await fs.writeFile(scriptPath, `${HANDOFF_SCRIPT}\n`, { mode: 0o700 }); - await fs.writeFile(runnerScriptPath, `${HANDOFF_COMMAND_RUNNER_SCRIPT}\n`, { mode: 0o700 }); await fs.writeFile(paramsPath, `${JSON.stringify(helperParams, null, 2)}\n`, { mode: 0o600 }); await fs.writeFile(metaPath, `${JSON.stringify(metaFile, null, 2)}\n`, { mode: 0o600 }); - const childEnv = { - ...stripSupervisorHintEnv(params.env ?? process.env), + const childEnv: NodeJS.ProcessEnv = { + ...serviceEnv, [CONTROL_PLANE_UPDATE_SENTINEL_META_ENV]: metaPath, OPENCLAW_UPDATE_RUN_HANDOFF: "1", }; + for (const key of SUPERVISOR_HINT_ENV_VARS) { + if (!SERVICE_IDENTITY_ENV_VARS.has(key)) { + delete childEnv[key]; + } + } const env = params.devTarget ? applyDevUpdateTargetEnv(childEnv, params.devTarget) : childEnv; - const spawnTarget = await resolveHandoffSpawn({ - supervisor: params.supervisor, - env, - execPath: params.execPath ?? process.execPath, - scriptPath, - paramsPath, - handoffId: params.handoffId, - }); - child = spawn(spawnTarget.command, spawnTarget.args, { - cwd: handoffCwd, + child = spawn(spawnCommand, spawnArgs, { + cwd: dir, env, detached: true, - stdio: ["ignore", "pipe", "ignore"], + stdio: ["pipe", "pipe", "ignore"], }); + owner.launcher = child; + child.stdin.on("error", () => child.stdin.destroy()).once("close", () => child.stdin.destroy()); + owner.launcherStartIdentity = child.pid ? getFileLockProcessStartTime(child.pid) : null; + if (owner.launcherStartIdentity == null) { + forceKillChildProcessTree(child); + throw new Error("managed update handoff process start identity is unavailable"); + } + child.once("exit", onExit); // systemd-run --scope remains synchronous until the helper exits, so this // child's exit owns the full handoff lifetime. The ready marker means the // helper owns the cross-process update lease before callers terminate the Gateway. - readiness = await waitForHandoffReady(child); + readiness = await waitForHandoffResponse(child); + if (`${readiness}\n` !== HANDOFF_READY_MARKER && !readiness.startsWith(HANDOFF_BUSY_MARKER)) { + throw new Error("managed update handoff returned an invalid readiness response"); + } + if (`${readiness}\n` === HANDOFF_READY_MARKER) { + const helper = readManagedServiceUpdateHandoffLease(rootIdentity); + if ( + helper?.owner !== params.handoffId || + !isPidAlive(helper.pid) || + getFileLockProcessStartTime(helper.pid)?.toString() !== helper.startIdentity + ) { + forceKillChildProcessTree(child); + throw new Error("managed update handoff helper lease identity is unavailable"); + } + owner.helper = helper; + } } catch (err) { + child?.removeListener("exit", onExit); await fs.rm(dir, { recursive: true, force: true }).catch(() => {}); throw err; } child.unref(); - return { - status: readiness.status === "ready" ? "started" : "joined", - ...(readiness.status === "ready" && child.pid ? { pid: child.pid } : {}), - command: commandLabel, - logPath, - ...(readiness.status === "joined" - ? readiness.handoffId - ? { handoffId: readiness.handoffId } - : {} - : params.handoffId - ? { handoffId: params.handoffId } - : {}), - }; + const result = { command: commandLabel, logPath }; + const handoffId = readiness.slice(HANDOFF_BUSY_MARKER.length).trim(); + return `${readiness}\n` === HANDOFF_READY_MARKER + ? { + ...result, + status: "started", + ...(child.pid ? { pid: child.pid } : {}), + handoffId: params.handoffId, + installRoot: rootIdentity, + } + : { + ...result, + status: "joined", + ...(handoffId ? { handoffId } : {}), + }; } export async function startManagedServiceUpdateHandoff( params: ManagedServiceUpdateHandoffParams, ): Promise { + if ( + !Number.isFinite(params.restartDrainTimeoutMs) || + !Number.isFinite(params.restartDelayMs ?? 0) + ) { + throw new Error("managed update handoff requires a finite restart deadline"); + } + if ( + params.supervisor === "systemd" && + (await findInstalledSystemdGatewayScope(params.env ?? process.env))?.scope === "system" + ) { + throw new Error( + "Managed update handoff requires a user-scope systemd unit; perform a manual system-service update.", + ); + } const root = resolveUpdateInstallRoot(params.root); - const handoffId = params.handoffId ?? randomUUID(); - return await spawnManagedServiceUpdateHandoff( + const active = activeManagedServiceUpdateHandoffs.get(root); + if (active?.flight && (!active.exited || active.claimed || active.cancelling)) { + const joined = await active.flight; + return { + status: "joined", + command: joined.command, + logPath: joined.logPath, + ...(joined.pid ? { pid: joined.pid } : {}), + ...(joined.handoffId ? { handoffId: joined.handoffId } : {}), + }; + } + const owner: ActiveManagedServiceUpdateHandoff = { handoffId: params.handoffId ?? randomUUID() }; + activeManagedServiceUpdateHandoffs.set(root, owner); + const flight = spawnManagedServiceUpdateHandoff( { ...params, - handoffId, + handoffId: owner.handoffId, meta: { ...params.meta, - handoffId: params.meta.handoffId ?? handoffId, + handoffId: params.meta.handoffId ?? owner.handoffId, }, }, root, + owner, ); + owner.flight = flight; + try { + return await flight; + } catch (err) { + if (activeManagedServiceUpdateHandoffs.get(root) === owner) { + activeManagedServiceUpdateHandoffs.delete(root); + } + throw err; + } +} + +export function claimManagedServiceUpdateHandoff( + identity: NonNullable, +): boolean { + const root = resolveUpdateInstallRoot(identity.installRoot); + const active = activeManagedServiceUpdateHandoffs.get(root); + const launcher = active?.launcher; + const helper = active?.helper; + const lease = readManagedServiceUpdateHandoffLease(root); + if ( + identity.kind !== "managed-update-handoff" || + active?.handoffId !== identity.handoffId || + !launcher?.pid || + !isPidAlive(launcher.pid) || + active.launcherStartIdentity == null || + getFileLockProcessStartTime(launcher.pid) !== active.launcherStartIdentity || + launcher.exitCode !== null || + launcher.signalCode !== null || + active.cancelling || + lease?.owner !== identity.handoffId || + helper?.owner !== identity.handoffId || + lease.pid !== helper.pid || + lease.startIdentity !== helper.startIdentity || + !isPidAlive(lease.pid) || + getFileLockProcessStartTime(lease.pid)?.toString() !== lease.startIdentity + ) { + return false; + } + active.claimed = true; + return true; +} + +function readManagedServiceUpdateHandoffLease( + root: string, + stale?: ActiveManagedServiceUpdateHandoff, +): { owner: string; pid: number; startIdentity: string } | null | undefined { + let db: DatabaseSync | undefined; + try { + db = openNodeSqliteDatabase(resolveManagedUpdateLeaseDatabasePath(), { readOnly: !stale }); + const lease = getNodeSqliteKysely<{ + managed_update_handoffs: { install_root: string; owner: string; payload_json: string }; + }>(db); + const row = executeSqliteQueryTakeFirstSync( + db, + lease + .selectFrom("managed_update_handoffs") + .select(["owner", "payload_json"]) + .where("install_root", "=", root), + ); + if (!row) { + return null; + } + const payload: unknown = JSON.parse(row.payload_json); + if ( + !isRecord(payload) || + Object.keys(payload).length !== 3 || + payload.version !== 1 || + typeof payload.pid !== "number" || + !Number.isInteger(payload.pid) || + payload.pid <= 0 || + typeof payload.startIdentity !== "string" || + !payload.startIdentity + ) { + return undefined; + } + const current = { owner: row.owner, pid: payload.pid, startIdentity: payload.startIdentity }; + if (!stale) { + return current; + } + const observedStart = getFileLockProcessStartTime(current.pid); + if ( + current.owner !== stale.handoffId || + current.pid !== stale.helper?.pid || + current.startIdentity !== stale.helper.startIdentity || + (!isPidDefinitelyDead(current.pid) && + (observedStart === null || String(observedStart) === current.startIdentity)) + ) { + return current; + } + const deleted = executeSqliteQueryTakeFirstSync( + db, + lease + .deleteFrom("managed_update_handoffs") + .where("install_root", "=", root) + .where("owner", "=", current.owner) + .where("payload_json", "=", row.payload_json) + .returning("owner"), + ); + return deleted ? null : undefined; + } catch { + return undefined; + } finally { + db?.close(); + } +} + +function sendManagedServiceUpdateHandoffCommand( + identity: NonNullable, + command: string, +): Promise { + const child = activeManagedServiceUpdateHandoffs.get( + resolveUpdateInstallRoot(identity.installRoot), + )?.launcher; + if (!child?.stdin || !child.stdout || child.stdin.destroyed) { + return Promise.resolve(null); + } + return waitForHandoffResponse(child, command).catch(() => null); +} + +export async function requestManagedServiceUpdateHandoffPark( + identity: NonNullable, +): Promise { + return ( + claimManagedServiceUpdateHandoff(identity) && + (await sendManagedServiceUpdateHandoffCommand(identity, "park")) === "parked" && + claimManagedServiceUpdateHandoff(identity) + ); +} + +export async function commitManagedServiceUpdateHandoff( + identity: NonNullable, + outcome: "update" | "restore" = "update", +): Promise { + return ( + claimManagedServiceUpdateHandoff(identity) && + (await sendManagedServiceUpdateHandoffCommand( + identity, + outcome === "update" ? "commit" : "restore-commit", + )) === "committed" + ); +} + +export async function cancelManagedServiceUpdateHandoff( + identity: NonNullable, +): Promise<"restored-in-process" | "restart-after-exit" | false> { + const root = resolveUpdateInstallRoot(identity.installRoot); + const active = activeManagedServiceUpdateHandoffs.get(root); + if ( + identity.kind !== "managed-update-handoff" || + active?.handoffId !== identity.handoffId || + active.cancelling + ) { + return false; + } + active.cancelling = true; + try { + const child = active.launcher; + if (child && !active.exited && child.exitCode === null && child.signalCode === null) { + const exited = new Promise((resolve) => { + child.once("exit", () => resolve()); + }); + const response = await sendManagedServiceUpdateHandoffCommand(identity, "cancel"); + if (response === "restore-after-exit") { + return "restart-after-exit"; + } + if (response !== "cancelled" && !active.exited && !child.stdin.destroyed) { + return false; + } + await exited; + } + if ( + readManagedServiceUpdateHandoffLease(root, active) !== null || + activeManagedServiceUpdateHandoffs.get(root) !== active + ) { + return false; + } + activeManagedServiceUpdateHandoffs.delete(root); + return "restored-in-process"; + } catch { + return false; + } finally { + active.cancelling = false; + } } export function buildManagedServiceHandoffUnavailableMessage(command: string): string { diff --git a/src/infra/update-startup.test.ts b/src/infra/update-startup.test.ts index a6f2bc66e94a..3539ece1dc10 100644 --- a/src/infra/update-startup.test.ts +++ b/src/infra/update-startup.test.ts @@ -54,6 +54,8 @@ const { pid: 12345, command: "openclaw update --yes --channel beta --timeout 2700", logPath: "/tmp/openclaw-handoff.log", + handoffId: "auto-handoff-id", + installRoot: "/opt/openclaw", })), versionMock: { value: "1.0.0" }, })); @@ -77,13 +79,8 @@ vi.mock("./openclaw-root.js", async () => { }; }); -vi.mock("./restart.js", () => ({ - resolveGatewayRestartDeferralTimeoutMs: (timeoutMs: unknown) => { - if (typeof timeoutMs !== "number" || !Number.isFinite(timeoutMs)) { - return 300_000; - } - return timeoutMs <= 0 ? undefined : Math.floor(timeoutMs); - }, +vi.mock("./restart.js", async () => ({ + ...(await vi.importActual("./restart.js")), scheduleGatewaySigusr1Restart: scheduleGatewaySigusr1RestartMock, })); @@ -313,6 +310,8 @@ describe("update-startup", () => { pid: 12345, command: "openclaw update --yes --channel beta --timeout 2700", logPath: "/tmp/openclaw-handoff.log", + handoffId: "auto-handoff-id", + installRoot: "/opt/openclaw", }); resetUpdateAvailableStateForTest(); }); @@ -1911,9 +1910,11 @@ describe("update-startup", () => { expect(log.info).not.toHaveBeenCalled(); }); - it("delegates configured auto-updates to an external supervisor", async () => { + it("keeps external auto-update supervision authoritative over native systemd markers", async () => { mockPackageUpdateStatus("beta", "2.0.0-beta.1"); process.env.OPENCLAW_SUPERVISOR_MODE = "external"; + process.env.OPENCLAW_SYSTEMD_UNIT = "openclaw-gateway.service"; + detectRespawnSupervisorMock.mockReturnValue("systemd"); const log = { info: vi.fn() }; const runAutoUpdate = createAutoUpdateSuccessMock(); @@ -1994,6 +1995,8 @@ describe("update-startup", () => { pid: 12345, command: "openclaw update --yes --channel beta --tag 2.0.0-beta.1 --timeout 2700", logPath: "/tmp/openclaw-handoff.log", + handoffId: "started-auto-handoff-id", + installRoot: await fs.realpath(installRoot), }); const log = { info: vi.fn() }; @@ -2036,6 +2039,11 @@ describe("update-startup", () => { expect(scheduleGatewaySigusr1RestartMock).toHaveBeenCalledWith({ delayMs: 0, reason: "update.auto", + successorOwner: { + kind: "managed-update-handoff", + handoffId: "started-auto-handoff-id", + installRoot: await fs.realpath(installRoot), + }, skipCooldown: true, skipDeferral: true, }); @@ -2130,6 +2138,7 @@ describe("update-startup", () => { expect.objectContaining({ root: "/opt/openclaw", timeoutMs: 45 * 60 * 1000, + restartDrainTimeoutMs: 300_000, channel: "beta", tag: "2.0.0-beta.1", restartDelayMs: 2000, @@ -2139,6 +2148,11 @@ describe("update-startup", () => { expect(scheduleGatewaySigusr1RestartMock).toHaveBeenCalledWith({ delayMs: 2000, reason: "update.auto", + successorOwner: { + kind: "managed-update-handoff", + handoffId: "auto-handoff-id", + installRoot: "/opt/openclaw", + }, skipCooldown: true, skipDeferral: true, }); diff --git a/src/infra/update-startup.ts b/src/infra/update-startup.ts index ca4d774ed1ee..c27def3f5bd4 100644 --- a/src/infra/update-startup.ts +++ b/src/infra/update-startup.ts @@ -39,6 +39,7 @@ import { import { resolveOpenClawPackageRoot } from "./openclaw-root.js"; import { readVerifiedGitUpdateReceipt, type VerifiedGitUpdateReceipt } from "./restart-sentinel.js"; import { + normalizeGatewayRestartDelayMs, resolveGatewayRestartDeferralTimeoutMs, scheduleGatewaySigusr1Restart, } from "./restart.js"; @@ -154,7 +155,6 @@ const AUTO_UPDATE_COMMAND_TIMEOUT_MS = 45 * 60 * 1000; const AUTO_STABLE_DELAY_HOURS_DEFAULT = 6; const AUTO_STABLE_JITTER_HOURS_DEFAULT = 12; const AUTO_BETA_CHECK_INTERVAL_HOURS_DEFAULT = 1; -const MANAGED_AUTO_UPDATE_SYSTEMD_RESTART_GRACE_MS = 2000; const DEV_COMMIT_LIMIT = 5; const DEV_COMMIT_SUBJECT_MAX_LENGTH = 120; const DEV_COMMIT_LOG_MAX_OUTPUT_BYTES = 8 * 1024; @@ -162,13 +162,7 @@ const DEV_COMMIT_LOG_MAX_OUTPUT_BYTES = 8 * 1024; type UpdateCheckStateDatabase = Pick; function shouldSkipCheck(allowInTests: boolean): boolean { - if (allowInTests) { - return false; - } - if (process.env.VITEST || process.env.NODE_ENV === "test") { - return true; - } - return false; + return !allowInTests && Boolean(process.env.VITEST || process.env.NODE_ENV === "test"); } function resolveAutoUpdatePolicy(cfg: OpenClawConfig): AutoUpdatePolicy { @@ -438,14 +432,12 @@ function resolveStableAutoApplyAtMs(params: { return firstSeenMs + baseDelayMs + jitterMs; } -function resolveManagedAutoUpdateRestartDelayMs(supervisor: RespawnSupervisor): number { - return supervisor === "systemd" ? MANAGED_AUTO_UPDATE_SYSTEMD_RESTART_GRACE_MS : 0; -} - async function startManagedServiceAutoUpdateHandoff( params: AutoUpdateRunParams & { supervisor: RespawnSupervisor }, ): Promise { - const restartDelayMs = resolveManagedAutoUpdateRestartDelayMs(params.supervisor); + const restartDelayMs = normalizeGatewayRestartDelayMs( + params.supervisor === "systemd" ? undefined : 0, + ); const handoffId = randomUUID(); try { if (!params.root?.trim()) { @@ -454,7 +446,9 @@ async function startManagedServiceAutoUpdateHandoff( const started = await startManagedServiceUpdateHandoff({ root: params.root, timeoutMs: params.timeoutMs, - restartDrainTimeoutMs: params.restartDrainTimeoutMs, + restartDrainTimeoutMs: + resolveGatewayRestartDeferralTimeoutMs(params.restartDrainTimeoutMs) ?? + resolveGatewayRestartDeferralTimeoutMs(), channel: params.channel, ...(params.packageTargetVersion ? { tag: params.packageTargetVersion } : {}), restartDelayMs, @@ -469,9 +463,11 @@ async function startManagedServiceAutoUpdateHandoff( // Pair helper creation with restart scheduling before any state persistence // can fail and leave an indefinite handoff waiting on a live parent. if (started.status === "started") { + const { handoffId: ownerId, installRoot } = started; scheduleGatewaySigusr1Restart({ delayMs: restartDelayMs, reason: "update.auto", + successorOwner: { kind: "managed-update-handoff", handoffId: ownerId, installRoot }, skipCooldown: true, skipDeferral: true, }); @@ -514,15 +510,7 @@ async function runAutoUpdateCommand(params: AutoUpdateRunParams): Promise { }); }); - it("returns null on unsupported platforms", () => { + it("reads a bounded Windows PowerShell process start identity", () => { + const execSpy = vi + .spyOn(childProcess, "execFileSync") + .mockReturnValue("2026-07-06T12:34:56.7890000Z\n"); + return withMockedPlatform("win32", async () => { + expect(getProcessStartTime(42)).toBeNull(); + expect(getFileLockProcessStartTime(42)).toBe(Date.UTC(2026, 6, 6, 12, 34, 56, 789)); + expect(execSpy).toHaveBeenCalledWith( + "powershell.exe", + [ + "-NoProfile", + "-NonInteractive", + "-Command", + "(Get-Process -Id 42).StartTime.ToString('o')", + ], + expect.objectContaining({ timeout: 1000, windowsHide: true }), + ); + execSpy.mockReturnValueOnce("invalid\n"); + expect(getFileLockProcessStartTime(42)).toBeNull(); + }); + }); + + it("returns null on unsupported platforms", () => { + return withMockedPlatform("freebsd", async () => { expect(getProcessStartTime(process.pid)).toBeNull(); expect(getFileLockProcessStartTime(process.pid)).toBeNull(); }); diff --git a/src/shared/pid-alive.ts b/src/shared/pid-alive.ts index 68ab91086f6b..4294ef65dc52 100644 --- a/src/shared/pid-alive.ts +++ b/src/shared/pid-alive.ts @@ -2,7 +2,7 @@ import childProcess from "node:child_process"; import fsSync from "node:fs"; -const DARWIN_PS_TIMEOUT_MS = 1000; +const PROCESS_START_TIMEOUT_MS = 1000; function isValidPid(pid: number): boolean { return Number.isInteger(pid) && pid > 0; @@ -40,10 +40,7 @@ export function isPidAlive(pid: number): boolean { return false; } } - if (isZombieProcess(pid)) { - return false; - } - return true; + return !isZombieProcess(pid); } /** Returns true only when the PID is invalid, missing, or known to be a Linux zombie. */ @@ -59,20 +56,27 @@ export function isPidDefinitelyDead(pid: number): boolean { return isZombieProcess(pid); } -function getDarwinProcessStartTime(pid: number): number | null { +function getPlatformProcessStartTime(pid: number): number | null { try { + const windows = process.platform === "win32"; + const command = windows ? `(Get-Process -Id ${pid}).StartTime.ToString('o')` : String(pid); + const args = windows + ? ["-NoProfile", "-NonInteractive", "-Command", command] + : ["-o", "lstart=", "-p", command]; const startedAt = childProcess - .execFileSync("/bin/ps", ["-o", "lstart=", "-p", String(pid)], { + .execFileSync(windows ? "powershell.exe" : "/bin/ps", args, { encoding: "utf8", env: { ...process.env, LC_ALL: "C", TZ: "UTC" }, stdio: ["ignore", "pipe", "ignore"], - timeout: DARWIN_PS_TIMEOUT_MS, + timeout: PROCESS_START_TIMEOUT_MS, + killSignal: "SIGKILL", + windowsHide: windows, }) .trim(); // Darwin's lstart output has no timezone. Force UTC for both ps and parsing so // a system timezone change cannot make a live lock owner look like PID reuse. - const startedAtMs = Date.parse(`${startedAt} UTC`); - return Number.isFinite(startedAtMs) ? Math.floor(startedAtMs / 1000) : null; + const startedAtMs = Date.parse(windows ? startedAt : `${startedAt} UTC`); + return Number.isFinite(startedAtMs) ? Math.floor(startedAtMs / (windows ? 1 : 1000)) : null; } catch { return null; } @@ -80,10 +84,7 @@ function getDarwinProcessStartTime(pid: number): number | null { /** Read the Linux procfs start identity used by Linux-owned runtime state. */ export function getProcessStartTime(pid: number): number | null { - if (!isValidPid(pid)) { - return null; - } - if (process.platform !== "linux") { + if (!isValidPid(pid) || process.platform !== "linux") { return null; } try { @@ -109,5 +110,7 @@ export function getFileLockProcessStartTime(pid: number): number | null { if (!isValidPid(pid)) { return null; } - return process.platform === "darwin" ? getDarwinProcessStartTime(pid) : getProcessStartTime(pid); + return process.platform === "darwin" || process.platform === "win32" + ? getPlatformProcessStartTime(pid) + : getProcessStartTime(pid); }