mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-25 03:45:46 -06:00
fix(update): keep managed handoff as successor owner (#128212)
* fix(update): make managed handoff own successor Park the exact native service before Gateway close and coalesce same-root handoff requests so only the detached updater can activate and restart. * refactor(update): reuse restart ownership helpers * test(gateway): isolate supervisor restart fixtures * fix(update): cancel handoff before restart fallback * fix(update): harden managed service handoff * fix(update): use canonical SQLite opener * fix(update): preserve lifecycle runtime boundaries
This commit is contained in:
committed by
GitHub
parent
b0e8a985ee
commit
e1a700840a
@@ -30,6 +30,12 @@ export {
|
||||
consumeGatewayRestartIntentSync,
|
||||
} from "../../infra/restart-intent.js";
|
||||
export { writeGatewayRestartHandoffSync } from "../../infra/restart-handoff.js";
|
||||
export {
|
||||
cancelManagedServiceUpdateHandoff,
|
||||
claimManagedServiceUpdateHandoff,
|
||||
commitManagedServiceUpdateHandoff,
|
||||
requestManagedServiceUpdateHandoffPark,
|
||||
} from "../../infra/update-managed-service-handoff.js";
|
||||
export { resetGatewaySuspendCoordinatorForLifecycleRestart } from "../../infra/gateway-suspend-coordinator.js";
|
||||
export { rotateAgentEventLifecycleGeneration } from "../../infra/agent-events.js";
|
||||
export { markUpdateRestartSentinelFailure } from "../../infra/restart-sentinel.js";
|
||||
|
||||
@@ -1,13 +1,16 @@
|
||||
// Gateway run loop tests cover foreground gateway lifecycle and restart behavior.
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { GatewayServer } from "../../gateway/server-public.js";
|
||||
import type { GatewayBonjourBeacon } from "../../infra/bonjour-discovery.js";
|
||||
import type { GatewayActiveWorkSnapshot } from "../../infra/gateway-active-work.js";
|
||||
import type { GatewayRestartIntent } from "../../infra/restart-intent.js";
|
||||
import { SUPERVISOR_HINT_ENV_VARS } from "../../infra/supervisor-markers.js";
|
||||
import { resolveGlobalMap } from "../../shared/global-singleton.js";
|
||||
import {
|
||||
GATEWAY_AGENT_MEDIA_MIGRATION_REQUIRED_REASON,
|
||||
OpenClawAgentDatabaseMediaMigrationRequiredError,
|
||||
} from "../../state/openclaw-agent-db-migration-required.js";
|
||||
import { captureEnv, deleteTestEnvValue } from "../../test-utils/env.js";
|
||||
import { pickBeaconHost, pickGatewayPort } from "./discover.js";
|
||||
|
||||
const acquireGatewayLock = vi.fn(async (_opts?: { port?: number }) => ({
|
||||
@@ -16,9 +19,21 @@ const acquireGatewayLock = vi.fn(async (_opts?: { port?: number }) => ({
|
||||
const consumeGatewayRestartIntentPayloadSync = vi.fn<
|
||||
() => { reason?: string; force?: boolean; waitMs?: number } | null
|
||||
>(() => null);
|
||||
const consumeGatewaySigusr1RestartIntent = vi.fn<
|
||||
() => { reason?: string; force?: boolean; waitMs?: number } | null
|
||||
>(() => null);
|
||||
const consumeGatewaySigusr1RestartIntent = vi.fn<() => GatewayRestartIntent | null>(() => null);
|
||||
const managedUpdateSuccessorOwner = {
|
||||
kind: "managed-update-handoff",
|
||||
handoffId: "handoff-under-test",
|
||||
installRoot: "/openclaw/install",
|
||||
} as const;
|
||||
type ManagedUpdateOwner = NonNullable<GatewayRestartIntent["successorOwner"]>;
|
||||
const cancelManagedServiceUpdateHandoff = vi.fn<
|
||||
(_identity: ManagedUpdateOwner) => Promise<false | "restored-in-process" | "restart-after-exit">
|
||||
>(async () => "restored-in-process");
|
||||
const claimManagedServiceUpdateHandoff = vi.fn((_identity: ManagedUpdateOwner) => true);
|
||||
const requestManagedServiceUpdateHandoffPark = vi.fn(async (_identity: ManagedUpdateOwner) => true);
|
||||
const commitManagedServiceUpdateHandoff = vi.fn(
|
||||
async (_identity: ManagedUpdateOwner, _outcome?: "update" | "restore") => true,
|
||||
);
|
||||
const consumeGatewaySigusr1RestartAuthorization = vi.fn(() => true);
|
||||
const consumeGatewayRestartIntentSync = vi.fn(() => false);
|
||||
const isGatewaySigusr1RestartExternallyAllowed = vi.fn(() => false);
|
||||
@@ -117,19 +132,17 @@ const reloadTaskRuntimeStateFromStore = vi.fn();
|
||||
const clearRuntimeConfigSnapshot = vi.fn();
|
||||
const restartGatewayProcessWithFreshPid = vi.fn<
|
||||
(_opts?: { env?: NodeJS.ProcessEnv }) => {
|
||||
mode: "spawned" | "supervised" | "disabled" | "failed";
|
||||
pid?: number;
|
||||
mode: "supervised" | "disabled" | "failed";
|
||||
detail?: string;
|
||||
handoffSpawned?: Promise<boolean>;
|
||||
}
|
||||
>(() => ({ mode: "disabled" }));
|
||||
const respawnGatewayProcessForUpdate = vi.fn<
|
||||
(_opts?: { env?: NodeJS.ProcessEnv }) => {
|
||||
mode: "spawned" | "supervised" | "disabled" | "failed";
|
||||
mode: "spawned" | "disabled" | "failed";
|
||||
pid?: number;
|
||||
detail?: string;
|
||||
child?: { kill: () => void };
|
||||
handoffSpawned?: Promise<boolean>;
|
||||
}
|
||||
>(() => ({ mode: "disabled", detail: "OPENCLAW_NO_RESPAWN" }));
|
||||
const markUpdateRestartSentinelFailure = vi.fn<(reason: string) => Promise<null>>(
|
||||
@@ -188,6 +201,19 @@ vi.mock("../../infra/restart-intent.js", () => ({
|
||||
consumeGatewayRestartIntentSync: () => consumeGatewayRestartIntentSync(),
|
||||
}));
|
||||
|
||||
vi.mock("../../infra/update-managed-service-handoff.js", () => ({
|
||||
cancelManagedServiceUpdateHandoff: (identity: ManagedUpdateOwner) =>
|
||||
cancelManagedServiceUpdateHandoff(identity),
|
||||
claimManagedServiceUpdateHandoff: (identity: ManagedUpdateOwner) =>
|
||||
claimManagedServiceUpdateHandoff(identity),
|
||||
requestManagedServiceUpdateHandoffPark: (identity: ManagedUpdateOwner) =>
|
||||
requestManagedServiceUpdateHandoffPark(identity),
|
||||
commitManagedServiceUpdateHandoff: (
|
||||
identity: ManagedUpdateOwner,
|
||||
outcome?: "update" | "restore",
|
||||
) => commitManagedServiceUpdateHandoff(identity, outcome),
|
||||
}));
|
||||
|
||||
vi.mock("../../infra/gateway-suspend-coordinator.js", () => ({
|
||||
resetGatewaySuspendCoordinatorForLifecycleRestart: () =>
|
||||
resetGatewaySuspendCoordinatorForLifecycleRestart(),
|
||||
@@ -473,8 +499,29 @@ function expectRestartHandoffCall(expected: {
|
||||
}
|
||||
|
||||
let gatewayWorkAdmissionActual: typeof import("../../process/gateway-work-admission.js");
|
||||
let supervisorEnvSnapshot: ReturnType<typeof captureEnv> | undefined;
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.useRealTimers();
|
||||
supervisorEnvSnapshot = captureEnv([...SUPERVISOR_HINT_ENV_VARS]);
|
||||
for (const key of SUPERVISOR_HINT_ENV_VARS) {
|
||||
deleteTestEnvValue(key);
|
||||
}
|
||||
|
||||
// clearAllMocks preserves queued one-shot results. A skipped lifecycle branch
|
||||
// must not shift a stale supervisor or respawn decision into the next case.
|
||||
consumeGatewaySigusr1RestartIntent.mockReset();
|
||||
consumeGatewaySigusr1RestartIntent.mockReturnValue(null);
|
||||
peekGatewaySigusr1RestartReason.mockReset();
|
||||
peekGatewaySigusr1RestartReason.mockReturnValue(undefined);
|
||||
restartGatewayProcessWithFreshPid.mockReset();
|
||||
restartGatewayProcessWithFreshPid.mockReturnValue({ mode: "disabled" });
|
||||
respawnGatewayProcessForUpdate.mockReset();
|
||||
respawnGatewayProcessForUpdate.mockReturnValue({
|
||||
mode: "disabled",
|
||||
detail: "OPENCLAW_NO_RESPAWN",
|
||||
});
|
||||
|
||||
gatewayWorkAdmissionActual = await vi.importActual("../../process/gateway-work-admission.js");
|
||||
gatewayWorkAdmissionActual.resetGatewayWorkAdmission();
|
||||
createGatewayActiveWorkSnapshot.mockReset();
|
||||
@@ -485,6 +532,23 @@ beforeEach(async () => {
|
||||
options?.onSnapshot?.(snapshot);
|
||||
return { drained: snapshot.idle, snapshot };
|
||||
});
|
||||
cancelManagedServiceUpdateHandoff.mockReset();
|
||||
cancelManagedServiceUpdateHandoff.mockResolvedValue("restored-in-process");
|
||||
claimManagedServiceUpdateHandoff.mockReset();
|
||||
claimManagedServiceUpdateHandoff.mockReturnValue(true);
|
||||
requestManagedServiceUpdateHandoffPark.mockReset();
|
||||
requestManagedServiceUpdateHandoffPark.mockResolvedValue(true);
|
||||
commitManagedServiceUpdateHandoff.mockReset();
|
||||
commitManagedServiceUpdateHandoff.mockResolvedValue(true);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
supervisorEnvSnapshot?.restore();
|
||||
supervisorEnvSnapshot = undefined;
|
||||
vi.useRealTimers();
|
||||
if (originalPlatformDescriptor) {
|
||||
Object.defineProperty(process, "platform", originalPlatformDescriptor);
|
||||
}
|
||||
});
|
||||
|
||||
describe("runGatewayLoop", () => {
|
||||
@@ -1501,9 +1565,14 @@ describe("runGatewayLoop", () => {
|
||||
await withIsolatedSignals(async ({ captureSignal }) => {
|
||||
const close = vi.fn(async () => {});
|
||||
const startupNeverReturns = new Promise<void>(() => {});
|
||||
let markStartupEntered: () => void = () => {};
|
||||
const startupEntered = new Promise<void>((resolve) => {
|
||||
markStartupEntered = resolve;
|
||||
});
|
||||
const { runtime, exited } = createRuntimeWithExitSignal();
|
||||
const completeBoot = vi.fn();
|
||||
const start = vi.fn(async () => {
|
||||
markStartupEntered();
|
||||
await startupNeverReturns;
|
||||
return createGatewayServer(close);
|
||||
});
|
||||
@@ -1515,6 +1584,7 @@ describe("runGatewayLoop", () => {
|
||||
completeBoot,
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
await startupEntered;
|
||||
const sigusr1 = captureSignal("SIGUSR1");
|
||||
|
||||
sigusr1();
|
||||
@@ -2033,11 +2103,12 @@ describe("runGatewayLoop", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("releases the lock before exiting on spawned restart", async () => {
|
||||
it("releases the lock before exiting on supervised restart", async () => {
|
||||
vi.clearAllMocks();
|
||||
peekGatewaySigusr1RestartReason.mockReturnValue(undefined);
|
||||
const originalTraceEnv = process.env.OPENCLAW_GATEWAY_RESTART_TRACE;
|
||||
process.env.OPENCLAW_GATEWAY_RESTART_TRACE = "1";
|
||||
process.env.OPENCLAW_SUPERVISOR_MODE = "external";
|
||||
|
||||
try {
|
||||
await withIsolatedSignals(async ({ captureSignal }) => {
|
||||
@@ -2046,11 +2117,7 @@ describe("runGatewayLoop", () => {
|
||||
release: lockRelease,
|
||||
});
|
||||
|
||||
// Override process-respawn to return "spawned" mode
|
||||
restartGatewayProcessWithFreshPid.mockReturnValueOnce({
|
||||
mode: "spawned",
|
||||
pid: 9999,
|
||||
});
|
||||
restartGatewayProcessWithFreshPid.mockReturnValueOnce({ mode: "supervised" });
|
||||
|
||||
const exitCallOrder: string[] = [];
|
||||
const { runtime, exited } = await createSignaledLoopHarness(exitCallOrder);
|
||||
@@ -2068,9 +2135,10 @@ describe("runGatewayLoop", () => {
|
||||
const [respawnOpts] = restartGatewayProcessWithFreshPid.mock.calls[0] ?? [];
|
||||
expect(respawnOpts?.env?.OPENCLAW_GATEWAY_RESTART_TRACE_STARTED_AT_MS).toMatch(/^\d/u);
|
||||
expect(respawnOpts?.env?.OPENCLAW_GATEWAY_RESTART_TRACE_LAST_AT_MS).toMatch(/^\d/u);
|
||||
expect(writeGatewayRestartHandoffSync).not.toHaveBeenCalled();
|
||||
expect(writeGatewayRestartHandoffSync).toHaveBeenCalledOnce();
|
||||
});
|
||||
} finally {
|
||||
delete process.env.OPENCLAW_SUPERVISOR_MODE;
|
||||
if (originalTraceEnv === undefined) {
|
||||
delete process.env.OPENCLAW_GATEWAY_RESTART_TRACE;
|
||||
} else {
|
||||
@@ -2363,11 +2431,12 @@ describe("runGatewayLoop", () => {
|
||||
async (reason) => {
|
||||
vi.clearAllMocks();
|
||||
peekGatewaySigusr1RestartReason.mockReturnValue(reason);
|
||||
respawnGatewayProcessForUpdate.mockReturnValueOnce({
|
||||
restartGatewayProcessWithFreshPid.mockReturnValueOnce({
|
||||
mode: "supervised",
|
||||
});
|
||||
try {
|
||||
setPlatform("freebsd");
|
||||
process.env.OPENCLAW_SUPERVISOR_MODE = "external";
|
||||
await withIsolatedSignals(async ({ captureSignal }) => {
|
||||
const { runtime, exited } = await createSignaledLoopHarness();
|
||||
const sigusr1 = captureSignal("SIGUSR1");
|
||||
@@ -2381,8 +2450,10 @@ describe("runGatewayLoop", () => {
|
||||
reason,
|
||||
supervisorMode: "external",
|
||||
});
|
||||
expect(respawnGatewayProcessForUpdate).not.toHaveBeenCalled();
|
||||
});
|
||||
} finally {
|
||||
delete process.env.OPENCLAW_SUPERVISOR_MODE;
|
||||
if (originalPlatformDescriptor) {
|
||||
Object.defineProperty(process, "platform", originalPlatformDescriptor);
|
||||
}
|
||||
@@ -2393,7 +2464,7 @@ describe("runGatewayLoop", () => {
|
||||
it("falls back in-process when a launchd update handoff fails to spawn", async () => {
|
||||
vi.clearAllMocks();
|
||||
peekGatewaySigusr1RestartReason.mockReturnValue("update.run");
|
||||
respawnGatewayProcessForUpdate.mockReturnValueOnce({
|
||||
restartGatewayProcessWithFreshPid.mockReturnValueOnce({
|
||||
mode: "supervised",
|
||||
handoffSpawned: Promise.resolve(false),
|
||||
});
|
||||
@@ -2431,7 +2502,7 @@ describe("runGatewayLoop", () => {
|
||||
vi.clearAllMocks();
|
||||
peekGatewaySigusr1RestartReason.mockReturnValue("update.run");
|
||||
process.env.OPENCLAW_SUPERVISOR_MODE = "external";
|
||||
respawnGatewayProcessForUpdate.mockReturnValueOnce({
|
||||
restartGatewayProcessWithFreshPid.mockReturnValueOnce({
|
||||
mode: "supervised",
|
||||
});
|
||||
writeGatewayRestartHandoffSync.mockReturnValueOnce(null);
|
||||
@@ -2461,19 +2532,12 @@ describe("runGatewayLoop", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("upgrades an accepted restart when a managed update arrives during shutdown", async () => {
|
||||
it("upgrades an accepted restart when an update arrives during shutdown", async () => {
|
||||
vi.clearAllMocks();
|
||||
consumeGatewayRestartIntentPayloadSync.mockReset();
|
||||
consumeGatewayRestartIntentPayloadSync.mockReturnValue(null);
|
||||
consumeGatewaySigusr1RestartIntent.mockReset();
|
||||
consumeGatewaySigusr1RestartIntent.mockReturnValue(null);
|
||||
consumeGatewaySigusr1RestartAuthorization.mockReset();
|
||||
consumeGatewaySigusr1RestartAuthorization.mockReturnValue(true);
|
||||
peekGatewaySigusr1RestartReason.mockReset();
|
||||
peekGatewaySigusr1RestartReason
|
||||
.mockReturnValueOnce("config.patch")
|
||||
.mockReturnValueOnce("update.auto");
|
||||
respawnGatewayProcessForUpdate.mockReturnValueOnce({ mode: "supervised" });
|
||||
restartGatewayProcessWithFreshPid.mockReturnValueOnce({ mode: "supervised" });
|
||||
|
||||
let releaseClose: () => void = () => {};
|
||||
const close = vi.fn<GatewayCloseFn>(
|
||||
@@ -2482,9 +2546,9 @@ describe("runGatewayLoop", () => {
|
||||
releaseClose = resolve;
|
||||
}),
|
||||
);
|
||||
|
||||
setPlatform("freebsd");
|
||||
process.env.OPENCLAW_SUPERVISOR_MODE = "external";
|
||||
try {
|
||||
setPlatform("freebsd");
|
||||
await withIsolatedSignals(async ({ captureSignal }) => {
|
||||
const { start, started } = createSignaledStart(close);
|
||||
const { runtime, exited } = createRuntimeWithExitSignal();
|
||||
@@ -2492,58 +2556,41 @@ describe("runGatewayLoop", () => {
|
||||
await waitForStart(started);
|
||||
const sigusr1 = captureSignal("SIGUSR1");
|
||||
|
||||
try {
|
||||
sigusr1();
|
||||
await waitForLoopCondition(
|
||||
() => close.mock.calls.length === 1,
|
||||
"restart close did not start",
|
||||
);
|
||||
expect(armShutdownHardExitWatchdog).toHaveBeenCalledTimes(1);
|
||||
sigusr1();
|
||||
await waitForLoopCondition(
|
||||
() =>
|
||||
gatewayLog.info.mock.calls.some(([message]) =>
|
||||
String(message).includes("upgrading to update.auto"),
|
||||
),
|
||||
"accepted restart was not upgraded",
|
||||
);
|
||||
expect(cancelShutdownHardExitWatchdog).toHaveBeenCalledTimes(1);
|
||||
expect(armShutdownHardExitWatchdog).toHaveBeenCalledTimes(2);
|
||||
sigusr1();
|
||||
await waitForLoopCondition(
|
||||
() => close.mock.calls.length === 1,
|
||||
"restart close did not start",
|
||||
);
|
||||
sigusr1();
|
||||
await waitForLoopCondition(
|
||||
() =>
|
||||
gatewayLog.info.mock.calls.some(([message]) =>
|
||||
String(message).includes("upgrading to update.auto"),
|
||||
),
|
||||
"accepted restart was not upgraded",
|
||||
);
|
||||
|
||||
releaseClose();
|
||||
await expect(exited).resolves.toBe(0);
|
||||
expect(cancelShutdownHardExitWatchdog).toHaveBeenCalledTimes(2);
|
||||
expect(respawnGatewayProcessForUpdate).toHaveBeenCalledTimes(1);
|
||||
expectRestartHandoffCall({
|
||||
restartKind: "update-process",
|
||||
reason: "update.auto",
|
||||
supervisorMode: "external",
|
||||
});
|
||||
} finally {
|
||||
releaseClose();
|
||||
await exited;
|
||||
}
|
||||
releaseClose();
|
||||
await expect(exited).resolves.toBe(0);
|
||||
expect(restartGatewayProcessWithFreshPid).toHaveBeenCalledOnce();
|
||||
expectRestartHandoffCall({
|
||||
restartKind: "update-process",
|
||||
reason: "update.auto",
|
||||
supervisorMode: "external",
|
||||
});
|
||||
});
|
||||
} finally {
|
||||
if (originalPlatformDescriptor) {
|
||||
Object.defineProperty(process, "platform", originalPlatformDescriptor);
|
||||
}
|
||||
releaseClose();
|
||||
delete process.env.OPENCLAW_SUPERVISOR_MODE;
|
||||
}
|
||||
});
|
||||
|
||||
it("reads a managed update upgrade after asynchronous lock release", async () => {
|
||||
it("reads an update upgrade after asynchronous lock release", async () => {
|
||||
vi.clearAllMocks();
|
||||
consumeGatewayRestartIntentPayloadSync.mockReset();
|
||||
consumeGatewayRestartIntentPayloadSync.mockReturnValue(null);
|
||||
consumeGatewaySigusr1RestartIntent.mockReset();
|
||||
consumeGatewaySigusr1RestartIntent.mockReturnValue(null);
|
||||
consumeGatewaySigusr1RestartAuthorization.mockReset();
|
||||
consumeGatewaySigusr1RestartAuthorization.mockReturnValue(true);
|
||||
peekGatewaySigusr1RestartReason.mockReset();
|
||||
peekGatewaySigusr1RestartReason
|
||||
.mockReturnValueOnce("config.patch")
|
||||
.mockReturnValueOnce("update.auto");
|
||||
respawnGatewayProcessForUpdate.mockReturnValueOnce({ mode: "supervised" });
|
||||
restartGatewayProcessWithFreshPid.mockReturnValueOnce({ mode: "supervised" });
|
||||
|
||||
let releaseLock: () => void = () => {};
|
||||
const lockReleaseBlocked = new Promise<void>((resolve) => {
|
||||
@@ -2553,31 +2600,157 @@ describe("runGatewayLoop", () => {
|
||||
await lockReleaseBlocked;
|
||||
});
|
||||
acquireGatewayLock.mockResolvedValueOnce({ release: lockRelease });
|
||||
process.env.OPENCLAW_SUPERVISOR_MODE = "external";
|
||||
try {
|
||||
await withIsolatedSignals(async ({ captureSignal }) => {
|
||||
const { runtime, exited } = await createSignaledLoopHarness();
|
||||
const sigusr1 = captureSignal("SIGUSR1");
|
||||
sigusr1();
|
||||
await waitForLoopCondition(
|
||||
() => lockRelease.mock.calls.length === 1,
|
||||
"restart did not reach lock release",
|
||||
);
|
||||
sigusr1();
|
||||
await waitForLoopCondition(
|
||||
() =>
|
||||
gatewayLog.info.mock.calls.some(([message]) =>
|
||||
String(message).includes("upgrading to update.auto"),
|
||||
),
|
||||
"lock-release restart was not upgraded",
|
||||
);
|
||||
|
||||
await withIsolatedSignals(async ({ captureSignal }) => {
|
||||
const { runtime, exited } = await createSignaledLoopHarness();
|
||||
const sigusr1 = captureSignal("SIGUSR1");
|
||||
|
||||
sigusr1();
|
||||
await waitForLoopCondition(
|
||||
() => lockRelease.mock.calls.length === 1,
|
||||
"restart did not reach lock release",
|
||||
);
|
||||
sigusr1();
|
||||
await waitForLoopCondition(
|
||||
() =>
|
||||
gatewayLog.info.mock.calls.some(([message]) =>
|
||||
String(message).includes("upgrading to update.auto"),
|
||||
),
|
||||
"lock-release restart was not upgraded",
|
||||
);
|
||||
|
||||
releaseLock();
|
||||
await expect(exited).resolves.toBe(0);
|
||||
expect(restartGatewayProcessWithFreshPid).toHaveBeenCalledOnce();
|
||||
expect(runtime.exit).toHaveBeenCalledWith(0);
|
||||
});
|
||||
} finally {
|
||||
releaseLock();
|
||||
await expect(exited).resolves.toBe(0);
|
||||
expect(respawnGatewayProcessForUpdate).toHaveBeenCalledTimes(1);
|
||||
expect(restartGatewayProcessWithFreshPid).not.toHaveBeenCalled();
|
||||
expect(runtime.exit).toHaveBeenCalledWith(0);
|
||||
delete process.env.OPENCLAW_SUPERVISOR_MODE;
|
||||
}
|
||||
});
|
||||
|
||||
it("recovers in process after exactly cancelling a replacement managed owner before exit", async () => {
|
||||
vi.clearAllMocks();
|
||||
const replacementOwner = { ...managedUpdateSuccessorOwner, handoffId: "replacement-handoff" };
|
||||
consumeGatewaySigusr1RestartIntent
|
||||
.mockReturnValueOnce({ reason: "update.run", successorOwner: managedUpdateSuccessorOwner })
|
||||
.mockReturnValueOnce({ reason: "update.auto", successorOwner: replacementOwner });
|
||||
cancelManagedServiceUpdateHandoff
|
||||
.mockResolvedValueOnce("restored-in-process")
|
||||
.mockResolvedValueOnce("restored-in-process");
|
||||
|
||||
let releaseCommit: () => void = () => {};
|
||||
const commitBlocked = new Promise<void>((resolve) => {
|
||||
releaseCommit = resolve;
|
||||
});
|
||||
commitManagedServiceUpdateHandoff.mockImplementationOnce(async () => {
|
||||
await commitBlocked;
|
||||
return true;
|
||||
});
|
||||
setPlatform("linux");
|
||||
process.env.OPENCLAW_SERVICE_MARKER = "openclaw";
|
||||
process.env.OPENCLAW_SERVICE_KIND = "gateway";
|
||||
try {
|
||||
await withIsolatedSignals(async ({ captureSignal }) => {
|
||||
const { start, runtime, exited } = await createSignaledLoopHarness();
|
||||
const sigusr1 = captureSignal("SIGUSR1");
|
||||
const sigint = captureSignal("SIGINT");
|
||||
sigusr1();
|
||||
await waitForLoopCondition(
|
||||
() => commitManagedServiceUpdateHandoff.mock.calls.length === 1,
|
||||
"managed owner did not reach its final helper commit",
|
||||
);
|
||||
sigusr1();
|
||||
await waitForLoopCondition(
|
||||
() => consumeGatewaySigusr1RestartIntent.mock.calls.length === 2,
|
||||
"replacement owner was not admitted before exit",
|
||||
);
|
||||
releaseCommit();
|
||||
await waitForLoopCondition(
|
||||
() => start.mock.calls.length === 2,
|
||||
"replacement managed owner cancellation did not reopen gateway admission",
|
||||
);
|
||||
|
||||
expect(requestManagedServiceUpdateHandoffPark).toHaveBeenCalledExactlyOnceWith(
|
||||
managedUpdateSuccessorOwner,
|
||||
);
|
||||
expect(cancelManagedServiceUpdateHandoff).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
managedUpdateSuccessorOwner,
|
||||
);
|
||||
expect(cancelManagedServiceUpdateHandoff).toHaveBeenNthCalledWith(2, replacementOwner);
|
||||
expect(commitManagedServiceUpdateHandoff).toHaveBeenCalledExactlyOnceWith(
|
||||
managedUpdateSuccessorOwner,
|
||||
"update",
|
||||
);
|
||||
expect(runtime.exit).not.toHaveBeenCalled();
|
||||
expect(start).toHaveBeenCalledTimes(2);
|
||||
expect(gatewayWorkAdmissionActual.isGatewayWorkAdmissionClosed()).toBe(false);
|
||||
|
||||
sigint();
|
||||
await expect(exited).resolves.toBe(0);
|
||||
});
|
||||
} finally {
|
||||
releaseCommit();
|
||||
delete process.env.OPENCLAW_SERVICE_MARKER;
|
||||
delete process.env.OPENCLAW_SERVICE_KIND;
|
||||
}
|
||||
});
|
||||
|
||||
it("reopens admission after a broken control pipe waits for the exact helper to exit", async () => {
|
||||
vi.clearAllMocks();
|
||||
consumeGatewaySigusr1RestartIntent.mockReturnValueOnce({
|
||||
reason: "update.run",
|
||||
successorOwner: managedUpdateSuccessorOwner,
|
||||
});
|
||||
requestManagedServiceUpdateHandoffPark.mockResolvedValueOnce(false);
|
||||
let releaseHelperExit: () => void = () => {};
|
||||
const helperExit = new Promise<void>((resolve) => {
|
||||
releaseHelperExit = resolve;
|
||||
});
|
||||
cancelManagedServiceUpdateHandoff.mockImplementationOnce(async () => {
|
||||
await helperExit;
|
||||
return "restored-in-process";
|
||||
});
|
||||
setPlatform("linux");
|
||||
process.env.OPENCLAW_SERVICE_MARKER = "openclaw";
|
||||
process.env.OPENCLAW_SERVICE_KIND = "gateway";
|
||||
|
||||
try {
|
||||
await withIsolatedSignals(async ({ captureSignal }) => {
|
||||
const { start, runtime, exited } = await createSignaledLoopHarness();
|
||||
const sigusr1 = captureSignal("SIGUSR1");
|
||||
const sigint = captureSignal("SIGINT");
|
||||
|
||||
sigusr1();
|
||||
await waitForLoopCondition(
|
||||
() => cancelManagedServiceUpdateHandoff.mock.calls.length === 1,
|
||||
"broken helper control pipe did not begin exact-owner cancellation",
|
||||
);
|
||||
expect(start).toHaveBeenCalledOnce();
|
||||
expect(gatewayWorkAdmissionActual.isGatewayWorkAdmissionClosed()).toBe(true);
|
||||
releaseHelperExit();
|
||||
await waitForLoopCondition(
|
||||
() => start.mock.calls.length === 2,
|
||||
"broken helper control pipe left the gateway permanently draining",
|
||||
);
|
||||
|
||||
expect(cancelManagedServiceUpdateHandoff).toHaveBeenCalledExactlyOnceWith(
|
||||
managedUpdateSuccessorOwner,
|
||||
);
|
||||
expect(commitManagedServiceUpdateHandoff).not.toHaveBeenCalled();
|
||||
expect(runtime.exit).not.toHaveBeenCalled();
|
||||
expect(gatewayWorkAdmissionActual.isGatewayWorkAdmissionClosed()).toBe(false);
|
||||
|
||||
sigint();
|
||||
await expect(exited).resolves.toBe(0);
|
||||
});
|
||||
} finally {
|
||||
releaseHelperExit();
|
||||
delete process.env.OPENCLAW_SERVICE_MARKER;
|
||||
delete process.env.OPENCLAW_SERVICE_KIND;
|
||||
}
|
||||
});
|
||||
|
||||
it("probes the configured gateway host for update respawn health", async () => {
|
||||
|
||||
+370
-324
@@ -1,4 +1,5 @@
|
||||
// In-process gateway run loop, restart signaling, drain, and update respawn handling.
|
||||
import type { ChildProcess } from "node:child_process";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import net from "node:net";
|
||||
import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
|
||||
@@ -17,6 +18,7 @@ import {
|
||||
type GatewayBootLifecycleCompletion,
|
||||
} from "../../infra/gateway-boot-lifecycle.js";
|
||||
import { acquireGatewayLock } from "../../infra/gateway-lock.js";
|
||||
import type { GatewayRestartIntent } from "../../infra/restart-intent.js";
|
||||
import type { GatewayRestartEmitter } from "../../infra/restart.js";
|
||||
import { flushLogger } from "../../logging/logger.js";
|
||||
import { createSubsystemLogger } from "../../logging/subsystem.js";
|
||||
@@ -42,17 +44,11 @@ const LOG_FLUSH_EXIT_TIMEOUT_MS = 4_000;
|
||||
const HARD_EXIT_WATCHDOG_GRACE_MS = 2_000;
|
||||
|
||||
type GatewayRunSignalAction = "stop" | "restart";
|
||||
type RestartDrainTimeoutMs = number | undefined;
|
||||
type RestartIntentOptions = {
|
||||
reason?: string;
|
||||
force?: boolean;
|
||||
waitMs?: number;
|
||||
};
|
||||
type GatewayRunSignalRequest = {
|
||||
action: GatewayRunSignalAction;
|
||||
signal: string;
|
||||
restartReason?: string;
|
||||
restartIntent?: RestartIntentOptions;
|
||||
restartIntent?: GatewayRestartIntent;
|
||||
};
|
||||
|
||||
type GatewayLifecycleRuntimeModule = typeof import("./lifecycle.runtime.js");
|
||||
@@ -70,20 +66,11 @@ const loadGatewayLifecycleRuntimeModule = () => gatewayLifecycleRuntimeLoader.lo
|
||||
async function waitForGatewayPortReady(host: string, port: number): Promise<boolean> {
|
||||
return await new Promise<boolean>((resolve) => {
|
||||
const socket = net.createConnection({ host, port });
|
||||
let settled = false;
|
||||
const finish = (value: boolean) => {
|
||||
if (settled) {
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
socket.removeAllListeners();
|
||||
socket.destroy();
|
||||
resolve(value);
|
||||
};
|
||||
const timer = setTimeout(() => {
|
||||
finish(false);
|
||||
}, UPDATE_RESPAWN_HEALTH_POLL_MS);
|
||||
socket.setTimeout(UPDATE_RESPAWN_HEALTH_POLL_MS, () => finish(false));
|
||||
socket.once("connect", () => finish(true));
|
||||
socket.once("error", () => finish(false));
|
||||
});
|
||||
@@ -141,6 +128,11 @@ export async function runGatewayLoop(params: {
|
||||
// here pulls the lifecycle re-export graph into memory, immune to later disk
|
||||
// rotation.
|
||||
const eagerLifecycleRuntime = await loadGatewayLifecycleRuntimeModule();
|
||||
const supervisorMode = eagerLifecycleRuntime.detectGatewayRespawnSupervisor(
|
||||
process.env,
|
||||
process.platform,
|
||||
{ includeLinuxOpenClawGatewayServiceMarker: true },
|
||||
);
|
||||
let lock = await acquireGatewayLock({ port: params.lockPort });
|
||||
let server: Awaited<ReturnType<typeof startGatewayServer>> | null = null;
|
||||
let shuttingDown = false;
|
||||
@@ -149,12 +141,22 @@ export async function runGatewayLoop(params: {
|
||||
// Defer lifecycle signals from that window until the loop can close and advance.
|
||||
let pendingStartupRequest: GatewayRunSignalRequest | null = null;
|
||||
let activeRestartRequest: GatewayRunSignalRequest | null = null;
|
||||
let committedGenericSuccessor: ChildProcess | true | null = null;
|
||||
let forceActiveRestartExit: (() => void) | null = null;
|
||||
let pendingStartupForceExitTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
let restartDrainingMarked = false;
|
||||
let startupFailedWithoutServerHandle = false;
|
||||
const processInstanceId = randomUUID();
|
||||
const waitForHealthyChild = params.waitForHealthyChild ?? waitForHealthyGatewayChild;
|
||||
const getManagedUpdateOwner = () =>
|
||||
(pendingStartupRequest ?? activeRestartRequest)?.restartIntent?.successorOwner;
|
||||
const sameManagedUpdateOwner = (
|
||||
left: GatewayRestartIntent["successorOwner"],
|
||||
right: GatewayRestartIntent["successorOwner"],
|
||||
) =>
|
||||
Boolean(
|
||||
left && right && left.handoffId === right.handoffId && left.installRoot === right.installRoot,
|
||||
);
|
||||
|
||||
const cleanupSignals = () => {
|
||||
process.removeListener("SIGTERM", onSigterm);
|
||||
@@ -165,7 +167,13 @@ export async function runGatewayLoop(params: {
|
||||
cleanupSignals();
|
||||
params.runtime.exit(code);
|
||||
};
|
||||
const exitProcessAfterLogFlush = async (code: number) => {
|
||||
const exitProcessAfterLogFlush = async (
|
||||
code: number,
|
||||
initialOwner?: GatewayRestartIntent["successorOwner"],
|
||||
initialOutcome: "update" | "restore" = "update",
|
||||
): Promise<void> => {
|
||||
let ownerToCommit = initialOwner;
|
||||
let commitOutcome = initialOutcome;
|
||||
// Graceful signal/restart paths call process.exit(), which skips beforeExit.
|
||||
let flushTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
const flushed = await Promise.race([
|
||||
@@ -174,69 +182,179 @@ export async function runGatewayLoop(params: {
|
||||
flushTimer = setTimeout(() => resolve(false), LOG_FLUSH_EXIT_TIMEOUT_MS);
|
||||
}),
|
||||
]);
|
||||
if (flushTimer) {
|
||||
clearTimeout(flushTimer);
|
||||
}
|
||||
clearTimeout(flushTimer);
|
||||
if (!flushed) {
|
||||
gatewayLog.warn(
|
||||
`log flush did not settle within ${LOG_FLUSH_EXIT_TIMEOUT_MS}ms; continuing shutdown`,
|
||||
);
|
||||
}
|
||||
exitProcess(code);
|
||||
for (;;) {
|
||||
const owner = getManagedUpdateOwner();
|
||||
if (!owner) {
|
||||
if (!ownerToCommit) {
|
||||
exitProcess(code);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (
|
||||
sameManagedUpdateOwner(owner, ownerToCommit) &&
|
||||
eagerLifecycleRuntime.claimManagedServiceUpdateHandoff(owner) &&
|
||||
(await eagerLifecycleRuntime.commitManagedServiceUpdateHandoff(owner, commitOutcome)) &&
|
||||
sameManagedUpdateOwner(getManagedUpdateOwner(), owner) &&
|
||||
eagerLifecycleRuntime.claimManagedServiceUpdateHandoff(owner)
|
||||
) {
|
||||
// Keep exact request ownership live through the synchronous exit call.
|
||||
exitProcess(code);
|
||||
return;
|
||||
}
|
||||
await markRestartHandoffUnavailable();
|
||||
const ownerToCancel = ownerToCommit ?? owner;
|
||||
const restoration = await cancelManagedUpdateHandoffBeforeRecovery(ownerToCancel);
|
||||
if (!restoration) {
|
||||
const child = committedGenericSuccessor === true ? null : committedGenericSuccessor;
|
||||
if (child && child.exitCode === null && child.signalCode === null) {
|
||||
const exited = new Promise<void>((resolve) => {
|
||||
child.once("exit", () => {
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
try {
|
||||
child.kill("SIGKILL");
|
||||
await exited;
|
||||
} catch {}
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (restoration === "restart-after-exit") {
|
||||
ownerToCommit = ownerToCancel;
|
||||
commitOutcome = "restore";
|
||||
const currentRequest = pendingStartupRequest ?? activeRestartRequest;
|
||||
if (
|
||||
currentRequest &&
|
||||
!sameManagedUpdateOwner(currentRequest.restartIntent?.successorOwner, ownerToCancel)
|
||||
) {
|
||||
currentRequest.restartIntent = {
|
||||
...currentRequest.restartIntent,
|
||||
successorOwner: ownerToCancel,
|
||||
};
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (!committedGenericSuccessor && initialOwner) {
|
||||
return reacquireAndResumeInProcessRestart(getManagedUpdateOwner() ?? owner);
|
||||
}
|
||||
exitProcess(code);
|
||||
return;
|
||||
}
|
||||
};
|
||||
const completeForcedStop = (reason: string) => {
|
||||
params.completeBoot?.({ outcome: "forced_stop", reason });
|
||||
};
|
||||
const writeStabilityBundle = async (reason: string, error?: unknown) => {
|
||||
const { writeDiagnosticStabilityBundleForFailureSync } =
|
||||
await loadGatewayLifecycleRuntimeModule();
|
||||
const result = writeDiagnosticStabilityBundleForFailureSync(reason, error);
|
||||
const writeStabilityBundle = (reason: string, error?: unknown) => {
|
||||
const result = eagerLifecycleRuntime.writeDiagnosticStabilityBundleForFailureSync(
|
||||
reason,
|
||||
error,
|
||||
);
|
||||
if ("message" in result) {
|
||||
gatewayLog.warn(result.message);
|
||||
}
|
||||
};
|
||||
const releaseLockIfHeld = async (): Promise<boolean> => {
|
||||
if (!lock) {
|
||||
return false;
|
||||
}
|
||||
await lock.release();
|
||||
const releaseLockIfHeld = async (): Promise<void> => {
|
||||
await lock?.release();
|
||||
lock = null;
|
||||
return true;
|
||||
};
|
||||
const reacquireLockForInProcessRestart = async (): Promise<boolean> => {
|
||||
const cancelManagedUpdateHandoffBeforeRecovery = async (
|
||||
initialOwner = getManagedUpdateOwner(),
|
||||
): Promise<false | "restored-in-process" | "restart-after-exit"> => {
|
||||
let owner = initialOwner;
|
||||
let requiresParentExit = false;
|
||||
try {
|
||||
lock = await acquireGatewayLock({ port: params.lockPort });
|
||||
return true;
|
||||
for (;;) {
|
||||
if (!owner) {
|
||||
return requiresParentExit ? "restart-after-exit" : "restored-in-process";
|
||||
}
|
||||
const restoration = await eagerLifecycleRuntime.cancelManagedServiceUpdateHandoff(owner);
|
||||
if (!restoration) {
|
||||
gatewayLog.error("managed update handoff cancellation unconfirmed; remaining draining");
|
||||
return false;
|
||||
}
|
||||
requiresParentExit ||= restoration === "restart-after-exit";
|
||||
const replacement = getManagedUpdateOwner();
|
||||
if (!replacement || sameManagedUpdateOwner(owner, replacement)) {
|
||||
return requiresParentExit ? "restart-after-exit" : "restored-in-process";
|
||||
}
|
||||
owner = replacement;
|
||||
}
|
||||
} catch (err) {
|
||||
gatewayLog.error(`failed to reacquire gateway lock for in-process restart: ${String(err)}`);
|
||||
exitProcess(1);
|
||||
gatewayLog.error(`managed update handoff cancellation failed: ${formatErrorMessage(err)}`);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
const confirmLaunchdHandoff = async (respawn: {
|
||||
handoffSpawned?: Promise<boolean>;
|
||||
}): Promise<boolean> => {
|
||||
const delay = new Promise<void>((resolve) => {
|
||||
setTimeout(resolve, LAUNCHD_SUPERVISED_RESTART_EXIT_DELAY_MS);
|
||||
});
|
||||
const spawned = respawn.handoffSpawned
|
||||
? await Promise.race([respawn.handoffSpawned, delay.then(() => true)])
|
||||
: false;
|
||||
// Preserve the crash-loop throttle window even when spawn settles early.
|
||||
await delay;
|
||||
return spawned;
|
||||
const forceExitAfterStabilityBundle = async (reason: string) => {
|
||||
try {
|
||||
writeStabilityBundle(reason);
|
||||
} finally {
|
||||
const owner = getManagedUpdateOwner();
|
||||
if (owner) {
|
||||
forceActiveRestartExit?.();
|
||||
}
|
||||
const restoration = await cancelManagedUpdateHandoffBeforeRecovery(owner);
|
||||
if (restoration) {
|
||||
params.completeBoot?.({ outcome: "forced_stop", reason });
|
||||
if (restoration === "restart-after-exit") {
|
||||
await exitProcessAfterLogFlush(1, owner, "restore");
|
||||
} else {
|
||||
exitProcess(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
const handleRestartAfterServerClose = async () => {
|
||||
const reacquireAndResumeInProcessRestart = async (
|
||||
alreadyCancelledOwner?: GatewayRestartIntent["successorOwner"],
|
||||
): Promise<void> => {
|
||||
for (;;) {
|
||||
const restartRequest = activeRestartRequest;
|
||||
const restartOwner = restartRequest?.restartIntent?.successorOwner;
|
||||
const restoration = sameManagedUpdateOwner(restartOwner, alreadyCancelledOwner)
|
||||
? "restored-in-process"
|
||||
: await cancelManagedUpdateHandoffBeforeRecovery(restartOwner);
|
||||
if (!restoration) {
|
||||
return;
|
||||
}
|
||||
if (restoration === "restart-after-exit") {
|
||||
await releaseLockIfHeld();
|
||||
return exitProcessAfterLogFlush(0, restartOwner, "restore");
|
||||
}
|
||||
if (activeRestartRequest !== restartRequest) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
lock = await acquireGatewayLock({ port: params.lockPort });
|
||||
} catch (err) {
|
||||
if (activeRestartRequest !== restartRequest) {
|
||||
continue;
|
||||
}
|
||||
gatewayLog.error(`failed to reacquire gateway lock for in-process restart: ${String(err)}`);
|
||||
exitProcess(1);
|
||||
return;
|
||||
}
|
||||
if (activeRestartRequest === restartRequest) {
|
||||
activeRestartRequest = null;
|
||||
shuttingDown = false;
|
||||
restartResolver?.();
|
||||
return;
|
||||
}
|
||||
await releaseLockIfHeld();
|
||||
}
|
||||
};
|
||||
const markRestartHandoffUnavailable = async (reason = "restart-handoff-unavailable") => {
|
||||
await eagerLifecycleRuntime.markUpdateRestartSentinelFailure(reason).catch((err: unknown) => {
|
||||
gatewayLog.warn(`failed to mark update restart ${reason}: ${String(err)}`);
|
||||
});
|
||||
};
|
||||
const handleRestartAfterServerClose = async (
|
||||
expectedOwner?: GatewayRestartIntent["successorOwner"],
|
||||
cancelled = false,
|
||||
): Promise<void> => {
|
||||
await releaseLockIfHeld();
|
||||
const {
|
||||
detectGatewayRespawnSupervisor,
|
||||
markUpdateRestartSentinelFailure,
|
||||
respawnGatewayProcessForUpdate,
|
||||
restartGatewayProcessWithFreshPid,
|
||||
writeGatewayRestartHandoffSync,
|
||||
} = await loadGatewayLifecycleRuntimeModule();
|
||||
// Lock release and lazy lifecycle loading may yield while a managed update
|
||||
// upgrades this restart. Keep the request live until a restart path commits.
|
||||
// Lock release may yield while a managed update upgrades this restart.
|
||||
const restartReason = activeRestartRequest?.restartReason;
|
||||
params.completeBoot?.({
|
||||
outcome: "planned_restart",
|
||||
@@ -244,214 +362,122 @@ export async function runGatewayLoop(params: {
|
||||
});
|
||||
const isUpdateRestart = isUpdateProcessRestartReason(restartReason);
|
||||
|
||||
if (isUpdateRestart) {
|
||||
const restartTraceHandoff = captureGatewayRestartTraceHandoff();
|
||||
const respawn = respawnGatewayProcessForUpdate({
|
||||
env: createGatewayRestartTraceHandoffEnv(restartTraceHandoff),
|
||||
});
|
||||
if (respawn.mode === "spawned") {
|
||||
const port = params.lockPort;
|
||||
const healthy =
|
||||
typeof port === "number"
|
||||
? await waitForHealthyChild(port, respawn.pid, params.healthHost ?? "127.0.0.1")
|
||||
: false;
|
||||
if (healthy) {
|
||||
activeRestartRequest = null;
|
||||
gatewayLog.info(
|
||||
`restart mode: update process respawn (spawned pid ${respawn.pid ?? "unknown"})`,
|
||||
);
|
||||
await exitProcessAfterLogFlush(0);
|
||||
return;
|
||||
}
|
||||
gatewayLog.warn(
|
||||
`update respawn child did not become healthy (${respawn.pid ?? "unknown"}); falling back to in-process restart`,
|
||||
);
|
||||
try {
|
||||
respawn.child?.kill();
|
||||
} catch {
|
||||
// Best-effort; parent fallback keeps the gateway reachable for recovery.
|
||||
}
|
||||
await markUpdateRestartSentinelFailure("restart-unhealthy").catch((err: unknown) => {
|
||||
gatewayLog.warn(`failed to mark update restart sentinel unhealthy: ${String(err)}`);
|
||||
});
|
||||
if (!(await reacquireLockForInProcessRestart())) {
|
||||
return;
|
||||
}
|
||||
shuttingDown = false;
|
||||
restartResolver?.();
|
||||
return;
|
||||
if (cancelled) {
|
||||
return reacquireAndResumeInProcessRestart(expectedOwner);
|
||||
}
|
||||
if (activeRestartRequest?.restartIntent?.successorOwner) {
|
||||
if (!expectedOwner) {
|
||||
gatewayLog.error("managed update handoff arrived after successor parking closed");
|
||||
await markRestartHandoffUnavailable();
|
||||
return reacquireAndResumeInProcessRestart();
|
||||
}
|
||||
if (respawn.mode === "supervised") {
|
||||
const supervisorMode = detectGatewayRespawnSupervisor(process.env, process.platform);
|
||||
markGatewayRestartTrace("restart.full-process-handoff", [
|
||||
["kind", "update-process"],
|
||||
["mode", respawn.mode],
|
||||
["supervisorMode", supervisorMode ?? "external"],
|
||||
]);
|
||||
const handoff = writeGatewayRestartHandoffSync({
|
||||
restartKind: "update-process",
|
||||
reason: restartReason,
|
||||
processInstanceId,
|
||||
supervisorMode: supervisorMode ?? "external",
|
||||
restartTrace: captureGatewayRestartTraceHandoff(),
|
||||
});
|
||||
if (supervisorMode === "external" && !handoff) {
|
||||
gatewayLog.warn(
|
||||
"external supervisor restart handoff could not be persisted; falling back to in-process restart",
|
||||
);
|
||||
await markUpdateRestartSentinelFailure("restart-handoff-unavailable").catch(
|
||||
(err: unknown) => {
|
||||
gatewayLog.warn(`failed to mark update restart handoff unavailable: ${String(err)}`);
|
||||
},
|
||||
);
|
||||
if (!(await reacquireLockForInProcessRestart())) {
|
||||
return;
|
||||
}
|
||||
activeRestartRequest = null;
|
||||
shuttingDown = false;
|
||||
restartResolver?.();
|
||||
return;
|
||||
gatewayLog.info("restart mode: managed update handoff owns successor");
|
||||
return exitProcessAfterLogFlush(0, expectedOwner);
|
||||
}
|
||||
|
||||
const respawnOptions = {
|
||||
env: createGatewayRestartTraceHandoffEnv(captureGatewayRestartTraceHandoff()),
|
||||
};
|
||||
const isStandaloneUpdate = isUpdateRestart && !supervisorMode;
|
||||
const respawn = isStandaloneUpdate
|
||||
? eagerLifecycleRuntime.respawnGatewayProcessForUpdate(respawnOptions)
|
||||
: eagerLifecycleRuntime.restartGatewayProcessWithFreshPid(respawnOptions);
|
||||
if (respawn.mode === "spawned") {
|
||||
const port = params.lockPort;
|
||||
const healthy =
|
||||
typeof port === "number"
|
||||
? await waitForHealthyChild(port, respawn.pid, params.healthHost ?? "127.0.0.1")
|
||||
: false;
|
||||
if (healthy) {
|
||||
committedGenericSuccessor = respawn.child ?? true;
|
||||
gatewayLog.info(
|
||||
`restart mode: update process respawn (spawned pid ${respawn.pid ?? "unknown"})`,
|
||||
);
|
||||
return exitProcessAfterLogFlush(0);
|
||||
}
|
||||
gatewayLog.warn(
|
||||
`update respawn child did not become healthy (${respawn.pid ?? "unknown"}); falling back to in-process restart`,
|
||||
);
|
||||
try {
|
||||
respawn.child?.kill();
|
||||
} catch {
|
||||
// Best-effort; parent fallback keeps the gateway reachable for recovery.
|
||||
}
|
||||
await markRestartHandoffUnavailable("restart-unhealthy");
|
||||
return reacquireAndResumeInProcessRestart();
|
||||
}
|
||||
if (respawn.mode === "supervised") {
|
||||
const restartKind = isUpdateRestart ? "update-process" : "full-process";
|
||||
markGatewayRestartTrace("restart.full-process-handoff", [
|
||||
["kind", restartKind],
|
||||
["mode", respawn.mode],
|
||||
["pid", "none"],
|
||||
["supervisorMode", supervisorMode ?? "none"],
|
||||
]);
|
||||
const handoff = eagerLifecycleRuntime.writeGatewayRestartHandoffSync({
|
||||
restartKind,
|
||||
reason: restartReason,
|
||||
processInstanceId,
|
||||
supervisorMode: supervisorMode ?? "external",
|
||||
restartTrace: captureGatewayRestartTraceHandoff(),
|
||||
});
|
||||
if (supervisorMode === "external" && !handoff) {
|
||||
gatewayLog.warn(
|
||||
"external supervisor restart handoff could not be persisted; falling back to in-process restart",
|
||||
);
|
||||
if (isUpdateRestart) {
|
||||
await markRestartHandoffUnavailable();
|
||||
}
|
||||
gatewayLog.info("restart mode: update process respawn (supervisor restart)");
|
||||
if (supervisorMode === "launchd" && !(await confirmLaunchdHandoff(respawn))) {
|
||||
return reacquireAndResumeInProcessRestart();
|
||||
}
|
||||
gatewayLog.info("restart mode: full process restart (supervisor restart)");
|
||||
if (supervisorMode === "launchd") {
|
||||
const delay = new Promise<void>((resolve) => {
|
||||
setTimeout(resolve, LAUNCHD_SUPERVISED_RESTART_EXIT_DELAY_MS);
|
||||
});
|
||||
const spawned = respawn.handoffSpawned
|
||||
? await Promise.race([respawn.handoffSpawned, delay.then(() => true)])
|
||||
: false;
|
||||
// Preserve the crash-loop throttle window even when spawn settles early.
|
||||
await delay;
|
||||
if (!spawned) {
|
||||
writeStabilityBundle("gateway.restart_handoff_spawn_failed");
|
||||
gatewayLog.warn(
|
||||
"launchd restart handoff failed to spawn; falling back to in-process restart",
|
||||
);
|
||||
await markUpdateRestartSentinelFailure("restart-handoff-unavailable").catch(
|
||||
(err: unknown) => {
|
||||
gatewayLog.warn(`failed to mark update restart handoff unavailable: ${String(err)}`);
|
||||
},
|
||||
);
|
||||
if (!(await reacquireLockForInProcessRestart())) {
|
||||
return;
|
||||
if (isUpdateRestart) {
|
||||
await markRestartHandoffUnavailable();
|
||||
}
|
||||
activeRestartRequest = null;
|
||||
shuttingDown = false;
|
||||
restartResolver?.();
|
||||
return;
|
||||
}
|
||||
activeRestartRequest = null;
|
||||
await exitProcessAfterLogFlush(0);
|
||||
return;
|
||||
}
|
||||
if (respawn.mode === "failed") {
|
||||
gatewayLog.warn(
|
||||
`update respawn failed (${respawn.detail ?? "unknown error"}); falling back to in-process restart`,
|
||||
);
|
||||
await markUpdateRestartSentinelFailure("restart-unhealthy").catch((err: unknown) => {
|
||||
gatewayLog.warn(`failed to mark update restart sentinel unhealthy: ${String(err)}`);
|
||||
});
|
||||
} else {
|
||||
gatewayLog.info(
|
||||
`restart mode: in-process restart (${respawn.detail ?? "OPENCLAW_NO_RESPAWN"})`,
|
||||
);
|
||||
}
|
||||
if (!(await reacquireLockForInProcessRestart())) {
|
||||
return;
|
||||
}
|
||||
activeRestartRequest = null;
|
||||
shuttingDown = false;
|
||||
restartResolver?.();
|
||||
return;
|
||||
}
|
||||
|
||||
// Release the lock BEFORE spawning so the child can acquire it immediately.
|
||||
const restartTraceHandoff = captureGatewayRestartTraceHandoff();
|
||||
const respawn = restartGatewayProcessWithFreshPid({
|
||||
env: createGatewayRestartTraceHandoffEnv(restartTraceHandoff),
|
||||
});
|
||||
if (respawn.mode === "spawned" || respawn.mode === "supervised") {
|
||||
const supervisorMode =
|
||||
respawn.mode === "supervised"
|
||||
? detectGatewayRespawnSupervisor(process.env, process.platform)
|
||||
: null;
|
||||
const modeLabel =
|
||||
respawn.mode === "spawned"
|
||||
? `spawned pid ${respawn.pid ?? "unknown"}`
|
||||
: "supervisor restart";
|
||||
markGatewayRestartTrace("restart.full-process-handoff", [
|
||||
["kind", "full-process"],
|
||||
["mode", respawn.mode],
|
||||
["pid", respawn.mode === "spawned" ? (respawn.pid ?? "unknown") : "none"],
|
||||
["supervisorMode", supervisorMode ?? "none"],
|
||||
]);
|
||||
if (respawn.mode === "supervised") {
|
||||
const handoff = writeGatewayRestartHandoffSync({
|
||||
restartKind: "full-process",
|
||||
reason: restartReason,
|
||||
processInstanceId,
|
||||
supervisorMode: supervisorMode ?? "external",
|
||||
restartTrace: captureGatewayRestartTraceHandoff(),
|
||||
});
|
||||
if (supervisorMode === "external" && !handoff) {
|
||||
gatewayLog.warn(
|
||||
"external supervisor restart handoff could not be persisted; falling back to in-process restart",
|
||||
);
|
||||
if (!(await reacquireLockForInProcessRestart())) {
|
||||
return;
|
||||
}
|
||||
activeRestartRequest = null;
|
||||
shuttingDown = false;
|
||||
restartResolver?.();
|
||||
return;
|
||||
return reacquireAndResumeInProcessRestart();
|
||||
}
|
||||
}
|
||||
gatewayLog.info(`restart mode: full process restart (${modeLabel})`);
|
||||
if (supervisorMode === "launchd" && !(await confirmLaunchdHandoff(respawn))) {
|
||||
await writeStabilityBundle("gateway.restart_handoff_spawn_failed");
|
||||
gatewayLog.warn(
|
||||
"launchd restart handoff failed to spawn; falling back to in-process restart",
|
||||
);
|
||||
if (!(await reacquireLockForInProcessRestart())) {
|
||||
return;
|
||||
}
|
||||
activeRestartRequest = null;
|
||||
shuttingDown = false;
|
||||
restartResolver?.();
|
||||
return;
|
||||
}
|
||||
activeRestartRequest = null;
|
||||
await exitProcessAfterLogFlush(0);
|
||||
return;
|
||||
committedGenericSuccessor = true;
|
||||
return exitProcessAfterLogFlush(0);
|
||||
}
|
||||
if (respawn.mode === "failed") {
|
||||
await writeStabilityBundle("gateway.restart_respawn_failed");
|
||||
if (!isStandaloneUpdate) {
|
||||
writeStabilityBundle("gateway.restart_respawn_failed");
|
||||
}
|
||||
gatewayLog.warn(
|
||||
`full process restart failed (${respawn.detail ?? "unknown error"}); falling back to in-process restart`,
|
||||
`${isStandaloneUpdate ? "update respawn" : "full process restart"} failed (${respawn.detail ?? "unknown error"}); falling back to in-process restart`,
|
||||
);
|
||||
if (isUpdateRestart) {
|
||||
await markRestartHandoffUnavailable("restart-unhealthy");
|
||||
}
|
||||
} else {
|
||||
gatewayLog.info(
|
||||
`restart mode: in-process restart (${respawn.detail ?? "OPENCLAW_NO_RESPAWN"})`,
|
||||
);
|
||||
}
|
||||
if (isUpdateProcessRestartReason(activeRestartRequest?.restartReason)) {
|
||||
await handleRestartAfterServerClose();
|
||||
return;
|
||||
if (!isUpdateRestart && isUpdateProcessRestartReason(activeRestartRequest?.restartReason)) {
|
||||
return handleRestartAfterServerClose();
|
||||
}
|
||||
if (!(await reacquireLockForInProcessRestart())) {
|
||||
return;
|
||||
}
|
||||
if (isUpdateProcessRestartReason(activeRestartRequest?.restartReason)) {
|
||||
await handleRestartAfterServerClose();
|
||||
return;
|
||||
}
|
||||
activeRestartRequest = null;
|
||||
shuttingDown = false;
|
||||
restartResolver?.();
|
||||
return reacquireAndResumeInProcessRestart();
|
||||
};
|
||||
const handleStopAfterServerClose = async () => {
|
||||
params.completeBoot?.({ outcome: "clean_stop", reason: "gateway.stop" });
|
||||
await releaseLockIfHeld();
|
||||
await exitProcessAfterLogFlush(0);
|
||||
};
|
||||
|
||||
const SUPERVISOR_STOP_TIMEOUT_MS = 30_000;
|
||||
const SHUTDOWN_TIMEOUT_MS = SUPERVISOR_STOP_TIMEOUT_MS - 5_000;
|
||||
const clearPendingStartupForceExitTimer = () => {
|
||||
if (!pendingStartupForceExitTimer) {
|
||||
return;
|
||||
}
|
||||
clearTimeout(pendingStartupForceExitTimer);
|
||||
clearTimeout(pendingStartupForceExitTimer ?? undefined);
|
||||
pendingStartupForceExitTimer = null;
|
||||
};
|
||||
const armPendingStartupForceExitTimer = () => {
|
||||
@@ -463,20 +489,13 @@ export async function runGatewayLoop(params: {
|
||||
gatewayLog.error(
|
||||
"startup restart request timed out before gateway returned a close handle; exiting for supervisor recovery",
|
||||
);
|
||||
void (async () => {
|
||||
try {
|
||||
await writeStabilityBundle("gateway.restart_startup_request_timeout");
|
||||
} finally {
|
||||
completeForcedStop("gateway.restart_startup_request_timeout");
|
||||
exitProcess(1);
|
||||
}
|
||||
})();
|
||||
void forceExitAfterStabilityBundle("gateway.restart_startup_request_timeout");
|
||||
}, SHUTDOWN_TIMEOUT_MS);
|
||||
pendingStartupForceExitTimer.unref?.();
|
||||
};
|
||||
const resolveRestartDrainTimeoutMs = async (
|
||||
restartIntent?: RestartIntentOptions,
|
||||
): Promise<RestartDrainTimeoutMs> => {
|
||||
const resolveRestartDrainTimeoutMs = (
|
||||
restartIntent?: GatewayRestartIntent,
|
||||
): number | undefined => {
|
||||
if (restartIntent?.force) {
|
||||
return 0;
|
||||
}
|
||||
@@ -484,8 +503,7 @@ export async function runGatewayLoop(params: {
|
||||
return restartIntent.waitMs > 0 ? Math.floor(restartIntent.waitMs) : undefined;
|
||||
}
|
||||
try {
|
||||
const { resolveGatewayRestartDeferralTimeoutMs } = await loadGatewayLifecycleRuntimeModule();
|
||||
return resolveGatewayRestartDeferralTimeoutMs();
|
||||
return eagerLifecycleRuntime.resolveGatewayRestartDeferralTimeoutMs();
|
||||
} catch {
|
||||
return DEFAULT_RESTART_DRAIN_TIMEOUT_MS;
|
||||
}
|
||||
@@ -515,21 +533,9 @@ export async function runGatewayLoop(params: {
|
||||
}
|
||||
forceExitTimer = setTimeout(() => {
|
||||
gatewayLog.error("shutdown timed out; exiting without full cleanup");
|
||||
void (async () => {
|
||||
try {
|
||||
await writeStabilityBundle(
|
||||
isRestart ? "gateway.restart_shutdown_timeout" : "gateway.stop_shutdown_timeout",
|
||||
);
|
||||
} finally {
|
||||
// Keep the in-process watchdog below the supervisor stop budget so this
|
||||
// path wins before launchd/systemd escalates to a hard kill. Exit
|
||||
// non-zero on any timeout so supervised installs restart cleanly.
|
||||
completeForcedStop(
|
||||
isRestart ? "gateway.restart_shutdown_timeout" : "gateway.stop_shutdown_timeout",
|
||||
);
|
||||
exitProcess(1);
|
||||
}
|
||||
})();
|
||||
void forceExitAfterStabilityBundle(
|
||||
isRestart ? "gateway.restart_shutdown_timeout" : "gateway.stop_shutdown_timeout",
|
||||
);
|
||||
}, forceExitMs);
|
||||
if (params.ownsProcessLifecycle === true) {
|
||||
hardExitWatchdog = armShutdownHardExitWatchdog({
|
||||
@@ -543,31 +549,36 @@ export async function runGatewayLoop(params: {
|
||||
}
|
||||
};
|
||||
const clearForceExitTimer = () => {
|
||||
if (forceExitTimer) {
|
||||
clearTimeout(forceExitTimer);
|
||||
forceExitTimer = null;
|
||||
}
|
||||
clearTimeout(forceExitTimer ?? undefined);
|
||||
forceExitTimer = null;
|
||||
hardExitWatchdog?.cancel();
|
||||
hardExitWatchdog = null;
|
||||
};
|
||||
if (isRestart) {
|
||||
forceActiveRestartExit = () => {
|
||||
clearForceExitTimer();
|
||||
armForceExitTimer(SHUTDOWN_TIMEOUT_MS);
|
||||
if (!getManagedUpdateOwner()) {
|
||||
armForceExitTimer(SHUTDOWN_TIMEOUT_MS);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
void (async () => {
|
||||
const restartDrainTimeoutMs = isRestart
|
||||
? await resolveRestartDrainTimeoutMs(restartIntent)
|
||||
: 0;
|
||||
let managedUpdateOwner: GatewayRestartIntent["successorOwner"];
|
||||
let managedUpdateCancellation:
|
||||
| false
|
||||
| "restored-in-process"
|
||||
| "restart-after-exit"
|
||||
| undefined;
|
||||
const restartDrainTimeoutMs = isRestart ? resolveRestartDrainTimeoutMs(restartIntent) : 0;
|
||||
const restartDrainDeadlineAt =
|
||||
isRestart && restartDrainTimeoutMs !== undefined
|
||||
? Date.now() + restartDrainTimeoutMs
|
||||
: undefined;
|
||||
// Managed helpers must reach native parking before either exit watchdog can arm.
|
||||
if (!isRestart) {
|
||||
armForceExitTimer(SHUTDOWN_TIMEOUT_MS);
|
||||
} else if (restartDrainTimeoutMs !== undefined) {
|
||||
} else if (restartDrainTimeoutMs !== undefined && !getManagedUpdateOwner()) {
|
||||
// Allow extra time for draining active turns on explicitly capped restarts.
|
||||
armForceExitTimer(restartDrainTimeoutMs + SHUTDOWN_TIMEOUT_MS);
|
||||
}
|
||||
@@ -576,21 +587,6 @@ export async function runGatewayLoop(params: {
|
||||
restartDrainTimeoutMs === undefined
|
||||
? "without a timeout"
|
||||
: `with timeout ${restartDrainTimeoutMs}ms`;
|
||||
const armCloseForceExitTimerForIndefiniteRestart = () => {
|
||||
if (isRestart && restartDrainTimeoutMs === undefined) {
|
||||
armForceExitTimer(SHUTDOWN_TIMEOUT_MS);
|
||||
}
|
||||
};
|
||||
const resolveRestartCloseDrainTimeoutMs = () => {
|
||||
if (!isRestart) {
|
||||
return null;
|
||||
}
|
||||
if (restartDrainTimeoutMs === undefined) {
|
||||
return Math.max(0, SHUTDOWN_TIMEOUT_MS - RESTART_CLOSE_REPLY_DRAIN_SHUTDOWN_RESERVE_MS);
|
||||
}
|
||||
return Math.max(0, (restartDrainDeadlineAt ?? Date.now()) - Date.now());
|
||||
};
|
||||
|
||||
try {
|
||||
// On restart, wait for the canonical process activity inventory before
|
||||
// tearing down the server so active work can settle.
|
||||
@@ -610,12 +606,6 @@ export async function runGatewayLoop(params: {
|
||||
markRestartAbortedMainSessions,
|
||||
waitForGatewayActiveWork,
|
||||
} = await loadGatewayLifecycleRuntimeModule();
|
||||
const collectActiveRestartSessionKeys = () => {
|
||||
return new Set<string>(listActiveEmbeddedRunSessionKeys());
|
||||
};
|
||||
const collectActiveRestartSessionIds = () => {
|
||||
return new Set<string>(listActiveEmbeddedRunSessionIds());
|
||||
};
|
||||
let activeRestartSessionKeysAtDrainStart = new Set<string>();
|
||||
let activeRestartSessionIdsAtDrainStart = new Set<string>();
|
||||
let hasMarkedActiveMainSessionsForRestart = false;
|
||||
@@ -627,11 +617,11 @@ export async function runGatewayLoop(params: {
|
||||
}
|
||||
const sessionKeys = new Set<string>([
|
||||
...activeRestartSessionKeysAtDrainStart,
|
||||
...collectActiveRestartSessionKeys(),
|
||||
...listActiveEmbeddedRunSessionKeys(),
|
||||
]);
|
||||
const sessionIds = new Set<string>([
|
||||
...activeRestartSessionIdsAtDrainStart,
|
||||
...collectActiveRestartSessionIds(),
|
||||
...listActiveEmbeddedRunSessionIds(),
|
||||
]);
|
||||
if (sessionKeys.size === 0 && sessionIds.size === 0) {
|
||||
return;
|
||||
@@ -643,9 +633,7 @@ export async function runGatewayLoop(params: {
|
||||
sessionIds,
|
||||
reason,
|
||||
});
|
||||
if (result.marked > 0) {
|
||||
hasMarkedActiveMainSessionsForRestart = true;
|
||||
}
|
||||
hasMarkedActiveMainSessionsForRestart = result.marked > 0;
|
||||
} catch (err) {
|
||||
gatewayLog.warn(
|
||||
`failed to mark interrupted main sessions for restart recovery: ${String(err)}`,
|
||||
@@ -662,8 +650,8 @@ export async function runGatewayLoop(params: {
|
||||
const initialSnapshot = createGatewayActiveWorkSnapshot();
|
||||
activeWorkAtDrainStart = initialSnapshot.counts.totalActive;
|
||||
activeRunsAtDrainStart = initialSnapshot.counts.embeddedRuns;
|
||||
activeRestartSessionKeysAtDrainStart = collectActiveRestartSessionKeys();
|
||||
activeRestartSessionIdsAtDrainStart = collectActiveRestartSessionIds();
|
||||
activeRestartSessionKeysAtDrainStart = new Set(listActiveEmbeddedRunSessionKeys());
|
||||
activeRestartSessionIdsAtDrainStart = new Set(listActiveEmbeddedRunSessionIds());
|
||||
|
||||
// Best-effort abort for compacting runs so transcript settlement does
|
||||
// not remain pending across restart boundaries.
|
||||
@@ -725,9 +713,7 @@ export async function runGatewayLoop(params: {
|
||||
["force", restartIntent?.force === true],
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
if (!isRestart) {
|
||||
} else {
|
||||
// Keep all process-owned work alive without spending the shutdown reserve
|
||||
// that server teardown and the supervisor watchdog need.
|
||||
try {
|
||||
@@ -746,8 +732,48 @@ export async function runGatewayLoop(params: {
|
||||
}
|
||||
}
|
||||
|
||||
armCloseForceExitTimerForIndefiniteRestart();
|
||||
const closeDrainTimeoutMs = resolveRestartCloseDrainTimeoutMs();
|
||||
if (isRestart && activeRestartRequest?.restartIntent?.successorOwner) {
|
||||
const owner = activeRestartRequest.restartIntent.successorOwner;
|
||||
managedUpdateOwner = owner;
|
||||
try {
|
||||
if (
|
||||
!sameManagedUpdateOwner(getManagedUpdateOwner(), owner) ||
|
||||
!(await eagerLifecycleRuntime.requestManagedServiceUpdateHandoffPark(owner)) ||
|
||||
!sameManagedUpdateOwner(getManagedUpdateOwner(), owner) ||
|
||||
!eagerLifecycleRuntime.claimManagedServiceUpdateHandoff(owner)
|
||||
) {
|
||||
throw new Error("managed update helper lost exact ownership during service parking");
|
||||
}
|
||||
} catch (err) {
|
||||
clearForceExitTimer();
|
||||
gatewayLog.error(
|
||||
`managed update handoff could not park ${supervisorMode}: ${String(err)}`,
|
||||
);
|
||||
await markRestartHandoffUnavailable();
|
||||
managedUpdateCancellation = await cancelManagedUpdateHandoffBeforeRecovery(owner);
|
||||
if (!managedUpdateCancellation) {
|
||||
return;
|
||||
}
|
||||
if (managedUpdateCancellation === "restart-after-exit") {
|
||||
await releaseLockIfHeld();
|
||||
await exitProcessAfterLogFlush(0, owner, "restore");
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
isRestart &&
|
||||
!forceExitTimer &&
|
||||
(!managedUpdateOwner || managedUpdateCancellation === "restored-in-process")
|
||||
) {
|
||||
armForceExitTimer(SHUTDOWN_TIMEOUT_MS);
|
||||
}
|
||||
const closeDrainTimeoutMs = !isRestart
|
||||
? null
|
||||
: restartDrainTimeoutMs === undefined
|
||||
? SHUTDOWN_TIMEOUT_MS - RESTART_CLOSE_REPLY_DRAIN_SHUTDOWN_RESERVE_MS
|
||||
: Math.max(0, (restartDrainDeadlineAt ?? Date.now()) - Date.now());
|
||||
await server?.close({
|
||||
reason: isRestart ? "gateway restarting" : "gateway stopping",
|
||||
restartExpectedMs: isRestart ? 1500 : null,
|
||||
@@ -756,17 +782,28 @@ export async function runGatewayLoop(params: {
|
||||
} catch (err) {
|
||||
gatewayLog.error(`shutdown step failed (gateway server close): ${formatErrorMessage(err)}`);
|
||||
} finally {
|
||||
server = null;
|
||||
const handoffClosed =
|
||||
managedUpdateCancellation !== false && managedUpdateCancellation !== "restart-after-exit";
|
||||
if (handoffClosed) {
|
||||
server = null;
|
||||
}
|
||||
if (isRestart) {
|
||||
try {
|
||||
await handleRestartAfterServerClose();
|
||||
if (handoffClosed) {
|
||||
await handleRestartAfterServerClose(
|
||||
managedUpdateOwner,
|
||||
managedUpdateCancellation === "restored-in-process",
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
clearForceExitTimer();
|
||||
forceActiveRestartExit = null;
|
||||
}
|
||||
} else {
|
||||
clearForceExitTimer();
|
||||
await handleStopAfterServerClose();
|
||||
params.completeBoot?.({ outcome: "clean_stop", reason: "gateway.stop" });
|
||||
await releaseLockIfHeld();
|
||||
await exitProcessAfterLogFlush(0);
|
||||
}
|
||||
}
|
||||
})();
|
||||
@@ -788,7 +825,7 @@ export async function runGatewayLoop(params: {
|
||||
action: GatewayRunSignalAction,
|
||||
signal: string,
|
||||
restartReason?: string,
|
||||
restartIntent?: RestartIntentOptions,
|
||||
restartIntent?: GatewayRestartIntent,
|
||||
) => {
|
||||
const acceptedRequest = { action, signal, restartReason, restartIntent };
|
||||
if (shuttingDown) {
|
||||
@@ -797,7 +834,12 @@ export async function runGatewayLoop(params: {
|
||||
action === "restart" &&
|
||||
isUpdateProcessRestartReason(restartReason) &&
|
||||
currentRestartRequest?.action === "restart" &&
|
||||
!isUpdateProcessRestartReason(currentRestartRequest.restartReason)
|
||||
(!isUpdateProcessRestartReason(currentRestartRequest.restartReason) ||
|
||||
(restartIntent?.successorOwner &&
|
||||
!sameManagedUpdateOwner(
|
||||
restartIntent.successorOwner,
|
||||
currentRestartRequest.restartIntent?.successorOwner,
|
||||
)))
|
||||
) {
|
||||
const upgradedRequest = {
|
||||
...currentRestartRequest,
|
||||
@@ -900,6 +942,10 @@ export async function runGatewayLoop(params: {
|
||||
if (restartIntent) {
|
||||
abortPendingChannelReloads();
|
||||
const authorized = consumeGatewaySigusr1RestartAuthorization();
|
||||
const processLocalIntent = authorized ? consumeGatewaySigusr1RestartIntent() : null;
|
||||
if (processLocalIntent?.successorOwner) {
|
||||
Object.assign(restartIntent, processLocalIntent);
|
||||
}
|
||||
markRestartDraining();
|
||||
if (authorized) {
|
||||
markGatewaySigusr1RestartHandled();
|
||||
@@ -1088,7 +1134,7 @@ export async function runGatewayLoop(params: {
|
||||
}
|
||||
const errMsg = formatErrorMessage(err);
|
||||
const errStack = err instanceof Error && err.stack ? `\n${err.stack}` : "";
|
||||
await writeStabilityBundle("gateway.restart_startup_failed", err);
|
||||
writeStabilityBundle("gateway.restart_startup_failed", err);
|
||||
gatewayLog.error(
|
||||
`gateway startup failed: ${errMsg}. ` +
|
||||
`Process will stay alive; fix the issue and restart.${errStack}`,
|
||||
|
||||
@@ -48,6 +48,7 @@ const startManagedServiceUpdateHandoffMock = vi.fn<
|
||||
command: "openclaw update --yes --timeout 1800",
|
||||
logPath: "/tmp/openclaw-update-run-handoff/handoff.log",
|
||||
handoffId: "handoff-1",
|
||||
installRoot: "/tmp/openclaw",
|
||||
}));
|
||||
const scheduleGatewaySigusr1RestartMock = vi.fn(() => ({ scheduled: true }));
|
||||
const logGatewayInfoMock = vi.fn();
|
||||
@@ -87,8 +88,8 @@ vi.mock("../../infra/restart-sentinel.js", async () => {
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("../../infra/restart.js", () => ({
|
||||
resolveGatewayRestartDeferralTimeoutMs: () => 300_000,
|
||||
vi.mock("../../infra/restart.js", async () => ({
|
||||
...(await vi.importActual<typeof import("../../infra/restart.js")>("../../infra/restart.js")),
|
||||
scheduleGatewaySigusr1Restart: scheduleGatewaySigusr1RestartMock,
|
||||
}));
|
||||
|
||||
|
||||
@@ -55,13 +55,14 @@ type ManagedServiceUpdateHandoffResult = Awaited<
|
||||
>
|
||||
>;
|
||||
const startManagedServiceUpdateHandoffMock = vi.fn<
|
||||
(params?: { handoffId?: string }) => Promise<ManagedServiceUpdateHandoffResult>
|
||||
(params?: { handoffId?: string; root?: string }) => Promise<ManagedServiceUpdateHandoffResult>
|
||||
>(async (params) => ({
|
||||
status: "started",
|
||||
pid: 12345,
|
||||
command: "openclaw update --yes --timeout 1800",
|
||||
logPath: "/tmp/openclaw-update-run-handoff/handoff.log",
|
||||
handoffId: params?.handoffId,
|
||||
handoffId: params?.handoffId ?? "handoff-default",
|
||||
installRoot: params?.root ?? "/tmp/openclaw",
|
||||
}));
|
||||
|
||||
const scheduleGatewaySigusr1RestartMock = vi.fn(() => ({ scheduled: true }));
|
||||
@@ -88,9 +89,7 @@ vi.mock("../../config/config.js", () => ({
|
||||
readConfigFileSnapshot: readConfigFileSnapshotMock,
|
||||
}));
|
||||
|
||||
vi.mock("../../config/commands.flags.js", () => ({
|
||||
isRestartEnabled: isRestartEnabledMock,
|
||||
}));
|
||||
vi.mock("../../config/commands.flags.js", () => ({ isRestartEnabled: isRestartEnabledMock }));
|
||||
|
||||
vi.mock("../../config/sessions.js", () => ({
|
||||
extractDeliveryInfo: (sessionKey: string | undefined) => {
|
||||
@@ -124,19 +123,12 @@ vi.mock("../../infra/restart-sentinel.js", async () => {
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("../../infra/restart.js", () => ({
|
||||
resolveGatewayRestartDeferralTimeoutMs: (timeoutMs: unknown) => {
|
||||
if (typeof timeoutMs !== "number" || !Number.isFinite(timeoutMs)) {
|
||||
return 300_000;
|
||||
}
|
||||
return timeoutMs <= 0 ? undefined : Math.floor(timeoutMs);
|
||||
},
|
||||
vi.mock("../../infra/restart.js", async () => ({
|
||||
...(await vi.importActual<typeof import("../../infra/restart.js")>("../../infra/restart.js")),
|
||||
scheduleGatewaySigusr1Restart: scheduleGatewaySigusr1RestartMock,
|
||||
}));
|
||||
|
||||
vi.mock("../../infra/package-json.js", () => ({
|
||||
readPackageVersion: readPackageVersionMock,
|
||||
}));
|
||||
vi.mock("../../infra/package-json.js", () => ({ readPackageVersion: readPackageVersionMock }));
|
||||
|
||||
vi.mock("../../version.js", () => ({
|
||||
get VERSION() {
|
||||
@@ -291,12 +283,13 @@ beforeEach(() => {
|
||||
recordLatestUpdateRestartSentinelMock.mockClear();
|
||||
startManagedServiceUpdateHandoffMock.mockClear();
|
||||
startManagedServiceUpdateHandoffMock.mockImplementation(
|
||||
async (params?: { handoffId?: string }) => ({
|
||||
async (params?: { handoffId?: string; root?: string }) => ({
|
||||
status: "started" as const,
|
||||
pid: 12345,
|
||||
command: "openclaw update --yes --timeout 1800",
|
||||
logPath: "/tmp/openclaw-update-run-handoff/handoff.log",
|
||||
handoffId: params?.handoffId,
|
||||
handoffId: params?.handoffId ?? "handoff-default",
|
||||
installRoot: params?.root ?? "/tmp/openclaw",
|
||||
}),
|
||||
);
|
||||
scheduleGatewaySigusr1RestartMock.mockClear();
|
||||
@@ -550,8 +543,21 @@ describe("update.run restart scheduling", () => {
|
||||
const [restartParams] = firstMockCall(
|
||||
scheduleGatewaySigusr1RestartMock,
|
||||
"gateway restart schedule",
|
||||
) as [{ delayMs?: number; reason?: string; skipCooldown?: boolean; skipDeferral?: boolean }];
|
||||
) as [
|
||||
{
|
||||
delayMs?: number;
|
||||
reason?: string;
|
||||
successorOwner?: unknown;
|
||||
skipCooldown?: boolean;
|
||||
skipDeferral?: boolean;
|
||||
},
|
||||
];
|
||||
expect(restartParams?.reason).toBe("update.run");
|
||||
expect(restartParams?.successorOwner).toEqual({
|
||||
kind: "managed-update-handoff",
|
||||
handoffId: handoffParams.handoffId,
|
||||
installRoot: "/tmp/openclaw-global",
|
||||
});
|
||||
expect(restartParams?.skipCooldown).toBe(true);
|
||||
expect(restartParams?.skipDeferral).toBe(true);
|
||||
expect(payload?.ok).toBe(true);
|
||||
@@ -673,6 +679,7 @@ describe("update.run restart scheduling", () => {
|
||||
expect(startManagedServiceUpdateHandoffMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
supervisor: "systemd",
|
||||
restartDrainTimeoutMs: 300_000,
|
||||
restartDelayMs: 2000,
|
||||
}),
|
||||
);
|
||||
@@ -680,12 +687,34 @@ describe("update.run restart scheduling", () => {
|
||||
expect.objectContaining({
|
||||
delayMs: 2000,
|
||||
reason: "update.run",
|
||||
successorOwner: {
|
||||
kind: "managed-update-handoff",
|
||||
handoffId: expect.any(String),
|
||||
installRoot: "/tmp/openclaw-global",
|
||||
},
|
||||
skipCooldown: true,
|
||||
skipDeferral: true,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it.each(["launchd", "systemd"] as const)(
|
||||
"normalizes overflow-sized %s handoff and restart delays together",
|
||||
async (supervisor) => {
|
||||
detectRespawnSupervisorMock.mockReturnValueOnce(supervisor);
|
||||
mockGlobalInstallSurface();
|
||||
|
||||
await invokeUpdateRun({ restartDelayMs: 2_147_153_648 });
|
||||
|
||||
expect(startManagedServiceUpdateHandoffMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ supervisor, restartDelayMs: 60_000 }),
|
||||
);
|
||||
expect(scheduleGatewaySigusr1RestartMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ delayMs: 60_000 }),
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("starts managed package handoff when the gateway cwd is unavailable", async () => {
|
||||
detectRespawnSupervisorMock.mockReturnValueOnce("launchd");
|
||||
mockGlobalInstallSurface();
|
||||
@@ -902,27 +931,29 @@ describe("update.run restart scheduling", () => {
|
||||
expect(payload?.handoff?.status).toBe("started");
|
||||
});
|
||||
|
||||
it("does not hand off systemd-supervised git/dev updates from generic systemd markers alone", async () => {
|
||||
it("hands marker-only systemd git/dev updates to the helper for exact ownership verification", async () => {
|
||||
detectRespawnSupervisorMock.mockReturnValueOnce("systemd");
|
||||
mockGitInstallSurface("/tmp/openclaw-git");
|
||||
|
||||
const payload = await withProcessEnv(
|
||||
{
|
||||
OPENCLAW_SYSTEMD_UNIT: undefined,
|
||||
INVOCATION_ID: "8a77e69a8f604bf0b7984879b9f17a7c",
|
||||
OPENCLAW_SERVICE_MARKER: "openclaw",
|
||||
OPENCLAW_SERVICE_KIND: "gateway",
|
||||
},
|
||||
() => captureUpdateRunPayload(),
|
||||
);
|
||||
|
||||
expect(runGatewayUpdateMock).not.toHaveBeenCalled();
|
||||
expect(startManagedServiceUpdateHandoffMock).not.toHaveBeenCalled();
|
||||
expect(scheduleGatewaySigusr1RestartMock).not.toHaveBeenCalled();
|
||||
expect(payload?.ok).toBe(false);
|
||||
expect(payload?.restart).toBeNull();
|
||||
expect(startManagedServiceUpdateHandoffMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ root: "/tmp/openclaw-git", supervisor: "systemd" }),
|
||||
);
|
||||
expect(scheduleGatewaySigusr1RestartMock).toHaveBeenCalledOnce();
|
||||
expect(payload?.ok).toBe(true);
|
||||
expect(payload?.result?.status).toBe("skipped");
|
||||
expect(payload?.result?.reason).toBe("managed-service-handoff-unavailable");
|
||||
expect(payload?.result?.reason).toBe("managed-service-handoff-started");
|
||||
expect(payload?.result?.mode).toBe("git");
|
||||
expect(payload?.handoff?.status).toBe("unavailable");
|
||||
expect(payload?.handoff?.status).toBe("started");
|
||||
});
|
||||
|
||||
it("returns a safe command when package updates cannot be handed off", async () => {
|
||||
@@ -961,11 +992,16 @@ describe("update.run restart scheduling", () => {
|
||||
expect(payload?.result?.mode).toBe("npm");
|
||||
});
|
||||
|
||||
it("delegates update.run without mutating or restarting under external supervision", async () => {
|
||||
it("keeps external update supervision authoritative even with native systemd markers", async () => {
|
||||
mockGlobalInstallSurface();
|
||||
detectRespawnSupervisorMock.mockReturnValue("systemd");
|
||||
|
||||
const payload = await withProcessEnv({ OPENCLAW_SUPERVISOR_MODE: "external" }, () =>
|
||||
captureUpdateRunPayload(),
|
||||
const payload = await withProcessEnv(
|
||||
{
|
||||
OPENCLAW_SUPERVISOR_MODE: "external",
|
||||
OPENCLAW_SYSTEMD_UNIT: "openclaw-gateway.service",
|
||||
},
|
||||
() => captureUpdateRunPayload(),
|
||||
);
|
||||
|
||||
expect(runGatewayUpdateMock).not.toHaveBeenCalled();
|
||||
|
||||
@@ -13,7 +13,6 @@ import { isRestartEnabled } from "../../config/commands.flags.js";
|
||||
import { readConfigFileSnapshot } from "../../config/config.js";
|
||||
import { extractDeliveryInfo } from "../../config/sessions.js";
|
||||
import type { OpenClawConfig } from "../../config/types.openclaw.js";
|
||||
import { GATEWAY_SERVICE_KIND, GATEWAY_SERVICE_MARKER } from "../../daemon/constants.js";
|
||||
import {
|
||||
EXTERNAL_SUPERVISOR_UPDATE_REQUIRED_REASON,
|
||||
isGatewayExternallySupervised,
|
||||
@@ -21,6 +20,7 @@ import {
|
||||
import { readPackageVersion } from "../../infra/package-json.js";
|
||||
import { type RestartSentinelPayload, writeRestartSentinel } from "../../infra/restart-sentinel.js";
|
||||
import {
|
||||
normalizeGatewayRestartDelayMs,
|
||||
resolveGatewayRestartDeferralTimeoutMs,
|
||||
scheduleGatewaySigusr1Restart,
|
||||
} from "../../infra/restart.js";
|
||||
@@ -114,47 +114,6 @@ async function readPreUpdateConfigForPostCoreFinalize(): Promise<
|
||||
};
|
||||
}
|
||||
|
||||
function resolveManagedServiceHandoffRestartDelayMs(
|
||||
restartDelayMs: number | undefined,
|
||||
supervisor: ReturnType<typeof detectRespawnSupervisor>,
|
||||
): number {
|
||||
const resolvedDelayMs = restartDelayMs ?? MANAGED_HANDOFF_RESTART_DELAY_MS;
|
||||
if (supervisor !== "systemd") {
|
||||
return resolvedDelayMs;
|
||||
}
|
||||
// systemd needs a short grace period after the handoff process starts before
|
||||
// the gateway exits, otherwise the service can restart before handoff state is durable.
|
||||
return Math.max(resolvedDelayMs, MANAGED_HANDOFF_RESTART_DELAY_MS);
|
||||
}
|
||||
|
||||
function hasManagedServiceHandoffContext(
|
||||
env: NodeJS.ProcessEnv,
|
||||
supervisor: ReturnType<typeof detectRespawnSupervisor>,
|
||||
): boolean {
|
||||
if (supervisor === "launchd") {
|
||||
return Boolean(
|
||||
env.OPENCLAW_LAUNCHD_LABEL?.trim() ||
|
||||
env.LAUNCH_JOB_LABEL?.trim() ||
|
||||
env.LAUNCH_JOB_NAME?.trim() ||
|
||||
env.XPC_SERVICE_NAME?.trim(),
|
||||
);
|
||||
}
|
||||
if (supervisor === "systemd") {
|
||||
// Ambient systemd markers only prove that a service manager started this
|
||||
// process. The detached CLI needs the durable unit name to stop the same
|
||||
// gateway before mutating the install root.
|
||||
return Boolean(env.OPENCLAW_SYSTEMD_UNIT?.trim());
|
||||
}
|
||||
if (supervisor === "schtasks") {
|
||||
return Boolean(
|
||||
env.OPENCLAW_WINDOWS_TASK_NAME?.trim() ||
|
||||
(env.OPENCLAW_SERVICE_MARKER?.trim() === GATEWAY_SERVICE_MARKER &&
|
||||
env.OPENCLAW_SERVICE_KIND?.trim() === GATEWAY_SERVICE_KIND),
|
||||
);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export const updateHandlers: GatewayRequestHandlers = {
|
||||
"update.status": async ({ params, respond, context }) => {
|
||||
if (!assertValidParams(params, validateUpdateStatusParams, "update.status", respond)) {
|
||||
@@ -269,8 +228,9 @@ export const updateHandlers: GatewayRequestHandlers = {
|
||||
threadId: requestedThreadId,
|
||||
note,
|
||||
continuationMessage,
|
||||
restartDelayMs,
|
||||
restartDelayMs: requestedRestartDelayMs,
|
||||
} = parseRestartRequestParams(params);
|
||||
const restartDelayMs = normalizeGatewayRestartDelayMs(requestedRestartDelayMs);
|
||||
const { deliveryContext: sessionDeliveryContext, threadId: sessionThreadId } =
|
||||
extractDeliveryInfo(sessionKey);
|
||||
const deliveryContext = requestedDeliveryContext ?? sessionDeliveryContext;
|
||||
@@ -312,16 +272,15 @@ export const updateHandlers: GatewayRequestHandlers = {
|
||||
installKind: status.installKind,
|
||||
git: status.git,
|
||||
}).channel;
|
||||
const supervisor = detectRespawnSupervisor(process.env, process.platform);
|
||||
const hasHandoffContext = supervisor
|
||||
? hasManagedServiceHandoffContext(process.env, supervisor)
|
||||
: false;
|
||||
const supervisor = detectRespawnSupervisor(process.env, process.platform, {
|
||||
includeLinuxOpenClawGatewayServiceMarker: true,
|
||||
});
|
||||
const requiresManagedServiceHandoff =
|
||||
installSurface.kind === "global" || (installSurface.kind === "git" && supervisor !== null);
|
||||
const managedGitPreflightFailure =
|
||||
installSurface.kind === "git" &&
|
||||
effectiveChannel === "dev" &&
|
||||
hasHandoffContext &&
|
||||
supervisor &&
|
||||
!isGatewayExternallySupervised()
|
||||
? await runGatewayUpdatePreflight(installRoot, timeoutMs, adoptedDevTarget)
|
||||
: undefined;
|
||||
@@ -386,15 +345,16 @@ export const updateHandlers: GatewayRequestHandlers = {
|
||||
...(handoffChannel ? { channel: handoffChannel } : {}),
|
||||
...(adoptedPackageTargetVersion ? { tag: adoptedPackageTargetVersion } : {}),
|
||||
});
|
||||
if (supervisor && hasHandoffContext) {
|
||||
if (supervisor) {
|
||||
try {
|
||||
const beforeVersion = await readPackageVersion(installRoot);
|
||||
const startedAt = Date.now();
|
||||
const handoffId = randomUUID();
|
||||
const managedRestartDelayMs = resolveManagedServiceHandoffRestartDelayMs(
|
||||
restartDelayMs,
|
||||
supervisor,
|
||||
);
|
||||
// systemd needs startup grace before the Gateway exits and its state becomes durable.
|
||||
const managedRestartDelayMs =
|
||||
supervisor === "systemd"
|
||||
? Math.max(restartDelayMs, MANAGED_HANDOFF_RESTART_DELAY_MS)
|
||||
: restartDelayMs;
|
||||
sentinelMeta.handoffId = handoffId;
|
||||
sentinelMeta.root = resolveUpdateInstallRoot(installRoot);
|
||||
// Managed services update from a detached helper so the running
|
||||
@@ -416,7 +376,7 @@ export const updateHandlers: GatewayRequestHandlers = {
|
||||
sentinelMeta.handoffId = started.handoffId ?? handoffId;
|
||||
// The owner pairs helper creation with parent exit before any
|
||||
// persistence can fail. Joiners leave both to the active owner.
|
||||
if (ownsManagedServiceHandoff) {
|
||||
if (started.status === "started") {
|
||||
handoff = {
|
||||
status: "started",
|
||||
...(started.pid ? { pid: started.pid } : {}),
|
||||
@@ -425,6 +385,11 @@ export const updateHandlers: GatewayRequestHandlers = {
|
||||
managedHandoffRestart = scheduleGatewaySigusr1Restart({
|
||||
delayMs: managedRestartDelayMs,
|
||||
reason: "update.run",
|
||||
successorOwner: {
|
||||
kind: "managed-update-handoff",
|
||||
handoffId: started.handoffId,
|
||||
installRoot: started.installRoot,
|
||||
},
|
||||
skipDeferral: true,
|
||||
skipCooldown: true,
|
||||
audit: {
|
||||
|
||||
@@ -9,8 +9,14 @@ import {
|
||||
tryBeginGatewayRootWorkAdmission,
|
||||
} from "../process/gateway-work-admission.js";
|
||||
type RestartModule = typeof import("./restart.js");
|
||||
const managedSuccessorOwner = {
|
||||
kind: "managed-update-handoff",
|
||||
handoffId: "managed-handoff",
|
||||
installRoot: "/canonical/install",
|
||||
} as const;
|
||||
|
||||
let consumeGatewaySigusr1RestartAuthorization: RestartModule["consumeGatewaySigusr1RestartAuthorization"];
|
||||
let consumeGatewaySigusr1RestartIntent: RestartModule["consumeGatewaySigusr1RestartIntent"];
|
||||
let deferGatewayRestartUntilIdle: RestartModule["deferGatewayRestartUntilIdle"];
|
||||
let isGatewaySigusr1RestartExternallyAllowed: RestartModule["isGatewaySigusr1RestartExternallyAllowed"];
|
||||
let markGatewaySigusr1RestartHandled: RestartModule["markGatewaySigusr1RestartHandled"];
|
||||
@@ -114,6 +120,7 @@ describe("infra runtime", () => {
|
||||
);
|
||||
({
|
||||
consumeGatewaySigusr1RestartAuthorization,
|
||||
consumeGatewaySigusr1RestartIntent,
|
||||
deferGatewayRestartUntilIdle,
|
||||
isGatewaySigusr1RestartExternallyAllowed,
|
||||
markGatewaySigusr1RestartHandled,
|
||||
@@ -445,17 +452,16 @@ describe("infra runtime", () => {
|
||||
const beforeEmit = vi.fn(async () => {
|
||||
await preparationBlocked;
|
||||
});
|
||||
let resolveSignal: () => void = () => {};
|
||||
const signalEmitted = new Promise<void>((resolve) => {
|
||||
resolveSignal = resolve;
|
||||
});
|
||||
const handler = () => resolveSignal();
|
||||
const staleEmitRestart = vi.fn(() => ({ status: "failed" as const }));
|
||||
const emitSpy = vi.spyOn(process, "emit");
|
||||
const handler = () => {};
|
||||
process.on("SIGUSR1", handler);
|
||||
try {
|
||||
scheduleGatewaySigusr1Restart({
|
||||
delayMs: 0,
|
||||
reason: "config.patch",
|
||||
emitHooks: { beforeEmit },
|
||||
sessionKey: "agent:main:session-A",
|
||||
emitHooks: { beforeEmit, emitRestart: staleEmitRestart },
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
await Promise.resolve();
|
||||
@@ -464,14 +470,71 @@ describe("infra runtime", () => {
|
||||
const update = scheduleGatewaySigusr1Restart({
|
||||
delayMs: 0,
|
||||
reason: "update.auto",
|
||||
successorOwner: managedSuccessorOwner,
|
||||
skipDeferral: true,
|
||||
});
|
||||
expect(update.coalesced).toBe(true);
|
||||
|
||||
releasePreparation();
|
||||
await signalEmitted;
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
|
||||
expect(staleEmitRestart).not.toHaveBeenCalled();
|
||||
expect(emitSpy).toHaveBeenCalledWith("SIGUSR1");
|
||||
expect(peekGatewaySigusr1RestartReason()).toBe("update.auto");
|
||||
expect(consumeGatewaySigusr1RestartIntent()).toEqual({
|
||||
reason: "update.auto",
|
||||
successorOwner: managedSuccessorOwner,
|
||||
});
|
||||
} finally {
|
||||
process.removeListener("SIGUSR1", handler);
|
||||
}
|
||||
});
|
||||
|
||||
it("retains managed successor ownership when an ordinary restart pulls the timer earlier", async () => {
|
||||
const handler = () => {};
|
||||
process.on("SIGUSR1", handler);
|
||||
try {
|
||||
scheduleGatewaySigusr1Restart({
|
||||
delayMs: 1_000,
|
||||
reason: "update.auto",
|
||||
successorOwner: managedSuccessorOwner,
|
||||
});
|
||||
scheduleGatewaySigusr1Restart({ delayMs: 0, reason: "config.patch" });
|
||||
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
|
||||
expect(peekGatewaySigusr1RestartReason()).toBe("update.auto");
|
||||
expect(consumeGatewaySigusr1RestartIntent()).toEqual({
|
||||
reason: "update.auto",
|
||||
successorOwner: managedSuccessorOwner,
|
||||
});
|
||||
} finally {
|
||||
process.removeListener("SIGUSR1", handler);
|
||||
}
|
||||
});
|
||||
|
||||
it("replaces stale managed successor ownership when its replacement coalesces", async () => {
|
||||
const replacementOwner = { ...managedSuccessorOwner, handoffId: "replacement-handoff" };
|
||||
const handler = () => {};
|
||||
process.on("SIGUSR1", handler);
|
||||
try {
|
||||
scheduleGatewaySigusr1Restart({
|
||||
delayMs: 1_000,
|
||||
reason: "update.auto",
|
||||
successorOwner: managedSuccessorOwner,
|
||||
});
|
||||
const replacement = scheduleGatewaySigusr1Restart({
|
||||
delayMs: 1_000,
|
||||
reason: "update.auto",
|
||||
successorOwner: replacementOwner,
|
||||
});
|
||||
|
||||
expect(replacement.coalesced).toBe(true);
|
||||
await vi.advanceTimersByTimeAsync(1_000);
|
||||
expect(consumeGatewaySigusr1RestartIntent()).toEqual({
|
||||
reason: "update.auto",
|
||||
successorOwner: replacementOwner,
|
||||
});
|
||||
} finally {
|
||||
process.removeListener("SIGUSR1", handler);
|
||||
}
|
||||
@@ -651,7 +714,7 @@ describe("infra runtime", () => {
|
||||
it("rejects coalesced emit hooks from a different session while preparation is in flight (#86742)", async () => {
|
||||
// Pins the CWE-200 in-flight preparation race: pendingRestartSessionKey
|
||||
// must stay alive through await beforeEmit(), otherwise a coalesced
|
||||
// different-session caller slips past updatePendingRestartEmitHooks
|
||||
// different-session caller slips past canReplacePendingRestartEmitHooks
|
||||
// and chains its own hooks while preparation runs.
|
||||
let releaseSessionAPrep: () => void = () => {};
|
||||
const sessionAPrepBlocked = new Promise<void>((resolve) => {
|
||||
@@ -1219,6 +1282,7 @@ describe("infra runtime", () => {
|
||||
const forced = scheduleGatewaySigusr1Restart({
|
||||
delayMs: 0,
|
||||
reason: "update.run",
|
||||
successorOwner: managedSuccessorOwner,
|
||||
skipDeferral: true,
|
||||
});
|
||||
|
||||
@@ -1226,6 +1290,10 @@ describe("infra runtime", () => {
|
||||
expect(emitSpy).toHaveBeenCalledWith("SIGUSR1");
|
||||
expect(staleBeforeEmit).not.toHaveBeenCalled();
|
||||
expect(peekGatewaySigusr1RestartReason()).toBe("update.run");
|
||||
expect(consumeGatewaySigusr1RestartIntent()).toEqual({
|
||||
reason: "update.run",
|
||||
successorOwner: managedSuccessorOwner,
|
||||
});
|
||||
} finally {
|
||||
process.removeListener("SIGUSR1", handler);
|
||||
}
|
||||
|
||||
@@ -354,22 +354,6 @@ describe("respawnGatewayProcessForUpdate", () => {
|
||||
expect(spawnMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("actively schedules launchd update relaunch before exiting", () => {
|
||||
clearSupervisorHints();
|
||||
setPlatform("darwin");
|
||||
process.env.OPENCLAW_LAUNCHD_LABEL = "ai.openclaw.gateway";
|
||||
process.env.OPENCLAW_NO_RESPAWN = "1";
|
||||
|
||||
const result = respawnGatewayProcessForUpdate();
|
||||
|
||||
expect(result.mode).toBe("supervised");
|
||||
expect(result.handoffSpawned).toBeInstanceOf(Promise);
|
||||
expect(scheduleLaunchdHandoffMock).toHaveBeenCalledWith({
|
||||
mode: "start-after-exit",
|
||||
waitForPid: process.pid,
|
||||
});
|
||||
});
|
||||
|
||||
it("allows detached respawn on unmanaged Windows during updates", () => {
|
||||
clearSupervisorHints();
|
||||
setPlatform("win32");
|
||||
@@ -397,18 +381,6 @@ describe("respawnGatewayProcessForUpdate", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("delegates update restarts to external supervision without spawning", () => {
|
||||
clearSupervisorHints();
|
||||
setPlatform("linux");
|
||||
process.env.OPENCLAW_SUPERVISOR_MODE = "external";
|
||||
|
||||
const result = respawnGatewayProcessForUpdate();
|
||||
|
||||
expect(result).toEqual({ mode: "supervised" });
|
||||
expect(spawnMock).not.toHaveBeenCalled();
|
||||
expect(triggerOpenClawRestartMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rewrites a pnpm-versioned OpenClaw entry before detached update respawn", () => {
|
||||
clearSupervisorHints();
|
||||
setPlatform("linux");
|
||||
@@ -497,19 +469,6 @@ describe("respawnGatewayProcessForUpdate", () => {
|
||||
expect(onCallOrder).toBeLessThan(unrefCallOrder);
|
||||
});
|
||||
|
||||
it("exits to a managed supervisor for updates even when respawn is disabled", () => {
|
||||
clearSupervisorHints();
|
||||
setPlatform("linux");
|
||||
process.env.OPENCLAW_NO_RESPAWN = "1";
|
||||
process.env.OPENCLAW_SERVICE_MARKER = "openclaw";
|
||||
process.env.OPENCLAW_SERVICE_KIND = "gateway";
|
||||
|
||||
const result = respawnGatewayProcessForUpdate();
|
||||
|
||||
expect(result).toEqual({ mode: "supervised" });
|
||||
expect(spawnMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns failed when update detached respawn throws", () => {
|
||||
delete process.env.OPENCLAW_NO_RESPAWN;
|
||||
clearSupervisorHints();
|
||||
|
||||
@@ -7,16 +7,16 @@ import { formatErrorMessage } from "./errors.js";
|
||||
import { triggerOpenClawRestart } from "./restart.js";
|
||||
import { detectGatewayRespawnSupervisor } from "./supervisor-markers.js";
|
||||
|
||||
type RespawnMode = "spawned" | "supervised" | "disabled" | "failed";
|
||||
|
||||
type GatewayRespawnResult = {
|
||||
mode: RespawnMode;
|
||||
pid?: number;
|
||||
mode: "supervised" | "disabled" | "failed";
|
||||
detail?: string;
|
||||
handoffSpawned?: Promise<boolean>;
|
||||
};
|
||||
|
||||
type GatewayUpdateRespawnResult = GatewayRespawnResult & {
|
||||
type GatewayUpdateRespawnResult = {
|
||||
mode: "spawned" | "disabled" | "failed";
|
||||
pid?: number;
|
||||
detail?: string;
|
||||
child?: ChildProcess;
|
||||
};
|
||||
type GatewayRespawnOptions = {
|
||||
@@ -35,39 +35,6 @@ function rewritePnpmVersionedOpenClawEntryPath(entryPath: string): string {
|
||||
);
|
||||
}
|
||||
|
||||
function spawnDetachedGatewayProcess(opts: GatewayRespawnOptions = {}): {
|
||||
child: ChildProcess;
|
||||
pid?: number;
|
||||
} {
|
||||
const [entryArg, ...entryArgs] = process.argv.slice(1);
|
||||
const args = [
|
||||
...process.execArgv,
|
||||
...(entryArg ? [rewritePnpmVersionedOpenClawEntryPath(entryArg)] : []),
|
||||
...entryArgs,
|
||||
];
|
||||
const child = spawn(process.execPath, args, {
|
||||
env: opts.env ? { ...process.env, ...opts.env } : process.env,
|
||||
detached: true,
|
||||
stdio: "inherit",
|
||||
});
|
||||
// Detached spawn failures can arrive asynchronously after spawn() returns.
|
||||
// Keep this listener before unref() so the parent does not crash during handoff.
|
||||
child.on("error", () => {});
|
||||
child.unref();
|
||||
return { child, pid: child.pid ?? undefined };
|
||||
}
|
||||
|
||||
function scheduleLaunchdRestartAfterExit(): GatewayRespawnResult {
|
||||
const handoff = scheduleDetachedLaunchdRestartHandoff({
|
||||
mode: "start-after-exit",
|
||||
waitForPid: process.pid,
|
||||
});
|
||||
if (!handoff.ok) {
|
||||
return { mode: "failed", detail: handoff.error };
|
||||
}
|
||||
return { mode: "supervised", handoffSpawned: handoff.value };
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempt to restart this process with a fresh PID.
|
||||
* - supervised environments (launchd/systemd/schtasks): caller should exit and let supervisor restart
|
||||
@@ -84,7 +51,13 @@ export function restartGatewayProcessWithFreshPid(
|
||||
const supervisor = detectGatewayRespawnSupervisor(process.env);
|
||||
if (supervisor) {
|
||||
if (supervisor === "launchd") {
|
||||
return scheduleLaunchdRestartAfterExit();
|
||||
const handoff = scheduleDetachedLaunchdRestartHandoff({
|
||||
mode: "start-after-exit",
|
||||
waitForPid: process.pid,
|
||||
});
|
||||
return handoff.ok
|
||||
? { mode: "supervised", handoffSpawned: handoff.value }
|
||||
: { mode: "failed", detail: handoff.error };
|
||||
}
|
||||
if (supervisor === "schtasks") {
|
||||
const restart = triggerOpenClawRestart();
|
||||
@@ -97,68 +70,46 @@ export function restartGatewayProcessWithFreshPid(
|
||||
}
|
||||
return { mode: "supervised" };
|
||||
}
|
||||
if (process.platform === "win32") {
|
||||
// Detached respawn is unsafe on Windows without an identified Scheduled Task:
|
||||
// the child becomes orphaned if the original process exits.
|
||||
return {
|
||||
mode: "disabled",
|
||||
detail: "win32: detached respawn unsupported without Scheduled Task markers",
|
||||
};
|
||||
}
|
||||
if (isContainerEnvironment()) {
|
||||
return {
|
||||
mode: "disabled",
|
||||
detail: "container: use in-process restart to keep PID 1 alive",
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
mode: "disabled",
|
||||
detail: "unmanaged: use in-process restart to keep custom supervisor PID tracking stable",
|
||||
};
|
||||
// Unmanaged Windows or containers cannot safely surrender their tracked process.
|
||||
const detail =
|
||||
process.platform === "win32"
|
||||
? "win32: detached respawn unsupported without Scheduled Task markers"
|
||||
: isContainerEnvironment()
|
||||
? "container: use in-process restart to keep PID 1 alive"
|
||||
: "unmanaged: use in-process restart to keep custom supervisor PID tracking stable";
|
||||
return { mode: "disabled", detail };
|
||||
}
|
||||
|
||||
/**
|
||||
* Update restarts must replace the OS process so the new code runs from a
|
||||
* fresh module graph after package files have changed on disk.
|
||||
*
|
||||
* Unlike the generic restart path, update mode allows detached respawn on
|
||||
* unmanaged Windows installs because there is no safe in-process fallback once
|
||||
* the installed package contents have been replaced.
|
||||
* The caller resolves supervisor ownership first; this path is only for an
|
||||
* unmanaged process whose installed package contents have been replaced.
|
||||
*/
|
||||
export function respawnGatewayProcessForUpdate(
|
||||
opts: GatewayRespawnOptions = {},
|
||||
): GatewayUpdateRespawnResult {
|
||||
const supervisor = detectGatewayRespawnSupervisor(process.env, process.platform, {
|
||||
includeLinuxOpenClawGatewayServiceMarker: true,
|
||||
});
|
||||
if (supervisor) {
|
||||
// Managed update handoffs require the original PID to exit before the
|
||||
// detached helper can mutate the install, even when respawn is disabled.
|
||||
if (supervisor === "launchd") {
|
||||
return scheduleLaunchdRestartAfterExit();
|
||||
}
|
||||
if (supervisor === "schtasks") {
|
||||
const restart = triggerOpenClawRestart();
|
||||
if (!restart.ok) {
|
||||
return {
|
||||
mode: "failed",
|
||||
detail: restart.detail ?? `${restart.method} restart failed`,
|
||||
};
|
||||
}
|
||||
}
|
||||
return { mode: "supervised" };
|
||||
}
|
||||
if (isTruthyEnvValue(process.env.OPENCLAW_NO_RESPAWN)) {
|
||||
return { mode: "disabled", detail: "OPENCLAW_NO_RESPAWN" };
|
||||
}
|
||||
try {
|
||||
const { child, pid } = spawnDetachedGatewayProcess(opts);
|
||||
return { mode: "spawned", pid, child };
|
||||
const [entryArg, ...entryArgs] = process.argv.slice(1);
|
||||
const args = [
|
||||
...process.execArgv,
|
||||
...(entryArg ? [rewritePnpmVersionedOpenClawEntryPath(entryArg)] : []),
|
||||
...entryArgs,
|
||||
];
|
||||
const child = spawn(process.execPath, args, {
|
||||
env: opts.env ? { ...process.env, ...opts.env } : process.env,
|
||||
detached: true,
|
||||
stdio: "inherit",
|
||||
});
|
||||
// Register before unref: late detached-spawn failures must not crash the parent.
|
||||
child.on("error", () => {});
|
||||
child.unref();
|
||||
return { mode: "spawned", pid: child.pid ?? undefined, child };
|
||||
} catch (err) {
|
||||
return {
|
||||
mode: "failed",
|
||||
detail: formatErrorMessage(err),
|
||||
};
|
||||
return { mode: "failed", detail: formatErrorMessage(err) };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -120,7 +120,7 @@ describe("gateway restart intent", () => {
|
||||
expect(readIntentRow(env)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("round-trips restart reason, force, and wait options", () => {
|
||||
it("round-trips restart options without persisting process-local successor identity", () => {
|
||||
const env = createIntentEnv();
|
||||
|
||||
expect(
|
||||
@@ -128,7 +128,15 @@ describe("gateway restart intent", () => {
|
||||
env,
|
||||
targetPid: process.pid,
|
||||
reason: "gateway.restart",
|
||||
intent: { force: true, waitMs: 12_345 },
|
||||
intent: {
|
||||
force: true,
|
||||
waitMs: 12_345,
|
||||
successorOwner: {
|
||||
kind: "managed-update-handoff",
|
||||
handoffId: "private-handoff",
|
||||
installRoot: "/private/install",
|
||||
},
|
||||
},
|
||||
}),
|
||||
).toBe(true);
|
||||
|
||||
|
||||
@@ -32,12 +32,14 @@ export type GatewayRestartIntent = {
|
||||
reason?: string;
|
||||
force?: boolean;
|
||||
waitMs?: number;
|
||||
// Process-local only: persisted restart requests cannot delegate successor ownership.
|
||||
successorOwner?: {
|
||||
kind: "managed-update-handoff";
|
||||
handoffId: string;
|
||||
installRoot: string;
|
||||
};
|
||||
};
|
||||
|
||||
function normalizeRestartIntentPid(pid: number | undefined): number | null {
|
||||
return asPositiveSafeInteger(pid) ?? null;
|
||||
}
|
||||
|
||||
export function normalizeRestartIntentReason(reason: string | undefined): string | undefined {
|
||||
const normalized = reason?.trim();
|
||||
return normalized ? truncateUtf16Safe(normalized, 200) : undefined;
|
||||
@@ -49,7 +51,7 @@ export function writeGatewayRestartIntentSync(opts: {
|
||||
intent?: GatewayRestartIntent;
|
||||
reason?: string;
|
||||
}): boolean {
|
||||
const targetPid = normalizeRestartIntentPid(opts.targetPid);
|
||||
const targetPid = asPositiveSafeInteger(opts.targetPid) ?? null;
|
||||
if (targetPid === null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -36,7 +36,12 @@ vi.mock("../config/paths.js", () => ({
|
||||
|
||||
const { cleanStaleGatewayProcessesSync, findGatewayPidsOnPortSync } =
|
||||
await import("./restart-stale-pids.js");
|
||||
const { triggerOpenClawRestart } = await import("./restart.js");
|
||||
const {
|
||||
normalizeGatewayRestartDelayMs,
|
||||
resetGatewayRestartStateForInProcessRestart,
|
||||
scheduleGatewaySigusr1Restart,
|
||||
triggerOpenClawRestart,
|
||||
} = await import("./restart.js");
|
||||
|
||||
const envSnapshot = captureFullEnv();
|
||||
|
||||
@@ -262,3 +267,34 @@ describe("triggerOpenClawRestart", () => {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("gateway restart delay normalization", () => {
|
||||
it.each([
|
||||
{ requested: undefined, effective: 2000 },
|
||||
{ requested: Number.NaN, effective: 2000 },
|
||||
{ requested: Number.POSITIVE_INFINITY, effective: 2000 },
|
||||
{ requested: -1, effective: 0 },
|
||||
{ requested: 1500.8, effective: 1500 },
|
||||
{ requested: 2_147_153_648, effective: 60_000 },
|
||||
])("normalizes $requested to $effective ms", ({ requested, effective }) => {
|
||||
expect(normalizeGatewayRestartDelayMs(requested)).toBe(effective);
|
||||
});
|
||||
|
||||
it("does not emit an overflow-sized restart before its effective 60-second delay", async () => {
|
||||
vi.useFakeTimers();
|
||||
const killSpy = vi.spyOn(process, "kill").mockImplementation(() => true);
|
||||
try {
|
||||
const restart = scheduleGatewaySigusr1Restart({
|
||||
delayMs: 2_147_153_648,
|
||||
skipCooldown: true,
|
||||
});
|
||||
|
||||
expect(restart.delayMs).toBe(60_000);
|
||||
await vi.advanceTimersByTimeAsync(59_999);
|
||||
expect(killSpy).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
resetGatewayRestartStateForInProcessRestart();
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
+119
-140
@@ -45,6 +45,7 @@ let lastRestartEmittedAt = 0;
|
||||
let pendingRestartTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
let pendingRestartDueAt = 0;
|
||||
let pendingRestartReason: string | undefined;
|
||||
let pendingRestartSuccessorOwner: GatewayRestartIntent["successorOwner"];
|
||||
let pendingRestartEmitHooks: RestartEmitHooks | undefined;
|
||||
let pendingRestartSessionKey: string | undefined;
|
||||
let pendingRestartSkipDeferral = false;
|
||||
@@ -63,12 +64,11 @@ function hasUnconsumedRestartSignal(): boolean {
|
||||
}
|
||||
|
||||
function clearPendingScheduledRestart(): void {
|
||||
if (pendingRestartTimer) {
|
||||
clearTimeout(pendingRestartTimer);
|
||||
}
|
||||
clearTimeout(pendingRestartTimer ?? undefined);
|
||||
pendingRestartTimer = null;
|
||||
pendingRestartDueAt = 0;
|
||||
pendingRestartReason = undefined;
|
||||
pendingRestartSuccessorOwner = undefined;
|
||||
pendingRestartEmitHooks = undefined;
|
||||
pendingRestartSessionKey = undefined;
|
||||
pendingRestartSkipDeferral = false;
|
||||
@@ -179,19 +179,12 @@ function formatRestartAudit(audit: RestartAuditInfo | undefined): string {
|
||||
const clientIp =
|
||||
typeof audit?.clientIp === "string" && audit.clientIp.trim() ? audit.clientIp.trim() : null;
|
||||
const changed = summarizeChangedPaths(audit?.changedPaths);
|
||||
const fields = [];
|
||||
if (actor) {
|
||||
fields.push(`actor=${actor}`);
|
||||
}
|
||||
if (deviceId) {
|
||||
fields.push(`device=${deviceId}`);
|
||||
}
|
||||
if (clientIp) {
|
||||
fields.push(`ip=${clientIp}`);
|
||||
}
|
||||
if (changed) {
|
||||
fields.push(`changedPaths=${changed}`);
|
||||
}
|
||||
const fields = [
|
||||
actor && `actor=${actor}`,
|
||||
deviceId && `device=${deviceId}`,
|
||||
clientIp && `ip=${clientIp}`,
|
||||
changed && `changedPaths=${changed}`,
|
||||
].filter(Boolean);
|
||||
return fields.length > 0 ? fields.join(" ") : "actor=<unknown>";
|
||||
}
|
||||
|
||||
@@ -297,10 +290,7 @@ export function requestGatewayRestartWithSignalAdmission(
|
||||
}
|
||||
|
||||
function resetSigusr1AuthorizationIfExpired(now = Date.now()) {
|
||||
if (sigusr1AuthorizedCount <= 0) {
|
||||
return;
|
||||
}
|
||||
if (now <= sigusr1AuthorizedUntil) {
|
||||
if (sigusr1AuthorizedCount <= 0 || now <= sigusr1AuthorizedUntil) {
|
||||
return;
|
||||
}
|
||||
sigusr1AuthorizedCount = 0;
|
||||
@@ -315,9 +305,8 @@ export function isGatewaySigusr1RestartExternallyAllowed() {
|
||||
return sigusr1ExternalAllowed;
|
||||
}
|
||||
|
||||
function authorizeGatewaySigusr1Restart(delayMs = 0) {
|
||||
const delay = Math.max(0, Math.floor(delayMs));
|
||||
const expiresAt = Date.now() + delay + SIGUSR1_AUTH_GRACE_MS;
|
||||
function authorizeGatewaySigusr1Restart() {
|
||||
const expiresAt = Date.now() + SIGUSR1_AUTH_GRACE_MS;
|
||||
sigusr1AuthorizedCount += 1;
|
||||
if (expiresAt > sigusr1AuthorizedUntil) {
|
||||
sigusr1AuthorizedUntil = expiresAt;
|
||||
@@ -406,44 +395,22 @@ type GatewayRestartEmitResult =
|
||||
| { status: "coalesced" }
|
||||
| { status: "failed" };
|
||||
|
||||
export function resolveGatewayRestartDeferralTimeoutMs(): number;
|
||||
export function resolveGatewayRestartDeferralTimeoutMs(timeoutMs: unknown): number | undefined;
|
||||
export function resolveGatewayRestartDeferralTimeoutMs(timeoutMs?: unknown): number | undefined {
|
||||
if (typeof timeoutMs !== "number" || !Number.isFinite(timeoutMs)) {
|
||||
return DEFAULT_RESTART_DEFERRAL_TIMEOUT_MS;
|
||||
}
|
||||
if (timeoutMs <= 0) {
|
||||
return undefined;
|
||||
}
|
||||
return Math.floor(timeoutMs);
|
||||
return timeoutMs > 0 ? Math.floor(timeoutMs) : undefined;
|
||||
}
|
||||
|
||||
function canReplacePendingRestartEmitHooks(
|
||||
hooks: RestartEmitHooks | undefined,
|
||||
sessionKey: string | undefined,
|
||||
): boolean {
|
||||
if (!hooks) {
|
||||
return true;
|
||||
}
|
||||
return pendingRestartSessionKey === undefined || pendingRestartSessionKey === sessionKey;
|
||||
}
|
||||
|
||||
// Returns true when the new hooks took ownership of the pending restart slot.
|
||||
// Coalesced callers from a different sessionKey are rejected to prevent the
|
||||
// cross-session continuation overwrite documented in #86742 (CWE-200).
|
||||
function updatePendingRestartEmitHooks(
|
||||
hooks: RestartEmitHooks | undefined,
|
||||
sessionKey: string | undefined,
|
||||
): boolean {
|
||||
if (!canReplacePendingRestartEmitHooks(hooks, sessionKey)) {
|
||||
return false;
|
||||
}
|
||||
if (!hooks) {
|
||||
return false;
|
||||
}
|
||||
pendingRestartEmitHooks = hooks;
|
||||
if (sessionKey !== undefined) {
|
||||
pendingRestartSessionKey = sessionKey;
|
||||
}
|
||||
return true;
|
||||
return (
|
||||
!hooks || pendingRestartSessionKey === undefined || pendingRestartSessionKey === sessionKey
|
||||
);
|
||||
}
|
||||
|
||||
async function rejectPreparedRestartHook(hooks: RestartEmitHooks | undefined): Promise<void> {
|
||||
@@ -556,24 +523,26 @@ async function emitPreparedGatewayRestartUnderAdmission(
|
||||
pendingRestartEmitHooks = undefined;
|
||||
}
|
||||
|
||||
// Slot settled and no awaits remain before emission — release ownership for
|
||||
// every emission attempt, not only hookless ones, so a later session can
|
||||
// claim continuation hooks for the next restart cycle.
|
||||
pendingRestartSessionKey = undefined;
|
||||
|
||||
// Track every successfully prepared hook set (parked + caller) so non-emitted
|
||||
// outcomes can roll back both the gateway-tool sentinel and reload preflight.
|
||||
const preparedHooksList: RestartEmitHooks[] = [];
|
||||
if (preparedParked) {
|
||||
preparedHooksList.push(preparedParked);
|
||||
}
|
||||
const preparedHooksList: RestartEmitHooks[] = preparedParked ? [preparedParked] : [];
|
||||
if (hooks && callerPrepared) {
|
||||
preparedHooksList.push(hooks);
|
||||
}
|
||||
// With caller hooks, emission stays the caller's (or falls back to the core
|
||||
// signal path if its preparation failed); parked hooks never own emission
|
||||
// when a caller is present.
|
||||
const emitOwner = hooks ? (callerPrepared ? hooks : undefined) : preparedParked;
|
||||
const emitOwner =
|
||||
hooks && callerPrepared
|
||||
? hooks
|
||||
: hooks || (pendingRestartSuccessorOwner && pendingRestartSessionKey === undefined)
|
||||
? undefined
|
||||
: preparedParked;
|
||||
|
||||
// Slot settled and no awaits remain before emission — release ownership for
|
||||
// every emission attempt, not only hookless ones, so a later session can
|
||||
// claim continuation hooks for the next restart cycle.
|
||||
pendingRestartSessionKey = undefined;
|
||||
|
||||
if (!isCurrent()) {
|
||||
await rejectPreparedRestartHooks(preparedHooksList);
|
||||
@@ -586,8 +555,15 @@ async function emitPreparedGatewayRestartUnderAdmission(
|
||||
? pendingRestartReason
|
||||
: undefined;
|
||||
const resolvedReason = preferredReason ?? reasonOverride;
|
||||
const successorOwner = pendingRestartSuccessorOwner ?? intent?.successorOwner;
|
||||
const resolvedIntent =
|
||||
preferredReason && intent ? { ...intent, reason: preferredReason } : intent;
|
||||
preferredReason || successorOwner
|
||||
? {
|
||||
...intent,
|
||||
...(resolvedReason ? { reason: resolvedReason } : {}),
|
||||
...(successorOwner ? { successorOwner } : {}),
|
||||
}
|
||||
: intent;
|
||||
const emitResult = emitOwner?.emitRestart
|
||||
? emitOwner.emitRestart(resolvedReason, resolvedIntent)
|
||||
: requestGatewayRestartWithSignalAdmission(resolvedReason, resolvedIntent);
|
||||
@@ -862,18 +838,11 @@ function formatSpawnDetail(result: {
|
||||
return "unknown error";
|
||||
}
|
||||
}
|
||||
const stderr = clean(result.stderr);
|
||||
if (stderr) {
|
||||
return stderr;
|
||||
}
|
||||
const stdout = clean(result.stdout);
|
||||
if (stdout) {
|
||||
return stdout;
|
||||
}
|
||||
if (typeof result.status === "number") {
|
||||
return `exit ${result.status}`;
|
||||
}
|
||||
return "unknown error";
|
||||
return (
|
||||
clean(result.stderr) ||
|
||||
clean(result.stdout) ||
|
||||
(typeof result.status === "number" ? `exit ${result.status}` : "unknown error")
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeSystemdUnit(raw?: string, profile?: string): string {
|
||||
@@ -1009,6 +978,12 @@ export type ScheduledRestart = {
|
||||
emitHooksQueued: boolean;
|
||||
};
|
||||
|
||||
export function normalizeGatewayRestartDelayMs(delayMs?: number): number {
|
||||
return typeof delayMs === "number" && Number.isFinite(delayMs)
|
||||
? Math.min(Math.max(Math.floor(delayMs), 0), 60_000)
|
||||
: 2000;
|
||||
}
|
||||
|
||||
export function scheduleGatewaySigusr1Restart(opts?: {
|
||||
delayMs?: number;
|
||||
reason?: string;
|
||||
@@ -1018,24 +993,35 @@ export function scheduleGatewaySigusr1Restart(opts?: {
|
||||
sessionKey?: string;
|
||||
skipDeferral?: boolean;
|
||||
skipCooldown?: boolean;
|
||||
successorOwner?: GatewayRestartIntent["successorOwner"];
|
||||
}): ScheduledRestart {
|
||||
const delayMsRaw =
|
||||
typeof opts?.delayMs === "number" && Number.isFinite(opts.delayMs)
|
||||
? Math.floor(opts.delayMs)
|
||||
: 2000;
|
||||
const delayMs = Math.min(Math.max(delayMsRaw, 0), 60_000);
|
||||
const delayMs = normalizeGatewayRestartDelayMs(opts?.delayMs);
|
||||
const reason = normalizeRestartIntentReason(opts?.reason);
|
||||
const hasSigusr1Listener = process.listenerCount("SIGUSR1") > 0;
|
||||
const mode = hasSigusr1Listener ? "emit" : process.platform === "win32" ? "supervisor" : "signal";
|
||||
const mode: ScheduledRestart["mode"] =
|
||||
process.listenerCount("SIGUSR1") > 0
|
||||
? "emit"
|
||||
: process.platform === "win32"
|
||||
? "supervisor"
|
||||
: "signal";
|
||||
const nowMs = Date.now();
|
||||
const skipCooldown = opts?.skipCooldown === true;
|
||||
const cooldownMsApplied = skipCooldown
|
||||
? 0
|
||||
: Math.max(0, lastRestartEmittedAt + RESTART_COOLDOWN_MS - nowMs);
|
||||
const cooldownMsApplied =
|
||||
opts?.skipCooldown === true
|
||||
? 0
|
||||
: Math.max(0, lastRestartEmittedAt + RESTART_COOLDOWN_MS - nowMs);
|
||||
const restartResultBase = {
|
||||
ok: true,
|
||||
pid: process.pid,
|
||||
signal: "SIGUSR1" as const,
|
||||
reason,
|
||||
mode,
|
||||
cooldownMsApplied,
|
||||
};
|
||||
const requestedDueAt = nowMs + delayMs + cooldownMsApplied;
|
||||
const skipDeferral = opts?.skipDeferral === true;
|
||||
let nextPendingEmitHooks = opts?.emitHooks;
|
||||
let nextPendingSessionKey = opts?.sessionKey;
|
||||
let nextPendingReason = reason;
|
||||
let nextPendingSuccessorOwner = opts?.successorOwner;
|
||||
|
||||
if (hasUnconsumedRestartSignal()) {
|
||||
if (shouldPreferRestartReason(reason, emittedRestartReason)) {
|
||||
@@ -1045,18 +1031,20 @@ export function scheduleGatewaySigusr1Restart(opts?: {
|
||||
emittedRestartIntent = { ...emittedRestartIntent, reason };
|
||||
}
|
||||
}
|
||||
if (opts?.successorOwner) {
|
||||
emittedRestartIntent = {
|
||||
...emittedRestartIntent,
|
||||
...(emittedRestartReason ? { reason: emittedRestartReason } : {}),
|
||||
successorOwner: opts.successorOwner,
|
||||
};
|
||||
}
|
||||
restartLog.warn(
|
||||
`restart request coalesced (already in-flight) reason=${reason ?? "unspecified"} ${formatRestartAudit(opts?.audit)}`,
|
||||
);
|
||||
return {
|
||||
ok: true,
|
||||
pid: process.pid,
|
||||
signal: "SIGUSR1",
|
||||
...restartResultBase,
|
||||
delayMs: 0,
|
||||
reason,
|
||||
mode,
|
||||
coalesced: true,
|
||||
cooldownMsApplied,
|
||||
// SIGUSR1 already emitted; the new caller's hooks cannot run for this cycle.
|
||||
emitHooksQueued: false,
|
||||
};
|
||||
@@ -1064,44 +1052,39 @@ export function scheduleGatewaySigusr1Restart(opts?: {
|
||||
|
||||
if (pendingRestartTimer || pendingRestartPreparing) {
|
||||
const remainingMs = pendingRestartPreparing ? 0 : Math.max(0, pendingRestartDueAt - nowMs);
|
||||
// Hookless forced restarts that own no sentinel may preserve an accepted
|
||||
// pending hook; update/handoff callers rely on the default clear path.
|
||||
const preservePendingHooks =
|
||||
opts?.preservePendingEmitHooksOnDeferralBypass === true &&
|
||||
opts?.emitHooks === undefined &&
|
||||
pendingRestartSessionKey !== undefined;
|
||||
if (pendingRestartPreparing && skipDeferral && activeDeferralPolls.size > 0) {
|
||||
restartLog.warn(
|
||||
`restart request bypassed active deferral reason=${reason ?? "unspecified"} pendingReason=${pendingRestartReason ?? "unspecified"} ${formatRestartAudit(opts?.audit)}`,
|
||||
);
|
||||
clearActiveDeferralPolls();
|
||||
pendingRestartReason = reason;
|
||||
// Hookless forced restarts that own no sentinel may preserve an accepted
|
||||
// pending hook; update/handoff callers rely on the default clear path.
|
||||
const preservePendingHooks =
|
||||
opts?.preservePendingEmitHooksOnDeferralBypass === true &&
|
||||
opts?.emitHooks === undefined &&
|
||||
pendingRestartSessionKey !== undefined;
|
||||
pendingRestartSuccessorOwner = opts?.successorOwner ?? pendingRestartSuccessorOwner;
|
||||
if (!preservePendingHooks) {
|
||||
pendingRestartEmitHooks = opts?.emitHooks;
|
||||
pendingRestartSessionKey = opts?.sessionKey;
|
||||
}
|
||||
void emitPreparedGatewayRestart(undefined, reason);
|
||||
return {
|
||||
ok: true,
|
||||
pid: process.pid,
|
||||
signal: "SIGUSR1",
|
||||
...restartResultBase,
|
||||
delayMs: 0,
|
||||
reason,
|
||||
mode,
|
||||
coalesced: false,
|
||||
cooldownMsApplied,
|
||||
emitHooksQueued: opts?.emitHooks !== undefined,
|
||||
};
|
||||
}
|
||||
const shouldUpgradeToSkipDeferral = skipDeferral && !pendingRestartSkipDeferral;
|
||||
const shouldPullEarlier =
|
||||
!pendingRestartPreparing &&
|
||||
(requestedDueAt < pendingRestartDueAt || shouldUpgradeToSkipDeferral);
|
||||
(requestedDueAt < pendingRestartDueAt || (skipDeferral && !pendingRestartSkipDeferral));
|
||||
if (shouldPullEarlier) {
|
||||
const preservePendingHooks =
|
||||
opts?.preservePendingEmitHooksOnDeferralBypass === true &&
|
||||
opts?.emitHooks === undefined &&
|
||||
pendingRestartSessionKey !== undefined;
|
||||
if (shouldPreferRestartReason(pendingRestartReason, reason)) {
|
||||
nextPendingReason = pendingRestartReason;
|
||||
}
|
||||
nextPendingSuccessorOwner ??= pendingRestartSuccessorOwner;
|
||||
if (
|
||||
!preservePendingHooks &&
|
||||
!canReplacePendingRestartEmitHooks(opts?.emitHooks, opts?.sessionKey)
|
||||
@@ -1109,79 +1092,75 @@ export function scheduleGatewaySigusr1Restart(opts?: {
|
||||
restartLog.warn(
|
||||
`restart continuation dropped: another session owns the pending restart (callerSessionKey=${opts?.sessionKey ?? "unspecified"} pendingSessionKey=${pendingRestartSessionKey ?? "unspecified"})`,
|
||||
);
|
||||
if (pendingRestartTimer) {
|
||||
clearTimeout(pendingRestartTimer);
|
||||
}
|
||||
clearTimeout(pendingRestartTimer ?? undefined);
|
||||
pendingRestartTimer = null;
|
||||
pendingRestartDueAt = requestedDueAt;
|
||||
pendingRestartReason = reason;
|
||||
pendingRestartReason = nextPendingReason;
|
||||
pendingRestartSuccessorOwner = nextPendingSuccessorOwner;
|
||||
pendingRestartSkipDeferral = pendingRestartSkipDeferral || skipDeferral;
|
||||
armPendingRestartTimer(requestedDueAt, nowMs);
|
||||
return {
|
||||
ok: true,
|
||||
pid: process.pid,
|
||||
signal: "SIGUSR1",
|
||||
...restartResultBase,
|
||||
delayMs: Math.max(0, requestedDueAt - nowMs),
|
||||
reason,
|
||||
mode,
|
||||
coalesced: true,
|
||||
cooldownMsApplied,
|
||||
emitHooksQueued: false,
|
||||
};
|
||||
}
|
||||
const preservedEmitHooks = preservePendingHooks ? pendingRestartEmitHooks : undefined;
|
||||
const preservedSessionKey = preservePendingHooks ? pendingRestartSessionKey : undefined;
|
||||
if (preservePendingHooks) {
|
||||
nextPendingEmitHooks = pendingRestartEmitHooks;
|
||||
nextPendingSessionKey = pendingRestartSessionKey;
|
||||
}
|
||||
restartLog.warn(
|
||||
`restart request rescheduled earlier reason=${reason ?? "unspecified"} pendingReason=${pendingRestartReason ?? "unspecified"} oldDelayMs=${remainingMs} newDelayMs=${Math.max(0, requestedDueAt - nowMs)} ${formatRestartAudit(opts?.audit)}`,
|
||||
);
|
||||
clearPendingScheduledRestart();
|
||||
if (preservePendingHooks) {
|
||||
nextPendingEmitHooks = preservedEmitHooks;
|
||||
nextPendingSessionKey = preservedSessionKey;
|
||||
}
|
||||
} else {
|
||||
if (shouldPreferRestartReason(reason, pendingRestartReason)) {
|
||||
const restartReasonPromoted = shouldPreferRestartReason(reason, pendingRestartReason);
|
||||
if (restartReasonPromoted) {
|
||||
pendingRestartReason = reason;
|
||||
}
|
||||
pendingRestartSuccessorOwner = opts?.successorOwner ?? pendingRestartSuccessorOwner;
|
||||
pendingRestartSkipDeferral = pendingRestartSkipDeferral || skipDeferral;
|
||||
restartLog.warn(
|
||||
`restart request coalesced (already scheduled) reason=${reason ?? "unspecified"} pendingReason=${pendingRestartReason ?? "unspecified"} delayMs=${remainingMs} ${formatRestartAudit(opts?.audit)}`,
|
||||
);
|
||||
const emitHooksQueued = updatePendingRestartEmitHooks(opts?.emitHooks, opts?.sessionKey);
|
||||
const emitHooksQueued =
|
||||
opts?.emitHooks !== undefined &&
|
||||
canReplacePendingRestartEmitHooks(opts.emitHooks, opts.sessionKey);
|
||||
if (
|
||||
emitHooksQueued ||
|
||||
(!preservePendingHooks &&
|
||||
opts?.emitHooks === undefined &&
|
||||
(restartReasonPromoted || opts?.successorOwner !== undefined))
|
||||
) {
|
||||
pendingRestartEmitHooks = opts?.emitHooks;
|
||||
pendingRestartSessionKey = opts?.emitHooks ? opts.sessionKey : undefined;
|
||||
}
|
||||
if (opts?.emitHooks && !emitHooksQueued) {
|
||||
restartLog.warn(
|
||||
`restart continuation dropped: another session owns the pending restart (callerSessionKey=${opts.sessionKey ?? "unspecified"} pendingSessionKey=${pendingRestartSessionKey ?? "unspecified"})`,
|
||||
);
|
||||
}
|
||||
return {
|
||||
ok: true,
|
||||
pid: process.pid,
|
||||
signal: "SIGUSR1",
|
||||
...restartResultBase,
|
||||
delayMs: remainingMs,
|
||||
reason,
|
||||
mode,
|
||||
coalesced: true,
|
||||
cooldownMsApplied,
|
||||
emitHooksQueued,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
pendingRestartDueAt = requestedDueAt;
|
||||
pendingRestartReason = reason;
|
||||
pendingRestartReason = nextPendingReason;
|
||||
pendingRestartSuccessorOwner = nextPendingSuccessorOwner;
|
||||
pendingRestartEmitHooks = nextPendingEmitHooks;
|
||||
pendingRestartSessionKey = nextPendingSessionKey;
|
||||
pendingRestartSkipDeferral = skipDeferral;
|
||||
armPendingRestartTimer(requestedDueAt, nowMs);
|
||||
return {
|
||||
ok: true,
|
||||
pid: process.pid,
|
||||
signal: "SIGUSR1",
|
||||
...restartResultBase,
|
||||
delayMs: Math.max(0, requestedDueAt - nowMs),
|
||||
reason,
|
||||
mode,
|
||||
coalesced: false,
|
||||
cooldownMsApplied,
|
||||
emitHooksQueued: opts?.emitHooks !== undefined,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -106,6 +106,15 @@ describe("detectRespawnSupervisor", () => {
|
||||
"win32",
|
||||
),
|
||||
).toBeNull();
|
||||
expect(
|
||||
detectRespawnSupervisor(
|
||||
{
|
||||
OPENCLAW_SERVICE_MARKER: "other",
|
||||
OPENCLAW_SERVICE_KIND: "gateway",
|
||||
},
|
||||
"win32",
|
||||
),
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it("ignores service markers on non-Windows platforms and unknown platforms", () => {
|
||||
|
||||
@@ -75,9 +75,7 @@ export function detectRespawnSupervisor(
|
||||
if (hasAnyHint(env, SUPERVISOR_HINTS.schtasks)) {
|
||||
return "schtasks";
|
||||
}
|
||||
const marker = env.OPENCLAW_SERVICE_MARKER?.trim();
|
||||
const serviceKind = env.OPENCLAW_SERVICE_KIND?.trim();
|
||||
return marker && serviceKind === "gateway" ? "schtasks" : null;
|
||||
return hasOpenClawGatewayServiceMarker(env) ? "schtasks" : null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -1,24 +1,33 @@
|
||||
// Managed-service handoff command tests cover immutable update target serialization.
|
||||
import { EventEmitter } from "node:events";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { PassThrough } from "node:stream";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { parseDevUpdateTargetEnv, type DevUpdateTarget } from "./update-dev-target.js";
|
||||
import { signalMockManagedUpdateHandoffReady } from "./update-managed-service-handoff.test-support.js";
|
||||
|
||||
const spawnMock = vi.hoisted(() => vi.fn());
|
||||
const tempDirs = new Set<string>();
|
||||
const mockedHandoffLeaseCleanups = new Set<() => void>();
|
||||
const MOCK_INSTALL_ROOT = path.join(os.tmpdir(), `openclaw-handoff-command-${process.pid}`);
|
||||
|
||||
function createReadyChild() {
|
||||
function createReadyChild(_command: string, args: string[]) {
|
||||
const child = Object.assign(new EventEmitter(), {
|
||||
pid: 24680,
|
||||
pid: process.pid,
|
||||
exitCode: null,
|
||||
signalCode: null,
|
||||
stdin: new PassThrough(),
|
||||
stdout: new PassThrough(),
|
||||
unref: vi.fn(),
|
||||
});
|
||||
process.nextTick(() => {
|
||||
child.stdout.write("OPENCLAW_UPDATE_HANDOFF_READY\n");
|
||||
signalMockManagedUpdateHandoffReady({
|
||||
child,
|
||||
paramsPath: args.at(-1) ?? "",
|
||||
cleanups: mockedHandoffLeaseCleanups,
|
||||
});
|
||||
});
|
||||
return child;
|
||||
}
|
||||
@@ -37,6 +46,9 @@ beforeEach(() => {
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
for (const cleanup of mockedHandoffLeaseCleanups) {
|
||||
cleanup();
|
||||
}
|
||||
await Promise.all([...tempDirs].map((dir) => fs.rm(dir, { recursive: true, force: true })));
|
||||
tempDirs.clear();
|
||||
vi.resetModules();
|
||||
@@ -47,18 +59,23 @@ async function startHandoffAndReadCommand(params: {
|
||||
tag?: string;
|
||||
devTarget?: DevUpdateTarget;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
restartDelayMs?: number;
|
||||
restartDrainTimeoutMs?: number;
|
||||
}): Promise<{
|
||||
command: string;
|
||||
commandArgv: string[] | undefined;
|
||||
parentExitTimeoutMs: number;
|
||||
parentExitDeadlineAt: number;
|
||||
spawnEnv: NodeJS.ProcessEnv | undefined;
|
||||
}> {
|
||||
const { startManagedServiceUpdateHandoff } = await import("./update-managed-service-handoff.js");
|
||||
const result = await startManagedServiceUpdateHandoff({
|
||||
root: "/tmp/openclaw",
|
||||
restartDrainTimeoutMs: 300_000,
|
||||
root: MOCK_INSTALL_ROOT,
|
||||
restartDrainTimeoutMs: params.restartDrainTimeoutMs ?? 300_000,
|
||||
...(params.restartDelayMs === undefined ? {} : { restartDelayMs: params.restartDelayMs }),
|
||||
channel: params.channel,
|
||||
...(params.tag ? { tag: params.tag } : {}),
|
||||
parentPid: 12345,
|
||||
parentPid: process.pid,
|
||||
execPath: "/usr/local/bin/node",
|
||||
argv1: "/opt/openclaw/openclaw.mjs",
|
||||
meta: {},
|
||||
@@ -75,22 +92,45 @@ async function startHandoffAndReadCommand(params: {
|
||||
tempDirs.add(path.dirname(paramsPath));
|
||||
const helperParams = JSON.parse(await fs.readFile(paramsPath, "utf-8")) as {
|
||||
commandArgv?: string[];
|
||||
parentExitTimeoutMs: number;
|
||||
parentExitDeadlineAt: number;
|
||||
};
|
||||
const metaPath = path.join(path.dirname(paramsPath), "sentinel-meta.json");
|
||||
const metaFile = JSON.parse(await fs.readFile(metaPath, "utf-8")) as {
|
||||
meta?: { root?: string };
|
||||
};
|
||||
expect(metaFile.meta?.root).toBe(
|
||||
await fs.realpath("/tmp/openclaw").catch(() => path.resolve("/tmp/openclaw")),
|
||||
await fs.realpath(MOCK_INSTALL_ROOT).catch(() => path.resolve(MOCK_INSTALL_ROOT)),
|
||||
);
|
||||
return {
|
||||
command: result.command,
|
||||
commandArgv: helperParams.commandArgv,
|
||||
parentExitTimeoutMs: helperParams.parentExitTimeoutMs,
|
||||
parentExitDeadlineAt: helperParams.parentExitDeadlineAt,
|
||||
spawnEnv: spawnCall?.[2]?.env,
|
||||
};
|
||||
}
|
||||
|
||||
describe("managed service update handoff command", () => {
|
||||
it.each([
|
||||
{ drain: 300_000, expected: 390_000 },
|
||||
{ drain: Number.MAX_SAFE_INTEGER, expected: 2_147_483_647 },
|
||||
])(
|
||||
"serializes a bounded timer-safe restart deadline for drain $drain",
|
||||
async ({ drain, expected }) => {
|
||||
const startedAt = Date.now();
|
||||
const result = await startHandoffAndReadCommand({
|
||||
channel: "beta",
|
||||
restartDelayMs: 60_000,
|
||||
restartDrainTimeoutMs: drain,
|
||||
});
|
||||
|
||||
expect(result.parentExitTimeoutMs).toBe(expected);
|
||||
expect(result.parentExitDeadlineAt).toBeGreaterThanOrEqual(startedAt + expected);
|
||||
expect(result.parentExitDeadlineAt).toBeLessThanOrEqual(Date.now() + expected);
|
||||
},
|
||||
);
|
||||
|
||||
it("serializes extended-stable into the detached CLI command", async () => {
|
||||
const result = await startHandoffAndReadCommand({ channel: "extended-stable" });
|
||||
|
||||
|
||||
@@ -5,20 +5,27 @@ import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { PassThrough } from "node:stream";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { isPidAlive } from "../shared/pid-alive.js";
|
||||
import { signalMockManagedUpdateHandoffReady } from "./update-managed-service-handoff.test-support.js";
|
||||
|
||||
const spawnMock = vi.hoisted(() => vi.fn());
|
||||
const tempDirs = new Set<string>();
|
||||
const handoffParents = new Map<string, import("node:child_process").ChildProcess>();
|
||||
const mockedHandoffLeaseCleanups = new Set<() => void>();
|
||||
|
||||
function createReadyChild() {
|
||||
function createReadyChild(_command: string, args: string[]) {
|
||||
const child = Object.assign(new EventEmitter(), {
|
||||
pid: 24680,
|
||||
pid: process.pid,
|
||||
exitCode: null,
|
||||
signalCode: null,
|
||||
stdin: new PassThrough(),
|
||||
stdout: new PassThrough(),
|
||||
unref: vi.fn(),
|
||||
});
|
||||
process.nextTick(() => {
|
||||
child.stdout.write("OPENCLAW_UPDATE_HANDOFF_READY\n");
|
||||
process.nextTick(signalMockManagedUpdateHandoffReady, {
|
||||
child,
|
||||
paramsPath: args.at(-1) ?? "",
|
||||
cleanups: mockedHandoffLeaseCleanups,
|
||||
});
|
||||
return child;
|
||||
}
|
||||
@@ -31,12 +38,24 @@ vi.mock("node:child_process", async () => {
|
||||
});
|
||||
});
|
||||
|
||||
vi.mock("../daemon/systemd-scope.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("../daemon/systemd-scope.js")>()),
|
||||
findInstalledSystemdGatewayScope: vi.fn(async () => null),
|
||||
}));
|
||||
|
||||
beforeEach(() => {
|
||||
spawnMock.mockReset();
|
||||
spawnMock.mockImplementation(createReadyChild);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
for (const parent of handoffParents.values()) {
|
||||
parent.stdin?.end();
|
||||
}
|
||||
handoffParents.clear();
|
||||
for (const cleanup of mockedHandoffLeaseCleanups) {
|
||||
cleanup();
|
||||
}
|
||||
await Promise.all([...tempDirs].map((dir) => fs.rm(dir, { recursive: true, force: true })));
|
||||
tempDirs.clear();
|
||||
vi.resetModules();
|
||||
@@ -51,20 +70,12 @@ async function pathExists(filePath: string): Promise<boolean> {
|
||||
}
|
||||
}
|
||||
|
||||
function processIsAlive(pid: number): boolean {
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function prepareConcurrentHandoffHelper(): Promise<{
|
||||
tmpDir: string;
|
||||
helperScriptPath: string;
|
||||
baseParams: Record<string, unknown>;
|
||||
}> {
|
||||
const { DatabaseSync } = await import("node:sqlite");
|
||||
const { startManagedServiceUpdateHandoff } = await import("./update-managed-service-handoff.js");
|
||||
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-handoff-concurrent-test-"));
|
||||
tempDirs.add(tmpDir);
|
||||
@@ -84,12 +95,20 @@ async function prepareConcurrentHandoffHelper(): Promise<{
|
||||
|
||||
const [, args] = spawnMock.mock.calls.at(-1) as unknown as [string, string[]];
|
||||
const helperScriptPath = args[0] ?? "";
|
||||
const baseParams = JSON.parse(await fs.readFile(args[1] ?? "", "utf-8")) as Record<
|
||||
string,
|
||||
unknown
|
||||
>;
|
||||
const database = new DatabaseSync(String(baseParams.updateLeaseDatabasePath));
|
||||
try {
|
||||
database
|
||||
.prepare("DELETE FROM managed_update_handoffs WHERE install_root = ? AND owner = ?")
|
||||
.run(String(baseParams.updateLeaseKey), String(baseParams.updateLeaseOwner));
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
tempDirs.add(path.dirname(helperScriptPath));
|
||||
return {
|
||||
tmpDir,
|
||||
helperScriptPath,
|
||||
baseParams: JSON.parse(await fs.readFile(args[1] ?? "", "utf-8")) as Record<string, unknown>,
|
||||
};
|
||||
return { tmpDir, helperScriptPath, baseParams };
|
||||
}
|
||||
|
||||
async function writeConcurrentHandoffParams(params: {
|
||||
@@ -101,13 +120,27 @@ async function writeConcurrentHandoffParams(params: {
|
||||
stateDatabasePath?: string;
|
||||
leaseDatabasePath?: string;
|
||||
}): Promise<string> {
|
||||
const { spawn } =
|
||||
await vi.importActual<typeof import("node:child_process")>("node:child_process");
|
||||
const { getFileLockProcessStartTime } = await import("../shared/pid-alive.js");
|
||||
const parent = spawn(process.execPath, ["-e", "process.stdin.resume()"], {
|
||||
stdio: ["pipe", "ignore", "ignore"],
|
||||
});
|
||||
const parentPid = parent.pid;
|
||||
const startIdentity = parentPid ? getFileLockProcessStartTime(parentPid) : null;
|
||||
if (!parentPid || startIdentity === null) {
|
||||
parent.kill("SIGKILL");
|
||||
throw new Error("expected a parent process with a stable start identity");
|
||||
}
|
||||
const paramsPath = path.join(params.tmpDir, `${params.name}.json`);
|
||||
handoffParents.set(paramsPath, parent);
|
||||
await fs.writeFile(
|
||||
paramsPath,
|
||||
`${JSON.stringify(
|
||||
{
|
||||
...params.baseParams,
|
||||
parentPid: 0,
|
||||
parentPid,
|
||||
parentStartIdentity: String(startIdentity),
|
||||
parentExitTimeoutMs: 5_000,
|
||||
handoffId: params.owner,
|
||||
updateLeaseOwner: params.owner,
|
||||
@@ -128,18 +161,42 @@ async function writeConcurrentHandoffParams(params: {
|
||||
return paramsPath;
|
||||
}
|
||||
|
||||
function driveHandoffProtocol(
|
||||
child: import("node:child_process").ChildProcess,
|
||||
paramsPath: string,
|
||||
): void {
|
||||
let buffered = "";
|
||||
child.stdout?.on("data", (chunk: Buffer | string) => {
|
||||
buffered += chunk.toString();
|
||||
let newline: number;
|
||||
while ((newline = buffered.indexOf("\n")) >= 0) {
|
||||
const line = buffered.slice(0, newline);
|
||||
buffered = buffered.slice(newline + 1);
|
||||
if (line === "OPENCLAW_UPDATE_HANDOFF_READY") {
|
||||
child.stdin?.write("park\n");
|
||||
} else if (line === "parked") {
|
||||
child.stdin?.write("commit\n");
|
||||
} else if (line === "committed") {
|
||||
handoffParents.get(paramsPath)?.stdin?.end();
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function runHelper(params: {
|
||||
execFile: typeof import("node:child_process").execFile;
|
||||
helperScriptPath: string;
|
||||
paramsPath: string;
|
||||
cwd: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
}): Promise<{ code: number | null; stdout: string; stderr: string }> {
|
||||
return await new Promise((resolve) => {
|
||||
params.execFile(
|
||||
const child = params.execFile(
|
||||
process.execPath,
|
||||
[params.helperScriptPath, params.paramsPath],
|
||||
{ cwd: params.cwd, encoding: "utf8" },
|
||||
{ cwd: params.cwd, encoding: "utf8", ...(params.env ? { env: params.env } : {}) },
|
||||
(error, stdout, stderr) => {
|
||||
handoffParents.get(params.paramsPath)?.stdin?.end();
|
||||
const childError = error as NodeJS.ErrnoException | null;
|
||||
resolve({
|
||||
code: typeof childError?.code === "number" ? childError.code : 0,
|
||||
@@ -148,15 +205,17 @@ async function runHelper(params: {
|
||||
});
|
||||
},
|
||||
);
|
||||
driveHandoffProtocol(child, params.paramsPath);
|
||||
});
|
||||
}
|
||||
|
||||
describe("managed service update handoff cross-process lease", () => {
|
||||
it("joins the durable owner reported by a replacement helper", async () => {
|
||||
const child = Object.assign(new EventEmitter(), {
|
||||
pid: 24681,
|
||||
pid: process.pid,
|
||||
exitCode: null,
|
||||
signalCode: null,
|
||||
stdin: new PassThrough(),
|
||||
stdout: new PassThrough(),
|
||||
unref: vi.fn(),
|
||||
});
|
||||
@@ -171,7 +230,7 @@ describe("managed service update handoff cross-process lease", () => {
|
||||
const result = await startManagedServiceUpdateHandoff({
|
||||
root: "/tmp/openclaw",
|
||||
restartDrainTimeoutMs: 300_000,
|
||||
parentPid: 12345,
|
||||
parentPid: process.pid,
|
||||
execPath: "/usr/local/bin/node",
|
||||
argv1: "/opt/openclaw/openclaw.mjs",
|
||||
handoffId: "replacement-handoff",
|
||||
@@ -187,6 +246,488 @@ describe("managed service update handoff cross-process lease", () => {
|
||||
expect(result).not.toHaveProperty("pid");
|
||||
});
|
||||
|
||||
it.runIf(process.platform !== "win32").each([
|
||||
{ label: "its exact dead helper", replacement: "exact", reclaimed: true },
|
||||
{ label: "a mismatched owner", replacement: "owner", reclaimed: false },
|
||||
{ label: "a mismatched helper identity", replacement: "identity", reclaimed: false },
|
||||
{ label: "another live process", replacement: "live", reclaimed: false },
|
||||
{ label: "an unknown malformed process identity", replacement: "unknown", reclaimed: false },
|
||||
] as const)(
|
||||
"claims a separate systemd scope helper and fences cancellation against $label",
|
||||
async ({ replacement, reclaimed }) => {
|
||||
const { DatabaseSync } = await import("node:sqlite");
|
||||
const { getFileLockProcessStartTime } = await import("../shared/pid-alive.js");
|
||||
const { spawn } =
|
||||
await vi.importActual<typeof import("node:child_process")>("node:child_process");
|
||||
const {
|
||||
cancelManagedServiceUpdateHandoff,
|
||||
claimManagedServiceUpdateHandoff,
|
||||
startManagedServiceUpdateHandoff,
|
||||
} = await import("./update-managed-service-handoff.js");
|
||||
const tmpDir = await fs.realpath(
|
||||
await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-handoff-scope-wrapper-")),
|
||||
);
|
||||
tempDirs.add(tmpDir);
|
||||
const helperExitPath = path.join(tmpDir, "nested-helper-exited");
|
||||
const launcherPath = path.join(tmpDir, "systemd-run");
|
||||
await fs.writeFile(
|
||||
launcherPath,
|
||||
`#!${process.execPath}
|
||||
const fs = require("node:fs");
|
||||
const { spawn } = require("node:child_process");
|
||||
const [command, scriptPath, paramsPath] = process.argv.slice(-3);
|
||||
const helper = spawn(command, [scriptPath, paramsPath], { stdio: ["pipe", "pipe", "ignore"] });
|
||||
let helperAlive = true;
|
||||
helper.stdin.on("error", () => {});
|
||||
helper.stdout.pipe(process.stdout, { end: false });
|
||||
helper.once("exit", () => {
|
||||
helperAlive = false;
|
||||
fs.writeFileSync(${JSON.stringify(helperExitPath)}, String(helper.pid));
|
||||
});
|
||||
process.stdin.on("data", (chunk) => {
|
||||
if (helperAlive) {
|
||||
helper.stdin.write(chunk);
|
||||
} else if (chunk.toString().includes("cancel\\n")) {
|
||||
process.stdout.write("cancelled\\n", () => process.exit(0));
|
||||
}
|
||||
});
|
||||
`,
|
||||
{ mode: 0o700 },
|
||||
);
|
||||
spawnMock.mockImplementationOnce(spawn);
|
||||
|
||||
const handoffId = `scope-wrapper-${replacement}-${path.basename(tmpDir)}`;
|
||||
let launcher: import("node:child_process").ChildProcess | undefined;
|
||||
let helperPid = 0;
|
||||
let leaseDatabasePath: string | undefined;
|
||||
let leaseOwner = handoffId;
|
||||
try {
|
||||
const started = await startManagedServiceUpdateHandoff({
|
||||
root: tmpDir,
|
||||
restartDrainTimeoutMs: 5_000,
|
||||
restartDelayMs: 0,
|
||||
parentPid: process.pid,
|
||||
execPath: process.execPath,
|
||||
argv1: "/opt/openclaw/openclaw.mjs",
|
||||
supervisor: "systemd",
|
||||
handoffId,
|
||||
meta: { handoffId },
|
||||
env: {
|
||||
OPENCLAW_STATE_DIR: tmpDir,
|
||||
PATH: `${tmpDir}${path.delimiter}${process.env.PATH ?? ""}`,
|
||||
},
|
||||
});
|
||||
if (started.status !== "started" || !started.pid) {
|
||||
throw new Error("expected the systemd scope wrapper to start a real handoff");
|
||||
}
|
||||
launcher = spawnMock.mock.results.at(-1)?.value as
|
||||
| import("node:child_process").ChildProcess
|
||||
| undefined;
|
||||
tempDirs.add(path.dirname(started.logPath));
|
||||
const [, args] = spawnMock.mock.calls.at(-1) as unknown as [string, string[]];
|
||||
const helperParams = JSON.parse(await fs.readFile(args.at(-1) ?? "", "utf8")) as {
|
||||
updateLeaseDatabasePath: string;
|
||||
updateLeaseKey: string;
|
||||
};
|
||||
leaseDatabasePath = helperParams.updateLeaseDatabasePath;
|
||||
const identity = {
|
||||
kind: "managed-update-handoff" as const,
|
||||
handoffId,
|
||||
installRoot: started.installRoot,
|
||||
};
|
||||
const leaseDatabase = new DatabaseSync(leaseDatabasePath);
|
||||
let originalRow: { owner: string; payload_json: string };
|
||||
try {
|
||||
const current = leaseDatabase
|
||||
.prepare(
|
||||
"SELECT owner, payload_json FROM managed_update_handoffs WHERE install_root = ?",
|
||||
)
|
||||
.get(helperParams.updateLeaseKey) as
|
||||
| { owner: string; payload_json: string }
|
||||
| undefined;
|
||||
if (!current) {
|
||||
throw new Error("expected the nested handoff helper to own its durable lease");
|
||||
}
|
||||
originalRow = current;
|
||||
const helperIdentity = JSON.parse(current.payload_json) as {
|
||||
pid: number;
|
||||
startIdentity: string;
|
||||
};
|
||||
helperPid = helperIdentity.pid;
|
||||
|
||||
expect(helperPid).not.toBe(started.pid);
|
||||
expect(launcher?.pid).toBe(started.pid);
|
||||
expect(isPidAlive(helperPid)).toBe(true);
|
||||
expect(claimManagedServiceUpdateHandoff(identity)).toBe(true);
|
||||
|
||||
process.kill(helperPid, "SIGKILL");
|
||||
await vi.waitFor(
|
||||
async () => {
|
||||
await expect(fs.readFile(helperExitPath, "utf8")).resolves.toBe(String(helperPid));
|
||||
expect(isPidAlive(helperPid)).toBe(false);
|
||||
expect(isPidAlive(started.pid!)).toBe(true);
|
||||
},
|
||||
{ interval: 10, timeout: 5_000 },
|
||||
);
|
||||
expect(claimManagedServiceUpdateHandoff(identity)).toBe(false);
|
||||
|
||||
let payload = current.payload_json;
|
||||
if (replacement === "owner") {
|
||||
leaseOwner = `${handoffId}-foreign`;
|
||||
} else if (replacement === "identity") {
|
||||
payload = JSON.stringify({
|
||||
version: 1,
|
||||
pid: helperPid,
|
||||
startIdentity: `${helperIdentity.startIdentity}-mismatched`,
|
||||
});
|
||||
} else if (replacement === "live") {
|
||||
const liveStartIdentity = getFileLockProcessStartTime(process.pid);
|
||||
if (liveStartIdentity === null) {
|
||||
throw new Error("expected the live replacement to have a stable process identity");
|
||||
}
|
||||
payload = JSON.stringify({
|
||||
version: 1,
|
||||
pid: process.pid,
|
||||
startIdentity: String(liveStartIdentity),
|
||||
});
|
||||
} else if (replacement === "unknown") {
|
||||
payload = JSON.stringify({ version: 1, pid: helperPid, startIdentity: null });
|
||||
}
|
||||
if (replacement !== "exact") {
|
||||
leaseDatabase
|
||||
.prepare(
|
||||
"UPDATE managed_update_handoffs SET owner = ?, payload_json = ? WHERE install_root = ? AND owner = ?",
|
||||
)
|
||||
.run(leaseOwner, payload, helperParams.updateLeaseKey, handoffId);
|
||||
originalRow = { owner: leaseOwner, payload_json: payload };
|
||||
}
|
||||
} finally {
|
||||
leaseDatabase.close();
|
||||
}
|
||||
|
||||
await expect(cancelManagedServiceUpdateHandoff(identity)).resolves.toBe(
|
||||
reclaimed ? "restored-in-process" : false,
|
||||
);
|
||||
const retained = new DatabaseSync(leaseDatabasePath, { readOnly: true });
|
||||
try {
|
||||
expect(
|
||||
retained
|
||||
.prepare(
|
||||
"SELECT owner, payload_json FROM managed_update_handoffs WHERE install_root = ?",
|
||||
)
|
||||
.get(helperParams.updateLeaseKey),
|
||||
).toEqual(reclaimed ? undefined : originalRow!);
|
||||
} finally {
|
||||
retained.close();
|
||||
}
|
||||
} finally {
|
||||
if (helperPid > 0 && isPidAlive(helperPid)) {
|
||||
process.kill(helperPid, "SIGKILL");
|
||||
}
|
||||
if (launcher?.pid && isPidAlive(launcher.pid)) {
|
||||
launcher.kill("SIGKILL");
|
||||
}
|
||||
if (leaseDatabasePath) {
|
||||
const cleanup = new DatabaseSync(leaseDatabasePath);
|
||||
try {
|
||||
cleanup
|
||||
.prepare("DELETE FROM managed_update_handoffs WHERE install_root = ? AND owner = ?")
|
||||
.run(tmpDir, leaseOwner);
|
||||
} finally {
|
||||
cleanup.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
["an unavailable parent start identity", null, false],
|
||||
["a reused parent start identity", "0", false],
|
||||
["an existing live owner with a null start identity", "current", true],
|
||||
])("rejects %s before announcing ownership", async (_label, identity, invalidOwner) => {
|
||||
const { execFile } =
|
||||
await vi.importActual<typeof import("node:child_process")>("node:child_process");
|
||||
const { getFileLockProcessStartTime } = await import("../shared/pid-alive.js");
|
||||
const { DatabaseSync } = await import("node:sqlite");
|
||||
const { tmpDir, helperScriptPath, baseParams } = await prepareConcurrentHandoffHelper();
|
||||
const markerPath = path.join(tmpDir, "invalid-parent-update-ran");
|
||||
const paramsPath = await writeConcurrentHandoffParams({
|
||||
tmpDir,
|
||||
baseParams,
|
||||
name: "invalid-parent",
|
||||
owner: "invalid-parent-owner",
|
||||
commandArgv: [
|
||||
process.execPath,
|
||||
"-e",
|
||||
`require("node:fs").writeFileSync(${JSON.stringify(markerPath)}, "ran")`,
|
||||
],
|
||||
});
|
||||
const params = JSON.parse(await fs.readFile(paramsPath, "utf8")) as Record<string, unknown>;
|
||||
const currentStartIdentity = getFileLockProcessStartTime(process.pid);
|
||||
if (currentStartIdentity === null) {
|
||||
throw new Error("expected the live parent to have a stable start identity");
|
||||
}
|
||||
await fs.writeFile(
|
||||
paramsPath,
|
||||
JSON.stringify({
|
||||
...params,
|
||||
parentPid: process.pid,
|
||||
parentStartIdentity: identity === "current" ? String(currentStartIdentity) : identity,
|
||||
}),
|
||||
);
|
||||
const leaseDatabasePath = String(params.updateLeaseDatabasePath);
|
||||
const leaseKey = String(params.updateLeaseKey);
|
||||
const invalidPayload = JSON.stringify({
|
||||
version: 1,
|
||||
pid: process.pid,
|
||||
startIdentity: null,
|
||||
});
|
||||
const invalidUpdatedAt = Date.now();
|
||||
if (invalidOwner) {
|
||||
await fs.mkdir(path.dirname(leaseDatabasePath), { recursive: true, mode: 0o700 });
|
||||
const db = new DatabaseSync(leaseDatabasePath);
|
||||
try {
|
||||
db.exec(
|
||||
"CREATE TABLE IF NOT EXISTS managed_update_handoffs (install_root TEXT NOT NULL PRIMARY KEY, owner TEXT NOT NULL, payload_json TEXT NOT NULL, updated_at INTEGER NOT NULL) STRICT;",
|
||||
);
|
||||
db.prepare(
|
||||
"INSERT INTO managed_update_handoffs (install_root, owner, payload_json, updated_at) VALUES (?, ?, ?, ?)",
|
||||
).run(leaseKey, "unverifiable-owner", invalidPayload, invalidUpdatedAt);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
if (process.platform !== "win32") {
|
||||
await fs.chmod(leaseDatabasePath, 0o600);
|
||||
}
|
||||
}
|
||||
try {
|
||||
const result = await runHelper({ execFile, helperScriptPath, paramsPath, cwd: tmpDir });
|
||||
|
||||
expect(result.code).toBe(1);
|
||||
expect(result.stdout).not.toContain("OPENCLAW_UPDATE_HANDOFF_READY");
|
||||
await expect(pathExists(markerPath)).resolves.toBe(false);
|
||||
if (await pathExists(leaseDatabasePath)) {
|
||||
const db = new DatabaseSync(leaseDatabasePath, { readOnly: true });
|
||||
try {
|
||||
const row = db
|
||||
.prepare(
|
||||
"SELECT owner, payload_json, updated_at FROM managed_update_handoffs WHERE install_root = ?",
|
||||
)
|
||||
.get(leaseKey);
|
||||
expect(row).toEqual(
|
||||
invalidOwner
|
||||
? {
|
||||
owner: "unverifiable-owner",
|
||||
payload_json: invalidPayload,
|
||||
updated_at: invalidUpdatedAt,
|
||||
}
|
||||
: undefined,
|
||||
);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
if (invalidOwner) {
|
||||
const db = new DatabaseSync(leaseDatabasePath);
|
||||
db.prepare("DELETE FROM managed_update_handoffs WHERE install_root = ?").run(leaseKey);
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("preserves a live durable owner when its current start identity cannot be observed", async () => {
|
||||
const { execFile } =
|
||||
await vi.importActual<typeof import("node:child_process")>("node:child_process");
|
||||
const { DatabaseSync } = await import("node:sqlite");
|
||||
const { getFileLockProcessStartTime } = await import("../shared/pid-alive.js");
|
||||
const { tmpDir, helperScriptPath, baseParams } = await prepareConcurrentHandoffHelper();
|
||||
const ownerStartIdentity = getFileLockProcessStartTime(process.pid);
|
||||
if (ownerStartIdentity === null) {
|
||||
throw new Error("expected the live lease owner to have a stable process identity");
|
||||
}
|
||||
const leaseDatabasePath = String(baseParams.updateLeaseDatabasePath);
|
||||
const leaseKey = String(baseParams.updateLeaseKey);
|
||||
const owner = "identity-probe-unavailable-owner";
|
||||
const payload = JSON.stringify({
|
||||
version: 1,
|
||||
pid: process.pid,
|
||||
startIdentity: String(ownerStartIdentity),
|
||||
});
|
||||
await fs.mkdir(path.dirname(leaseDatabasePath), { recursive: true, mode: 0o700 });
|
||||
const database = new DatabaseSync(leaseDatabasePath);
|
||||
try {
|
||||
database.exec(
|
||||
"CREATE TABLE IF NOT EXISTS managed_update_handoffs (install_root TEXT NOT NULL PRIMARY KEY, owner TEXT NOT NULL, payload_json TEXT NOT NULL, updated_at INTEGER NOT NULL) STRICT;",
|
||||
);
|
||||
database
|
||||
.prepare(
|
||||
"INSERT INTO managed_update_handoffs (install_root, owner, payload_json, updated_at) VALUES (?, ?, ?, ?)",
|
||||
)
|
||||
.run(leaseKey, owner, payload, Date.now());
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
if (process.platform !== "win32") {
|
||||
await fs.chmod(leaseDatabasePath, 0o600);
|
||||
}
|
||||
const preloadPath = path.join(tmpDir, "unavailable-owner-identity.cjs");
|
||||
await fs.writeFile(
|
||||
preloadPath,
|
||||
`const fs = require("node:fs");
|
||||
const childProcess = require("node:child_process");
|
||||
const ownerPid = ${process.pid};
|
||||
const readFileSync = fs.readFileSync;
|
||||
fs.readFileSync = function(filePath, ...args) {
|
||||
if (filePath === "/proc/" + ownerPid + "/stat") {
|
||||
throw Object.assign(new Error("identity probe unavailable"), { code: "EIO" });
|
||||
}
|
||||
return readFileSync.call(this, filePath, ...args);
|
||||
};
|
||||
const spawnSync = childProcess.spawnSync;
|
||||
childProcess.spawnSync = function(command, args, options) {
|
||||
if (args.some((argument) => String(argument).includes(String(ownerPid)))) {
|
||||
return { status: null, stdout: "", error: new Error("identity probe unavailable") };
|
||||
}
|
||||
return spawnSync.call(this, command, args, options);
|
||||
};
|
||||
`,
|
||||
);
|
||||
const markerPath = path.join(tmpDir, "identity-probe-unavailable-updater-ran");
|
||||
const paramsPath = await writeConcurrentHandoffParams({
|
||||
tmpDir,
|
||||
baseParams,
|
||||
name: "identity-probe-unavailable",
|
||||
owner: "competing-owner",
|
||||
commandArgv: [
|
||||
process.execPath,
|
||||
"-e",
|
||||
`require("node:fs").writeFileSync(${JSON.stringify(markerPath)}, "ran")`,
|
||||
],
|
||||
});
|
||||
try {
|
||||
const result = await runHelper({
|
||||
execFile,
|
||||
helperScriptPath,
|
||||
paramsPath,
|
||||
cwd: tmpDir,
|
||||
env: { ...process.env, NODE_OPTIONS: `--require ${preloadPath}` },
|
||||
});
|
||||
|
||||
expect(result, result.stderr).toMatchObject({
|
||||
code: 0,
|
||||
stdout: expect.stringContaining(`HANDOFF_BUSY ${owner}`),
|
||||
});
|
||||
const retained = new DatabaseSync(leaseDatabasePath, { readOnly: true });
|
||||
try {
|
||||
expect(
|
||||
retained
|
||||
.prepare(
|
||||
"SELECT owner, payload_json FROM managed_update_handoffs WHERE install_root = ?",
|
||||
)
|
||||
.get(leaseKey),
|
||||
).toEqual({ owner, payload_json: payload });
|
||||
} finally {
|
||||
retained.close();
|
||||
}
|
||||
await expect(pathExists(markerPath)).resolves.toBe(false);
|
||||
} finally {
|
||||
const cleanup = new DatabaseSync(leaseDatabasePath);
|
||||
cleanup
|
||||
.prepare("DELETE FROM managed_update_handoffs WHERE install_root = ? AND owner = ?")
|
||||
.run(leaseKey, owner);
|
||||
cleanup.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("releases exact helper ownership when cancellation cannot record its terminal sentinel", async () => {
|
||||
const { spawn } =
|
||||
await vi.importActual<typeof import("node:child_process")>("node:child_process");
|
||||
const { DatabaseSync } = await import("node:sqlite");
|
||||
const { tmpDir, helperScriptPath, baseParams } = await prepareConcurrentHandoffHelper();
|
||||
const markerPath = path.join(tmpDir, "sentinel-failure-updater-ran");
|
||||
const owner = "sentinel-failure-owner";
|
||||
const paramsPath = await writeConcurrentHandoffParams({
|
||||
tmpDir,
|
||||
baseParams,
|
||||
name: "sentinel-failure",
|
||||
owner,
|
||||
commandArgv: [
|
||||
process.execPath,
|
||||
"-e",
|
||||
`require("node:fs").writeFileSync(${JSON.stringify(markerPath)}, "ran")`,
|
||||
],
|
||||
});
|
||||
const stateDatabasePath = String(baseParams.stateDatabasePath);
|
||||
await fs.mkdir(path.dirname(stateDatabasePath), { recursive: true });
|
||||
const stateDb = new DatabaseSync(stateDatabasePath);
|
||||
try {
|
||||
stateDb.exec(
|
||||
[
|
||||
"CREATE TABLE gateway_restart_sentinel (",
|
||||
"sentinel_key TEXT NOT NULL PRIMARY KEY, version INTEGER NOT NULL,",
|
||||
"kind TEXT NOT NULL, status TEXT NOT NULL, ts INTEGER NOT NULL,",
|
||||
"session_key TEXT, thread_id TEXT, delivery_channel TEXT, delivery_to TEXT,",
|
||||
"delivery_account_id TEXT, message TEXT, continuation_json TEXT, doctor_hint TEXT,",
|
||||
"stats_json TEXT, payload_json TEXT NOT NULL, updated_at_ms INTEGER NOT NULL",
|
||||
") STRICT;",
|
||||
"CREATE TRIGGER reject_update_sentinel_write",
|
||||
"BEFORE INSERT ON gateway_restart_sentinel",
|
||||
"WHEN NEW.sentinel_key = 'current'",
|
||||
"BEGIN SELECT RAISE(ABORT, 'sentinel write rejected'); END;",
|
||||
].join(" "),
|
||||
);
|
||||
} finally {
|
||||
stateDb.close();
|
||||
}
|
||||
const leaseDatabasePath = String(baseParams.updateLeaseDatabasePath);
|
||||
const leaseKey = String(baseParams.updateLeaseKey);
|
||||
const helper = spawn(process.execPath, [helperScriptPath, paramsPath], {
|
||||
cwd: tmpDir,
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
});
|
||||
let output = "";
|
||||
helper.stdout.on("data", (chunk: Buffer | string) => {
|
||||
output += chunk.toString();
|
||||
if (output.includes("OPENCLAW_UPDATE_HANDOFF_READY") && !output.includes("cancelled")) {
|
||||
helper.stdin.write("cancel\n");
|
||||
}
|
||||
});
|
||||
const exited = new Promise<number | null>((resolve) => {
|
||||
helper.once("close", resolve);
|
||||
});
|
||||
|
||||
try {
|
||||
await expect(exited).resolves.toBe(0);
|
||||
expect(output).toContain("cancelled");
|
||||
const db = new DatabaseSync(leaseDatabasePath, { readOnly: true });
|
||||
try {
|
||||
expect(
|
||||
db
|
||||
.prepare("SELECT owner FROM managed_update_handoffs WHERE install_root = ?")
|
||||
.get(leaseKey),
|
||||
).toBeUndefined();
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
const log = await fs.readFile(path.join(tmpDir, "sentinel-failure.log"), "utf8");
|
||||
expect(log).toContain("failed to write update sentinel failure");
|
||||
expect(log).toContain("sentinel write rejected");
|
||||
await expect(pathExists(markerPath)).resolves.toBe(false);
|
||||
} finally {
|
||||
if (helper.exitCode === null && helper.signalCode === null) {
|
||||
helper.kill("SIGKILL");
|
||||
}
|
||||
const db = new DatabaseSync(leaseDatabasePath);
|
||||
db.prepare("DELETE FROM managed_update_handoffs WHERE install_root = ? AND owner = ?").run(
|
||||
leaseKey,
|
||||
owner,
|
||||
);
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
it.runIf(process.platform === "win32")(
|
||||
"reclaims a reused Windows PID with a different creation identity",
|
||||
async () => {
|
||||
@@ -330,8 +871,9 @@ describe("managed service update handoff cross-process lease", () => {
|
||||
|
||||
const first = spawn(process.execPath, [helperScriptPath, firstParamsPath], {
|
||||
cwd: tmpDir,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
});
|
||||
driveHandoffProtocol(first, firstParamsPath);
|
||||
let firstStdout = "";
|
||||
first.stdout.on("data", (chunk) => (firstStdout += chunk));
|
||||
let orphanPid = 0;
|
||||
@@ -344,7 +886,7 @@ describe("managed service update handoff cross-process lease", () => {
|
||||
{ interval: 10, timeout: 5_000 },
|
||||
);
|
||||
orphanPid = Number(await fs.readFile(orphanPidPath, "utf8"));
|
||||
expect(processIsAlive(orphanPid)).toBe(true);
|
||||
expect(isPidAlive(orphanPid)).toBe(true);
|
||||
first.kill("SIGKILL");
|
||||
await new Promise<void>((resolve) => {
|
||||
first.once("close", () => resolve());
|
||||
@@ -363,7 +905,7 @@ describe("managed service update handoff cross-process lease", () => {
|
||||
await expect(pathExists(secondStartedPath)).resolves.toBe(false);
|
||||
|
||||
await fs.writeFile(releaseOrphanPath, "release");
|
||||
await vi.waitFor(() => expect(processIsAlive(orphanPid)).toBe(false), {
|
||||
await vi.waitFor(() => expect(isPidAlive(orphanPid)).toBe(false), {
|
||||
interval: 20,
|
||||
timeout: 5_000,
|
||||
});
|
||||
@@ -384,7 +926,7 @@ describe("managed service update handoff cross-process lease", () => {
|
||||
if (first.exitCode === null) {
|
||||
first.kill("SIGKILL");
|
||||
}
|
||||
if (orphanPid > 0 && processIsAlive(orphanPid)) {
|
||||
if (orphanPid > 0 && isPidAlive(orphanPid)) {
|
||||
process.kill(orphanPid, "SIGKILL");
|
||||
}
|
||||
}
|
||||
@@ -437,8 +979,9 @@ describe("managed service update handoff cross-process lease", () => {
|
||||
|
||||
const first = spawn(process.execPath, [helperScriptPath, firstParamsPath], {
|
||||
cwd: tmpDir,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
});
|
||||
driveHandoffProtocol(first, firstParamsPath);
|
||||
let firstStdout = "";
|
||||
let firstStderr = "";
|
||||
first.stdout.on("data", (chunk) => (firstStdout += chunk));
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
export type ManagedServiceManagerBoundaryOptions = {
|
||||
cancelAfterPark?: boolean;
|
||||
parentExitTimeoutMs?: number;
|
||||
launchdFault?: "wrong-parent" | "missing-restored-pid" | "dead-restored-pid";
|
||||
launchdTeardown?: {
|
||||
bootoutDelayMs?: number;
|
||||
clockEachCommandMs?: number;
|
||||
loadedPrints?: number;
|
||||
pendingBootstrapFailures?: number;
|
||||
pendingOperationInProgress?: number;
|
||||
};
|
||||
lateCommand?: "park" | "commit";
|
||||
overdueCommit?: boolean;
|
||||
systemdFault?: "start-failed" | "dead-restored-pid";
|
||||
};
|
||||
|
||||
export type ManagedServiceCommandTiming = {
|
||||
action: string;
|
||||
startedAtMs: number;
|
||||
timeoutMs: number;
|
||||
};
|
||||
|
||||
export function createManagedServiceManagerFixtureScript(params: {
|
||||
kind: "systemd" | "launchd";
|
||||
parentPid: number;
|
||||
statePath: string;
|
||||
commandsPath: string;
|
||||
options?: ManagedServiceManagerBoundaryOptions;
|
||||
}): string {
|
||||
const { commandsPath, kind, options, parentPid, statePath } = params;
|
||||
return `#!${process.execPath}
|
||||
const fs = require("node:fs");
|
||||
const args = process.argv.slice(2);
|
||||
const statePath = ${JSON.stringify(statePath)};
|
||||
const state = fs.existsSync(statePath) ? JSON.parse(fs.readFileSync(statePath, "utf8")) : {};
|
||||
fs.appendFileSync(${JSON.stringify(commandsPath)}, args.join(" ") + "\\n");
|
||||
const action = args.find((arg) => ["show", "stop", "reset-failed", "start", "print", "disable", "bootout", "enable", "bootstrap", "kickstart"].includes(arg));
|
||||
if (${JSON.stringify(kind)} === "systemd") {
|
||||
if (action === "stop") state.parked = true;
|
||||
if (action === "stop" && ${Boolean(options?.lateCommand)}) setTimeout(() => {}, 200);
|
||||
if (action === "reset-failed") state.reset = true;
|
||||
if (action === "start" && ${JSON.stringify(options?.systemdFault)} === "start-failed") {
|
||||
state.startFailed = true;
|
||||
process.stderr.write("start limit hit\\n");
|
||||
process.exitCode = 1;
|
||||
} else if (action === "start") state.restored = true;
|
||||
if (action === "show") {
|
||||
const active = !state.parked || state.restored;
|
||||
const restoredPid = ${JSON.stringify(options?.systemdFault)} === "dead-restored-pid" ? 2147483647 : ${process.pid};
|
||||
process.stdout.write([
|
||||
"Id=openclaw-gateway.service",
|
||||
"LoadState=loaded",
|
||||
"ActiveState=" + (active ? "active" : "inactive"),
|
||||
"MainPID=" + (state.restored ? restoredPid : active ? ${parentPid} : 0),
|
||||
"ExecMainStartTimestampMonotonic=" + (state.restored ? "222" : "111"),
|
||||
].join("\\n") + "\\n");
|
||||
}
|
||||
} else {
|
||||
if (action === "disable") state.disabled = true;
|
||||
if (action === "bootout") {
|
||||
state.parked = true;
|
||||
state.loadedPrintsRemaining = ${options?.launchdTeardown?.loadedPrints ?? 0};
|
||||
state.pendingBootstrapFailures = ${options?.launchdTeardown?.pendingBootstrapFailures ?? 0};
|
||||
state.pendingOperationInProgress = ${options?.launchdTeardown?.pendingOperationInProgress ?? 0};
|
||||
const delay = ${options?.launchdTeardown?.bootoutDelayMs ?? 0};
|
||||
if (delay) setTimeout(() => {
|
||||
state.bootoutCompleted = true;
|
||||
fs.writeFileSync(statePath, JSON.stringify(state));
|
||||
}, delay);
|
||||
}
|
||||
if (action === "enable") state.disabled = false;
|
||||
if (action === "bootstrap" || action === "kickstart") {
|
||||
state.bootstrapAttempts = (state.bootstrapAttempts || 0) + 1;
|
||||
if (state.pendingOperationInProgress > 0) {
|
||||
state.pendingOperationInProgress -= 1;
|
||||
state.operationInProgressObserved = (state.operationInProgressObserved || 0) + 1;
|
||||
process.stderr.write("Bootstrap failed: 37: Operation already in progress\\n");
|
||||
process.exitCode = 37;
|
||||
} else if (!state.unloaded) {
|
||||
process.stderr.write("Bootstrap failed: 37: Operation already in progress\\n");
|
||||
process.exitCode = 37;
|
||||
} else if (action === "bootstrap" && state.pendingBootstrapFailures > 0) {
|
||||
state.pendingBootstrapFailures -= 1;
|
||||
process.stderr.write("Bootstrap failed: 5: Input/output error\\n");
|
||||
process.exitCode = 5;
|
||||
} else state.restored = true;
|
||||
}
|
||||
if (action === "print") {
|
||||
let parentAlive = false;
|
||||
try { process.kill(${parentPid}, 0); parentAlive = true; } catch {}
|
||||
if (state.parked && !state.restored && !parentAlive) {
|
||||
if (state.loadedPrintsRemaining > 0) {
|
||||
state.loadedPrintsRemaining -= 1;
|
||||
state.loadedPrintsObserved = (state.loadedPrintsObserved || 0) + 1;
|
||||
} else {
|
||||
state.unloaded = true;
|
||||
process.stderr.write("Could not find service\\n");
|
||||
fs.writeFileSync(statePath, JSON.stringify(state));
|
||||
process.exit(113);
|
||||
}
|
||||
}
|
||||
const fault = ${JSON.stringify(options?.launchdFault)};
|
||||
if (state.restored && fault === "missing-restored-pid") {
|
||||
process.stdout.write("state = running\\n");
|
||||
} else {
|
||||
const restoredPid = fault === "dead-restored-pid" ? 2147483647 : ${process.pid};
|
||||
const currentPid = fault === "wrong-parent" ? ${process.pid} : ${parentPid};
|
||||
process.stdout.write("state = running\\npid = " + (state.restored ? restoredPid : currentPid) + "\\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
fs.writeFileSync(statePath, JSON.stringify(state));
|
||||
`;
|
||||
}
|
||||
|
||||
export function createManagedServiceLaunchdClockPreload(params: {
|
||||
commandTimingsPath: string;
|
||||
clockEachCommandMs: number;
|
||||
}): string {
|
||||
return [
|
||||
'const fs = require("node:fs");',
|
||||
'const children = require("node:child_process");',
|
||||
"const actualSpawn = children.spawn;",
|
||||
"const actualSetTimeout = global.setTimeout;",
|
||||
"const startedAt = Date.now();",
|
||||
"let elapsed = 0;",
|
||||
"Date.now = () => startedAt + elapsed;",
|
||||
"global.setTimeout = (callback, delay, ...args) => {",
|
||||
" if (delay === 500) {",
|
||||
" elapsed += delay;",
|
||||
" return actualSetTimeout(callback, 0, ...args);",
|
||||
" }",
|
||||
" return actualSetTimeout(callback, delay, ...args);",
|
||||
"};",
|
||||
"children.spawn = (command, args, options) => {",
|
||||
' if (command === "launchctl") {',
|
||||
" const timeoutMs = options.timeout;",
|
||||
" const startedAtMs = Date.now();",
|
||||
` fs.appendFileSync(${JSON.stringify(params.commandTimingsPath)}, JSON.stringify({ action: args[0], startedAtMs, timeoutMs }) + "\\n");`,
|
||||
` elapsed += Math.min(${params.clockEachCommandMs}, timeoutMs);`,
|
||||
" }",
|
||||
" return actualSpawn(command, args, options);",
|
||||
"};",
|
||||
].join("\n");
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -5,7 +5,7 @@ import { EventEmitter } from "node:events";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { PassThrough } from "node:stream";
|
||||
import { PassThrough, type Readable } from "node:stream";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js";
|
||||
import {
|
||||
@@ -14,20 +14,40 @@ import {
|
||||
} from "../state/openclaw-state-db.js";
|
||||
import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js";
|
||||
import { claimOpenClawStateOwnership } from "../state/openclaw-state-ownership-operations.js";
|
||||
import { executeSqliteQuerySync, getNodeSqliteKysely } from "./kysely-sync.js";
|
||||
import {
|
||||
executeSqliteQuerySync,
|
||||
executeSqliteQueryTakeFirstSync,
|
||||
getNodeSqliteKysely,
|
||||
} from "./kysely-sync.js";
|
||||
import { signalMockManagedUpdateHandoffReady } from "./update-managed-service-handoff.test-support.js";
|
||||
|
||||
const { spawnMock } = vi.hoisted(() => ({ spawnMock: vi.fn() }));
|
||||
|
||||
function createSpawnMock() {
|
||||
return Object.assign(new EventEmitter(), {
|
||||
pid: 24680,
|
||||
pid: process.pid,
|
||||
exitCode: null,
|
||||
signalCode: null,
|
||||
stdin: new PassThrough(),
|
||||
stdout: new PassThrough(),
|
||||
unref: vi.fn(),
|
||||
});
|
||||
}
|
||||
|
||||
async function waitForHandoffLine(output: Readable | null, expected: string): Promise<void> {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const onData = (chunk: Buffer | string) => {
|
||||
if (!chunk.toString().includes(`${expected}\n`)) {
|
||||
return;
|
||||
}
|
||||
output?.removeListener("data", onData);
|
||||
resolve();
|
||||
};
|
||||
output?.on("data", onData);
|
||||
output?.once("end", () => reject(new Error(`helper exited before ${expected}`)));
|
||||
});
|
||||
}
|
||||
|
||||
vi.mock("node:child_process", async () => {
|
||||
const { mockNodeChildProcessModule } =
|
||||
await import("../gateway/server-methods/node-child-process.test-support.js");
|
||||
@@ -37,82 +57,170 @@ vi.mock("node:child_process", async () => {
|
||||
});
|
||||
|
||||
const tempDirs = new Set<string>();
|
||||
const mockedHandoffLeaseCleanups = new Set<() => void>();
|
||||
type GatewayRestartSentinelDatabase = Pick<OpenClawStateKyselyDatabase, "gateway_restart_sentinel">;
|
||||
|
||||
beforeEach(() => {
|
||||
spawnMock.mockReset();
|
||||
spawnMock.mockImplementation(() => {
|
||||
spawnMock.mockImplementation((_command: string, args: string[]) => {
|
||||
const child = createSpawnMock();
|
||||
process.nextTick(() => {
|
||||
child.stdout.write("OPENCLAW_UPDATE_HANDOFF_READY\n");
|
||||
signalMockManagedUpdateHandoffReady({
|
||||
child,
|
||||
paramsPath: args.at(-1) ?? "",
|
||||
cleanups: mockedHandoffLeaseCleanups,
|
||||
});
|
||||
});
|
||||
return child;
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
for (const cleanup of mockedHandoffLeaseCleanups) {
|
||||
cleanup();
|
||||
}
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
await Promise.all([...tempDirs].map((dir) => fs.rm(dir, { recursive: true, force: true })));
|
||||
tempDirs.clear();
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
function writeRestartSentinelRow(
|
||||
env: NodeJS.ProcessEnv,
|
||||
sentinel: {
|
||||
version: 1;
|
||||
revision: number;
|
||||
payload: {
|
||||
kind: string;
|
||||
status: string;
|
||||
ts: number;
|
||||
stats: Record<string, unknown>;
|
||||
};
|
||||
},
|
||||
): void {
|
||||
function writeRestartSentinelRow(env: NodeJS.ProcessEnv, sentinel: unknown): void {
|
||||
const { db } = openOpenClawStateDatabase({ env });
|
||||
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
|
||||
const payload =
|
||||
sentinel && typeof sentinel === "object" && (sentinel as { version?: unknown }).version === 1
|
||||
? (sentinel as { payload?: unknown }).payload
|
||||
: null;
|
||||
if (!payload || typeof payload !== "object") {
|
||||
throw new Error("expected versioned restart sentinel payload");
|
||||
}
|
||||
const record = payload as {
|
||||
kind?: unknown;
|
||||
status?: unknown;
|
||||
ts?: unknown;
|
||||
sessionKey?: unknown;
|
||||
threadId?: unknown;
|
||||
deliveryContext?: { channel?: unknown; to?: unknown; accountId?: unknown };
|
||||
message?: unknown;
|
||||
continuation?: unknown;
|
||||
doctorHint?: unknown;
|
||||
stats?: unknown;
|
||||
};
|
||||
const revision =
|
||||
typeof (sentinel as { revision?: unknown }).revision === "number"
|
||||
? (sentinel as { revision: number }).revision
|
||||
: Date.now();
|
||||
executeSqliteQuerySync(
|
||||
db,
|
||||
stateDb.insertInto("gateway_restart_sentinel").values({
|
||||
sentinel_key: record.kind === "revision-floor" ? "revision-floor" : "current",
|
||||
version: 1,
|
||||
kind: typeof record.kind === "string" ? record.kind : "update",
|
||||
status: typeof record.status === "string" ? record.status : "skipped",
|
||||
ts: typeof record.ts === "number" ? record.ts : Date.now(),
|
||||
session_key: typeof record.sessionKey === "string" ? record.sessionKey : null,
|
||||
thread_id: typeof record.threadId === "string" ? record.threadId : null,
|
||||
delivery_channel:
|
||||
typeof record.deliveryContext?.channel === "string" ? record.deliveryContext.channel : null,
|
||||
delivery_to:
|
||||
typeof record.deliveryContext?.to === "string" ? record.deliveryContext.to : null,
|
||||
delivery_account_id:
|
||||
typeof record.deliveryContext?.accountId === "string"
|
||||
? record.deliveryContext.accountId
|
||||
: null,
|
||||
message: typeof record.message === "string" ? record.message : null,
|
||||
continuation_json: record.continuation ? JSON.stringify(record.continuation) : null,
|
||||
doctor_hint: typeof record.doctorHint === "string" ? record.doctorHint : null,
|
||||
stats_json: record.stats ? JSON.stringify(record.stats) : null,
|
||||
payload_json: JSON.stringify(payload),
|
||||
updated_at_ms: revision,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
function replaceRestartSentinelRow(env: NodeJS.ProcessEnv, sentinel: unknown): void {
|
||||
const { db } = openOpenClawStateDatabase({ env });
|
||||
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
|
||||
executeSqliteQuerySync(
|
||||
db,
|
||||
stateDb.insertInto("gateway_restart_sentinel").values({
|
||||
sentinel_key: "current",
|
||||
version: sentinel.version,
|
||||
kind: sentinel.payload.kind,
|
||||
status: sentinel.payload.status,
|
||||
ts: sentinel.payload.ts,
|
||||
session_key: null,
|
||||
thread_id: null,
|
||||
delivery_channel: null,
|
||||
delivery_to: null,
|
||||
delivery_account_id: null,
|
||||
message: null,
|
||||
continuation_json: null,
|
||||
doctor_hint: null,
|
||||
stats_json: JSON.stringify(sentinel.payload.stats),
|
||||
payload_json: JSON.stringify(sentinel.payload),
|
||||
updated_at_ms: sentinel.revision,
|
||||
}),
|
||||
stateDb.deleteFrom("gateway_restart_sentinel").where("sentinel_key", "=", "current"),
|
||||
);
|
||||
writeRestartSentinelRow(env, sentinel);
|
||||
}
|
||||
|
||||
function readRestartSentinelPayload(env: NodeJS.ProcessEnv, key = "current"): unknown {
|
||||
const { db } = openOpenClawStateDatabase({ env });
|
||||
const stateDb = getNodeSqliteKysely<GatewayRestartSentinelDatabase>(db);
|
||||
const row = executeSqliteQueryTakeFirstSync(
|
||||
db,
|
||||
stateDb
|
||||
.selectFrom("gateway_restart_sentinel")
|
||||
.select(["version", "payload_json", "updated_at_ms"])
|
||||
.where("sentinel_key", "=", key),
|
||||
);
|
||||
return row
|
||||
? { version: row.version, payload: JSON.parse(row.payload_json), revision: row.updated_at_ms }
|
||||
: null;
|
||||
}
|
||||
|
||||
async function createLegacyRestartSentinelTable(env: NodeJS.ProcessEnv): Promise<void> {
|
||||
const sqlite = await import("node:sqlite");
|
||||
const stateDatabasePath = resolveOpenClawStateSqlitePath(env);
|
||||
await fs.mkdir(path.dirname(stateDatabasePath), { recursive: true });
|
||||
const db = new sqlite.DatabaseSync(stateDatabasePath);
|
||||
try {
|
||||
db.exec(`
|
||||
CREATE TABLE gateway_restart_sentinel (
|
||||
sentinel_key TEXT NOT NULL PRIMARY KEY,
|
||||
version INTEGER NOT NULL,
|
||||
kind TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
ts INTEGER NOT NULL,
|
||||
session_key TEXT,
|
||||
thread_id TEXT,
|
||||
payload_json TEXT NOT NULL,
|
||||
updated_at_ms INTEGER NOT NULL
|
||||
);
|
||||
`);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function runOwnershipHelper(params: {
|
||||
handoffId?: string;
|
||||
metaHandoffId?: string;
|
||||
prepareStateDatabase: (env: NodeJS.ProcessEnv) => Promise<void> | void;
|
||||
whileHelperRunning?: (context: { logPath: string }) => Promise<void> | void;
|
||||
prepareStateDatabase?: (env: NodeJS.ProcessEnv) => Promise<void> | void;
|
||||
sentinel?: unknown;
|
||||
deepStatePath?: boolean;
|
||||
commandDelayMs?: number;
|
||||
commandExitCode?: number;
|
||||
runnerFault?: "closed-stdin";
|
||||
whileHelperRunning?: (context: {
|
||||
env: NodeJS.ProcessEnv;
|
||||
logPath: string;
|
||||
}) => Promise<void> | void;
|
||||
}) {
|
||||
const { execFile } =
|
||||
const { spawn } =
|
||||
await vi.importActual<typeof import("node:child_process")>("node:child_process");
|
||||
const { getFileLockProcessStartTime } = await import("../shared/pid-alive.js");
|
||||
const { startManagedServiceUpdateHandoff } = await import("./update-managed-service-handoff.js");
|
||||
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-handoff-ownership-test-"));
|
||||
tempDirs.add(tmpDir);
|
||||
const env = { OPENCLAW_STATE_DIR: tmpDir } as NodeJS.ProcessEnv;
|
||||
let stateDir = tmpDir;
|
||||
while (
|
||||
params.deepStatePath &&
|
||||
resolveOpenClawStateSqlitePath({ OPENCLAW_STATE_DIR: stateDir }).length <= 260
|
||||
) {
|
||||
stateDir = path.join(stateDir, `segment-${"x".repeat(24)}`);
|
||||
}
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir } as NodeJS.ProcessEnv;
|
||||
|
||||
await startManagedServiceUpdateHandoff({
|
||||
root: tmpDir,
|
||||
timeoutMs: 1_800_000,
|
||||
restartDrainTimeoutMs: 300_000,
|
||||
restartDelayMs: 500,
|
||||
parentPid: process.pid,
|
||||
execPath: "/usr/local/bin/node",
|
||||
argv1: "/opt/openclaw/openclaw.mjs",
|
||||
@@ -124,6 +232,8 @@ async function runOwnershipHelper(params: {
|
||||
continuationMessage: "continue after restart",
|
||||
},
|
||||
});
|
||||
const mockedLauncher = spawnMock.mock.results.at(-1)?.value as ReturnType<typeof createSpawnMock>;
|
||||
mockedLauncher.emit("exit", 0, null);
|
||||
|
||||
const [, args, spawnOptions] = spawnMock.mock.calls.at(-1) as unknown as [
|
||||
string,
|
||||
@@ -136,17 +246,63 @@ async function runOwnershipHelper(params: {
|
||||
string,
|
||||
unknown
|
||||
>;
|
||||
await params.prepareStateDatabase(env);
|
||||
await params.prepareStateDatabase?.(env);
|
||||
if (params.sentinel !== undefined) {
|
||||
writeRestartSentinelRow(env, params.sentinel);
|
||||
}
|
||||
const parent = spawn(process.execPath, ["-e", "process.stdin.resume()"], {
|
||||
stdio: ["pipe", "ignore", "ignore"],
|
||||
});
|
||||
const parentPid = parent.pid;
|
||||
const startIdentity = parentPid ? getFileLockProcessStartTime(parentPid) : null;
|
||||
if (!parentPid || startIdentity === null) {
|
||||
parent.kill("SIGKILL");
|
||||
throw new Error("expected a parent process with a stable start identity");
|
||||
}
|
||||
const helperParamsPath = path.join(tmpDir, "helper-params.json");
|
||||
const logPath = path.join(tmpDir, "handoff.log");
|
||||
const updaterPath = path.join(tmpDir, "updater-ran");
|
||||
const runnerClosedPath = path.join(tmpDir, "runner-closed-stdin");
|
||||
const preloadPath = path.join(tmpDir, "runner-fault-preload.cjs");
|
||||
if (params.runnerFault === "closed-stdin") {
|
||||
await fs.writeFile(
|
||||
preloadPath,
|
||||
`const fs = require("node:fs");
|
||||
const childProcess = require("node:child_process");
|
||||
const originalSpawn = childProcess.spawn;
|
||||
const closedPath = ${JSON.stringify(runnerClosedPath)};
|
||||
childProcess.spawn = function(command, args, options) {
|
||||
if (command !== process.execPath || args[0] !== "-e" ||
|
||||
!args[1].includes('process.stdin.once("data"')) {
|
||||
return originalSpawn.apply(this, arguments);
|
||||
}
|
||||
const runnerScript = 'const fs=require("node:fs");fs.closeSync(0);' +
|
||||
'fs.writeFileSync(' + JSON.stringify(closedPath) + ',String(process.pid));' +
|
||||
'setTimeout(() => process.exit(0),5000)';
|
||||
const child = originalSpawn.call(this, command, ["-e", runnerScript, args[2]], options);
|
||||
const deadline = Date.now() + 5000;
|
||||
const waitWord = new Int32Array(new SharedArrayBuffer(4));
|
||||
while (!fs.existsSync(closedPath) && Date.now() < deadline) {
|
||||
Atomics.wait(waitWord, 0, 0, 10);
|
||||
}
|
||||
if (!fs.existsSync(closedPath)) throw new Error("runner did not close stdin");
|
||||
return child;
|
||||
};
|
||||
`,
|
||||
);
|
||||
}
|
||||
await fs.writeFile(
|
||||
helperParamsPath,
|
||||
`${JSON.stringify(
|
||||
{
|
||||
...helperParams,
|
||||
parentPid: 0,
|
||||
parentExitTimeoutMs: 5_000,
|
||||
commandArgv: [process.execPath, "-e", "process.exit(7)"],
|
||||
parentPid,
|
||||
parentStartIdentity: String(startIdentity),
|
||||
commandArgv: [
|
||||
process.execPath,
|
||||
"-e",
|
||||
`require("node:fs").writeFileSync(${JSON.stringify(updaterPath)},"ran");setTimeout(() => process.exit(${params.commandExitCode ?? 1}), ${params.commandDelayMs ?? 0})`,
|
||||
],
|
||||
logPath,
|
||||
sensitivePaths: [],
|
||||
},
|
||||
@@ -155,27 +311,55 @@ async function runOwnershipHelper(params: {
|
||||
)}\n`,
|
||||
);
|
||||
|
||||
const helper = spawn(process.execPath, [helperScriptPath, helperParamsPath], {
|
||||
cwd: tmpDir,
|
||||
env: {
|
||||
...spawnOptions.env,
|
||||
...(params.runnerFault ? { NODE_OPTIONS: `--require ${preloadPath}` } : {}),
|
||||
},
|
||||
stdio: ["pipe", "pipe", params.runnerFault ? "pipe" : "ignore"],
|
||||
});
|
||||
const helperInput = helper.stdin;
|
||||
if (!helperInput) {
|
||||
throw new Error("expected the managed update helper to expose its control pipe");
|
||||
}
|
||||
let stderr = "";
|
||||
helper.stderr?.on("data", (chunk: Buffer | string) => {
|
||||
stderr += chunk.toString();
|
||||
});
|
||||
const resultPromise = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>(
|
||||
(resolve) => {
|
||||
execFile(
|
||||
process.execPath,
|
||||
[helperScriptPath, helperParamsPath],
|
||||
{ cwd: tmpDir, env: spawnOptions.env },
|
||||
(err) => {
|
||||
const childError = err as (NodeJS.ErrnoException & { signal?: NodeJS.Signals }) | null;
|
||||
resolve({
|
||||
code: typeof childError?.code === "number" ? childError.code : 0,
|
||||
signal: childError?.signal ?? null,
|
||||
});
|
||||
},
|
||||
);
|
||||
helper.once("close", (code, signal) => resolve({ code, signal }));
|
||||
},
|
||||
);
|
||||
await params.whileHelperRunning?.({ logPath });
|
||||
return { result: await resultPromise, env, logPath };
|
||||
await waitForHandoffLine(helper.stdout, "OPENCLAW_UPDATE_HANDOFF_READY");
|
||||
const parked = waitForHandoffLine(helper.stdout, "parked");
|
||||
helperInput.write("park\n");
|
||||
await parked;
|
||||
const committed = waitForHandoffLine(helper.stdout, "committed");
|
||||
helperInput.write("commit\n");
|
||||
await committed;
|
||||
parent.stdin.end();
|
||||
await params.whileHelperRunning?.({ env, logPath });
|
||||
const result = await resultPromise;
|
||||
if (params.runnerFault) {
|
||||
const runnerPid = Number(await fs.readFile(runnerClosedPath, "utf8"));
|
||||
try {
|
||||
process.kill(runnerPid, "SIGKILL");
|
||||
} catch {}
|
||||
}
|
||||
return {
|
||||
result,
|
||||
env,
|
||||
logPath,
|
||||
stderr,
|
||||
updaterPath,
|
||||
leaseDatabasePath: String(helperParams.updateLeaseDatabasePath),
|
||||
leaseKey: String(helperParams.updateLeaseKey),
|
||||
};
|
||||
}
|
||||
|
||||
describe("managed service update handoff external ownership", () => {
|
||||
describe("managed service update handoff state ownership and sentinel persistence", () => {
|
||||
it("refuses fallback writes to externally owned state without the supervisor marker", async () => {
|
||||
let before:
|
||||
| {
|
||||
@@ -188,6 +372,7 @@ describe("managed service update handoff external ownership", () => {
|
||||
}
|
||||
| undefined;
|
||||
const { result, env, logPath } = await runOwnershipHelper({
|
||||
commandExitCode: 7,
|
||||
prepareStateDatabase: async (stateEnv) => {
|
||||
const externalEnv = { ...stateEnv, OPENCLAW_SUPERVISOR_MODE: "external" };
|
||||
claimOpenClawStateOwnership("gateway-supervisor", { env: externalEnv });
|
||||
@@ -247,6 +432,7 @@ describe("managed service update handoff external ownership", () => {
|
||||
let helperResult: Awaited<ReturnType<typeof runOwnershipHelper>> | undefined;
|
||||
try {
|
||||
helperResult = await runOwnershipHelper({
|
||||
commandExitCode: 7,
|
||||
handoffId: "handoff-ownership-race",
|
||||
metaHandoffId: "handoff-ownership-race",
|
||||
prepareStateDatabase: async (stateEnv) => {
|
||||
@@ -321,4 +507,249 @@ describe("managed service update handoff external ownership", () => {
|
||||
/race-supervisor.*OPENCLAW_SUPERVISOR_MODE=external/u,
|
||||
);
|
||||
});
|
||||
|
||||
it("writes a fallback update failure when no restart sentinel row exists", async () => {
|
||||
const { result, env } = await runOwnershipHelper({
|
||||
handoffId: "handoff-123",
|
||||
metaHandoffId: "handoff-123",
|
||||
});
|
||||
|
||||
expect(result).toEqual({ code: 1, signal: null });
|
||||
expect(readRestartSentinelPayload(env)).toMatchObject({
|
||||
version: 1,
|
||||
payload: {
|
||||
kind: "update",
|
||||
status: "error",
|
||||
sessionKey: "agent:test:webchat:dm:user-123",
|
||||
stats: {
|
||||
handoffId: "handoff-123",
|
||||
reason: "managed-service-handoff-failed",
|
||||
},
|
||||
},
|
||||
});
|
||||
if (process.platform !== "win32") {
|
||||
const mode = (await fs.stat(resolveOpenClawStateSqlitePath(env))).mode & 0o777;
|
||||
expect(mode).toBe(0o600);
|
||||
}
|
||||
});
|
||||
|
||||
it("reclaims the exact runner lease and records failure when its stdin closes before the update gate", async () => {
|
||||
const { DatabaseSync } = await import("node:sqlite");
|
||||
const { result, env, stderr, updaterPath, leaseDatabasePath, leaseKey } =
|
||||
await runOwnershipHelper({
|
||||
handoffId: "handoff-runner-closed-stdin",
|
||||
metaHandoffId: "handoff-runner-closed-stdin",
|
||||
runnerFault: "closed-stdin",
|
||||
});
|
||||
|
||||
expect(result).toEqual({ code: 1, signal: null });
|
||||
expect(stderr).not.toMatch(/Unhandled 'error' event|Error: write EPIPE/u);
|
||||
await expect(fs.access(updaterPath)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
|
||||
const leaseDatabase = new DatabaseSync(leaseDatabasePath, { readOnly: true });
|
||||
try {
|
||||
expect(
|
||||
leaseDatabase
|
||||
.prepare("SELECT owner FROM managed_update_handoffs WHERE install_root = ?")
|
||||
.get(leaseKey),
|
||||
).toBeUndefined();
|
||||
} finally {
|
||||
leaseDatabase.close();
|
||||
}
|
||||
|
||||
expect(readRestartSentinelPayload(env)).toMatchObject({
|
||||
payload: {
|
||||
kind: "update",
|
||||
status: "error",
|
||||
stats: {
|
||||
handoffId: "handoff-runner-closed-stdin",
|
||||
reason: expect.stringMatching(/^managed-service-handoff-(?:helper|spawn)-failed$/u),
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it.runIf(process.platform === "win32")(
|
||||
"writes fallback state through the detached helper beyond MAX_PATH",
|
||||
async () => {
|
||||
const { result, env } = await runOwnershipHelper({
|
||||
deepStatePath: true,
|
||||
handoffId: "handoff-windows-long-path",
|
||||
metaHandoffId: "handoff-windows-long-path",
|
||||
});
|
||||
const statePath = resolveOpenClawStateSqlitePath(env);
|
||||
expect(statePath.startsWith("\\\\?\\")).toBe(false);
|
||||
expect(statePath.length).toBeGreaterThan(260);
|
||||
expect(result).toEqual({ code: 1, signal: null });
|
||||
expect(readRestartSentinelPayload(env)).toMatchObject({
|
||||
payload: { status: "error" },
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("waits for a concurrent state writer before persisting the fallback failure", async () => {
|
||||
let lockReleased: Promise<void> | undefined;
|
||||
const { result, env } = await runOwnershipHelper({
|
||||
handoffId: "handoff-locked",
|
||||
metaHandoffId: "handoff-locked",
|
||||
prepareStateDatabase: async (stateEnv) => {
|
||||
openOpenClawStateDatabase({ env: stateEnv });
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
const sqlite = await import("node:sqlite");
|
||||
const lock = new sqlite.DatabaseSync(resolveOpenClawStateSqlitePath(stateEnv));
|
||||
lock.exec("BEGIN IMMEDIATE;");
|
||||
lockReleased = new Promise((resolve) => {
|
||||
setTimeout(() => {
|
||||
lock.exec("COMMIT;");
|
||||
lock.close();
|
||||
resolve();
|
||||
}, 200);
|
||||
});
|
||||
},
|
||||
});
|
||||
await lockReleased;
|
||||
|
||||
expect(result).toEqual({ code: 1, signal: null });
|
||||
expect(readRestartSentinelPayload(env)).toMatchObject({
|
||||
version: 1,
|
||||
payload: {
|
||||
status: "error",
|
||||
stats: {
|
||||
handoffId: "handoff-locked",
|
||||
reason: "managed-service-handoff-failed",
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("repairs legacy restart sentinel columns before writing fallback failures", async () => {
|
||||
const { result, env } = await runOwnershipHelper({
|
||||
handoffId: "handoff-123",
|
||||
metaHandoffId: "handoff-123",
|
||||
prepareStateDatabase: createLegacyRestartSentinelTable,
|
||||
});
|
||||
|
||||
expect(result).toEqual({ code: 1, signal: null });
|
||||
expect(readRestartSentinelPayload(env)).toMatchObject({
|
||||
version: 1,
|
||||
payload: {
|
||||
kind: "update",
|
||||
status: "error",
|
||||
stats: {
|
||||
reason: "managed-service-handoff-failed",
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("does not overwrite a restart sentinel owned by another startup task", async () => {
|
||||
const unrelatedSentinel = {
|
||||
version: 1,
|
||||
payload: {
|
||||
kind: "config",
|
||||
status: "skipped",
|
||||
message: "preserve this restart task",
|
||||
stats: { reason: "config-restart-pending" },
|
||||
},
|
||||
};
|
||||
const { result, env } = await runOwnershipHelper({
|
||||
sentinel: unrelatedSentinel,
|
||||
});
|
||||
|
||||
expect(result).toEqual({ code: 1, signal: null });
|
||||
expect(readRestartSentinelPayload(env)).toMatchObject(unrelatedSentinel);
|
||||
});
|
||||
|
||||
it("does not overwrite a newer pending update handoff sentinel", async () => {
|
||||
const newerSentinel = {
|
||||
version: 1,
|
||||
payload: {
|
||||
kind: "update",
|
||||
status: "skipped",
|
||||
message: "new handoff still pending",
|
||||
stats: {
|
||||
mode: "npm",
|
||||
handoffId: "newer-handoff",
|
||||
reason: "managed-service-handoff-started",
|
||||
steps: [],
|
||||
durationMs: 0,
|
||||
},
|
||||
},
|
||||
};
|
||||
const { result, env } = await runOwnershipHelper({
|
||||
handoffId: "old-handoff",
|
||||
metaHandoffId: "old-handoff",
|
||||
sentinel: newerSentinel,
|
||||
});
|
||||
|
||||
expect(result).toEqual({ code: 1, signal: null });
|
||||
expect(readRestartSentinelPayload(env)).toMatchObject(newerSentinel);
|
||||
});
|
||||
|
||||
it("preserves a newer sentinel written while the detached helper is active", async () => {
|
||||
const oldSentinel = {
|
||||
version: 1,
|
||||
revision: 100,
|
||||
payload: {
|
||||
kind: "update",
|
||||
status: "skipped",
|
||||
ts: 100,
|
||||
stats: {
|
||||
handoffId: "old-handoff",
|
||||
reason: "managed-service-handoff-started",
|
||||
},
|
||||
},
|
||||
};
|
||||
const newerSentinel = {
|
||||
version: 1,
|
||||
revision: 200,
|
||||
payload: {
|
||||
kind: "restart",
|
||||
status: "ok",
|
||||
ts: 200,
|
||||
},
|
||||
};
|
||||
const { env } = await runOwnershipHelper({
|
||||
handoffId: "old-handoff",
|
||||
metaHandoffId: "old-handoff",
|
||||
sentinel: oldSentinel,
|
||||
commandDelayMs: 200,
|
||||
whileHelperRunning: async ({ env: stateEnv }) => {
|
||||
await new Promise<void>((resolve) => {
|
||||
setTimeout(resolve, 50);
|
||||
});
|
||||
replaceRestartSentinelRow(stateEnv, newerSentinel);
|
||||
},
|
||||
});
|
||||
|
||||
expect(readRestartSentinelPayload(env)).toMatchObject({
|
||||
payload: newerSentinel.payload,
|
||||
revision: 200,
|
||||
});
|
||||
});
|
||||
|
||||
it("advances the durable revision floor even when it is ahead of the clock", async () => {
|
||||
const futureRevision = Date.now() + 60_000;
|
||||
const { env } = await runOwnershipHelper({
|
||||
handoffId: "handoff-future-revision",
|
||||
metaHandoffId: "handoff-future-revision",
|
||||
sentinel: {
|
||||
version: 1,
|
||||
revision: futureRevision,
|
||||
payload: {
|
||||
kind: "revision-floor",
|
||||
status: "skipped",
|
||||
ts: 123,
|
||||
stats: {
|
||||
handoffId: "handoff-future-revision",
|
||||
reason: "managed-service-handoff-started",
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(readRestartSentinelPayload(env, "revision-floor")).toMatchObject({
|
||||
revision: futureRevision + 1,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,775 @@
|
||||
// Process-local handoff sharing complements the durable cross-process lease.
|
||||
import { EventEmitter } from "node:events";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { PassThrough } from "node:stream";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
|
||||
import {
|
||||
signalMockManagedUpdateHandoffReady,
|
||||
type MockManagedUpdateHandoffLeaseFailure,
|
||||
} from "./update-managed-service-handoff.test-support.js";
|
||||
|
||||
const spawnMock = vi.hoisted(() => vi.fn());
|
||||
const forceKillChildProcessTreeMock = vi.hoisted(() => vi.fn());
|
||||
const findInstalledSystemdGatewayScopeMock = vi.hoisted(() =>
|
||||
vi.fn(
|
||||
async (_env: NodeJS.ProcessEnv) =>
|
||||
null as {
|
||||
scope: "user" | "system";
|
||||
unitName: string;
|
||||
unitPath: string;
|
||||
} | null,
|
||||
),
|
||||
);
|
||||
const tempRoots = useAutoCleanupTempDirTracker(afterEach);
|
||||
const mockedHandoffLeaseCleanups = new Set<() => void>();
|
||||
const MOCK_INSTALL_ROOT = path.join(os.tmpdir(), `openclaw-handoff-single-flight-${process.pid}`);
|
||||
|
||||
function createReadyChild(
|
||||
pid: number,
|
||||
paramsPath: string,
|
||||
failure?: MockManagedUpdateHandoffLeaseFailure,
|
||||
) {
|
||||
const child = Object.assign(new EventEmitter(), {
|
||||
pid,
|
||||
exitCode: null,
|
||||
signalCode: null,
|
||||
stdin: new PassThrough(),
|
||||
stdout: new PassThrough(),
|
||||
unref: vi.fn(),
|
||||
});
|
||||
process.nextTick(() => {
|
||||
signalMockManagedUpdateHandoffReady({
|
||||
child,
|
||||
paramsPath,
|
||||
cleanups: mockedHandoffLeaseCleanups,
|
||||
...(child.pid === process.pid ? {} : { startIdentity: 17 }),
|
||||
failure,
|
||||
});
|
||||
});
|
||||
return child;
|
||||
}
|
||||
|
||||
vi.mock("node:child_process", async () => {
|
||||
const { mockNodeChildProcessModule } =
|
||||
await import("../gateway/server-methods/node-child-process.test-support.js");
|
||||
return mockNodeChildProcessModule({
|
||||
spawn: spawnMock as unknown as typeof import("node:child_process").spawn,
|
||||
});
|
||||
});
|
||||
|
||||
vi.mock("../daemon/systemd-scope.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("../daemon/systemd-scope.js")>()),
|
||||
findInstalledSystemdGatewayScope: findInstalledSystemdGatewayScopeMock,
|
||||
}));
|
||||
|
||||
vi.mock("../process/child-process-tree.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("../process/child-process-tree.js")>()),
|
||||
forceKillChildProcessTree: forceKillChildProcessTreeMock,
|
||||
}));
|
||||
|
||||
beforeEach(async () => {
|
||||
let pid = 24680;
|
||||
const liveChildren = new Set<number>();
|
||||
const processIdentity = await import("../shared/pid-alive.js");
|
||||
const parentStartIdentity = processIdentity.getFileLockProcessStartTime(process.pid);
|
||||
vi.spyOn(processIdentity, "getFileLockProcessStartTime").mockImplementation((targetPid) =>
|
||||
targetPid === process.pid ? parentStartIdentity : liveChildren.has(targetPid) ? 17 : null,
|
||||
);
|
||||
vi.spyOn(processIdentity, "isPidAlive").mockImplementation(
|
||||
(targetPid) => targetPid === process.pid || liveChildren.has(targetPid),
|
||||
);
|
||||
forceKillChildProcessTreeMock.mockReset();
|
||||
forceKillChildProcessTreeMock.mockImplementation((child: ReturnType<typeof createReadyChild>) => {
|
||||
child.stdout.destroy();
|
||||
});
|
||||
findInstalledSystemdGatewayScopeMock.mockReset();
|
||||
findInstalledSystemdGatewayScopeMock.mockResolvedValue(null);
|
||||
spawnMock.mockReset();
|
||||
spawnMock.mockImplementation((_command: string, args: string[]) => {
|
||||
const child = createReadyChild(pid++, args.at(-1) ?? "");
|
||||
liveChildren.add(child.pid);
|
||||
child.once("exit", () => liveChildren.delete(child.pid));
|
||||
return child;
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
for (const cleanup of mockedHandoffLeaseCleanups) {
|
||||
cleanup();
|
||||
}
|
||||
const handoffDirs = spawnMock.mock.calls.flatMap((call) => {
|
||||
const args = call[1] as string[] | undefined;
|
||||
const scriptPath = args?.[0];
|
||||
return scriptPath ? [path.dirname(scriptPath)] : [];
|
||||
});
|
||||
await Promise.all(handoffDirs.map((dir) => fs.rm(dir, { recursive: true, force: true })));
|
||||
vi.restoreAllMocks();
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
const baseParams = {
|
||||
restartDrainTimeoutMs: 300_000,
|
||||
parentPid: process.pid,
|
||||
execPath: "/usr/local/bin/node",
|
||||
argv1: "/opt/openclaw/openclaw.mjs",
|
||||
};
|
||||
|
||||
describe("managed service update handoff single-flight", () => {
|
||||
it.each([
|
||||
["does not exist", "absent"],
|
||||
["has malformed helper identity", "malformed"],
|
||||
["belongs to a different owner", "wrong-owner"],
|
||||
["identifies a dead helper", "dead-helper"],
|
||||
] as const)("rejects READY when the durable helper lease %s", async (_label, failure) => {
|
||||
spawnMock.mockImplementationOnce((_command: string, args: string[]) =>
|
||||
createReadyChild(process.pid, args.at(-1) ?? "", failure),
|
||||
);
|
||||
const { claimManagedServiceUpdateHandoff, startManagedServiceUpdateHandoff } =
|
||||
await import("./update-managed-service-handoff.js");
|
||||
const identity = {
|
||||
kind: "managed-update-handoff" as const,
|
||||
handoffId: `invalid-ready-${failure}`,
|
||||
installRoot: `${MOCK_INSTALL_ROOT}-${failure}`,
|
||||
};
|
||||
|
||||
await expect(
|
||||
startManagedServiceUpdateHandoff({
|
||||
...baseParams,
|
||||
root: identity.installRoot,
|
||||
handoffId: identity.handoffId,
|
||||
meta: {},
|
||||
}),
|
||||
).rejects.toThrow(/lease|helper|identity/u);
|
||||
|
||||
const child = spawnMock.mock.results[0]?.value as ReturnType<typeof createReadyChild>;
|
||||
expect(forceKillChildProcessTreeMock).toHaveBeenCalledExactlyOnceWith(child);
|
||||
expect(child.unref).not.toHaveBeenCalled();
|
||||
expect(claimManagedServiceUpdateHandoff(identity)).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects system-scope systemd before spawning or reserving handoff ownership", async () => {
|
||||
findInstalledSystemdGatewayScopeMock.mockResolvedValueOnce({
|
||||
scope: "system",
|
||||
unitName: "openclaw-gateway.service",
|
||||
unitPath: "/etc/systemd/system/openclaw-gateway.service",
|
||||
});
|
||||
const { claimManagedServiceUpdateHandoff, startManagedServiceUpdateHandoff } =
|
||||
await import("./update-managed-service-handoff.js");
|
||||
const root = `${MOCK_INSTALL_ROOT}-system-scope`;
|
||||
|
||||
await expect(
|
||||
startManagedServiceUpdateHandoff({
|
||||
...baseParams,
|
||||
root,
|
||||
handoffId: "system-handoff",
|
||||
supervisor: "systemd",
|
||||
env: { OPENCLAW_SYSTEMD_UNIT: "openclaw-gateway.service" },
|
||||
meta: {},
|
||||
}),
|
||||
).rejects.toThrow(/user-scope systemd unit.*manual system-service update/);
|
||||
expect(spawnMock).not.toHaveBeenCalled();
|
||||
expect(
|
||||
claimManagedServiceUpdateHandoff({
|
||||
kind: "managed-update-handoff",
|
||||
handoffId: "system-handoff",
|
||||
installRoot: root,
|
||||
}),
|
||||
).toBe(false);
|
||||
|
||||
await expect(
|
||||
startManagedServiceUpdateHandoff({ ...baseParams, root, meta: {} }),
|
||||
).resolves.toMatchObject({ status: "started" });
|
||||
expect(spawnMock).toHaveBeenCalledOnce();
|
||||
const owner = spawnMock.mock.results[0]?.value as ReturnType<typeof createReadyChild>;
|
||||
owner.emit("exit", 0, null);
|
||||
});
|
||||
|
||||
it("shares one same-root helper until its lifecycle ends", async () => {
|
||||
const { startManagedServiceUpdateHandoff } =
|
||||
await import("./update-managed-service-handoff.js");
|
||||
const first = startManagedServiceUpdateHandoff({
|
||||
...baseParams,
|
||||
root: MOCK_INSTALL_ROOT,
|
||||
handoffId: "handoff-first",
|
||||
meta: { handoffId: "handoff-first" },
|
||||
});
|
||||
const second = startManagedServiceUpdateHandoff({
|
||||
...baseParams,
|
||||
root: MOCK_INSTALL_ROOT,
|
||||
handoffId: "handoff-second",
|
||||
meta: { handoffId: "handoff-second" },
|
||||
});
|
||||
|
||||
const outcomes = await Promise.all([first, second]);
|
||||
expect(outcomes).toEqual([
|
||||
expect.objectContaining({ status: "started", handoffId: "handoff-first" }),
|
||||
expect.objectContaining({ status: "joined", handoffId: "handoff-first" }),
|
||||
]);
|
||||
expect(outcomes[1]).not.toHaveProperty("installRoot");
|
||||
expect(spawnMock).toHaveBeenCalledOnce();
|
||||
|
||||
const owner = spawnMock.mock.results[0]?.value as ReturnType<typeof createReadyChild>;
|
||||
owner.emit("exit", 0, null);
|
||||
const next = startManagedServiceUpdateHandoff({
|
||||
...baseParams,
|
||||
root: MOCK_INSTALL_ROOT,
|
||||
handoffId: "handoff-next",
|
||||
meta: { handoffId: "handoff-next" },
|
||||
});
|
||||
|
||||
await expect(next).resolves.toMatchObject({
|
||||
status: "started",
|
||||
handoffId: "handoff-next",
|
||||
});
|
||||
expect(spawnMock).toHaveBeenCalledTimes(2);
|
||||
const nextOwner = spawnMock.mock.results[1]?.value as ReturnType<typeof createReadyChild>;
|
||||
nextOwner.emit("exit", 0, null);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["has exited before its ChildProcess notification", "dead"],
|
||||
["reuses its PID for another process", "reused"],
|
||||
["no longer exposes its process start identity", "unknown"],
|
||||
])("rejects a helper claim when its operating-system process %s", async (_label, failure) => {
|
||||
vi.restoreAllMocks();
|
||||
const { spawn } =
|
||||
await vi.importActual<typeof import("node:child_process")>("node:child_process");
|
||||
spawnMock.mockImplementation(spawn);
|
||||
const processIdentity = await import("../shared/pid-alive.js");
|
||||
const root = await fs.realpath(tempRoots.make("openclaw-helper-process-identity-"));
|
||||
const parent = spawn(process.execPath, ["-e", "process.stdin.resume()"], {
|
||||
stdio: ["pipe", "ignore", "ignore"],
|
||||
});
|
||||
try {
|
||||
const {
|
||||
cancelManagedServiceUpdateHandoff,
|
||||
claimManagedServiceUpdateHandoff,
|
||||
startManagedServiceUpdateHandoff,
|
||||
} = await import("./update-managed-service-handoff.js");
|
||||
const started = await startManagedServiceUpdateHandoff({
|
||||
root,
|
||||
restartDrainTimeoutMs: 300_000,
|
||||
parentPid: parent.pid,
|
||||
execPath: process.execPath,
|
||||
argv1: process.argv[1],
|
||||
env: { ...process.env, OPENCLAW_STATE_DIR: root },
|
||||
meta: {},
|
||||
});
|
||||
if (started.status !== "started" || !started.pid) {
|
||||
throw new Error("expected real detached handoff ownership");
|
||||
}
|
||||
const identity = { kind: "managed-update-handoff" as const, ...started };
|
||||
expect(claimManagedServiceUpdateHandoff(identity)).toBe(true);
|
||||
if (failure === "dead") {
|
||||
vi.spyOn(processIdentity, "isPidAlive").mockReturnValue(false);
|
||||
} else {
|
||||
const helperStartIdentity = processIdentity.getFileLockProcessStartTime(started.pid);
|
||||
if (helperStartIdentity === null) {
|
||||
throw new Error("expected the real detached helper to have a process identity");
|
||||
}
|
||||
vi.spyOn(processIdentity, "getFileLockProcessStartTime").mockReturnValue(
|
||||
failure === "reused" ? helperStartIdentity + 1 : null,
|
||||
);
|
||||
}
|
||||
expect(claimManagedServiceUpdateHandoff(identity)).toBe(false);
|
||||
vi.restoreAllMocks();
|
||||
await expect(cancelManagedServiceUpdateHandoff(identity)).resolves.toBe(
|
||||
"restored-in-process",
|
||||
);
|
||||
} finally {
|
||||
vi.restoreAllMocks();
|
||||
parent.stdin?.end();
|
||||
}
|
||||
});
|
||||
|
||||
it("terminates the exact helper when its initial start identity is unavailable", async () => {
|
||||
const processIdentity = await import("../shared/pid-alive.js");
|
||||
vi.mocked(processIdentity.getFileLockProcessStartTime)
|
||||
.mockReturnValueOnce(17)
|
||||
.mockReturnValueOnce(null);
|
||||
const { claimManagedServiceUpdateHandoff, startManagedServiceUpdateHandoff } =
|
||||
await import("./update-managed-service-handoff.js");
|
||||
const identity = {
|
||||
kind: "managed-update-handoff" as const,
|
||||
handoffId: "identity-unavailable",
|
||||
installRoot: `${MOCK_INSTALL_ROOT}-identity-unavailable`,
|
||||
};
|
||||
|
||||
await expect(
|
||||
startManagedServiceUpdateHandoff({
|
||||
...baseParams,
|
||||
root: identity.installRoot,
|
||||
handoffId: identity.handoffId,
|
||||
meta: {},
|
||||
}),
|
||||
).rejects.toThrow("process start identity is unavailable");
|
||||
|
||||
const child = spawnMock.mock.results[0]?.value as ReturnType<typeof createReadyChild>;
|
||||
expect(forceKillChildProcessTreeMock).toHaveBeenCalledExactlyOnceWith(child);
|
||||
expect(child.unref).not.toHaveBeenCalled();
|
||||
expect(claimManagedServiceUpdateHandoff(identity)).toBe(false);
|
||||
});
|
||||
|
||||
it("reclaims only its exact dead detached helper before reopening the install root", async () => {
|
||||
vi.restoreAllMocks();
|
||||
const { spawn } =
|
||||
await vi.importActual<typeof import("node:child_process")>("node:child_process");
|
||||
const { DatabaseSync } = await import("node:sqlite");
|
||||
const processIdentity = await import("../shared/pid-alive.js");
|
||||
const { getFileLockProcessStartTime } = processIdentity;
|
||||
spawnMock.mockImplementation(spawn);
|
||||
const root = await fs.realpath(tempRoots.make("openclaw-dead-handoff-owner-"));
|
||||
const markerPath = path.join(root, "updater-ran");
|
||||
const updaterPath = path.join(root, "updater.cjs");
|
||||
await fs.writeFile(
|
||||
updaterPath,
|
||||
`require("node:fs").writeFileSync(${JSON.stringify(markerPath)}, "ran")`,
|
||||
);
|
||||
const parent = spawn(process.execPath, ["-e", "process.stdin.resume()"], {
|
||||
stdio: ["pipe", "ignore", "ignore"],
|
||||
});
|
||||
const { cancelManagedServiceUpdateHandoff, startManagedServiceUpdateHandoff } =
|
||||
await import("./update-managed-service-handoff.js");
|
||||
let leaseDatabasePath: string | undefined;
|
||||
let deadOwner: string | undefined;
|
||||
let replacement: Awaited<ReturnType<typeof startManagedServiceUpdateHandoff>> | undefined;
|
||||
try {
|
||||
const start = () =>
|
||||
startManagedServiceUpdateHandoff({
|
||||
root,
|
||||
restartDrainTimeoutMs: 300_000,
|
||||
parentPid: parent.pid,
|
||||
execPath: process.execPath,
|
||||
argv1: updaterPath,
|
||||
env: { ...process.env, OPENCLAW_STATE_DIR: root },
|
||||
meta: {},
|
||||
});
|
||||
const started = await start();
|
||||
if (started.status !== "started" || !started.pid) {
|
||||
throw new Error("expected real detached handoff ownership");
|
||||
}
|
||||
deadOwner = started.handoffId;
|
||||
const helperStartIdentity = getFileLockProcessStartTime(started.pid);
|
||||
if (helperStartIdentity === null) {
|
||||
throw new Error("expected the detached helper to have a stable process identity");
|
||||
}
|
||||
const helper = spawnMock.mock.results[0]?.value as import("node:child_process").ChildProcess;
|
||||
const [, args] = spawnMock.mock.calls[0] as [string, string[]];
|
||||
leaseDatabasePath = (
|
||||
JSON.parse(await fs.readFile(args[1] ?? "", "utf8")) as {
|
||||
updateLeaseDatabasePath: string;
|
||||
}
|
||||
).updateLeaseDatabasePath;
|
||||
const readLease = () => {
|
||||
const db = new DatabaseSync(leaseDatabasePath!, { readOnly: true });
|
||||
try {
|
||||
return db
|
||||
.prepare(
|
||||
"SELECT owner, payload_json FROM managed_update_handoffs WHERE install_root = ?",
|
||||
)
|
||||
.get(root) as { owner: string; payload_json: string } | undefined;
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
};
|
||||
const initialLease = readLease();
|
||||
expect(initialLease?.owner).toBe(started.handoffId);
|
||||
expect(JSON.parse(initialLease?.payload_json ?? "null")).toEqual({
|
||||
version: 1,
|
||||
pid: started.pid,
|
||||
startIdentity: String(helperStartIdentity),
|
||||
});
|
||||
|
||||
const helperExited = new Promise<void>((resolve) => {
|
||||
helper.once("exit", () => resolve());
|
||||
});
|
||||
helper.kill("SIGKILL");
|
||||
await helperExited;
|
||||
expect(readLease()).toEqual(initialLease);
|
||||
|
||||
const ownStartIdentity = getFileLockProcessStartTime(process.pid);
|
||||
if (ownStartIdentity === null || !initialLease) {
|
||||
throw new Error("expected complete live-parent and dead-helper lease identities");
|
||||
}
|
||||
const originalPayload = {
|
||||
version: 1,
|
||||
pid: started.pid,
|
||||
startIdentity: String(helperStartIdentity),
|
||||
};
|
||||
const rejectedOwners = [
|
||||
{
|
||||
label: "replacement owner",
|
||||
owner: "replacement-owner",
|
||||
payload: originalPayload,
|
||||
},
|
||||
{
|
||||
label: "different live process",
|
||||
owner: started.handoffId,
|
||||
payload: {
|
||||
...originalPayload,
|
||||
pid: process.pid,
|
||||
startIdentity: String(ownStartIdentity),
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "different recorded start identity",
|
||||
owner: started.handoffId,
|
||||
payload: { ...originalPayload, startIdentity: `${helperStartIdentity}-reused` },
|
||||
},
|
||||
{
|
||||
label: "malformed process identity",
|
||||
owner: started.handoffId,
|
||||
payload: { ...originalPayload, startIdentity: null },
|
||||
},
|
||||
{
|
||||
label: "noncanonical process identity",
|
||||
owner: started.handoffId,
|
||||
payload: { ...originalPayload, unexpected: true },
|
||||
},
|
||||
];
|
||||
const writeLease = (owner: string, payload: unknown) => {
|
||||
if (!leaseDatabasePath) {
|
||||
throw new Error("expected the detached helper lease database path");
|
||||
}
|
||||
const db = new DatabaseSync(leaseDatabasePath);
|
||||
try {
|
||||
db.prepare(
|
||||
"UPDATE managed_update_handoffs SET owner = ?, payload_json = ? WHERE install_root = ?",
|
||||
).run(owner, JSON.stringify(payload), root);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
};
|
||||
const identity = { kind: "managed-update-handoff" as const, ...started };
|
||||
for (const rejected of rejectedOwners) {
|
||||
writeLease(rejected.owner, rejected.payload);
|
||||
await expect(cancelManagedServiceUpdateHandoff(identity), rejected.label).resolves.toBe(
|
||||
false,
|
||||
);
|
||||
expect(readLease(), rejected.label).toEqual({
|
||||
owner: rejected.owner,
|
||||
payload_json: JSON.stringify(rejected.payload),
|
||||
});
|
||||
}
|
||||
writeLease(started.handoffId, originalPayload);
|
||||
const unknownDeath = vi.spyOn(processIdentity, "isPidDefinitelyDead").mockReturnValue(false);
|
||||
await expect(cancelManagedServiceUpdateHandoff(identity)).resolves.toBe(false);
|
||||
expect(readLease()).toEqual(initialLease);
|
||||
unknownDeath.mockRestore();
|
||||
|
||||
await expect(cancelManagedServiceUpdateHandoff(identity)).resolves.toBe(
|
||||
"restored-in-process",
|
||||
);
|
||||
expect(readLease()).toBeUndefined();
|
||||
await expect(fs.access(markerPath)).rejects.toThrow();
|
||||
|
||||
replacement = await start();
|
||||
if (replacement.status !== "started") {
|
||||
throw new Error("expected cancellation to reopen the install root");
|
||||
}
|
||||
expect(replacement.installRoot).toBe(root);
|
||||
expect(replacement.handoffId).not.toBe(started.handoffId);
|
||||
await expect(
|
||||
cancelManagedServiceUpdateHandoff({ kind: "managed-update-handoff", ...replacement }),
|
||||
).resolves.toBe("restored-in-process");
|
||||
replacement = undefined;
|
||||
expect(readLease()).toBeUndefined();
|
||||
await expect(fs.access(markerPath)).rejects.toThrow();
|
||||
} finally {
|
||||
if (replacement?.status === "started") {
|
||||
await cancelManagedServiceUpdateHandoff({
|
||||
kind: "managed-update-handoff",
|
||||
...replacement,
|
||||
});
|
||||
}
|
||||
if (leaseDatabasePath && deadOwner) {
|
||||
const db = new DatabaseSync(leaseDatabasePath);
|
||||
db.prepare("DELETE FROM managed_update_handoffs WHERE install_root = ? AND owner = ?").run(
|
||||
root,
|
||||
deadOwner,
|
||||
);
|
||||
db.close();
|
||||
}
|
||||
parent.stdin?.end();
|
||||
}
|
||||
});
|
||||
|
||||
it("waits for the exact helper to release its lease after an immediate control-pipe EPIPE", async () => {
|
||||
vi.restoreAllMocks();
|
||||
const { spawn } =
|
||||
await vi.importActual<typeof import("node:child_process")>("node:child_process");
|
||||
const { DatabaseSync } = await import("node:sqlite");
|
||||
spawnMock.mockImplementation(spawn);
|
||||
const root = await fs.realpath(tempRoots.make("openclaw-handoff-control-epipe-"));
|
||||
const parent = spawn(process.execPath, ["-e", "process.stdin.resume()"], {
|
||||
stdio: ["pipe", "ignore", "ignore"],
|
||||
});
|
||||
const {
|
||||
cancelManagedServiceUpdateHandoff,
|
||||
requestManagedServiceUpdateHandoffPark,
|
||||
startManagedServiceUpdateHandoff,
|
||||
} = await import("./update-managed-service-handoff.js");
|
||||
const started = await startManagedServiceUpdateHandoff({
|
||||
root,
|
||||
restartDrainTimeoutMs: 300_000,
|
||||
parentPid: parent.pid,
|
||||
execPath: process.execPath,
|
||||
argv1: process.argv[1],
|
||||
env: { ...process.env, OPENCLAW_STATE_DIR: root },
|
||||
meta: {},
|
||||
});
|
||||
if (started.status !== "started") {
|
||||
throw new Error("expected real detached helper ownership");
|
||||
}
|
||||
const identity = { kind: "managed-update-handoff" as const, ...started };
|
||||
const helper = spawnMock.mock.results[0]?.value as import("node:child_process").ChildProcess;
|
||||
const input = helper.stdin;
|
||||
if (!input) {
|
||||
throw new Error("expected the detached helper control pipe");
|
||||
}
|
||||
const [, args] = spawnMock.mock.calls[0] as [string, string[]];
|
||||
const { updateLeaseDatabasePath } = JSON.parse(await fs.readFile(args[1] ?? "", "utf8")) as {
|
||||
updateLeaseDatabasePath: string;
|
||||
};
|
||||
let controlDestroyed = false;
|
||||
const destroyed = vi
|
||||
.spyOn(input, "destroyed", "get")
|
||||
.mockImplementation(() => controlDestroyed);
|
||||
const write = vi.spyOn(input, "write").mockImplementation(((
|
||||
_chunk: unknown,
|
||||
callback: unknown,
|
||||
) => {
|
||||
controlDestroyed = true;
|
||||
const error = Object.assign(new Error("write EPIPE"), { code: "EPIPE" });
|
||||
if (typeof callback === "function") {
|
||||
callback(error);
|
||||
}
|
||||
input.emit("error", error);
|
||||
return false;
|
||||
}) as typeof input.write);
|
||||
|
||||
try {
|
||||
await expect(requestManagedServiceUpdateHandoffPark(identity)).resolves.toBe(false);
|
||||
expect(helper.exitCode).toBeNull();
|
||||
let cancellationSettled = false;
|
||||
const cancellation = cancelManagedServiceUpdateHandoff(identity).then((result) => {
|
||||
cancellationSettled = true;
|
||||
return result;
|
||||
});
|
||||
await new Promise<void>((resolve) => {
|
||||
setImmediate(resolve);
|
||||
});
|
||||
|
||||
expect(cancellationSettled).toBe(false);
|
||||
expect(helper.exitCode).toBeNull();
|
||||
write.mockRestore();
|
||||
destroyed.mockRestore();
|
||||
input.end();
|
||||
|
||||
await expect(cancellation).resolves.toBe("restored-in-process");
|
||||
expect(helper.exitCode !== null || helper.signalCode !== null).toBe(true);
|
||||
const database = new DatabaseSync(updateLeaseDatabasePath, { readOnly: true });
|
||||
try {
|
||||
expect(
|
||||
database
|
||||
.prepare("SELECT owner FROM managed_update_handoffs WHERE install_root = ?")
|
||||
.get(root),
|
||||
).toBeUndefined();
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
} finally {
|
||||
write.mockRestore();
|
||||
destroyed.mockRestore();
|
||||
input.end();
|
||||
parent.stdin?.end();
|
||||
}
|
||||
});
|
||||
|
||||
it("joins canonical aliases while distinct install roots remain independent", async () => {
|
||||
const tempDir = tempRoots.make("openclaw-handoff-root-");
|
||||
const root = path.join(tempDir, "install");
|
||||
const alias = path.join(tempDir, "install-alias");
|
||||
const otherRoot = path.join(tempDir, "other");
|
||||
await fs.mkdir(root);
|
||||
await fs.mkdir(otherRoot);
|
||||
await fs.symlink(root, alias, "dir");
|
||||
const { startManagedServiceUpdateHandoff } =
|
||||
await import("./update-managed-service-handoff.js");
|
||||
|
||||
const owner = await startManagedServiceUpdateHandoff({
|
||||
...baseParams,
|
||||
root,
|
||||
handoffId: "handoff-root",
|
||||
meta: {},
|
||||
});
|
||||
await expect(
|
||||
startManagedServiceUpdateHandoff({ ...baseParams, root: alias, meta: {} }),
|
||||
).resolves.toMatchObject({ status: "joined", handoffId: "handoff-root" });
|
||||
const other = await startManagedServiceUpdateHandoff({
|
||||
...baseParams,
|
||||
root: otherRoot,
|
||||
handoffId: "handoff-other",
|
||||
meta: {},
|
||||
});
|
||||
|
||||
expect(owner).toMatchObject({
|
||||
status: "started",
|
||||
handoffId: "handoff-root",
|
||||
installRoot: await fs.realpath(root),
|
||||
});
|
||||
expect(other).toMatchObject({ status: "started", handoffId: "handoff-other" });
|
||||
expect(spawnMock).toHaveBeenCalledTimes(2);
|
||||
for (const result of spawnMock.mock.results) {
|
||||
(result.value as ReturnType<typeof createReadyChild>).emit("exit", 0, null);
|
||||
}
|
||||
});
|
||||
|
||||
it.each([
|
||||
["releases the root for another owner", false, false, false, false],
|
||||
["retains a claimed exited owner until release is confirmed", false, false, true, false],
|
||||
["treats control-pipe disconnect as cancellation", false, false, true, true],
|
||||
["wins a concurrent parent exit without running the updater", true, false, false, false],
|
||||
["refuses recovery when another owner replaces the completed helper", false, true, true, false],
|
||||
])("cancellation %s", async (_label, exitParent, replaceOwner, claimBeforeExit, disconnect) => {
|
||||
vi.restoreAllMocks();
|
||||
const { spawn } =
|
||||
await vi.importActual<typeof import("node:child_process")>("node:child_process");
|
||||
const { DatabaseSync } = await import("node:sqlite");
|
||||
const { getFileLockProcessStartTime } = await import("../shared/pid-alive.js");
|
||||
const replacementStartIdentity = getFileLockProcessStartTime(process.pid);
|
||||
if (replacementStartIdentity === null) {
|
||||
throw new Error("expected the replacement owner to have a stable process identity");
|
||||
}
|
||||
spawnMock.mockImplementation(spawn);
|
||||
const root = await fs.realpath(tempRoots.make("openclaw-cancel-owner-"));
|
||||
const markerPath = path.join(root, "updater-ran");
|
||||
const updaterPath = path.join(root, "updater.cjs");
|
||||
await fs.writeFile(
|
||||
updaterPath,
|
||||
`require("node:fs").writeFileSync(${JSON.stringify(markerPath)}, "ran")`,
|
||||
);
|
||||
const parent = spawn(process.execPath, ["-e", "process.stdin.resume()"], {
|
||||
stdio: ["pipe", "ignore", "ignore"],
|
||||
});
|
||||
const {
|
||||
cancelManagedServiceUpdateHandoff,
|
||||
claimManagedServiceUpdateHandoff,
|
||||
startManagedServiceUpdateHandoff,
|
||||
} = await import("./update-managed-service-handoff.js");
|
||||
const start = () =>
|
||||
startManagedServiceUpdateHandoff({
|
||||
root,
|
||||
restartDrainTimeoutMs: 300_000,
|
||||
parentPid: parent.pid,
|
||||
execPath: process.execPath,
|
||||
argv1: updaterPath,
|
||||
env: { ...process.env, OPENCLAW_STATE_DIR: root },
|
||||
meta: {},
|
||||
});
|
||||
const started = await start();
|
||||
if (started.status !== "started") {
|
||||
throw new Error("expected handoff ownership");
|
||||
}
|
||||
const identity = { kind: "managed-update-handoff" as const, ...started };
|
||||
if (claimBeforeExit) {
|
||||
expect(claimManagedServiceUpdateHandoff(identity)).toBe(true);
|
||||
expect(claimManagedServiceUpdateHandoff(identity)).toBe(true);
|
||||
}
|
||||
await expect(
|
||||
cancelManagedServiceUpdateHandoff({ ...identity, handoffId: "joined" }),
|
||||
).resolves.toBe(false);
|
||||
const child = spawnMock.mock.results[0]?.value as import("node:child_process").ChildProcess;
|
||||
const [, args] = spawnMock.mock.calls[0] as [string, string[]];
|
||||
const helper = JSON.parse(await fs.readFile(args[1] ?? "", "utf8")) as {
|
||||
updateLeaseDatabasePath: string;
|
||||
stateDatabasePath: string;
|
||||
};
|
||||
let joinedAfterExit: ReturnType<typeof start> | undefined;
|
||||
child.once("exit", () => {
|
||||
if (!exitParent) {
|
||||
joinedAfterExit = start();
|
||||
}
|
||||
if (replaceOwner) {
|
||||
const replacement = new DatabaseSync(helper.updateLeaseDatabasePath);
|
||||
replacement
|
||||
.prepare(
|
||||
"INSERT INTO managed_update_handoffs (install_root, owner, payload_json, updated_at) VALUES (?, ?, ?, ?)",
|
||||
)
|
||||
.run(
|
||||
root,
|
||||
"replacement",
|
||||
JSON.stringify({
|
||||
version: 1,
|
||||
pid: process.pid,
|
||||
startIdentity: String(replacementStartIdentity),
|
||||
}),
|
||||
Date.now(),
|
||||
);
|
||||
replacement.close();
|
||||
}
|
||||
});
|
||||
let cancellation: ReturnType<typeof cancelManagedServiceUpdateHandoff>;
|
||||
if (claimBeforeExit) {
|
||||
const exited = new Promise<void>((resolve) => {
|
||||
child.once("exit", () => resolve());
|
||||
});
|
||||
if (disconnect) {
|
||||
child.stdin?.end();
|
||||
} else {
|
||||
child.stdin?.write("cancel\n");
|
||||
}
|
||||
await exited;
|
||||
expect(claimManagedServiceUpdateHandoff(identity)).toBe(false);
|
||||
expect(joinedAfterExit).toBeDefined();
|
||||
cancellation = cancelManagedServiceUpdateHandoff(identity);
|
||||
} else {
|
||||
cancellation = cancelManagedServiceUpdateHandoff(identity);
|
||||
if (exitParent) {
|
||||
parent.stdin?.end();
|
||||
}
|
||||
}
|
||||
await expect(cancellation).resolves.toBe(replaceOwner ? false : "restored-in-process");
|
||||
expect(child.exitCode !== null || child.signalCode !== null).toBe(true);
|
||||
const sentinel = new DatabaseSync(helper.stateDatabasePath, { readOnly: true });
|
||||
const terminal = sentinel
|
||||
.prepare("SELECT payload_json FROM gateway_restart_sentinel WHERE sentinel_key = 'current'")
|
||||
.get() as { payload_json: string };
|
||||
sentinel.close();
|
||||
expect(JSON.parse(terminal.payload_json)).toMatchObject({
|
||||
status: "error",
|
||||
stats: { reason: "managed-service-handoff-cancelled" },
|
||||
});
|
||||
if (joinedAfterExit) {
|
||||
await expect(joinedAfterExit).resolves.toMatchObject({
|
||||
status: "joined",
|
||||
handoffId: started.handoffId,
|
||||
});
|
||||
expect(spawnMock).toHaveBeenCalledOnce();
|
||||
}
|
||||
await expect(fs.access(markerPath)).rejects.toThrow();
|
||||
if (!replaceOwner && !exitParent) {
|
||||
const priorSentinel = new DatabaseSync(helper.stateDatabasePath);
|
||||
priorSentinel
|
||||
.prepare("DELETE FROM gateway_restart_sentinel WHERE sentinel_key = 'current'")
|
||||
.run();
|
||||
priorSentinel.close();
|
||||
const next = await start();
|
||||
if (next.status !== "started") {
|
||||
throw new Error("expected replacement ownership");
|
||||
}
|
||||
await expect(
|
||||
cancelManagedServiceUpdateHandoff({ kind: "managed-update-handoff", ...next }),
|
||||
).resolves.toBe("restored-in-process");
|
||||
}
|
||||
if (replaceOwner) {
|
||||
const replacement = new DatabaseSync(helper.updateLeaseDatabasePath);
|
||||
replacement.prepare("DELETE FROM managed_update_handoffs WHERE install_root = ?").run(root);
|
||||
replacement.close();
|
||||
}
|
||||
parent.stdin?.end();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,84 @@
|
||||
import type { EventEmitter } from "node:events";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import type { Writable } from "node:stream";
|
||||
import { getFileLockProcessStartTime } from "../shared/pid-alive.js";
|
||||
|
||||
export type MockManagedUpdateHandoffLeaseFailure =
|
||||
| "absent"
|
||||
| "malformed"
|
||||
| "wrong-owner"
|
||||
| "dead-helper";
|
||||
|
||||
export function signalMockManagedUpdateHandoffReady(params: {
|
||||
child: EventEmitter & { pid: number; stdout: Pick<Writable, "destroyed" | "write"> };
|
||||
paramsPath: string;
|
||||
cleanups: Set<() => void>;
|
||||
startIdentity?: number;
|
||||
failure?: MockManagedUpdateHandoffLeaseFailure;
|
||||
}): void {
|
||||
const { child, cleanups, failure } = params;
|
||||
if (child.stdout.destroyed) {
|
||||
return;
|
||||
}
|
||||
const lease = JSON.parse(fs.readFileSync(params.paramsPath, "utf8")) as {
|
||||
updateLeaseDatabasePath: string;
|
||||
updateLeaseKey: string;
|
||||
updateLeaseOwner: string;
|
||||
};
|
||||
const startIdentity = params.startIdentity ?? getFileLockProcessStartTime(child.pid);
|
||||
if (startIdentity === null) {
|
||||
throw new Error("expected the mocked handoff child to have a live process identity");
|
||||
}
|
||||
fs.mkdirSync(path.dirname(lease.updateLeaseDatabasePath), { recursive: true, mode: 0o700 });
|
||||
const owner =
|
||||
failure === "wrong-owner" ? `${lease.updateLeaseOwner}-replacement` : lease.updateLeaseOwner;
|
||||
const payload = JSON.stringify({
|
||||
version: 1,
|
||||
pid: failure === "dead-helper" ? child.pid + 1_000_000 : child.pid,
|
||||
startIdentity: failure === "malformed" ? null : String(startIdentity),
|
||||
});
|
||||
const db = new DatabaseSync(lease.updateLeaseDatabasePath);
|
||||
try {
|
||||
if (process.platform !== "win32") {
|
||||
fs.chmodSync(lease.updateLeaseDatabasePath, 0o600);
|
||||
}
|
||||
db.exec("PRAGMA busy_timeout = 5000;");
|
||||
db.exec(
|
||||
"CREATE TABLE IF NOT EXISTS managed_update_handoffs " +
|
||||
"(install_root TEXT NOT NULL PRIMARY KEY, owner TEXT NOT NULL, " +
|
||||
"payload_json TEXT NOT NULL, updated_at INTEGER NOT NULL) STRICT",
|
||||
);
|
||||
if (failure !== "absent") {
|
||||
db.prepare(
|
||||
"INSERT INTO managed_update_handoffs " +
|
||||
"(install_root, owner, payload_json, updated_at) VALUES (?, ?, ?, ?) " +
|
||||
"ON CONFLICT(install_root) DO UPDATE SET updated_at = excluded.updated_at " +
|
||||
"WHERE owner = excluded.owner AND payload_json = excluded.payload_json",
|
||||
).run(lease.updateLeaseKey, owner, payload, Date.now());
|
||||
}
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
if (failure !== "absent") {
|
||||
const cleanup = () => {
|
||||
cleanups.delete(cleanup);
|
||||
const cleanupDb = new DatabaseSync(lease.updateLeaseDatabasePath);
|
||||
try {
|
||||
cleanupDb.exec("PRAGMA busy_timeout = 5000;");
|
||||
cleanupDb
|
||||
.prepare(
|
||||
"DELETE FROM managed_update_handoffs " +
|
||||
"WHERE install_root = ? AND owner = ? AND payload_json = ?",
|
||||
)
|
||||
.run(lease.updateLeaseKey, owner, payload);
|
||||
} finally {
|
||||
cleanupDb.close();
|
||||
}
|
||||
};
|
||||
cleanups.add(cleanup);
|
||||
child.once("exit", cleanup);
|
||||
}
|
||||
child.stdout.write("OPENCLAW_UPDATE_HANDOFF_READY\n");
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -54,6 +54,8 @@ const {
|
||||
pid: 12345,
|
||||
command: "openclaw update --yes --channel beta --timeout 2700",
|
||||
logPath: "/tmp/openclaw-handoff.log",
|
||||
handoffId: "auto-handoff-id",
|
||||
installRoot: "/opt/openclaw",
|
||||
})),
|
||||
versionMock: { value: "1.0.0" },
|
||||
}));
|
||||
@@ -77,13 +79,8 @@ vi.mock("./openclaw-root.js", async () => {
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("./restart.js", () => ({
|
||||
resolveGatewayRestartDeferralTimeoutMs: (timeoutMs: unknown) => {
|
||||
if (typeof timeoutMs !== "number" || !Number.isFinite(timeoutMs)) {
|
||||
return 300_000;
|
||||
}
|
||||
return timeoutMs <= 0 ? undefined : Math.floor(timeoutMs);
|
||||
},
|
||||
vi.mock("./restart.js", async () => ({
|
||||
...(await vi.importActual<typeof import("./restart.js")>("./restart.js")),
|
||||
scheduleGatewaySigusr1Restart: scheduleGatewaySigusr1RestartMock,
|
||||
}));
|
||||
|
||||
@@ -313,6 +310,8 @@ describe("update-startup", () => {
|
||||
pid: 12345,
|
||||
command: "openclaw update --yes --channel beta --timeout 2700",
|
||||
logPath: "/tmp/openclaw-handoff.log",
|
||||
handoffId: "auto-handoff-id",
|
||||
installRoot: "/opt/openclaw",
|
||||
});
|
||||
resetUpdateAvailableStateForTest();
|
||||
});
|
||||
@@ -1911,9 +1910,11 @@ describe("update-startup", () => {
|
||||
expect(log.info).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("delegates configured auto-updates to an external supervisor", async () => {
|
||||
it("keeps external auto-update supervision authoritative over native systemd markers", async () => {
|
||||
mockPackageUpdateStatus("beta", "2.0.0-beta.1");
|
||||
process.env.OPENCLAW_SUPERVISOR_MODE = "external";
|
||||
process.env.OPENCLAW_SYSTEMD_UNIT = "openclaw-gateway.service";
|
||||
detectRespawnSupervisorMock.mockReturnValue("systemd");
|
||||
const log = { info: vi.fn() };
|
||||
const runAutoUpdate = createAutoUpdateSuccessMock();
|
||||
|
||||
@@ -1994,6 +1995,8 @@ describe("update-startup", () => {
|
||||
pid: 12345,
|
||||
command: "openclaw update --yes --channel beta --tag 2.0.0-beta.1 --timeout 2700",
|
||||
logPath: "/tmp/openclaw-handoff.log",
|
||||
handoffId: "started-auto-handoff-id",
|
||||
installRoot: await fs.realpath(installRoot),
|
||||
});
|
||||
const log = { info: vi.fn() };
|
||||
|
||||
@@ -2036,6 +2039,11 @@ describe("update-startup", () => {
|
||||
expect(scheduleGatewaySigusr1RestartMock).toHaveBeenCalledWith({
|
||||
delayMs: 0,
|
||||
reason: "update.auto",
|
||||
successorOwner: {
|
||||
kind: "managed-update-handoff",
|
||||
handoffId: "started-auto-handoff-id",
|
||||
installRoot: await fs.realpath(installRoot),
|
||||
},
|
||||
skipCooldown: true,
|
||||
skipDeferral: true,
|
||||
});
|
||||
@@ -2130,6 +2138,7 @@ describe("update-startup", () => {
|
||||
expect.objectContaining({
|
||||
root: "/opt/openclaw",
|
||||
timeoutMs: 45 * 60 * 1000,
|
||||
restartDrainTimeoutMs: 300_000,
|
||||
channel: "beta",
|
||||
tag: "2.0.0-beta.1",
|
||||
restartDelayMs: 2000,
|
||||
@@ -2139,6 +2148,11 @@ describe("update-startup", () => {
|
||||
expect(scheduleGatewaySigusr1RestartMock).toHaveBeenCalledWith({
|
||||
delayMs: 2000,
|
||||
reason: "update.auto",
|
||||
successorOwner: {
|
||||
kind: "managed-update-handoff",
|
||||
handoffId: "auto-handoff-id",
|
||||
installRoot: "/opt/openclaw",
|
||||
},
|
||||
skipCooldown: true,
|
||||
skipDeferral: true,
|
||||
});
|
||||
|
||||
+11
-23
@@ -39,6 +39,7 @@ import {
|
||||
import { resolveOpenClawPackageRoot } from "./openclaw-root.js";
|
||||
import { readVerifiedGitUpdateReceipt, type VerifiedGitUpdateReceipt } from "./restart-sentinel.js";
|
||||
import {
|
||||
normalizeGatewayRestartDelayMs,
|
||||
resolveGatewayRestartDeferralTimeoutMs,
|
||||
scheduleGatewaySigusr1Restart,
|
||||
} from "./restart.js";
|
||||
@@ -154,7 +155,6 @@ const AUTO_UPDATE_COMMAND_TIMEOUT_MS = 45 * 60 * 1000;
|
||||
const AUTO_STABLE_DELAY_HOURS_DEFAULT = 6;
|
||||
const AUTO_STABLE_JITTER_HOURS_DEFAULT = 12;
|
||||
const AUTO_BETA_CHECK_INTERVAL_HOURS_DEFAULT = 1;
|
||||
const MANAGED_AUTO_UPDATE_SYSTEMD_RESTART_GRACE_MS = 2000;
|
||||
const DEV_COMMIT_LIMIT = 5;
|
||||
const DEV_COMMIT_SUBJECT_MAX_LENGTH = 120;
|
||||
const DEV_COMMIT_LOG_MAX_OUTPUT_BYTES = 8 * 1024;
|
||||
@@ -162,13 +162,7 @@ const DEV_COMMIT_LOG_MAX_OUTPUT_BYTES = 8 * 1024;
|
||||
type UpdateCheckStateDatabase = Pick<OpenClawStateKyselyDatabase, "update_check_state">;
|
||||
|
||||
function shouldSkipCheck(allowInTests: boolean): boolean {
|
||||
if (allowInTests) {
|
||||
return false;
|
||||
}
|
||||
if (process.env.VITEST || process.env.NODE_ENV === "test") {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
return !allowInTests && Boolean(process.env.VITEST || process.env.NODE_ENV === "test");
|
||||
}
|
||||
|
||||
function resolveAutoUpdatePolicy(cfg: OpenClawConfig): AutoUpdatePolicy {
|
||||
@@ -438,14 +432,12 @@ function resolveStableAutoApplyAtMs(params: {
|
||||
return firstSeenMs + baseDelayMs + jitterMs;
|
||||
}
|
||||
|
||||
function resolveManagedAutoUpdateRestartDelayMs(supervisor: RespawnSupervisor): number {
|
||||
return supervisor === "systemd" ? MANAGED_AUTO_UPDATE_SYSTEMD_RESTART_GRACE_MS : 0;
|
||||
}
|
||||
|
||||
async function startManagedServiceAutoUpdateHandoff(
|
||||
params: AutoUpdateRunParams & { supervisor: RespawnSupervisor },
|
||||
): Promise<AutoUpdateRunResult> {
|
||||
const restartDelayMs = resolveManagedAutoUpdateRestartDelayMs(params.supervisor);
|
||||
const restartDelayMs = normalizeGatewayRestartDelayMs(
|
||||
params.supervisor === "systemd" ? undefined : 0,
|
||||
);
|
||||
const handoffId = randomUUID();
|
||||
try {
|
||||
if (!params.root?.trim()) {
|
||||
@@ -454,7 +446,9 @@ async function startManagedServiceAutoUpdateHandoff(
|
||||
const started = await startManagedServiceUpdateHandoff({
|
||||
root: params.root,
|
||||
timeoutMs: params.timeoutMs,
|
||||
restartDrainTimeoutMs: params.restartDrainTimeoutMs,
|
||||
restartDrainTimeoutMs:
|
||||
resolveGatewayRestartDeferralTimeoutMs(params.restartDrainTimeoutMs) ??
|
||||
resolveGatewayRestartDeferralTimeoutMs(),
|
||||
channel: params.channel,
|
||||
...(params.packageTargetVersion ? { tag: params.packageTargetVersion } : {}),
|
||||
restartDelayMs,
|
||||
@@ -469,9 +463,11 @@ async function startManagedServiceAutoUpdateHandoff(
|
||||
// Pair helper creation with restart scheduling before any state persistence
|
||||
// can fail and leave an indefinite handoff waiting on a live parent.
|
||||
if (started.status === "started") {
|
||||
const { handoffId: ownerId, installRoot } = started;
|
||||
scheduleGatewaySigusr1Restart({
|
||||
delayMs: restartDelayMs,
|
||||
reason: "update.auto",
|
||||
successorOwner: { kind: "managed-update-handoff", handoffId: ownerId, installRoot },
|
||||
skipCooldown: true,
|
||||
skipDeferral: true,
|
||||
});
|
||||
@@ -514,15 +510,7 @@ async function runAutoUpdateCommand(params: AutoUpdateRunParams): Promise<AutoUp
|
||||
}
|
||||
}
|
||||
if (supervisor) {
|
||||
return await startManagedServiceAutoUpdateHandoff({
|
||||
channel: params.channel,
|
||||
timeoutMs: params.timeoutMs,
|
||||
restartDrainTimeoutMs: params.restartDrainTimeoutMs,
|
||||
root: params.root,
|
||||
...(params.packageTargetVersion ? { packageTargetVersion: params.packageTargetVersion } : {}),
|
||||
...(params.devTarget ? { devTarget: params.devTarget } : {}),
|
||||
supervisor,
|
||||
});
|
||||
return await startManagedServiceAutoUpdateHandoff({ ...params, supervisor });
|
||||
}
|
||||
|
||||
const targetArgs = [
|
||||
|
||||
@@ -223,8 +223,31 @@ describe("process start times", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("returns null on unsupported platforms", () => {
|
||||
it("reads a bounded Windows PowerShell process start identity", () => {
|
||||
const execSpy = vi
|
||||
.spyOn(childProcess, "execFileSync")
|
||||
.mockReturnValue("2026-07-06T12:34:56.7890000Z\n");
|
||||
|
||||
return withMockedPlatform("win32", async () => {
|
||||
expect(getProcessStartTime(42)).toBeNull();
|
||||
expect(getFileLockProcessStartTime(42)).toBe(Date.UTC(2026, 6, 6, 12, 34, 56, 789));
|
||||
expect(execSpy).toHaveBeenCalledWith(
|
||||
"powershell.exe",
|
||||
[
|
||||
"-NoProfile",
|
||||
"-NonInteractive",
|
||||
"-Command",
|
||||
"(Get-Process -Id 42).StartTime.ToString('o')",
|
||||
],
|
||||
expect.objectContaining({ timeout: 1000, windowsHide: true }),
|
||||
);
|
||||
execSpy.mockReturnValueOnce("invalid\n");
|
||||
expect(getFileLockProcessStartTime(42)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
it("returns null on unsupported platforms", () => {
|
||||
return withMockedPlatform("freebsd", async () => {
|
||||
expect(getProcessStartTime(process.pid)).toBeNull();
|
||||
expect(getFileLockProcessStartTime(process.pid)).toBeNull();
|
||||
});
|
||||
|
||||
+18
-15
@@ -2,7 +2,7 @@
|
||||
import childProcess from "node:child_process";
|
||||
import fsSync from "node:fs";
|
||||
|
||||
const DARWIN_PS_TIMEOUT_MS = 1000;
|
||||
const PROCESS_START_TIMEOUT_MS = 1000;
|
||||
|
||||
function isValidPid(pid: number): boolean {
|
||||
return Number.isInteger(pid) && pid > 0;
|
||||
@@ -40,10 +40,7 @@ export function isPidAlive(pid: number): boolean {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (isZombieProcess(pid)) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
return !isZombieProcess(pid);
|
||||
}
|
||||
|
||||
/** Returns true only when the PID is invalid, missing, or known to be a Linux zombie. */
|
||||
@@ -59,20 +56,27 @@ export function isPidDefinitelyDead(pid: number): boolean {
|
||||
return isZombieProcess(pid);
|
||||
}
|
||||
|
||||
function getDarwinProcessStartTime(pid: number): number | null {
|
||||
function getPlatformProcessStartTime(pid: number): number | null {
|
||||
try {
|
||||
const windows = process.platform === "win32";
|
||||
const command = windows ? `(Get-Process -Id ${pid}).StartTime.ToString('o')` : String(pid);
|
||||
const args = windows
|
||||
? ["-NoProfile", "-NonInteractive", "-Command", command]
|
||||
: ["-o", "lstart=", "-p", command];
|
||||
const startedAt = childProcess
|
||||
.execFileSync("/bin/ps", ["-o", "lstart=", "-p", String(pid)], {
|
||||
.execFileSync(windows ? "powershell.exe" : "/bin/ps", args, {
|
||||
encoding: "utf8",
|
||||
env: { ...process.env, LC_ALL: "C", TZ: "UTC" },
|
||||
stdio: ["ignore", "pipe", "ignore"],
|
||||
timeout: DARWIN_PS_TIMEOUT_MS,
|
||||
timeout: PROCESS_START_TIMEOUT_MS,
|
||||
killSignal: "SIGKILL",
|
||||
windowsHide: windows,
|
||||
})
|
||||
.trim();
|
||||
// Darwin's lstart output has no timezone. Force UTC for both ps and parsing so
|
||||
// a system timezone change cannot make a live lock owner look like PID reuse.
|
||||
const startedAtMs = Date.parse(`${startedAt} UTC`);
|
||||
return Number.isFinite(startedAtMs) ? Math.floor(startedAtMs / 1000) : null;
|
||||
const startedAtMs = Date.parse(windows ? startedAt : `${startedAt} UTC`);
|
||||
return Number.isFinite(startedAtMs) ? Math.floor(startedAtMs / (windows ? 1 : 1000)) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
@@ -80,10 +84,7 @@ function getDarwinProcessStartTime(pid: number): number | null {
|
||||
|
||||
/** Read the Linux procfs start identity used by Linux-owned runtime state. */
|
||||
export function getProcessStartTime(pid: number): number | null {
|
||||
if (!isValidPid(pid)) {
|
||||
return null;
|
||||
}
|
||||
if (process.platform !== "linux") {
|
||||
if (!isValidPid(pid) || process.platform !== "linux") {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
@@ -109,5 +110,7 @@ export function getFileLockProcessStartTime(pid: number): number | null {
|
||||
if (!isValidPid(pid)) {
|
||||
return null;
|
||||
}
|
||||
return process.platform === "darwin" ? getDarwinProcessStartTime(pid) : getProcessStartTime(pid);
|
||||
return process.platform === "darwin" || process.platform === "win32"
|
||||
? getPlatformProcessStartTime(pid)
|
||||
: getProcessStartTime(pid);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user