From fc2a7be0bc959ab9c60ba85597ba3b50940dfc4a Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sat, 6 Jun 2026 07:19:40 -0700 Subject: [PATCH] fix(update): hand off supervised auto-updates --- src/cli/update-cli/update-command.ts | 8 +- src/gateway/server-methods/update.test.ts | 11 +- src/gateway/server-methods/update.ts | 16 ++- .../update-managed-service-handoff.test.ts | 13 +- .../update-managed-service-handoff.ts | 33 +++-- src/infra/update-startup.test.ts | 114 +++++++++++++++++- src/infra/update-startup.ts | 94 ++++++++++++++- 7 files changed, 252 insertions(+), 37 deletions(-) rename src/{gateway/server-methods => infra}/update-managed-service-handoff.test.ts (96%) rename src/{gateway/server-methods => infra}/update-managed-service-handoff.ts (93%) diff --git a/src/cli/update-cli/update-command.ts b/src/cli/update-cli/update-command.ts index ffc83e30e535..40c4e51d5cc5 100644 --- a/src/cli/update-cli/update-command.ts +++ b/src/cli/update-cli/update-command.ts @@ -828,12 +828,6 @@ function gatewayAncestryBlockMessage(pid: unknown): string | undefined { return isGatewayAncestorPid(pid) ? formatGatewayAncestryBlockMessage(pid) : undefined; } -function gatewayRuntimeAncestryBlockMessage( - runtime: { pid?: unknown } | null | undefined, -): string | undefined { - return gatewayAncestryBlockMessage(runtime?.pid); -} - function serviceControlStdoutForMode(jsonMode: boolean): NodeJS.WritableStream { return jsonMode ? JSON_MODE_SERVICE_STDOUT : process.stdout; } @@ -900,7 +894,7 @@ async function maybeStopManagedServiceBeforePackageUpdate(params: { }; } - const blockMessage = gatewayRuntimeAncestryBlockMessage(serviceState.runtime); + const blockMessage = gatewayAncestryBlockMessage(serviceState.runtime?.pid); if (blockMessage) { return { stopped: false, diff --git a/src/gateway/server-methods/update.test.ts b/src/gateway/server-methods/update.test.ts index af0416c11cfa..873aa64e72db 100644 --- a/src/gateway/server-methods/update.test.ts +++ b/src/gateway/server-methods/update.test.ts @@ -125,11 +125,14 @@ vi.mock("./restart-request.js", () => ({ }), })); -vi.mock("./update-managed-service-handoff.js", () => ({ +vi.mock("../../infra/update-managed-service-handoff.js", () => ({ startManagedServiceUpdateHandoff: startManagedServiceUpdateHandoffMock, - formatManagedServiceUpdateCommand: (timeoutMs?: number) => - timeoutMs - ? `openclaw update --yes --timeout ${Math.ceil(timeoutMs / 1000)}` + formatManagedServiceUpdateCommand: (params?: { + timeoutMs?: number; + channel?: "stable" | "beta" | "dev"; + }) => + params?.timeoutMs + ? `openclaw update --yes --timeout ${Math.ceil(params.timeoutMs / 1000)}` : "openclaw update --yes", buildManagedServiceHandoffUnavailableMessage: (command: string) => `Run \`${command}\` from a shell outside the gateway service.`, diff --git a/src/gateway/server-methods/update.ts b/src/gateway/server-methods/update.ts index 4f568d0b13cf..56a7d7d743a1 100644 --- a/src/gateway/server-methods/update.ts +++ b/src/gateway/server-methods/update.ts @@ -15,6 +15,11 @@ import { scheduleGatewaySigusr1Restart } from "../../infra/restart.js"; import { detectRespawnSupervisor } from "../../infra/supervisor-markers.js"; import { normalizeUpdateChannel } from "../../infra/update-channels.js"; import { CONTROL_PLANE_UPDATE_HANDOFF_STARTED_REASON } from "../../infra/update-control-plane-sentinel.js"; +import { + buildManagedServiceHandoffUnavailableMessage, + formatManagedServiceUpdateCommand, + startManagedServiceUpdateHandoff, +} from "../../infra/update-managed-service-handoff.js"; import { buildUpdateRestartSentinelPayload, type UpdateRestartSentinelMeta, @@ -27,11 +32,6 @@ import { } from "../server-restart-sentinel.js"; import { parseRestartRequestParams } from "./restart-request.js"; import type { GatewayRequestHandlers } from "./types.js"; -import { - buildManagedServiceHandoffUnavailableMessage, - formatManagedServiceUpdateCommand, - startManagedServiceUpdateHandoff, -} from "./update-managed-service-handoff.js"; import { assertValidParams } from "./validation.js"; const SYSTEMD_HANDOFF_RESTART_GRACE_MS = 2000; @@ -146,7 +146,10 @@ export const updateHandlers: GatewayRequestHandlers = { durationMs: 0, }; } else if (installSurface.kind === "global") { - const command = formatManagedServiceUpdateCommand(timeoutMs); + const command = formatManagedServiceUpdateCommand({ + timeoutMs, + channel: configChannel ?? undefined, + }); if (supervisor) { try { const startedAt = Date.now(); @@ -157,6 +160,7 @@ export const updateHandlers: GatewayRequestHandlers = { const started = await startManagedServiceUpdateHandoff({ root, timeoutMs, + channel: configChannel ?? undefined, restartDelayMs, meta: sentinelMeta, handoffId, diff --git a/src/gateway/server-methods/update-managed-service-handoff.test.ts b/src/infra/update-managed-service-handoff.test.ts similarity index 96% rename from src/gateway/server-methods/update-managed-service-handoff.test.ts rename to src/infra/update-managed-service-handoff.test.ts index 2ab6432761b0..9329b2626f19 100644 --- a/src/gateway/server-methods/update-managed-service-handoff.test.ts +++ b/src/infra/update-managed-service-handoff.test.ts @@ -5,12 +5,12 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { SUPERVISOR_HINT_ENV_VARS } from "../../infra/supervisor-markers.js"; -import { CONTROL_PLANE_UPDATE_SENTINEL_META_ENV } from "../../infra/update-control-plane-sentinel.js"; +import { SUPERVISOR_HINT_ENV_VARS } from "./supervisor-markers.js"; +import { CONTROL_PLANE_UPDATE_SENTINEL_META_ENV } from "./update-control-plane-sentinel.js"; import { cleanupStaleManagedServiceUpdateHandoffs, MANAGED_SERVICE_UPDATE_HANDOFF_TEMP_PREFIX, -} from "../../infra/update-managed-service-handoff-cleanup.js"; +} from "./update-managed-service-handoff-cleanup.js"; const { spawnMock } = vi.hoisted(() => ({ spawnMock: vi.fn(() => ({ @@ -20,7 +20,9 @@ const { spawnMock } = vi.hoisted(() => ({ })); vi.mock("node:child_process", async () => { - const { mockNodeChildProcessModule } = await import("./node-child-process.test-support.js"); + const { mockNodeChildProcessModule } = await import( + "../gateway/server-methods/node-child-process.test-support.js" + ); return mockNodeChildProcessModule({ spawn: spawnMock as unknown as typeof import("node:child_process").spawn, }); @@ -205,6 +207,7 @@ describe("managed service update handoff", () => { execPath: "/usr/local/bin/node", argv1: "/opt/openclaw/openclaw.mjs", handoffId: "handoff-123", + channel: "beta", supervisor: "systemd", env: { PATH: binDir, @@ -249,6 +252,8 @@ describe("managed service update handoff", () => { "update", "--yes", "--json", + "--channel", + "beta", "--timeout", "1800", ]); diff --git a/src/gateway/server-methods/update-managed-service-handoff.ts b/src/infra/update-managed-service-handoff.ts similarity index 93% rename from src/gateway/server-methods/update-managed-service-handoff.ts rename to src/infra/update-managed-service-handoff.ts index 2ea93766bb97..48a7043a3240 100644 --- a/src/gateway/server-methods/update-managed-service-handoff.ts +++ b/src/infra/update-managed-service-handoff.ts @@ -4,17 +4,17 @@ import { spawn } from "node:child_process"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { resolveRestartSentinelPath } from "../../infra/restart-sentinel.js"; +import { resolveRestartSentinelPath } from "./restart-sentinel.js"; import { SUPERVISOR_HINT_ENV_VARS, type RespawnSupervisor, -} from "../../infra/supervisor-markers.js"; +} from "./supervisor-markers.js"; import { CONTROL_PLANE_UPDATE_SENTINEL_META_ENV, type ControlPlaneUpdateSentinelMetaFile, -} from "../../infra/update-control-plane-sentinel.js"; -import { MANAGED_SERVICE_UPDATE_HANDOFF_TEMP_PREFIX } from "../../infra/update-managed-service-handoff-cleanup.js"; -import type { UpdateRestartSentinelMeta } from "../../infra/update-restart-sentinel-payload.js"; +} from "./update-control-plane-sentinel.js"; +import { MANAGED_SERVICE_UPDATE_HANDOFF_TEMP_PREFIX } from "./update-managed-service-handoff-cleanup.js"; +import type { UpdateRestartSentinelMeta } from "./update-restart-sentinel-payload.js"; const PARENT_EXIT_GRACE_MS = 60_000; const SYSTEMD_RUN_CANDIDATE_PATHS = ["/usr/bin/systemd-run", "/bin/systemd-run"] as const; @@ -269,10 +269,14 @@ function isNodeLikeRuntime(execPath: string | undefined): boolean { function resolveUpdateCliArgv(params: { timeoutMs?: number; + channel?: "stable" | "beta" | "dev"; execPath?: string; argv1?: string; }): string[] { const updateArgs = ["update", "--yes", "--json"]; + if (params.channel) { + updateArgs.push("--channel", params.channel); + } if (typeof params.timeoutMs === "number" && Number.isFinite(params.timeoutMs)) { updateArgs.push("--timeout", String(Math.max(1, Math.ceil(params.timeoutMs / 1000)))); } @@ -288,10 +292,16 @@ function resolveUpdateCliArgv(params: { return ["openclaw", ...updateArgs]; } -export function formatManagedServiceUpdateCommand(timeoutMs?: number): string { +export function formatManagedServiceUpdateCommand(params?: { + timeoutMs?: number; + channel?: "stable" | "beta" | "dev"; +}): string { const args = ["openclaw", "update", "--yes"]; - if (typeof timeoutMs === "number" && Number.isFinite(timeoutMs)) { - args.push("--timeout", String(Math.max(1, Math.ceil(timeoutMs / 1000)))); + if (params?.channel) { + args.push("--channel", params.channel); + } + if (typeof params?.timeoutMs === "number" && Number.isFinite(params.timeoutMs)) { + args.push("--timeout", String(Math.max(1, Math.ceil(params.timeoutMs / 1000)))); } return args.join(" "); } @@ -410,6 +420,7 @@ async function resolveHandoffSpawn(params: { export async function startManagedServiceUpdateHandoff(params: { root: string; timeoutMs?: number; + channel?: "stable" | "beta" | "dev"; restartDelayMs?: number; meta: UpdateRestartSentinelMeta; handoffId?: string; @@ -426,10 +437,14 @@ export async function startManagedServiceUpdateHandoff(params: { const logPath = path.join(dir, "handoff.log"); const commandArgv = resolveUpdateCliArgv({ timeoutMs: params.timeoutMs, + channel: params.channel, execPath: params.execPath ?? process.execPath, argv1: params.argv1 ?? process.argv[1], }); - const commandLabel = formatManagedServiceUpdateCommand(params.timeoutMs); + const commandLabel = formatManagedServiceUpdateCommand({ + timeoutMs: params.timeoutMs, + channel: params.channel, + }); const handoffCwd = await resolveManagedServiceHandoffCwd(params.root); const metaFile: ControlPlaneUpdateSentinelMetaFile = { version: 1, diff --git a/src/infra/update-startup.test.ts b/src/infra/update-startup.test.ts index d8963d1aedf9..7a36f2de0dd3 100644 --- a/src/infra/update-startup.test.ts +++ b/src/infra/update-startup.test.ts @@ -7,6 +7,21 @@ import { createSuiteTempRootTracker } from "../test-helpers/temp-dir.js"; import { captureEnv } from "../test-utils/env.js"; import type { UpdateCheckResult } from "./update-check.js"; +const { + detectRespawnSupervisorMock, + scheduleGatewaySigusr1RestartMock, + startManagedServiceUpdateHandoffMock, +} = vi.hoisted(() => ({ + detectRespawnSupervisorMock: vi.fn(), + scheduleGatewaySigusr1RestartMock: vi.fn(() => ({ scheduled: true })), + startManagedServiceUpdateHandoffMock: vi.fn(async () => ({ + status: "started" as const, + pid: 12345, + command: "openclaw update --yes --channel beta --timeout 2700", + logPath: "/tmp/openclaw-handoff.log", + })), +})); + vi.mock("./openclaw-root.js", async () => { const actual = await vi.importActual("./openclaw-root.js"); return { @@ -15,6 +30,19 @@ vi.mock("./openclaw-root.js", async () => { }; }); +vi.mock("./restart.js", () => ({ + scheduleGatewaySigusr1Restart: scheduleGatewaySigusr1RestartMock, +})); + +vi.mock("./supervisor-markers.js", async () => { + const actual = + await vi.importActual("./supervisor-markers.js"); + return { + ...actual, + detectRespawnSupervisor: detectRespawnSupervisorMock, + }; +}); + vi.mock("./update-check.js", async () => { const parse = (value: string) => value.split(".").map((part) => Number.parseInt(part, 10)); const compareSemverStrings = (a: string, b: string) => { @@ -45,6 +73,10 @@ vi.mock("../process/exec.js", () => ({ runCommandWithTimeout: vi.fn(), })); +vi.mock("./update-managed-service-handoff.js", () => ({ + startManagedServiceUpdateHandoff: startManagedServiceUpdateHandoffMock, +})); + describe("update-startup", () => { const suiteRootTracker = createSuiteTempRootTracker({ prefix: "openclaw-update-check-suite-" }); let tempDir: string; @@ -79,6 +111,13 @@ describe("update-startup", () => { envSnapshot = captureEnv([ "OPENCLAW_NO_AUTO_UPDATE", "OPENCLAW_STATE_DIR", + "OPENCLAW_SERVICE_KIND", + "OPENCLAW_SERVICE_MARKER", + "OPENCLAW_GATEWAY_SERVICE_PID", + "OPENCLAW_LAUNCHD_LABEL", + "OPENCLAW_SYSTEMD_UNIT", + "OPENCLAW_WINDOWS_TASK_NAME", + "INVOCATION_ID", "NODE_ENV", "VITEST", ]); @@ -106,6 +145,16 @@ describe("update-startup", () => { vi.mocked(checkUpdateStatus).mockClear(); vi.mocked(resolveNpmChannelTag).mockClear(); vi.mocked(runCommandWithTimeout).mockClear(); + detectRespawnSupervisorMock.mockReset(); + detectRespawnSupervisorMock.mockReturnValue(null); + scheduleGatewaySigusr1RestartMock.mockClear(); + startManagedServiceUpdateHandoffMock.mockClear(); + startManagedServiceUpdateHandoffMock.mockResolvedValue({ + status: "started", + pid: 12345, + command: "openclaw update --yes --channel beta --timeout 2700", + logPath: "/tmp/openclaw-handoff.log", + }); resetUpdateAvailableStateForTest(); }); @@ -496,6 +545,8 @@ describe("update-startup", () => { } expect(runCommandWithTimeout).toHaveBeenCalledTimes(1); + expect(startManagedServiceUpdateHandoffMock).not.toHaveBeenCalled(); + expect(scheduleGatewaySigusr1RestartMock).not.toHaveBeenCalled(); const [argv, options] = requireFirstRunCommandCall(); expect(argv).toEqual([ process.execPath, @@ -506,11 +557,64 @@ describe("update-startup", () => { "beta", "--json", ]); - expect(options).toEqual({ - timeoutMs: 45 * 60 * 1000, - env: { - OPENCLAW_AUTO_UPDATE: "1", - }, + expect(typeof options).toBe("object"); + if (typeof options !== "object") { + throw new Error("expected command options object"); + } + expect(options.timeoutMs).toBe(45 * 60 * 1000); + expect(options.env).toEqual({ OPENCLAW_AUTO_UPDATE: "1" }); + }); + + it("hands supervised auto-updates to a detached service handoff before restarting", async () => { + mockPackageInstallStatus(); + mockNpmChannelTag("beta", "2.0.0-beta.1"); + detectRespawnSupervisorMock.mockReturnValue("launchd"); + const log = { info: vi.fn() }; + + await runGatewayUpdateCheck({ + cfg: createBetaAutoUpdateConfig(), + log, + isNixMode: false, + allowInTests: true, + }); + + expect(runCommandWithTimeout).not.toHaveBeenCalled(); + expect(startManagedServiceUpdateHandoffMock).toHaveBeenCalledWith( + expect.objectContaining({ + root: "/opt/openclaw", + timeoutMs: 45 * 60 * 1000, + channel: "beta", + restartDelayMs: 0, + supervisor: "launchd", + handoffId: expect.any(String), + meta: { + handoffId: expect.any(String), + note: "background auto-update", + }, + }), + ); + const handoffCalls = startManagedServiceUpdateHandoffMock.mock.calls as unknown as Array< + [ + { + handoffId?: string; + meta?: { handoffId?: string }; + }, + ] + >; + const [handoffParams] = handoffCalls[0] ?? []; + expect(handoffParams?.meta?.handoffId).toBe(handoffParams?.handoffId); + expect(scheduleGatewaySigusr1RestartMock).toHaveBeenCalledWith({ + delayMs: 0, + reason: "update.auto", + skipCooldown: true, + skipDeferral: true, + }); + expect(log.info).toHaveBeenCalledWith("auto-update handoff started", { + channel: "beta", + version: "2.0.0-beta.1", + tag: "beta", + command: "openclaw update --yes --channel beta --timeout 2700", + logPath: "/tmp/openclaw-handoff.log", }); }); diff --git a/src/infra/update-startup.ts b/src/infra/update-startup.ts index b491eba27f70..bac200224418 100644 --- a/src/infra/update-startup.ts +++ b/src/infra/update-startup.ts @@ -1,6 +1,7 @@ // Runs startup update checks and optional auto-update handoff. import { createHash, randomUUID } from "node:crypto"; import fs from "node:fs/promises"; +import os from "node:os"; import path from "node:path"; import { asDateTimestampMs, @@ -15,8 +16,12 @@ import { VERSION } from "../version.js"; import { isTruthyEnvValue } from "./env.js"; import { writeJson } from "./json-files.js"; import { resolveOpenClawPackageRoot } from "./openclaw-root.js"; +import { scheduleGatewaySigusr1Restart } from "./restart.js"; +import { detectRespawnSupervisor, type RespawnSupervisor } from "./supervisor-markers.js"; import { normalizeUpdateChannel, DEFAULT_PACKAGE_CHANNEL } from "./update-channels.js"; import { compareSemverStrings, resolveNpmChannelTag, checkUpdateStatus } from "./update-check.js"; +import { CONTROL_PLANE_UPDATE_HANDOFF_STARTED_REASON } from "./update-control-plane-sentinel.js"; +import { startManagedServiceUpdateHandoff } from "./update-managed-service-handoff.js"; type UpdateCheckState = { lastCheckedAt?: string; @@ -47,6 +52,9 @@ type AutoUpdateRunResult = { stdout?: string; stderr?: string; reason?: string; + command?: string; + logPath?: string; + restartDelayMs?: number; }; export type UpdateAvailable = { @@ -72,6 +80,7 @@ const AUTO_UPDATE_COMMAND_TIMEOUT_MS = 45 * 60 * 1000; const AUTO_STABLE_DELAY_HOURS_DEFAULT = 6; const AUTO_STABLE_JITTER_HOURS_DEFAULT = 12; const AUTO_BETA_CHECK_INTERVAL_HOURS_DEFAULT = 1; +const MANAGED_AUTO_UPDATE_SYSTEMD_RESTART_GRACE_MS = 2000; function shouldSkipCheck(allowInTests: boolean): boolean { if (allowInTests) { @@ -248,11 +257,74 @@ function resolveStableAutoApplyAtMs(params: { return firstSeenMs + baseDelayMs + jitterMs; } +function resolveAutoUpdateHandoffRoot(root: string | undefined): string { + if (root?.trim()) { + return root; + } + try { + return process.cwd(); + } catch { + return os.homedir(); + } +} + +function resolveManagedAutoUpdateRestartDelayMs(supervisor: RespawnSupervisor): number { + return supervisor === "systemd" ? MANAGED_AUTO_UPDATE_SYSTEMD_RESTART_GRACE_MS : 0; +} + +async function startManagedServiceAutoUpdateHandoff(params: { + channel: "stable" | "beta"; + timeoutMs: number; + root?: string; + supervisor: RespawnSupervisor; +}): Promise { + const restartDelayMs = resolveManagedAutoUpdateRestartDelayMs(params.supervisor); + const handoffId = randomUUID(); + try { + const started = await startManagedServiceUpdateHandoff({ + root: resolveAutoUpdateHandoffRoot(params.root), + timeoutMs: params.timeoutMs, + channel: params.channel, + restartDelayMs, + supervisor: params.supervisor, + handoffId, + meta: { + handoffId, + note: "background auto-update", + }, + }); + return { + ok: true, + code: 0, + reason: CONTROL_PLANE_UPDATE_HANDOFF_STARTED_REASON, + command: started.command, + logPath: started.logPath, + restartDelayMs, + }; + } catch (err) { + return { + ok: false, + code: null, + reason: String(err), + }; + } +} + async function runAutoUpdateCommand(params: { channel: "stable" | "beta"; timeoutMs: number; root?: string; }): Promise { + const supervisor = detectRespawnSupervisor(process.env, process.platform); + if (supervisor) { + return await startManagedServiceAutoUpdateHandoff({ + channel: params.channel, + timeoutMs: params.timeoutMs, + root: params.root, + supervisor, + }); + } + const baseArgs = ["update", "--yes", "--channel", params.channel, "--json"]; const execPath = process.execPath?.trim(); const argv1 = process.argv[1]?.trim(); @@ -386,6 +458,7 @@ export async function runGatewayUpdateCheck(params: { ...state, lastCheckedAt: resolveUpdateCheckTimestamp(now), }; + let pendingAutoUpdateRestartDelayMs: number | null = null; if (status.installKind !== "package") { delete nextState.lastAvailableVersion; @@ -484,9 +557,18 @@ export async function runGatewayUpdateCheck(params: { const outcome = await runAuto({ channel, timeoutMs: AUTO_UPDATE_COMMAND_TIMEOUT_MS, - root: root ?? undefined, + root: root ?? status.root ?? undefined, }); - if (outcome.ok) { + if (outcome.ok && outcome.reason === CONTROL_PLANE_UPDATE_HANDOFF_STARTED_REASON) { + pendingAutoUpdateRestartDelayMs = outcome.restartDelayMs ?? 0; + params.log.info("auto-update handoff started", { + channel, + version: resolved.version, + tag, + ...(outcome.command ? { command: outcome.command } : {}), + ...(outcome.logPath ? { logPath: outcome.logPath } : {}), + }); + } else if (outcome.ok) { nextState.autoLastSuccessVersion = resolved.version; nextState.autoLastSuccessAt = resolveUpdateCheckTimestamp(now); params.log.info("auto-update applied", { @@ -515,6 +597,14 @@ export async function runGatewayUpdateCheck(params: { } await writeState(statePath, nextState); + if (pendingAutoUpdateRestartDelayMs !== null) { + scheduleGatewaySigusr1Restart({ + delayMs: pendingAutoUpdateRestartDelayMs, + reason: "update.auto", + skipCooldown: true, + skipDeferral: true, + }); + } } export function scheduleGatewayUpdateCheck(params: {