diff --git a/src/daemon/launchd.test.ts b/src/daemon/launchd.test.ts index dc7aff8b0ef8..8cc661256e57 100644 --- a/src/daemon/launchd.test.ts +++ b/src/daemon/launchd.test.ts @@ -40,6 +40,7 @@ const state = vi.hoisted(() => ({ bootstrapError: "", bootstrapCode: 1, bootstrapLoadsServiceOnFailure: false, + bootstrapTransient: false, kickstartError: "", kickstartCode: 1, kickstartFailuresRemaining: 0, @@ -223,6 +224,23 @@ async function runStopLaunchAgentWithFakeTimers(args: Parameters[0]) { + vi.useFakeTimers(); + try { + const restartPromise = restartLaunchAgent(args) + .then((value) => ({ ok: true as const, value })) + .catch((error: unknown) => ({ ok: false as const, error })); + await vi.runAllTimersAsync(); + const result = await restartPromise; + if (!result.ok) { + throw result.error; + } + return result.value; + } finally { + vi.useRealTimers(); + } +} + function expectLaunchctlEnableBootstrapOrder(env: Record) { const domain = typeof process.getuid === "function" ? `gui/${process.getuid()}` : "gui/501"; const label = "ai.openclaw.gateway"; @@ -343,11 +361,18 @@ function executeLaunchctlMock(file: string, args: string[]) { } if (call[0] === "bootstrap") { if (state.bootstrapError) { + const detail = state.bootstrapError; + // Transient failures clear after one attempt so recovery paths that retry + // a bootstrap can be exercised the way launchd behaves once a booted-out + // job finishes tearing down. + if (state.bootstrapTransient) { + state.bootstrapError = ""; + } if (state.bootstrapLoadsServiceOnFailure) { state.serviceLoaded = true; state.serviceRunning = true; } - return { stdout: "", stderr: state.bootstrapError, code: state.bootstrapCode }; + return { stdout: "", stderr: detail, code: state.bootstrapCode }; } state.serviceLoaded = true; state.serviceRunning = true; @@ -523,6 +548,7 @@ beforeEach(() => { state.bootstrapError = ""; state.bootstrapCode = 1; state.bootstrapLoadsServiceOnFailure = false; + state.bootstrapTransient = false; state.kickstartError = ""; state.kickstartCode = 1; state.kickstartFailuresRemaining = 0; @@ -2552,6 +2578,28 @@ describe("launchd install", () => { expect(onMutation.mock.calls).toEqual([[{ mode: "enable" }], [{ mode: "bootstrap" }]]); }); + it("fails an already-loaded bootstrap immediately instead of waiting out the teardown deadline", async () => { + const env = createDefaultLaunchdEnv(); + const onMutation = vi.fn(); + state.kickstartError = "Could not find service"; + state.kickstartFailuresRemaining = 1; + // launchd answers EIO for a label that is still registered. `startLaunchAgent` + // never booted the job out, so there is no teardown to wait for: retrying + // until the teardown deadline would stall the start for no gain. Real timers + // here so a reintroduced retry loop blows the test timeout rather than + // passing under vi.runAllTimersAsync(). + state.bootstrapError = + "Could not bootstrap service: 5: Input/output error: already exists in domain for gui/501"; + state.bootstrapCode = 5; + + await expect(startLaunchAgent({ env, stdout: new PassThrough(), onMutation })).rejects.toThrow( + "launchctl bootstrap failed: Could not bootstrap service: 5: Input/output error", + ); + + expect(countMatching(state.launchctlCalls, (call) => call[0] === "bootstrap")).toBe(1); + expect(onMutation).not.toHaveBeenCalledWith({ mode: "bootstrap" }); + }); + it("audits enable but not kickstart when the later launch fails", async () => { const env = createDefaultLaunchdEnv(); const onMutation = vi.fn(); @@ -2647,14 +2695,113 @@ describe("launchd install", () => { restartLaunchAgent({ env, stdout: new PassThrough(), onMutation }), ).rejects.toThrow("launchctl bootstrap failed: Operation not permitted"); + // The trailing enable comes from the post-failure recovery attempt, which + // cannot report a bootstrap mutation because this bootstrap keeps failing. expect(onMutation.mock.calls).toEqual([ [{ mode: "enable" }], [{ mode: "bootout" }], [{ mode: "enable" }], + [{ mode: "enable" }], ]); expect(onMutation).not.toHaveBeenCalledWith({ mode: "bootstrap" }); }); + it("reloads the LaunchAgent through a transient reload bootstrap failure", async () => { + const env = { + ...createDefaultLaunchdEnv(), + OPENCLAW_GATEWAY_PORT: "18789", + }; + setLaunchAgentPlist({ + env, + label: "ai.openclaw.gateway", + programArguments: ["node", "gateway.js"], + }); + // launchd answers EIO while the just-booted-out job is still tearing down. + state.bootstrapError = "Bootstrap failed: 5: Input/output error"; + state.bootstrapCode = 5; + state.bootstrapTransient = true; + const onMutation = vi.fn(); + + const result = await runRestartLaunchAgentWithFakeTimers({ + env, + stdout: new PassThrough(), + onMutation, + }); + + // Teardown is transient, so the retry must land a real bootstrap rather than + // surfacing an error the operator has to recover from by hand. + expect(result).toEqual({ outcome: "completed" }); + expect(state.serviceLoaded).toBe(true); + expect(onMutation).toHaveBeenCalledWith({ mode: "bootstrap" }); + }); + + it("reports the LaunchAgent as unloaded when bootstrap teardown never clears", async () => { + const env = { + ...createDefaultLaunchdEnv(), + OPENCLAW_GATEWAY_PORT: "18789", + }; + setLaunchAgentPlist({ + env, + label: "ai.openclaw.gateway", + programArguments: ["node", "gateway.js"], + }); + // EIO that never clears must stay bounded instead of retrying forever, and + // the restore attempt fails with it, so the label really does stay absent. + state.bootstrapError = "Bootstrap failed: 5: Input/output error"; + state.bootstrapCode = 5; + + const error = await runRestartLaunchAgentWithFakeTimers({ + env, + stdout: new PassThrough(), + }).catch((caught: unknown) => caught); + + // bootout already removed the job, so the operator has to learn both why the + // bootstrap failed and that nothing is left for KeepAlive to respawn. + const domain = typeof process.getuid === "function" ? `gui/${process.getuid()}` : "gui/501"; + expect(state.serviceLoaded).toBe(false); + expect(error).toBeInstanceOf(Error); + const message = (error as Error).message; + expect(message).toContain( + "launchctl bootstrap failed: Bootstrap failed: 5: Input/output error", + ); + expect(message).toContain(`LaunchAgent ${domain}/ai.openclaw.gateway is not loaded`); + expect(message).toContain("The gateway is down and launchd has no job left to respawn it."); + expect(message).toContain("openclaw gateway start"); + }); + + it("does not wait out the teardown deadline when the reload bootstrap reports already-loaded", async () => { + const env = { + ...createDefaultLaunchdEnv(), + OPENCLAW_GATEWAY_PORT: "18789", + }; + setLaunchAgentPlist({ + env, + label: "ai.openclaw.gateway", + programArguments: ["node", "gateway.js"], + }); + // Same EIO code as a pending teardown, but the label is still registered + // rather than draining, so there is nothing to wait for. Real timers here: + // a retry loop would blow the test timeout instead of quietly passing. + state.bootstrapError = + "Could not bootstrap service: 5: Input/output error: already exists in domain for gui/501"; + state.bootstrapCode = 5; + state.bootstrapLoadsServiceOnFailure = true; + + const error = await restartLaunchAgent({ env, stdout: new PassThrough() }).catch( + (caught: unknown) => caught, + ); + + expect(countMatching(state.launchctlCalls, (call) => call[0] === "bootstrap")).toBe(1); + expect(error).toBeInstanceOf(Error); + const message = (error as Error).message; + expect(message).toContain( + "launchctl bootstrap failed: Could not bootstrap service: 5: Input/output error", + ); + // The label is still registered, so the recovery probe finds it and the + // failure must not claim the gateway was left unloaded. + expect(message).not.toContain("is not loaded"); + }); + it("completes reload when the mutation observer fails after bootout", async () => { const env = { ...createDefaultLaunchdEnv(), diff --git a/src/daemon/launchd.ts b/src/daemon/launchd.ts index 50cd6211516e..b7799c28371d 100644 --- a/src/daemon/launchd.ts +++ b/src/daemon/launchd.ts @@ -80,6 +80,11 @@ const OPENCLAW_NODE_RUNTIME_NAMES = new Set(["bun", "bun.exe", "node", "node.exe const OPENCLAW_SCRIPT_NAMES = new Set(["openclaw.mjs"]); const LAUNCH_AGENT_STOP_PORT_RELEASE_TIMEOUT_MS = LAUNCH_AGENT_EXIT_TIMEOUT_SECONDS * 1_000; const LAUNCH_AGENT_STOP_PORT_RELEASE_POLL_MS = 100; +// launchd reserves the label until the outgoing job actually exits, and it +// SIGKILLs that job once ExitTimeOut elapses. Bound the bootstrap retry by that +// same deadline plus slack so a drain-on-SIGTERM gateway cannot outlast it. +const LAUNCH_AGENT_BOOTSTRAP_TEARDOWN_TIMEOUT_MS = (LAUNCH_AGENT_EXIT_TIMEOUT_SECONDS + 10) * 1_000; +const LAUNCH_AGENT_BOOTSTRAP_TEARDOWN_POLL_MS = 500; const LAUNCHCTL_PROTECTED_PID_TIMEOUT_MS = 2_000; export type StaleOpenClawUpdateLaunchdJob = { @@ -635,6 +640,10 @@ async function bootstrapLaunchAgentOrThrow(params: { actionHint: string; onMutation?: (mode: "enable" | "bootstrap") => void; skipEnable?: boolean; + // Opt-in for callers that just issued `bootout` on this label. Only those can + // race a pending teardown, so start/install/recovery paths keep failing fast + // on an unrelated EIO instead of waiting out the teardown deadline. + retryPendingTeardown?: boolean; }) { // `disable` state survives bootout and plist rewrites; explicit start/repair // paths must clear it before asking launchd to load the job again. @@ -644,27 +653,38 @@ async function bootstrapLaunchAgentOrThrow(params: { params.onMutation?.("enable"); } } - const boot = await execLaunchctl(["bootstrap", params.domain, params.plistPath]); - if (boot.code === 0) { - params.onMutation?.("bootstrap"); - return; - } - const detail = (boot.stderr || boot.stdout).trim(); - if (isUnsupportedGuiDomain(detail)) { - throwBootstrapGuiSessionError({ - detail, - domain: params.domain, - actionHint: params.actionHint, - }); - } - if (isLaunchctlOperationAlreadyInProgress(detail)) { - const state = await probeLaunchAgentState(params.serviceTarget); - if (state.state === "running" || state.state === "stopped") { + const teardownDeadline = Date.now() + LAUNCH_AGENT_BOOTSTRAP_TEARDOWN_TIMEOUT_MS; + for (;;) { + const boot = await execLaunchctl(["bootstrap", params.domain, params.plistPath]); + if (boot.code === 0) { params.onMutation?.("bootstrap"); return; } + const detail = (boot.stderr || boot.stdout).trim(); + if (isUnsupportedGuiDomain(detail)) { + throwBootstrapGuiSessionError({ + detail, + domain: params.domain, + actionHint: params.actionHint, + }); + } + if (isLaunchctlOperationAlreadyInProgress(detail)) { + const state = await probeLaunchAgentState(params.serviceTarget); + if (state.state === "running" || state.state === "stopped") { + params.onMutation?.("bootstrap"); + return; + } + } + const remainingMs = teardownDeadline - Date.now(); + if ( + !params.retryPendingTeardown || + !isLaunchctlBootstrapPendingTeardown(boot) || + remainingMs <= 0 + ) { + throw new Error(`launchctl bootstrap failed: ${detail}`); + } + await sleep(Math.min(LAUNCH_AGENT_BOOTSTRAP_TEARDOWN_POLL_MS, remainingMs)); } - throw new Error(`launchctl bootstrap failed: ${detail}`); } async function ensureLaunchAgentPlistReadable(plistPath: string): Promise { @@ -1012,6 +1032,25 @@ function isLaunchctlOperationAlreadyInProgress(detail: string): boolean { ); } +function isLaunchctlBootstrapPendingTeardown(res: { + stdout: string; + stderr: string; + code: number; +}): boolean { + // `bootout` returns once launchd accepts the request, not once the job is gone, + // so bootstrapping the same label mid-teardown answers EIO. The plist is valid + // here, so this is a timing conflict to retry rather than a real I/O fault. + // + // launchd answers the same EIO for a label that is simply still registered + // ("already exists in domain"). That job is not tearing down, so waiting for a + // teardown that never comes only delays the failure. + if (isLaunchctlAlreadyLoaded(res)) { + return false; + } + const normalized = normalizeLowercaseStringOrEmpty(res.stderr || res.stdout); + return normalized.includes("bootstrap failed: 5") || normalized.includes("input/output error"); +} + async function bootoutLaunchAgentOrThrow(params: { serviceTarget: string; warning: string; @@ -1407,15 +1446,17 @@ async function rewriteLaunchAgentPlistForRestart({ return true; } +type LaunchAgentRestoreResult = { loaded: true } | { loaded: false; detail: string }; + async function ensureLaunchAgentLoadedAfterFailure(params: { domain: string; serviceTarget: string; plistPath: string; onMutation?: (mode: "enable" | "bootstrap") => void; -}): Promise { +}): Promise { const probe = await execLaunchctl(["print", params.serviceTarget]); if (probe.code === 0) { - return; + return { loaded: true }; } try { await bootstrapLaunchAgentOrThrow({ @@ -1425,11 +1466,29 @@ async function ensureLaunchAgentLoadedAfterFailure(params: { actionHint: "openclaw gateway start", onMutation: params.onMutation, }); - } catch { - // Best-effort only. Preserve the original kickstart failure below. + return { loaded: true }; + } catch (error) { + // A failed restore is not recoverable by launchd: the label is gone, so + // KeepAlive has nothing to respawn. Report it instead of dropping it. + return { loaded: false, detail: error instanceof Error ? error.message : String(error) }; } } +function formatLaunchAgentLeftUnloadedError(params: { + domain: string; + serviceTarget: string; + plistPath: string; + failure: string; + restoreDetail: string; +}): string { + return [ + params.failure, + `LaunchAgent ${params.serviceTarget} is not loaded and could not be restored: ${params.restoreDetail}`, + "The gateway is down and launchd has no job left to respawn it.", + `Fix: run \`openclaw gateway start\`, or \`launchctl bootstrap ${params.domain} ${params.plistPath}\`.`, + ].join("\n"); +} + export async function startLaunchAgent({ stdout, env, @@ -1560,13 +1619,39 @@ export async function restartLaunchAgent({ if (bootout.code === 0) { reportMutation("bootout"); } - await bootstrapLaunchAgentOrThrow({ - domain, - serviceTarget, - plistPath, - actionHint: "openclaw gateway restart", - onMutation: reportMutation, - }); + try { + await bootstrapLaunchAgentOrThrow({ + domain, + serviceTarget, + plistPath, + actionHint: "openclaw gateway restart", + onMutation: reportMutation, + retryPendingTeardown: true, + }); + } catch (error) { + // bootout already removed the job from the domain, so a failed bootstrap + // leaves the gateway down with no KeepAlive respawn to recover it. Restore + // the job before surfacing the original failure, as the kickstart path does. + const restored = await ensureLaunchAgentLoadedAfterFailure({ + domain, + serviceTarget, + plistPath, + onMutation: reportMutation, + }); + if (restored.loaded) { + throw error; + } + throw new Error( + formatLaunchAgentLeftUnloadedError({ + domain, + serviceTarget, + plistPath, + failure: error instanceof Error ? error.message : String(error), + restoreDetail: restored.detail, + }), + { cause: error }, + ); + } writeLaunchAgentActionLine(stdout, "Restarted LaunchAgent", serviceTarget); return { outcome: "completed" }; } @@ -1579,13 +1664,25 @@ export async function restartLaunchAgent({ } if (!isLaunchctlNotLoaded(start)) { - await ensureLaunchAgentLoadedAfterFailure({ + const restored = await ensureLaunchAgentLoadedAfterFailure({ domain, serviceTarget, plistPath, onMutation: reportMutation, }); - throw new Error(`launchctl kickstart failed: ${start.stderr || start.stdout}`.trim()); + const failure = `launchctl kickstart failed: ${start.stderr || start.stdout}`.trim(); + if (restored.loaded) { + throw new Error(failure); + } + throw new Error( + formatLaunchAgentLeftUnloadedError({ + domain, + serviceTarget, + plistPath, + failure, + restoreDetail: restored.detail, + }), + ); } // If the service was previously booted out, re-register the rewritten plist and retry.