fix: gateway restart fails and leaves the macOS LaunchAgent unloaded when the gateway drains active work (#116875)

* fix(daemon): retry launchd bootstrap while a booted-out job tears down

launchctl bootout returns once launchd accepts the request, not once the
job has exited, so the reload path could bootstrap into a label the
outgoing gateway still held and fail with EIO. bootout had already
deregistered the job, leaving KeepAlive with nothing to respawn and the
gateway silently down until a manual bootstrap.

Retry bootstrap while the teardown is still pending, bounded by the
plist ExitTimeOut, and restore the job if bootstrap still fails.

* fix(daemon): report an unrestored LaunchAgent and scope the bootstrap retry

The reload path swallowed a failed restore, so a bootstrap that failed after
bootout surfaced only the original error while the LaunchAgent stayed booted
out. Combine the restore failure into the thrown error and say plainly that
the job is gone and KeepAlive has nothing left to respawn.

The teardown retry also applied to every bootstrapLaunchAgentOrThrow caller.
launchd answers the same EIO for a label that is merely still registered, so
start, install, and recovery paths burned the full teardown deadline on an
already-loaded service before failing anyway. Gate the retry on callers that
just issued bootout, and defer to the existing isLaunchctlAlreadyLoaded model
instead of treating that response as a teardown to wait out.

src/daemon/launchd.test.ts

@@ -40,6 +40,7 @@ const state = vi.hoisted(() => ({
   bootstrapError: "",
   bootstrapCode: 1,
   bootstrapLoadsServiceOnFailure: false,
+  bootstrapTransient: false,
   kickstartError: "",
   kickstartCode: 1,
   kickstartFailuresRemaining: 0,
@@ -223,6 +224,23 @@ async function runStopLaunchAgentWithFakeTimers(args: Parameters<typeof stopLaun
   }
 }
 
+async function runRestartLaunchAgentWithFakeTimers(args: Parameters<typeof restartLaunchAgent>[0]) {
+  vi.useFakeTimers();
+  try {
+    const restartPromise = restartLaunchAgent(args)
+      .then((value) => ({ ok: true as const, value }))
+      .catch((error: unknown) => ({ ok: false as const, error }));
+    await vi.runAllTimersAsync();
+    const result = await restartPromise;
+    if (!result.ok) {
+      throw result.error;
+    }
+    return result.value;
+  } finally {
+    vi.useRealTimers();
+  }
+}
+
 function expectLaunchctlEnableBootstrapOrder(env: Record<string, string | undefined>) {
   const domain = typeof process.getuid === "function" ? `gui/${process.getuid()}` : "gui/501";
   const label = "ai.openclaw.gateway";
@@ -343,11 +361,18 @@ function executeLaunchctlMock(file: string, args: string[]) {
   }
   if (call[0] === "bootstrap") {
     if (state.bootstrapError) {
+      const detail = state.bootstrapError;
+      // Transient failures clear after one attempt so recovery paths that retry
+      // a bootstrap can be exercised the way launchd behaves once a booted-out
+      // job finishes tearing down.
+      if (state.bootstrapTransient) {
+        state.bootstrapError = "";
+      }
       if (state.bootstrapLoadsServiceOnFailure) {
         state.serviceLoaded = true;
         state.serviceRunning = true;
       }
-      return { stdout: "", stderr: state.bootstrapError, code: state.bootstrapCode };
+      return { stdout: "", stderr: detail, code: state.bootstrapCode };
     }
     state.serviceLoaded = true;
     state.serviceRunning = true;
@@ -523,6 +548,7 @@ beforeEach(() => {
   state.bootstrapError = "";
   state.bootstrapCode = 1;
   state.bootstrapLoadsServiceOnFailure = false;
+  state.bootstrapTransient = false;
   state.kickstartError = "";
   state.kickstartCode = 1;
   state.kickstartFailuresRemaining = 0;
@@ -2552,6 +2578,28 @@ describe("launchd install", () => {
     expect(onMutation.mock.calls).toEqual([[{ mode: "enable" }], [{ mode: "bootstrap" }]]);
   });
 
+  it("fails an already-loaded bootstrap immediately instead of waiting out the teardown deadline", async () => {
+    const env = createDefaultLaunchdEnv();
+    const onMutation = vi.fn();
+    state.kickstartError = "Could not find service";
+    state.kickstartFailuresRemaining = 1;
+    // launchd answers EIO for a label that is still registered. `startLaunchAgent`
+    // never booted the job out, so there is no teardown to wait for: retrying
+    // until the teardown deadline would stall the start for no gain. Real timers
+    // here so a reintroduced retry loop blows the test timeout rather than
+    // passing under vi.runAllTimersAsync().
+    state.bootstrapError =
+      "Could not bootstrap service: 5: Input/output error: already exists in domain for gui/501";
+    state.bootstrapCode = 5;
+
+    await expect(startLaunchAgent({ env, stdout: new PassThrough(), onMutation })).rejects.toThrow(
+      "launchctl bootstrap failed: Could not bootstrap service: 5: Input/output error",
+    );
+
+    expect(countMatching(state.launchctlCalls, (call) => call[0] === "bootstrap")).toBe(1);
+    expect(onMutation).not.toHaveBeenCalledWith({ mode: "bootstrap" });
+  });
+
   it("audits enable but not kickstart when the later launch fails", async () => {
     const env = createDefaultLaunchdEnv();
     const onMutation = vi.fn();
@@ -2647,14 +2695,113 @@ describe("launchd install", () => {
       restartLaunchAgent({ env, stdout: new PassThrough(), onMutation }),
     ).rejects.toThrow("launchctl bootstrap failed: Operation not permitted");
 
+    // The trailing enable comes from the post-failure recovery attempt, which
+    // cannot report a bootstrap mutation because this bootstrap keeps failing.
     expect(onMutation.mock.calls).toEqual([
       [{ mode: "enable" }],
       [{ mode: "bootout" }],
       [{ mode: "enable" }],
+      [{ mode: "enable" }],
     ]);
     expect(onMutation).not.toHaveBeenCalledWith({ mode: "bootstrap" });
   });
 
+  it("reloads the LaunchAgent through a transient reload bootstrap failure", async () => {
+    const env = {
+      ...createDefaultLaunchdEnv(),
+      OPENCLAW_GATEWAY_PORT: "18789",
+    };
+    setLaunchAgentPlist({
+      env,
+      label: "ai.openclaw.gateway",
+      programArguments: ["node", "gateway.js"],
+    });
+    // launchd answers EIO while the just-booted-out job is still tearing down.
+    state.bootstrapError = "Bootstrap failed: 5: Input/output error";
+    state.bootstrapCode = 5;
+    state.bootstrapTransient = true;
+    const onMutation = vi.fn();
+
+    const result = await runRestartLaunchAgentWithFakeTimers({
+      env,
+      stdout: new PassThrough(),
+      onMutation,
+    });
+
+    // Teardown is transient, so the retry must land a real bootstrap rather than
+    // surfacing an error the operator has to recover from by hand.
+    expect(result).toEqual({ outcome: "completed" });
+    expect(state.serviceLoaded).toBe(true);
+    expect(onMutation).toHaveBeenCalledWith({ mode: "bootstrap" });
+  });
+
+  it("reports the LaunchAgent as unloaded when bootstrap teardown never clears", async () => {
+    const env = {
+      ...createDefaultLaunchdEnv(),
+      OPENCLAW_GATEWAY_PORT: "18789",
+    };
+    setLaunchAgentPlist({
+      env,
+      label: "ai.openclaw.gateway",
+      programArguments: ["node", "gateway.js"],
+    });
+    // EIO that never clears must stay bounded instead of retrying forever, and
+    // the restore attempt fails with it, so the label really does stay absent.
+    state.bootstrapError = "Bootstrap failed: 5: Input/output error";
+    state.bootstrapCode = 5;
+
+    const error = await runRestartLaunchAgentWithFakeTimers({
+      env,
+      stdout: new PassThrough(),
+    }).catch((caught: unknown) => caught);
+
+    // bootout already removed the job, so the operator has to learn both why the
+    // bootstrap failed and that nothing is left for KeepAlive to respawn.
+    const domain = typeof process.getuid === "function" ? `gui/${process.getuid()}` : "gui/501";
+    expect(state.serviceLoaded).toBe(false);
+    expect(error).toBeInstanceOf(Error);
+    const message = (error as Error).message;
+    expect(message).toContain(
+      "launchctl bootstrap failed: Bootstrap failed: 5: Input/output error",
+    );
+    expect(message).toContain(`LaunchAgent ${domain}/ai.openclaw.gateway is not loaded`);
+    expect(message).toContain("The gateway is down and launchd has no job left to respawn it.");
+    expect(message).toContain("openclaw gateway start");
+  });
+
+  it("does not wait out the teardown deadline when the reload bootstrap reports already-loaded", async () => {
+    const env = {
+      ...createDefaultLaunchdEnv(),
+      OPENCLAW_GATEWAY_PORT: "18789",
+    };
+    setLaunchAgentPlist({
+      env,
+      label: "ai.openclaw.gateway",
+      programArguments: ["node", "gateway.js"],
+    });
+    // Same EIO code as a pending teardown, but the label is still registered
+    // rather than draining, so there is nothing to wait for. Real timers here:
+    // a retry loop would blow the test timeout instead of quietly passing.
+    state.bootstrapError =
+      "Could not bootstrap service: 5: Input/output error: already exists in domain for gui/501";
+    state.bootstrapCode = 5;
+    state.bootstrapLoadsServiceOnFailure = true;
+
+    const error = await restartLaunchAgent({ env, stdout: new PassThrough() }).catch(
+      (caught: unknown) => caught,
+    );
+
+    expect(countMatching(state.launchctlCalls, (call) => call[0] === "bootstrap")).toBe(1);
+    expect(error).toBeInstanceOf(Error);
+    const message = (error as Error).message;
+    expect(message).toContain(
+      "launchctl bootstrap failed: Could not bootstrap service: 5: Input/output error",
+    );
+    // The label is still registered, so the recovery probe finds it and the
+    // failure must not claim the gateway was left unloaded.
+    expect(message).not.toContain("is not loaded");
+  });
+
   it("completes reload when the mutation observer fails after bootout", async () => {
     const env = {
       ...createDefaultLaunchdEnv(),

src/daemon/launchd.ts

@@ -80,6 +80,11 @@ const OPENCLAW_NODE_RUNTIME_NAMES = new Set(["bun", "bun.exe", "node", "node.exe
 const OPENCLAW_SCRIPT_NAMES = new Set(["openclaw.mjs"]);
 const LAUNCH_AGENT_STOP_PORT_RELEASE_TIMEOUT_MS = LAUNCH_AGENT_EXIT_TIMEOUT_SECONDS * 1_000;
 const LAUNCH_AGENT_STOP_PORT_RELEASE_POLL_MS = 100;
+// launchd reserves the label until the outgoing job actually exits, and it
+// SIGKILLs that job once ExitTimeOut elapses. Bound the bootstrap retry by that
+// same deadline plus slack so a drain-on-SIGTERM gateway cannot outlast it.
+const LAUNCH_AGENT_BOOTSTRAP_TEARDOWN_TIMEOUT_MS = (LAUNCH_AGENT_EXIT_TIMEOUT_SECONDS + 10) * 1_000;
+const LAUNCH_AGENT_BOOTSTRAP_TEARDOWN_POLL_MS = 500;
 const LAUNCHCTL_PROTECTED_PID_TIMEOUT_MS = 2_000;
 
 export type StaleOpenClawUpdateLaunchdJob = {
@@ -635,6 +640,10 @@ async function bootstrapLaunchAgentOrThrow(params: {
   actionHint: string;
   onMutation?: (mode: "enable" | "bootstrap") => void;
   skipEnable?: boolean;
+  // Opt-in for callers that just issued `bootout` on this label. Only those can
+  // race a pending teardown, so start/install/recovery paths keep failing fast
+  // on an unrelated EIO instead of waiting out the teardown deadline.
+  retryPendingTeardown?: boolean;
 }) {
   // `disable` state survives bootout and plist rewrites; explicit start/repair
   // paths must clear it before asking launchd to load the job again.
@@ -644,27 +653,38 @@ async function bootstrapLaunchAgentOrThrow(params: {
       params.onMutation?.("enable");
     }
   }
-  const boot = await execLaunchctl(["bootstrap", params.domain, params.plistPath]);
-  if (boot.code === 0) {
-    params.onMutation?.("bootstrap");
-    return;
-  }
-  const detail = (boot.stderr || boot.stdout).trim();
-  if (isUnsupportedGuiDomain(detail)) {
-    throwBootstrapGuiSessionError({
-      detail,
-      domain: params.domain,
-      actionHint: params.actionHint,
-    });
-  }
-  if (isLaunchctlOperationAlreadyInProgress(detail)) {
-    const state = await probeLaunchAgentState(params.serviceTarget);
-    if (state.state === "running" || state.state === "stopped") {
+  const teardownDeadline = Date.now() + LAUNCH_AGENT_BOOTSTRAP_TEARDOWN_TIMEOUT_MS;
+  for (;;) {
+    const boot = await execLaunchctl(["bootstrap", params.domain, params.plistPath]);
+    if (boot.code === 0) {
       params.onMutation?.("bootstrap");
       return;
     }
+    const detail = (boot.stderr || boot.stdout).trim();
+    if (isUnsupportedGuiDomain(detail)) {
+      throwBootstrapGuiSessionError({
+        detail,
+        domain: params.domain,
+        actionHint: params.actionHint,
+      });
+    }
+    if (isLaunchctlOperationAlreadyInProgress(detail)) {
+      const state = await probeLaunchAgentState(params.serviceTarget);
+      if (state.state === "running" || state.state === "stopped") {
+        params.onMutation?.("bootstrap");
+        return;
+      }
+    }
+    const remainingMs = teardownDeadline - Date.now();
+    if (
+      !params.retryPendingTeardown ||
+      !isLaunchctlBootstrapPendingTeardown(boot) ||
+      remainingMs <= 0
+    ) {
+      throw new Error(`launchctl bootstrap failed: ${detail}`);
+    }
+    await sleep(Math.min(LAUNCH_AGENT_BOOTSTRAP_TEARDOWN_POLL_MS, remainingMs));
   }
-  throw new Error(`launchctl bootstrap failed: ${detail}`);
 }
 
 async function ensureLaunchAgentPlistReadable(plistPath: string): Promise<void> {
@@ -1012,6 +1032,25 @@ function isLaunchctlOperationAlreadyInProgress(detail: string): boolean {
   );
 }
 
+function isLaunchctlBootstrapPendingTeardown(res: {
+  stdout: string;
+  stderr: string;
+  code: number;
+}): boolean {
+  // `bootout` returns once launchd accepts the request, not once the job is gone,
+  // so bootstrapping the same label mid-teardown answers EIO. The plist is valid
+  // here, so this is a timing conflict to retry rather than a real I/O fault.
+  //
+  // launchd answers the same EIO for a label that is simply still registered
+  // ("already exists in domain"). That job is not tearing down, so waiting for a
+  // teardown that never comes only delays the failure.
+  if (isLaunchctlAlreadyLoaded(res)) {
+    return false;
+  }
+  const normalized = normalizeLowercaseStringOrEmpty(res.stderr || res.stdout);
+  return normalized.includes("bootstrap failed: 5") || normalized.includes("input/output error");
+}
+
 async function bootoutLaunchAgentOrThrow(params: {
   serviceTarget: string;
   warning: string;
@@ -1407,15 +1446,17 @@ async function rewriteLaunchAgentPlistForRestart({
   return true;
 }
 
+type LaunchAgentRestoreResult = { loaded: true } | { loaded: false; detail: string };
+
 async function ensureLaunchAgentLoadedAfterFailure(params: {
   domain: string;
   serviceTarget: string;
   plistPath: string;
   onMutation?: (mode: "enable" | "bootstrap") => void;
-}): Promise<void> {
+}): Promise<LaunchAgentRestoreResult> {
   const probe = await execLaunchctl(["print", params.serviceTarget]);
   if (probe.code === 0) {
-    return;
+    return { loaded: true };
   }
   try {
     await bootstrapLaunchAgentOrThrow({
@@ -1425,11 +1466,29 @@ async function ensureLaunchAgentLoadedAfterFailure(params: {
       actionHint: "openclaw gateway start",
       onMutation: params.onMutation,
     });
-  } catch {
-    // Best-effort only. Preserve the original kickstart failure below.
+    return { loaded: true };
+  } catch (error) {
+    // A failed restore is not recoverable by launchd: the label is gone, so
+    // KeepAlive has nothing to respawn. Report it instead of dropping it.
+    return { loaded: false, detail: error instanceof Error ? error.message : String(error) };
   }
 }
 
+function formatLaunchAgentLeftUnloadedError(params: {
+  domain: string;
+  serviceTarget: string;
+  plistPath: string;
+  failure: string;
+  restoreDetail: string;
+}): string {
+  return [
+    params.failure,
+    `LaunchAgent ${params.serviceTarget} is not loaded and could not be restored: ${params.restoreDetail}`,
+    "The gateway is down and launchd has no job left to respawn it.",
+    `Fix: run \`openclaw gateway start\`, or \`launchctl bootstrap ${params.domain} ${params.plistPath}\`.`,
+  ].join("\n");
+}
+
 export async function startLaunchAgent({
   stdout,
   env,
@@ -1560,13 +1619,39 @@ export async function restartLaunchAgent({
     if (bootout.code === 0) {
       reportMutation("bootout");
     }
-    await bootstrapLaunchAgentOrThrow({
-      domain,
-      serviceTarget,
-      plistPath,
-      actionHint: "openclaw gateway restart",
-      onMutation: reportMutation,
-    });
+    try {
+      await bootstrapLaunchAgentOrThrow({
+        domain,
+        serviceTarget,
+        plistPath,
+        actionHint: "openclaw gateway restart",
+        onMutation: reportMutation,
+        retryPendingTeardown: true,
+      });
+    } catch (error) {
+      // bootout already removed the job from the domain, so a failed bootstrap
+      // leaves the gateway down with no KeepAlive respawn to recover it. Restore
+      // the job before surfacing the original failure, as the kickstart path does.
+      const restored = await ensureLaunchAgentLoadedAfterFailure({
+        domain,
+        serviceTarget,
+        plistPath,
+        onMutation: reportMutation,
+      });
+      if (restored.loaded) {
+        throw error;
+      }
+      throw new Error(
+        formatLaunchAgentLeftUnloadedError({
+          domain,
+          serviceTarget,
+          plistPath,
+          failure: error instanceof Error ? error.message : String(error),
+          restoreDetail: restored.detail,
+        }),
+        { cause: error },
+      );
+    }
     writeLaunchAgentActionLine(stdout, "Restarted LaunchAgent", serviceTarget);
     return { outcome: "completed" };
   }
@@ -1579,13 +1664,25 @@ export async function restartLaunchAgent({
   }
 
   if (!isLaunchctlNotLoaded(start)) {
-    await ensureLaunchAgentLoadedAfterFailure({
+    const restored = await ensureLaunchAgentLoadedAfterFailure({
       domain,
       serviceTarget,
       plistPath,
       onMutation: reportMutation,
     });
-    throw new Error(`launchctl kickstart failed: ${start.stderr || start.stdout}`.trim());
+    const failure = `launchctl kickstart failed: ${start.stderr || start.stdout}`.trim();
+    if (restored.loaded) {
+      throw new Error(failure);
+    }
+    throw new Error(
+      formatLaunchAgentLeftUnloadedError({
+        domain,
+        serviceTarget,
+        plistPath,
+        failure,
+        restoreDetail: restored.detail,
+      }),
+    );
   }
 
   // If the service was previously booted out, re-register the rewritten plist and retry.