fix(gateway): recover channel autostart after crash loops (#118311)

* fix(gateway): recover channel autostart after crash loops

* docs(gateway): clarify crash-loop recovery steps

* test(gateway): type crash-loop recovery context

* test(gateway): correct recovery mock context

* test(gateway): keep recovery monitor coverage compact

* chore: leave crash-loop note to release flow

---------

Co-authored-by: Peter Steinberger <steipete@mac-studio-sf2.local>

src/cli/gateway-cli/run.option-collisions.test.ts

@@ -123,6 +123,10 @@ const bootLifecycle = vi.hoisted(() => ({
   record: vi.fn(
     (_env?: NodeJS.ProcessEnv, _nowMs?: number, _reason?: string): string | undefined => "boot-id",
   ),
+  recover: vi.fn(
+    (_bootId?: string, _env?: NodeJS.ProcessEnv, _nowMs?: number): string | undefined =>
+      "recovered-boot-id",
+  ),
   complete: vi.fn(),
 }));
 const netState = vi.hoisted(() => ({
@@ -317,6 +321,8 @@ vi.mock("../../infra/gateway-boot-lifecycle.js", () => ({
     bootLifecycle.inspect(env, nowMs),
   recordGatewayBootStart: (env?: NodeJS.ProcessEnv, nowMs?: number, reason?: string) =>
     bootLifecycle.record(env, nowMs, reason),
+  recordGatewayCrashLoopRecovery: (bootId?: string, env?: NodeJS.ProcessEnv, nowMs?: number) =>
+    bootLifecycle.recover(bootId, env, nowMs),
   completeGatewayBootLifecycle: (bootId: string | undefined, completion: unknown) =>
     bootLifecycle.complete(bootId, completion),
 }));
@@ -409,6 +415,7 @@ describe("gateway run option collisions", () => {
     bootLifecycle.decisions.length = 0;
     bootLifecycle.inspect.mockClear();
     bootLifecycle.record.mockClear();
+    bootLifecycle.recover.mockClear();
     bootLifecycle.complete.mockClear();
     startGatewayServer.mockClear();
     setGatewayWsLogStyle.mockClear();
@@ -467,6 +474,7 @@ describe("gateway run option collisions", () => {
       auth?: { mode?: string; token?: string; password?: string };
       bind?: string;
       channelAutostartSuppression?: { reason?: string; message?: string };
+      tryRecoverChannelAutostartSuppression?: () => boolean;
       ambientEnvTriggers?: "allow" | "suppress";
       startupConfigSnapshotRead?: { snapshot?: Record<string, unknown> };
       startupStartedAt?: number;
@@ -1594,6 +1602,55 @@ describe("gateway run option collisions", () => {
     expect(gatewayLogMessages.some((message) => message.includes("breaker recovered"))).toBe(true);
   });
 
+  it("recovers channel autostart only after the full breaker window drains", async () => {
+    runGatewayLoop.mockImplementationOnce(
+      async ({
+        beginBoot,
+        start,
+      }: {
+        beginBoot?: (startedAtMs: number) => Promise<void> | void;
+        start: GatewayLoopStart;
+      }) => {
+        await beginBoot?.(1000);
+        await start({ startupStartedAt: 1000 });
+      },
+    );
+    bootLifecycle.decisions.push({
+      tripped: true,
+      uncleanBoots: 3,
+      windowMs: 300_000,
+      shouldWriteStabilityBundle: false,
+      recovered: false,
+    });
+
+    await runGatewayCli(["gateway", "run", "--allow-unconfigured"]);
+
+    const recover = gatewayStartOptions().tryRecoverChannelAutostartSuppression;
+    expect(recover).toBeTypeOf("function");
+    bootLifecycle.decisions.push(
+      {
+        tripped: false,
+        uncleanBoots: 1,
+        windowMs: 300_000,
+        shouldWriteStabilityBundle: false,
+        recovered: true,
+      },
+      {
+        tripped: false,
+        uncleanBoots: 0,
+        windowMs: 300_000,
+        shouldWriteStabilityBundle: false,
+        recovered: true,
+      },
+    );
+
+    expect(recover?.()).toBe(false);
+    expect(bootLifecycle.recover).not.toHaveBeenCalled();
+    expect(recover?.()).toBe(true);
+    expect(bootLifecycle.recover).toHaveBeenCalledWith("boot-id", process.env, undefined);
+    expect(gatewayLogMessages.some((message) => message.includes("breaker recovered"))).toBe(true);
+  });
+
   it("skips failure bundles but exits nonzero for unconfirmed gateway lock conflicts", async () => {
     const port = await getFreePort();
     configState.snapshot = {

src/cli/gateway-cli/run.ts

@@ -43,6 +43,7 @@ import {
   GATEWAY_CRASH_LOOP_RECOVERED_REASON,
   inspectGatewayCrashLoopBreaker,
   recordGatewayBootStart,
+  recordGatewayCrashLoopRecovery,
   type GatewayCrashLoopBreakerDecision,
   type GatewayBootLifecycleCompletion,
 } from "../../infra/gateway-boot-lifecycle.js";
@@ -1136,6 +1137,22 @@ async function runGatewayCommandOnce(opts: GatewayRunOpts, hooks: GatewayRunRunt
   let crashLoopDecision: GatewayCrashLoopBreakerDecision | undefined;
   let channelAutostartSuppression: { reason: "crash-loop-breaker"; message: string } | undefined;
   let activeBootId: string | undefined;
+  const tryRecoverChannelAutostartSuppression = () => {
+    const decision = inspectGatewayCrashLoopBreaker(process.env);
+    // The current safe-mode boot remains an open row until the full window has
+    // drained. Requiring zero prevents a near-expiry history from restoring
+    // channels before this process itself has proven stable for the whole window.
+    if (!decision.recovered || decision.uncleanBoots !== 0) {
+      return false;
+    }
+    const recoveredBootId = recordGatewayCrashLoopRecovery(activeBootId, process.env);
+    if (!recoveredBootId) {
+      return false;
+    }
+    activeBootId = recoveredBootId;
+    gatewayLog.info("gateway restart-loop breaker recovered; channel auto-start restored");
+    return true;
+  };
   const beginBoot = async (startedAtMs: number) => {
     // run-loop calls beginBoot before every startGatewayServer invocation, so
     // in-process restarts re-evaluate breaker state instead of reusing stale mode.
@@ -1194,6 +1211,7 @@ async function runGatewayCommandOnce(opts: GatewayRunOpts, hooks: GatewayRunRunt
             : {}),
           ...(envSidecarStartupMode !== "start" ? { sidecarStartup: envSidecarStartupMode } : {}),
           ...(channelAutostartSuppression ? { channelAutostartSuppression } : {}),
+          ...(channelAutostartSuppression ? { tryRecoverChannelAutostartSuppression } : {}),
           ...(devMode
             ? {
                 ambientEnvTriggers: devAmbientEnvTriggers,