fix(gateway): prevent crash loops from state DB schema migration errors (#116733)

src/cli/gateway-cli/run.option-collisions.test.ts

@@ -6,6 +6,7 @@ import { CONFIG_AUDIT_STORE_LABEL } from "../../config/io.audit.js";
 import type { ConfigFileSnapshot } from "../../config/types.js";
 import { GATEWAY_SERVICE_RUNTIME_PID_ENV } from "../../daemon/constants.js";
 import { SUPERVISOR_HINT_ENV_VARS } from "../../infra/supervisor-markers.js";
+import { OpenClawStateDatabaseSchemaMigrationRequiredError } from "../../state/openclaw-state-db-schema-migration-required.js";
 import {
   captureEnv,
   deleteTestEnvValue,
@@ -36,9 +37,14 @@ const findVerifiedGatewayListenerPidsOnPortSync = vi.fn((_port: number) => [] as
 const formatGatewayPidList = vi.fn((pids: number[]) => pids.join(", "));
 const isTerminalInteractive = vi.fn(() => true);
 const offerInvalidConfigRecovery = vi.fn(async () => ({ status: "declined" as const }));
+const parkCurrentLaunchAgentForMaintenance = vi.fn(async () => false);
 const ensureDevGatewayConfig = vi.fn(async (_opts?: unknown) => {});
 type GatewayLoopStart = (params?: { startupStartedAt?: number }) => Promise<unknown>;
-const runGatewayLoop = vi.fn(async ({ start }: { start: GatewayLoopStart }) => {
+type GatewayLoopParams = {
+  start: GatewayLoopStart;
+  completeBoot?: (completion: unknown) => void;
+};
+const runGatewayLoop = vi.fn(async ({ start }: GatewayLoopParams) => {
   await start();
 });
 const normalizeStateDirEnv = vi.fn((_env?: NodeJS.ProcessEnv) => undefined);
@@ -114,7 +120,9 @@ const bootLifecycle = vi.hoisted(() => ({
         recovered: false,
       },
   ),
-  record: vi.fn((_env?: NodeJS.ProcessEnv, _nowMs?: number, _reason?: string) => "boot-id"),
+  record: vi.fn(
+    (_env?: NodeJS.ProcessEnv, _nowMs?: number, _reason?: string): string | undefined => "boot-id",
+  ),
   complete: vi.fn(),
 }));
 const netState = vi.hoisted(() => ({
@@ -261,6 +269,11 @@ vi.mock("../../gateway/server.js", () => ({
   startGatewayServer: (port: number, opts?: unknown) => startGatewayServer(port, opts),
 }));
 
+vi.mock("../../daemon/launchd.js", async (importOriginal) => ({
+  ...(await importOriginal<typeof import("../../daemon/launchd.js")>()),
+  parkCurrentLaunchAgentForMaintenance: () => parkCurrentLaunchAgentForMaintenance(),
+}));
+
 vi.mock("../../gateway/ws-logging.js", () => ({
   setGatewayWsLogStyle: (style: string) => setGatewayWsLogStyle(style),
 }));
@@ -408,6 +421,8 @@ describe("gateway run option collisions", () => {
     isTerminalInteractive.mockReset();
     isTerminalInteractive.mockReturnValue(true);
     offerInvalidConfigRecovery.mockClear();
+    parkCurrentLaunchAgentForMaintenance.mockReset();
+    parkCurrentLaunchAgentForMaintenance.mockResolvedValue(false);
     cleanStaleGatewayProcessesSync.mockClear();
     waitForPortBindable.mockClear();
     ensureDevGatewayConfig.mockClear();
@@ -1601,6 +1616,52 @@ describe("gateway run option collisions", () => {
     expect(writeDiagnosticStabilityBundleForFailureSync).not.toHaveBeenCalled();
   });
 
+  it("exits 78 and parks launchd for a repairable shared-state schema", async () => {
+    bootLifecycle.record.mockReturnValueOnce(undefined);
+    runGatewayLoop.mockImplementationOnce(async ({ start, completeBoot }: GatewayLoopParams) => {
+      try {
+        await start();
+      } catch (error) {
+        completeBoot?.({ outcome: "startup_failed", reason: "schema migration required" });
+        throw error;
+      }
+    });
+    startGatewayServer.mockRejectedValueOnce(
+      new OpenClawStateDatabaseSchemaMigrationRequiredError(
+        "agent-databases-composite-primary-key",
+        "/tmp/openclaw.sqlite",
+      ),
+    );
+    parkCurrentLaunchAgentForMaintenance.mockResolvedValueOnce(true);
+
+    await expect(runGatewayCli(["gateway", "run", "--allow-unconfigured"])).rejects.toThrow(
+      "__exit__:78",
+    );
+
+    expect(parkCurrentLaunchAgentForMaintenance).toHaveBeenCalledOnce();
+    expect(bootLifecycle.complete).toHaveBeenCalledWith(undefined, {
+      outcome: "startup_failed",
+      reason: "schema migration required",
+    });
+    expect(runtimeErrors.join("\n")).toContain(
+      "state database schema migration required (agent-databases-composite-primary-key)",
+    );
+  });
+
+  it("does not park launchd for a nonrepairable shared-state schema", async () => {
+    startGatewayServer.mockRejectedValueOnce(
+      new Error(
+        "OpenClaw state database /tmp/openclaw.sqlite has a noncanonical agent database registry schema that cannot be repaired automatically.",
+      ),
+    );
+
+    await expect(runGatewayCli(["gateway", "run", "--allow-unconfigured"])).rejects.toThrow(
+      "__exit__:1",
+    );
+
+    expect(parkCurrentLaunchAgentForMaintenance).not.toHaveBeenCalled();
+  });
+
   it.each([
     "gateway already running (pid 4242); lock timeout after 5000ms",
     "another gateway instance is already listening on ws://127.0.0.1",

src/cli/gateway-cli/run.ts

@@ -59,6 +59,7 @@ import { withDiagnosticPhase } from "../../logging/diagnostic-phase.js";
 import { createSubsystemLogger } from "../../logging/subsystem.js";
 import { defaultRuntime } from "../../runtime.js";
 import { findOpenClawAgentDatabaseMediaMigrationRequiredError } from "../../state/openclaw-agent-db-migration-required.js";
+import { findOpenClawStateDatabaseSchemaMigrationRequiredError } from "../../state/openclaw-state-db-schema-migration-required.js";
 import { printClawBanner, type ClawBannerResult } from "../claw-banner.js";
 import { formatCliCommand } from "../command-format.js";
 import { formatInvalidConfigPort, formatInvalidPortOption } from "../error-format.js";
@@ -475,7 +476,9 @@ function resolveGatewayLockErrorExitCode(err: unknown): number {
 }
 
 function resolveGatewayStartupFailureExitCode(err: unknown): number {
-  return isInvalidConfigError(err) || findOpenClawAgentDatabaseMediaMigrationRequiredError(err)
+  return isInvalidConfigError(err) ||
+    findOpenClawAgentDatabaseMediaMigrationRequiredError(err) ||
+    findOpenClawStateDatabaseSchemaMigrationRequiredError(err)
     ? EXIT_CONFIG_ERROR
     : 1;
 }
@@ -1144,6 +1147,8 @@ async function runGatewayCommandOnce(opts: GatewayRunOpts, hooks: GatewayRunRunt
       : crashLoopDecision.recovered
         ? GATEWAY_CRASH_LOOP_RECOVERED_REASON
         : undefined;
+    // Shared-state schema failures make this write fail open, so no lifecycle
+    // row exists for Doctor to reconcile after it repairs the schema.
     activeBootId = recordGatewayBootStart(process.env, startedAtMs, bootStartReason);
     channelAutostartSuppression = undefined;
     if (crashLoopDecision.recovered) {
@@ -1248,6 +1253,20 @@ async function runGatewayCommandOnce(opts: GatewayRunOpts, hooks: GatewayRunRunt
         );
       }
     }
+    if (findOpenClawStateDatabaseSchemaMigrationRequiredError(err)) {
+      try {
+        const { parkCurrentLaunchAgentForMaintenance } = await import("../../daemon/launchd.js");
+        if (await parkCurrentLaunchAgentForMaintenance()) {
+          gatewayLog.error(
+            `gateway requires state database schema migration; parked the managed LaunchAgent. Run ${formatCliCommand("openclaw doctor --fix")} to repair and restart it.`,
+          );
+        }
+      } catch (parkError) {
+        gatewayLog.error(
+          `failed to park the managed LaunchAgent after state schema migration-required startup: ${formatErrorMessage(parkError)}`,
+        );
+      }
+    }
     await maybeWriteGatewayStartupFailureBundle(err);
     defaultRuntime.error(
       `Gateway failed to start: ${formatErrorMessage(err)}. Run ${formatCliCommand("openclaw gateway status --deep")} for diagnostics.`,

src/state/openclaw-state-db-schema-migration-required.test.ts

@@ -0,0 +1,52 @@
+import { describe, expect, it } from "vitest";
+import {
+  findOpenClawStateDatabaseSchemaMigrationRequiredError,
+  OpenClawStateDatabaseSchemaMigrationRequiredError,
+} from "./openclaw-state-db-schema-migration-required.js";
+
+describe("state database schema migration error classification", () => {
+  it("recognizes a rehydrated exact migration error through its cause chain", () => {
+    const original = new OpenClawStateDatabaseSchemaMigrationRequiredError(
+      "agent-databases-composite-primary-key",
+      "/tmp/openclaw.sqlite",
+    );
+    const rehydrated = new Error("startup failed", {
+      cause: new Error(original.message),
+    });
+
+    expect(findOpenClawStateDatabaseSchemaMigrationRequiredError(rehydrated)).toMatchObject({
+      kind: "agent-databases-composite-primary-key",
+      pathname: "/tmp/openclaw.sqlite",
+    });
+  });
+
+  it("recognizes an exact instance of the typed error", () => {
+    const original = new OpenClawStateDatabaseSchemaMigrationRequiredError(
+      "audit-events-v2",
+      "/tmp/openclaw.sqlite",
+    );
+
+    expect(findOpenClawStateDatabaseSchemaMigrationRequiredError(original)).toBe(original);
+  });
+
+  it("does not classify similar operator guidance as the migration error", () => {
+    expect(
+      findOpenClawStateDatabaseSchemaMigrationRequiredError(
+        new Error(
+          "OpenClaw state database /tmp/openclaw.sqlite is stale; run openclaw doctor --fix.",
+        ),
+      ),
+    ).toBeUndefined();
+  });
+
+  it("does not classify the agent DB media migration error", () => {
+    // Ensure the state-DB classifier does not accidentally match agent-DB messages.
+    expect(
+      findOpenClawStateDatabaseSchemaMigrationRequiredError(
+        new Error(
+          "OpenClaw agent database /tmp/openclaw-agent.sqlite uses schema version 5; run openclaw doctor --fix to migrate persisted media before using it.",
+        ),
+      ),
+    ).toBeUndefined();
+  });
+});

src/state/openclaw-state-db-schema-migration-required.ts

@@ -0,0 +1,57 @@
+const GATEWAY_STATE_SCHEMA_MIGRATION_REQUIRED_REASON = "gateway.state_schema_migration_required";
+
+type OpenClawStateDatabaseSchemaMigrationRequiredKind =
+  | "agent-databases-composite-primary-key"
+  | "audit-events-v2";
+
+export class OpenClawStateDatabaseSchemaMigrationRequiredError extends Error {
+  readonly code = GATEWAY_STATE_SCHEMA_MIGRATION_REQUIRED_REASON;
+
+  constructor(
+    readonly kind: OpenClawStateDatabaseSchemaMigrationRequiredKind,
+    readonly pathname: string,
+  ) {
+    super(
+      `OpenClaw state database schema migration required (${kind}) at ${pathname}; run openclaw doctor --fix to migrate it.`,
+    );
+    this.name = "OpenClawStateDatabaseSchemaMigrationRequiredError";
+  }
+}
+
+const STATE_SCHEMA_MIGRATION_REQUIRED_MESSAGE =
+  /^OpenClaw state database schema migration required \((agent-databases-composite-primary-key|audit-events-v2)\) at (.+); run openclaw doctor --fix to migrate it\.$/u;
+
+function parseStateSchemaMigrationRequiredMessage(
+  message: unknown,
+): OpenClawStateDatabaseSchemaMigrationRequiredError | undefined {
+  if (typeof message !== "string") {
+    return undefined;
+  }
+  const match = STATE_SCHEMA_MIGRATION_REQUIRED_MESSAGE.exec(message);
+  const kind = match?.[1] as OpenClawStateDatabaseSchemaMigrationRequiredKind | undefined;
+  const pathname = match?.[2];
+  if (!kind || !pathname) {
+    return undefined;
+  }
+  return new OpenClawStateDatabaseSchemaMigrationRequiredError(kind, pathname);
+}
+
+export function findOpenClawStateDatabaseSchemaMigrationRequiredError(
+  error: unknown,
+): OpenClawStateDatabaseSchemaMigrationRequiredError | undefined {
+  let current = error;
+  const seen = new Set<unknown>();
+  while (current && typeof current === "object" && !seen.has(current)) {
+    if (current instanceof OpenClawStateDatabaseSchemaMigrationRequiredError) {
+      return current;
+    }
+    const errorLike = current as { cause?: unknown; message?: unknown };
+    const parsed = parseStateSchemaMigrationRequiredMessage(errorLike.message);
+    if (parsed) {
+      return parsed;
+    }
+    seen.add(current);
+    current = errorLike.cause;
+  }
+  return undefined;
+}

src/state/openclaw-state-db-schema-repair.ts

@@ -19,6 +19,7 @@ import {
   tableHasColumn,
   tablePrimaryKeyColumns,
 } from "./openclaw-state-db-schema-helpers.js";
+import { OpenClawStateDatabaseSchemaMigrationRequiredError } from "./openclaw-state-db-schema-migration-required.js";
 import * as sessionWatchMigration from "./openclaw-state-db-session-watch-migration.js";
 
 export function dropLegacyStateTables(db: DatabaseSync): void {
@@ -147,15 +148,19 @@ export function markCurrentStateSchemaVersion(
 export function assertCanonicalStateSchemaShape(db: DatabaseSync, pathname: string): void {
   operatorApprovalMigration.assertCanonicalOperatorApprovalKinds(db, pathname);
   if (!hasCanonicalAgentDatabasesPrimaryKey(db)) {
+    if (canRepairAgentDatabasesPrimaryKey(db)) {
+      throw new OpenClawStateDatabaseSchemaMigrationRequiredError(
+        "agent-databases-composite-primary-key",
+        pathname,
+      );
+    }
     throw new Error(
-      `OpenClaw state database ${pathname} has a legacy agent database registry schema; run openclaw doctor --fix to migrate it.`,
+      `OpenClaw state database ${pathname} has a noncanonical agent database registry schema that cannot be repaired automatically; restore the canonical agent_databases shape before retrying.`,
     );
   }
   if (!hasCanonicalAuditEventsSchema(db)) {
     if (canRepairLegacyAuditEventsSchema(db)) {
-      throw new Error(
-        `OpenClaw state database ${pathname} has a legacy audit event schema; run openclaw doctor --fix to migrate it.`,
-      );
+      throw new OpenClawStateDatabaseSchemaMigrationRequiredError("audit-events-v2", pathname);
     }
     throw new Error(
       `OpenClaw state database ${pathname} has a noncanonical audit event schema that cannot be repaired automatically; restore the canonical audit_events shape before retrying.`,

src/state/openclaw-state-db.test.ts

@@ -19,6 +19,10 @@ import { readSqliteNumberPragma } from "../infra/sqlite-pragma.test-support.js";
 import { loadTaskRegistryStateFromSqlite } from "../tasks/task-registry.store.sqlite.js";
 import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js";
 import { VERSION } from "../version.js";
+import {
+  findOpenClawStateDatabaseSchemaMigrationRequiredError,
+  OpenClawStateDatabaseSchemaMigrationRequiredError,
+} from "./openclaw-state-db-schema-migration-required.js";
 import type { DB as OpenClawStateKyselyDatabase } from "./openclaw-state-db.generated.js";
 import {
   assertOpenClawStateDatabaseForMaintenance,
@@ -53,6 +57,23 @@ function createTempStateDir(): string {
   return makeTempDir(stateDbTempDirs, "openclaw-state-db-");
 }
 
+function expectStateSchemaMigrationRequired(
+  run: () => unknown,
+  expected: {
+    kind: OpenClawStateDatabaseSchemaMigrationRequiredError["kind"];
+    pathname: string;
+  },
+): void {
+  let caught: unknown;
+  try {
+    run();
+  } catch (error) {
+    caught = error;
+  }
+  expect(caught).toBeInstanceOf(OpenClawStateDatabaseSchemaMigrationRequiredError);
+  expect(findOpenClawStateDatabaseSchemaMigrationRequiredError(caught)).toMatchObject(expected);
+}
+
 function replaceManagedImageRecordsWithLegacyTable(
   database: DatabaseSync,
   options: { withRow: boolean },
@@ -1957,6 +1978,79 @@ INSERT INTO macos_port_guardian_records VALUES (4242, 18789, '/usr/bin/ssh', 're
     );
   });
 
+  it("classifies the released agent registry primary key as Doctor-repairable", () => {
+    const stateDir = createTempStateDir();
+    const options = { env: { OPENCLAW_STATE_DIR: stateDir } };
+    const databasePath = openOpenClawStateDatabase(options).path;
+    closeOpenClawStateDatabaseForTest();
+
+    const { DatabaseSync } = requireNodeSqlite();
+    const legacy = new DatabaseSync(databasePath);
+    legacy.exec(`
+      ALTER TABLE agent_databases RENAME TO agent_databases_current;
+      CREATE TABLE agent_databases (
+        agent_id TEXT NOT NULL PRIMARY KEY,
+        path TEXT NOT NULL,
+        schema_version INTEGER NOT NULL,
+        last_seen_at INTEGER NOT NULL,
+        size_bytes INTEGER
+      );
+      INSERT INTO agent_databases (
+        agent_id,
+        path,
+        schema_version,
+        last_seen_at,
+        size_bytes
+      )
+      SELECT
+        agent_id,
+        path,
+        schema_version,
+        last_seen_at,
+        size_bytes
+      FROM agent_databases_current;
+      DROP TABLE agent_databases_current;
+    `);
+    legacy.close();
+
+    expectStateSchemaMigrationRequired(() => openOpenClawStateDatabase(options), {
+      kind: "agent-databases-composite-primary-key",
+      pathname: databasePath,
+    });
+  });
+
+  it("keeps an unrecognized agent registry schema fail-closed and nonrepairable", () => {
+    const stateDir = createTempStateDir();
+    const options = { env: { OPENCLAW_STATE_DIR: stateDir } };
+    const databasePath = openOpenClawStateDatabase(options).path;
+    closeOpenClawStateDatabaseForTest();
+
+    const { DatabaseSync } = requireNodeSqlite();
+    const malformed = new DatabaseSync(databasePath);
+    malformed.exec(`
+      DROP TABLE agent_databases;
+      CREATE TABLE agent_databases (
+        agent_id TEXT NOT NULL PRIMARY KEY,
+        path TEXT NOT NULL,
+        schema_version INTEGER NOT NULL,
+        last_seen_at INTEGER NOT NULL
+      );
+    `);
+    malformed.close();
+
+    let caught: unknown;
+    try {
+      openOpenClawStateDatabase(options);
+    } catch (error) {
+      caught = error;
+    }
+    expect(findOpenClawStateDatabaseSchemaMigrationRequiredError(caught)).toBeUndefined();
+    expect(caught).toBeInstanceOf(Error);
+    expect((caught as Error).message).toContain(
+      "noncanonical agent database registry schema that cannot be repaired automatically",
+    );
+  });
+
   it("migrates the released audit ledger to message-compatible attribution exactly once", () => {
     const stateDir = createTempStateDir();
     const databasePath = createLegacyAuditStateDatabase(stateDir);
@@ -1966,7 +2060,10 @@ INSERT INTO macos_port_guardian_records VALUES (4242, 18789, '/usr/bin/ssh', 're
       { kind: "audit-events-v2", path: databasePath },
       { kind: "strict-tables-v3", path: databasePath },
     ]);
-    expect(() => openOpenClawStateDatabase(options)).toThrow(/legacy audit event schema/);
+    expectStateSchemaMigrationRequired(() => openOpenClawStateDatabase(options), {
+      kind: "audit-events-v2",
+      pathname: databasePath,
+    });
 
     expect(repairOpenClawStateDatabaseSchema(options)).toEqual({
       changes: [