From daff73b7ad9cd9bb655b26cb836f7a6e2bd20fad Mon Sep 17 00:00:00 2001 From: Peter Lee Date: Fri, 31 Jul 2026 14:34:49 -0500 Subject: [PATCH] fix(gateway): prevent crash loops from state DB schema migration errors (#116733) --- .../gateway-cli/run.option-collisions.test.ts | 65 +++++++++++- src/cli/gateway-cli/run.ts | 21 +++- ...state-db-schema-migration-required.test.ts | 52 ++++++++++ ...claw-state-db-schema-migration-required.ts | 57 +++++++++++ src/state/openclaw-state-db-schema-repair.ts | 13 ++- src/state/openclaw-state-db.test.ts | 99 ++++++++++++++++++- 6 files changed, 299 insertions(+), 8 deletions(-) create mode 100644 src/state/openclaw-state-db-schema-migration-required.test.ts create mode 100644 src/state/openclaw-state-db-schema-migration-required.ts diff --git a/src/cli/gateway-cli/run.option-collisions.test.ts b/src/cli/gateway-cli/run.option-collisions.test.ts index 230fea49e6be..fad5c30b1e20 100644 --- a/src/cli/gateway-cli/run.option-collisions.test.ts +++ b/src/cli/gateway-cli/run.option-collisions.test.ts @@ -6,6 +6,7 @@ import { CONFIG_AUDIT_STORE_LABEL } from "../../config/io.audit.js"; import type { ConfigFileSnapshot } from "../../config/types.js"; import { GATEWAY_SERVICE_RUNTIME_PID_ENV } from "../../daemon/constants.js"; import { SUPERVISOR_HINT_ENV_VARS } from "../../infra/supervisor-markers.js"; +import { OpenClawStateDatabaseSchemaMigrationRequiredError } from "../../state/openclaw-state-db-schema-migration-required.js"; import { captureEnv, deleteTestEnvValue, @@ -36,9 +37,14 @@ const findVerifiedGatewayListenerPidsOnPortSync = vi.fn((_port: number) => [] as const formatGatewayPidList = vi.fn((pids: number[]) => pids.join(", ")); const isTerminalInteractive = vi.fn(() => true); const offerInvalidConfigRecovery = vi.fn(async () => ({ status: "declined" as const })); +const parkCurrentLaunchAgentForMaintenance = vi.fn(async () => false); const ensureDevGatewayConfig = vi.fn(async (_opts?: unknown) => {}); type GatewayLoopStart = (params?: { startupStartedAt?: number }) => Promise; -const runGatewayLoop = vi.fn(async ({ start }: { start: GatewayLoopStart }) => { +type GatewayLoopParams = { + start: GatewayLoopStart; + completeBoot?: (completion: unknown) => void; +}; +const runGatewayLoop = vi.fn(async ({ start }: GatewayLoopParams) => { await start(); }); const normalizeStateDirEnv = vi.fn((_env?: NodeJS.ProcessEnv) => undefined); @@ -114,7 +120,9 @@ const bootLifecycle = vi.hoisted(() => ({ recovered: false, }, ), - record: vi.fn((_env?: NodeJS.ProcessEnv, _nowMs?: number, _reason?: string) => "boot-id"), + record: vi.fn( + (_env?: NodeJS.ProcessEnv, _nowMs?: number, _reason?: string): string | undefined => "boot-id", + ), complete: vi.fn(), })); const netState = vi.hoisted(() => ({ @@ -261,6 +269,11 @@ vi.mock("../../gateway/server.js", () => ({ startGatewayServer: (port: number, opts?: unknown) => startGatewayServer(port, opts), })); +vi.mock("../../daemon/launchd.js", async (importOriginal) => ({ + ...(await importOriginal()), + parkCurrentLaunchAgentForMaintenance: () => parkCurrentLaunchAgentForMaintenance(), +})); + vi.mock("../../gateway/ws-logging.js", () => ({ setGatewayWsLogStyle: (style: string) => setGatewayWsLogStyle(style), })); @@ -408,6 +421,8 @@ describe("gateway run option collisions", () => { isTerminalInteractive.mockReset(); isTerminalInteractive.mockReturnValue(true); offerInvalidConfigRecovery.mockClear(); + parkCurrentLaunchAgentForMaintenance.mockReset(); + parkCurrentLaunchAgentForMaintenance.mockResolvedValue(false); cleanStaleGatewayProcessesSync.mockClear(); waitForPortBindable.mockClear(); ensureDevGatewayConfig.mockClear(); @@ -1601,6 +1616,52 @@ describe("gateway run option collisions", () => { expect(writeDiagnosticStabilityBundleForFailureSync).not.toHaveBeenCalled(); }); + it("exits 78 and parks launchd for a repairable shared-state schema", async () => { + bootLifecycle.record.mockReturnValueOnce(undefined); + runGatewayLoop.mockImplementationOnce(async ({ start, completeBoot }: GatewayLoopParams) => { + try { + await start(); + } catch (error) { + completeBoot?.({ outcome: "startup_failed", reason: "schema migration required" }); + throw error; + } + }); + startGatewayServer.mockRejectedValueOnce( + new OpenClawStateDatabaseSchemaMigrationRequiredError( + "agent-databases-composite-primary-key", + "/tmp/openclaw.sqlite", + ), + ); + parkCurrentLaunchAgentForMaintenance.mockResolvedValueOnce(true); + + await expect(runGatewayCli(["gateway", "run", "--allow-unconfigured"])).rejects.toThrow( + "__exit__:78", + ); + + expect(parkCurrentLaunchAgentForMaintenance).toHaveBeenCalledOnce(); + expect(bootLifecycle.complete).toHaveBeenCalledWith(undefined, { + outcome: "startup_failed", + reason: "schema migration required", + }); + expect(runtimeErrors.join("\n")).toContain( + "state database schema migration required (agent-databases-composite-primary-key)", + ); + }); + + it("does not park launchd for a nonrepairable shared-state schema", async () => { + startGatewayServer.mockRejectedValueOnce( + new Error( + "OpenClaw state database /tmp/openclaw.sqlite has a noncanonical agent database registry schema that cannot be repaired automatically.", + ), + ); + + await expect(runGatewayCli(["gateway", "run", "--allow-unconfigured"])).rejects.toThrow( + "__exit__:1", + ); + + expect(parkCurrentLaunchAgentForMaintenance).not.toHaveBeenCalled(); + }); + it.each([ "gateway already running (pid 4242); lock timeout after 5000ms", "another gateway instance is already listening on ws://127.0.0.1", diff --git a/src/cli/gateway-cli/run.ts b/src/cli/gateway-cli/run.ts index 7308909ab9b3..3f039de99ea6 100644 --- a/src/cli/gateway-cli/run.ts +++ b/src/cli/gateway-cli/run.ts @@ -59,6 +59,7 @@ import { withDiagnosticPhase } from "../../logging/diagnostic-phase.js"; import { createSubsystemLogger } from "../../logging/subsystem.js"; import { defaultRuntime } from "../../runtime.js"; import { findOpenClawAgentDatabaseMediaMigrationRequiredError } from "../../state/openclaw-agent-db-migration-required.js"; +import { findOpenClawStateDatabaseSchemaMigrationRequiredError } from "../../state/openclaw-state-db-schema-migration-required.js"; import { printClawBanner, type ClawBannerResult } from "../claw-banner.js"; import { formatCliCommand } from "../command-format.js"; import { formatInvalidConfigPort, formatInvalidPortOption } from "../error-format.js"; @@ -475,7 +476,9 @@ function resolveGatewayLockErrorExitCode(err: unknown): number { } function resolveGatewayStartupFailureExitCode(err: unknown): number { - return isInvalidConfigError(err) || findOpenClawAgentDatabaseMediaMigrationRequiredError(err) + return isInvalidConfigError(err) || + findOpenClawAgentDatabaseMediaMigrationRequiredError(err) || + findOpenClawStateDatabaseSchemaMigrationRequiredError(err) ? EXIT_CONFIG_ERROR : 1; } @@ -1144,6 +1147,8 @@ async function runGatewayCommandOnce(opts: GatewayRunOpts, hooks: GatewayRunRunt : crashLoopDecision.recovered ? GATEWAY_CRASH_LOOP_RECOVERED_REASON : undefined; + // Shared-state schema failures make this write fail open, so no lifecycle + // row exists for Doctor to reconcile after it repairs the schema. activeBootId = recordGatewayBootStart(process.env, startedAtMs, bootStartReason); channelAutostartSuppression = undefined; if (crashLoopDecision.recovered) { @@ -1248,6 +1253,20 @@ async function runGatewayCommandOnce(opts: GatewayRunOpts, hooks: GatewayRunRunt ); } } + if (findOpenClawStateDatabaseSchemaMigrationRequiredError(err)) { + try { + const { parkCurrentLaunchAgentForMaintenance } = await import("../../daemon/launchd.js"); + if (await parkCurrentLaunchAgentForMaintenance()) { + gatewayLog.error( + `gateway requires state database schema migration; parked the managed LaunchAgent. Run ${formatCliCommand("openclaw doctor --fix")} to repair and restart it.`, + ); + } + } catch (parkError) { + gatewayLog.error( + `failed to park the managed LaunchAgent after state schema migration-required startup: ${formatErrorMessage(parkError)}`, + ); + } + } await maybeWriteGatewayStartupFailureBundle(err); defaultRuntime.error( `Gateway failed to start: ${formatErrorMessage(err)}. Run ${formatCliCommand("openclaw gateway status --deep")} for diagnostics.`, diff --git a/src/state/openclaw-state-db-schema-migration-required.test.ts b/src/state/openclaw-state-db-schema-migration-required.test.ts new file mode 100644 index 000000000000..2e2c2ae782a7 --- /dev/null +++ b/src/state/openclaw-state-db-schema-migration-required.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from "vitest"; +import { + findOpenClawStateDatabaseSchemaMigrationRequiredError, + OpenClawStateDatabaseSchemaMigrationRequiredError, +} from "./openclaw-state-db-schema-migration-required.js"; + +describe("state database schema migration error classification", () => { + it("recognizes a rehydrated exact migration error through its cause chain", () => { + const original = new OpenClawStateDatabaseSchemaMigrationRequiredError( + "agent-databases-composite-primary-key", + "/tmp/openclaw.sqlite", + ); + const rehydrated = new Error("startup failed", { + cause: new Error(original.message), + }); + + expect(findOpenClawStateDatabaseSchemaMigrationRequiredError(rehydrated)).toMatchObject({ + kind: "agent-databases-composite-primary-key", + pathname: "/tmp/openclaw.sqlite", + }); + }); + + it("recognizes an exact instance of the typed error", () => { + const original = new OpenClawStateDatabaseSchemaMigrationRequiredError( + "audit-events-v2", + "/tmp/openclaw.sqlite", + ); + + expect(findOpenClawStateDatabaseSchemaMigrationRequiredError(original)).toBe(original); + }); + + it("does not classify similar operator guidance as the migration error", () => { + expect( + findOpenClawStateDatabaseSchemaMigrationRequiredError( + new Error( + "OpenClaw state database /tmp/openclaw.sqlite is stale; run openclaw doctor --fix.", + ), + ), + ).toBeUndefined(); + }); + + it("does not classify the agent DB media migration error", () => { + // Ensure the state-DB classifier does not accidentally match agent-DB messages. + expect( + findOpenClawStateDatabaseSchemaMigrationRequiredError( + new Error( + "OpenClaw agent database /tmp/openclaw-agent.sqlite uses schema version 5; run openclaw doctor --fix to migrate persisted media before using it.", + ), + ), + ).toBeUndefined(); + }); +}); diff --git a/src/state/openclaw-state-db-schema-migration-required.ts b/src/state/openclaw-state-db-schema-migration-required.ts new file mode 100644 index 000000000000..d0198fbcb7a2 --- /dev/null +++ b/src/state/openclaw-state-db-schema-migration-required.ts @@ -0,0 +1,57 @@ +const GATEWAY_STATE_SCHEMA_MIGRATION_REQUIRED_REASON = "gateway.state_schema_migration_required"; + +type OpenClawStateDatabaseSchemaMigrationRequiredKind = + | "agent-databases-composite-primary-key" + | "audit-events-v2"; + +export class OpenClawStateDatabaseSchemaMigrationRequiredError extends Error { + readonly code = GATEWAY_STATE_SCHEMA_MIGRATION_REQUIRED_REASON; + + constructor( + readonly kind: OpenClawStateDatabaseSchemaMigrationRequiredKind, + readonly pathname: string, + ) { + super( + `OpenClaw state database schema migration required (${kind}) at ${pathname}; run openclaw doctor --fix to migrate it.`, + ); + this.name = "OpenClawStateDatabaseSchemaMigrationRequiredError"; + } +} + +const STATE_SCHEMA_MIGRATION_REQUIRED_MESSAGE = + /^OpenClaw state database schema migration required \((agent-databases-composite-primary-key|audit-events-v2)\) at (.+); run openclaw doctor --fix to migrate it\.$/u; + +function parseStateSchemaMigrationRequiredMessage( + message: unknown, +): OpenClawStateDatabaseSchemaMigrationRequiredError | undefined { + if (typeof message !== "string") { + return undefined; + } + const match = STATE_SCHEMA_MIGRATION_REQUIRED_MESSAGE.exec(message); + const kind = match?.[1] as OpenClawStateDatabaseSchemaMigrationRequiredKind | undefined; + const pathname = match?.[2]; + if (!kind || !pathname) { + return undefined; + } + return new OpenClawStateDatabaseSchemaMigrationRequiredError(kind, pathname); +} + +export function findOpenClawStateDatabaseSchemaMigrationRequiredError( + error: unknown, +): OpenClawStateDatabaseSchemaMigrationRequiredError | undefined { + let current = error; + const seen = new Set(); + while (current && typeof current === "object" && !seen.has(current)) { + if (current instanceof OpenClawStateDatabaseSchemaMigrationRequiredError) { + return current; + } + const errorLike = current as { cause?: unknown; message?: unknown }; + const parsed = parseStateSchemaMigrationRequiredMessage(errorLike.message); + if (parsed) { + return parsed; + } + seen.add(current); + current = errorLike.cause; + } + return undefined; +} diff --git a/src/state/openclaw-state-db-schema-repair.ts b/src/state/openclaw-state-db-schema-repair.ts index 7dcc77f290c9..88de779fe6d4 100644 --- a/src/state/openclaw-state-db-schema-repair.ts +++ b/src/state/openclaw-state-db-schema-repair.ts @@ -19,6 +19,7 @@ import { tableHasColumn, tablePrimaryKeyColumns, } from "./openclaw-state-db-schema-helpers.js"; +import { OpenClawStateDatabaseSchemaMigrationRequiredError } from "./openclaw-state-db-schema-migration-required.js"; import * as sessionWatchMigration from "./openclaw-state-db-session-watch-migration.js"; export function dropLegacyStateTables(db: DatabaseSync): void { @@ -147,15 +148,19 @@ export function markCurrentStateSchemaVersion( export function assertCanonicalStateSchemaShape(db: DatabaseSync, pathname: string): void { operatorApprovalMigration.assertCanonicalOperatorApprovalKinds(db, pathname); if (!hasCanonicalAgentDatabasesPrimaryKey(db)) { + if (canRepairAgentDatabasesPrimaryKey(db)) { + throw new OpenClawStateDatabaseSchemaMigrationRequiredError( + "agent-databases-composite-primary-key", + pathname, + ); + } throw new Error( - `OpenClaw state database ${pathname} has a legacy agent database registry schema; run openclaw doctor --fix to migrate it.`, + `OpenClaw state database ${pathname} has a noncanonical agent database registry schema that cannot be repaired automatically; restore the canonical agent_databases shape before retrying.`, ); } if (!hasCanonicalAuditEventsSchema(db)) { if (canRepairLegacyAuditEventsSchema(db)) { - throw new Error( - `OpenClaw state database ${pathname} has a legacy audit event schema; run openclaw doctor --fix to migrate it.`, - ); + throw new OpenClawStateDatabaseSchemaMigrationRequiredError("audit-events-v2", pathname); } throw new Error( `OpenClaw state database ${pathname} has a noncanonical audit event schema that cannot be repaired automatically; restore the canonical audit_events shape before retrying.`, diff --git a/src/state/openclaw-state-db.test.ts b/src/state/openclaw-state-db.test.ts index de6ffd62ed53..85beb9ca293d 100644 --- a/src/state/openclaw-state-db.test.ts +++ b/src/state/openclaw-state-db.test.ts @@ -19,6 +19,10 @@ import { readSqliteNumberPragma } from "../infra/sqlite-pragma.test-support.js"; import { loadTaskRegistryStateFromSqlite } from "../tasks/task-registry.store.sqlite.js"; import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js"; import { VERSION } from "../version.js"; +import { + findOpenClawStateDatabaseSchemaMigrationRequiredError, + OpenClawStateDatabaseSchemaMigrationRequiredError, +} from "./openclaw-state-db-schema-migration-required.js"; import type { DB as OpenClawStateKyselyDatabase } from "./openclaw-state-db.generated.js"; import { assertOpenClawStateDatabaseForMaintenance, @@ -53,6 +57,23 @@ function createTempStateDir(): string { return makeTempDir(stateDbTempDirs, "openclaw-state-db-"); } +function expectStateSchemaMigrationRequired( + run: () => unknown, + expected: { + kind: OpenClawStateDatabaseSchemaMigrationRequiredError["kind"]; + pathname: string; + }, +): void { + let caught: unknown; + try { + run(); + } catch (error) { + caught = error; + } + expect(caught).toBeInstanceOf(OpenClawStateDatabaseSchemaMigrationRequiredError); + expect(findOpenClawStateDatabaseSchemaMigrationRequiredError(caught)).toMatchObject(expected); +} + function replaceManagedImageRecordsWithLegacyTable( database: DatabaseSync, options: { withRow: boolean }, @@ -1957,6 +1978,79 @@ INSERT INTO macos_port_guardian_records VALUES (4242, 18789, '/usr/bin/ssh', 're ); }); + it("classifies the released agent registry primary key as Doctor-repairable", () => { + const stateDir = createTempStateDir(); + const options = { env: { OPENCLAW_STATE_DIR: stateDir } }; + const databasePath = openOpenClawStateDatabase(options).path; + closeOpenClawStateDatabaseForTest(); + + const { DatabaseSync } = requireNodeSqlite(); + const legacy = new DatabaseSync(databasePath); + legacy.exec(` + ALTER TABLE agent_databases RENAME TO agent_databases_current; + CREATE TABLE agent_databases ( + agent_id TEXT NOT NULL PRIMARY KEY, + path TEXT NOT NULL, + schema_version INTEGER NOT NULL, + last_seen_at INTEGER NOT NULL, + size_bytes INTEGER + ); + INSERT INTO agent_databases ( + agent_id, + path, + schema_version, + last_seen_at, + size_bytes + ) + SELECT + agent_id, + path, + schema_version, + last_seen_at, + size_bytes + FROM agent_databases_current; + DROP TABLE agent_databases_current; + `); + legacy.close(); + + expectStateSchemaMigrationRequired(() => openOpenClawStateDatabase(options), { + kind: "agent-databases-composite-primary-key", + pathname: databasePath, + }); + }); + + it("keeps an unrecognized agent registry schema fail-closed and nonrepairable", () => { + const stateDir = createTempStateDir(); + const options = { env: { OPENCLAW_STATE_DIR: stateDir } }; + const databasePath = openOpenClawStateDatabase(options).path; + closeOpenClawStateDatabaseForTest(); + + const { DatabaseSync } = requireNodeSqlite(); + const malformed = new DatabaseSync(databasePath); + malformed.exec(` + DROP TABLE agent_databases; + CREATE TABLE agent_databases ( + agent_id TEXT NOT NULL PRIMARY KEY, + path TEXT NOT NULL, + schema_version INTEGER NOT NULL, + last_seen_at INTEGER NOT NULL + ); + `); + malformed.close(); + + let caught: unknown; + try { + openOpenClawStateDatabase(options); + } catch (error) { + caught = error; + } + expect(findOpenClawStateDatabaseSchemaMigrationRequiredError(caught)).toBeUndefined(); + expect(caught).toBeInstanceOf(Error); + expect((caught as Error).message).toContain( + "noncanonical agent database registry schema that cannot be repaired automatically", + ); + }); + it("migrates the released audit ledger to message-compatible attribution exactly once", () => { const stateDir = createTempStateDir(); const databasePath = createLegacyAuditStateDatabase(stateDir); @@ -1966,7 +2060,10 @@ INSERT INTO macos_port_guardian_records VALUES (4242, 18789, '/usr/bin/ssh', 're { kind: "audit-events-v2", path: databasePath }, { kind: "strict-tables-v3", path: databasePath }, ]); - expect(() => openOpenClawStateDatabase(options)).toThrow(/legacy audit event schema/); + expectStateSchemaMigrationRequired(() => openOpenClawStateDatabase(options), { + kind: "audit-events-v2", + pathname: databasePath, + }); expect(repairOpenClawStateDatabaseSchema(options)).toEqual({ changes: [