From e5fb023740310b1c2c8bfd5e41020e032de49864 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 25 Jul 2026 14:58:30 -0700 Subject: [PATCH] fix(state): prevent gateway crash after model catalog upgrade (#113875) * fix(state): preserve lazy additive tables on upgrade * chore: keep release notes in PR --- src/infra/sqlite-schema-contract.test.ts | 20 +++++++++++ src/infra/sqlite-schema-contract.ts | 10 ++++++ src/model-catalog/remote-store.test.ts | 42 +++++++++++++++++++++- src/state/openclaw-state-db-contract.ts | 3 ++ src/state/openclaw-state-db-maintenance.ts | 2 ++ src/state/openclaw-state-db.ts | 38 ++++++++++++++++++-- 6 files changed, 111 insertions(+), 4 deletions(-) diff --git a/src/infra/sqlite-schema-contract.test.ts b/src/infra/sqlite-schema-contract.test.ts index 04cf2740864c..acbfba45ae16 100644 --- a/src/infra/sqlite-schema-contract.test.ts +++ b/src/infra/sqlite-schema-contract.test.ts @@ -145,6 +145,26 @@ describe("assertSqliteSchemaContains", () => { } }); + it("accepts only allowlisted missing lazy-additive tables", () => { + const migratedSchema = CANONICAL_SCHEMA.replace( + / {2}CREATE TABLE events \([\s\S]*?\n {2}\);\n/u, + "", + ); + const database = createDatabase(migratedSchema); + try { + expect(() => assertSqliteSchemaContains(database, "test database", CANONICAL_SCHEMA)).toThrow( + "missing table events", + ); + expect(() => + assertSqliteSchemaContains(database, "test database", CANONICAL_SCHEMA, { + allowedMissingTables: ["events"], + }), + ).not.toThrow(); + } finally { + database.close(); + } + }); + it("accepts equivalent foreign keys declared in migration order", () => { const migratedSchema = CANONICAL_SCHEMA.replace( ` FOREIGN KEY (parent_id) REFERENCES parents(id) ON DELETE CASCADE, diff --git a/src/infra/sqlite-schema-contract.ts b/src/infra/sqlite-schema-contract.ts index 10b63cad1db0..745c18d371d6 100644 --- a/src/infra/sqlite-schema-contract.ts +++ b/src/infra/sqlite-schema-contract.ts @@ -67,6 +67,12 @@ export type CanonicalSqliteNamedIndexContract = { }; export type SqliteSchemaCompatibility = { + /** + * Canonical additive tables that may be absent until their owning feature + * performs its one-time lazy ensure. Present tables still require the exact + * canonical shape. + */ + allowedMissingTables?: readonly string[]; /** * Exact definitions produced by supported additive migrations when SQLite * requires a temporary default that the clean schema does not retain. @@ -99,11 +105,15 @@ export function assertSqliteSchemaContains( compatibility: SqliteSchemaCompatibility = {}, ): void { const expected = getSqliteSchemaContract(schemaSql); + const allowedMissingTables = new Set(compatibility.allowedMissingTables ?? []); const mismatches: string[] = []; for (const [tableName, expectedTable] of expected) { const actualTable = collectSqliteTableContract(database, tableName); if (!actualTable) { + if (allowedMissingTables.has(tableName)) { + continue; + } mismatches.push(`missing table ${tableName}`); continue; } diff --git a/src/model-catalog/remote-store.test.ts b/src/model-catalog/remote-store.test.ts index a891475570d4..96216a349c23 100644 --- a/src/model-catalog/remote-store.test.ts +++ b/src/model-catalog/remote-store.test.ts @@ -2,7 +2,11 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; -import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; +import { requireNodeSqlite } from "../infra/node-sqlite.js"; +import { + closeOpenClawStateDatabaseForTest, + openOpenClawStateDatabase, +} from "../state/openclaw-state-db.js"; import { markRemoteModelCatalogChecked, readRemoteModelCatalog, @@ -18,6 +22,42 @@ afterEach(() => { }); describe("remote model catalog store", () => { + it("lazily adds the cache table to an existing current-schema database", () => { + const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-catalog-"))); + roots.push(root); + const options = { path: path.join(root, "state.sqlite") }; + openOpenClawStateDatabase(options); + closeOpenClawStateDatabaseForTest(); + + const { DatabaseSync } = requireNodeSqlite(); + const preCatalog = new DatabaseSync(options.path); + preCatalog.exec("DROP TABLE model_catalog_remote;"); + preCatalog.exec("DROP INDEX idx_task_runs_status;"); + preCatalog.close(); + + const reopened = openOpenClawStateDatabase(options); + expect( + reopened.db + .prepare("SELECT name FROM sqlite_schema WHERE type = 'table' AND name = ?") + .get("model_catalog_remote"), + ).toBeUndefined(); + expect( + reopened.db + .prepare("SELECT name FROM sqlite_schema WHERE type = 'index' AND name = ?") + .get("idx_task_runs_status"), + ).toEqual({ name: "idx_task_runs_status" }); + closeOpenClawStateDatabaseForTest(); + + expect(readRemoteModelCatalog(options)).toBeUndefined(); + const upgraded = new DatabaseSync(options.path, { readOnly: true }); + expect( + upgraded + .prepare("SELECT name FROM sqlite_schema WHERE type = 'table' AND name = ?") + .get("model_catalog_remote"), + ).toEqual({ name: "model_catalog_remote" }); + upgraded.close(); + }); + it("lazily ensures twice and upserts the single slot", () => { const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-catalog-"))); roots.push(root); diff --git a/src/state/openclaw-state-db-contract.ts b/src/state/openclaw-state-db-contract.ts index 1b1068fc4e25..f7597cfe4e55 100644 --- a/src/state/openclaw-state-db-contract.ts +++ b/src/state/openclaw-state-db-contract.ts @@ -5,6 +5,9 @@ import type { SqliteWalMaintenance } from "../infra/sqlite-wal.js"; // v5 records durable cloud-worker result refs on pending workspace fences. export const OPENCLAW_STATE_SCHEMA_VERSION = 6; export const OPENCLAW_STATE_STRICT_SCHEMA_VERSION = 3; +// Added after v6 shipped. The cache stays optional until its feature-local +// lazy ensure runs; fold it into the next natural schema-version bump. +export const LAZY_ADDITIVE_STATE_TABLES = ["model_catalog_remote"] as const; /** Maximum time one synchronous SQLite call may wait for a lock. */ export const OPENCLAW_SQLITE_BUSY_TIMEOUT_MS = 5_000; /** User-facing guide for schema refusals; lives here so error sites avoid import cycles. */ diff --git a/src/state/openclaw-state-db-maintenance.ts b/src/state/openclaw-state-db-maintenance.ts index db333df091cd..975f3ae9e616 100644 --- a/src/state/openclaw-state-db-maintenance.ts +++ b/src/state/openclaw-state-db-maintenance.ts @@ -11,6 +11,7 @@ import { } from "../infra/sqlite-user-version.js"; import { OPENCLAW_DATABASE_SCHEMA_DOCS_URL, + LAZY_ADDITIVE_STATE_TABLES, OPENCLAW_STATE_SCHEMA_VERSION, type OpenClawStateDatabaseOptions, } from "./openclaw-state-db-contract.js"; @@ -18,6 +19,7 @@ import { resolveOpenClawStateSqlitePath } from "./openclaw-state-db.paths.js"; import { OPENCLAW_STATE_SCHEMA_SQL } from "./openclaw-state-schema.generated.js"; const OPENCLAW_STATE_MAINTENANCE_SCHEMA_COMPATIBILITY = { + allowedMissingTables: LAZY_ADDITIVE_STATE_TABLES, allowedColumnDefinitions: { "diagnostic_events.sequence": ["sequence INTEGER NOT NULL DEFAULT 0"], "commitments.attempts": ["attempts INTEGER NOT NULL DEFAULT 0"], diff --git a/src/state/openclaw-state-db.ts b/src/state/openclaw-state-db.ts index 76c2c0895d4c..971811278440 100644 --- a/src/state/openclaw-state-db.ts +++ b/src/state/openclaw-state-db.ts @@ -40,6 +40,7 @@ import { import { repairAuditEventsSchema } from "./openclaw-state-db-audit-migration.js"; import { OPENCLAW_DATABASE_SCHEMA_DOCS_URL, + LAZY_ADDITIVE_STATE_TABLES, OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, OPENCLAW_STATE_SCHEMA_VERSION, OPENCLAW_STATE_STRICT_SCHEMA_VERSION, @@ -135,6 +136,31 @@ export function clearOpenClawStateDatabaseOpenFailure(pathname: string): void { type OpenClawStateMetadataDatabase = Pick; const stateDbLog = createSubsystemLogger("state/db"); +function executeCanonicalStateSchema( + database: DatabaseSync, + options: { includeLazyAdditiveTables: boolean }, +): void { + if (options.includeLazyAdditiveTables) { + database.exec(OPENCLAW_STATE_SCHEMA_SQL); + return; + } + + // Current-version databases may lack lazy cache tables, but the remaining + // canonical DDL must still run so doctor can restore indexes and triggers. + let eagerSchema = OPENCLAW_STATE_SCHEMA_SQL; + for (const tableName of LAZY_ADDITIVE_STATE_TABLES) { + const startMarker = `CREATE TABLE IF NOT EXISTS ${tableName} (`; + const start = eagerSchema.indexOf(startMarker); + const endMarker = "\n) STRICT;"; + const end = start >= 0 ? eagerSchema.indexOf(endMarker, start) : -1; + if (start < 0 || end < 0) { + throw new Error(`lazy additive state schema block is missing for ${tableName}`); + } + eagerSchema = `${eagerSchema.slice(0, start)}${eagerSchema.slice(end + endMarker.length)}`; + } + database.exec(eagerSchema); +} + export function repairOpenClawStateDatabaseSchema(options: OpenClawStateDatabaseOptions = {}): { changes: string[]; warnings: string[]; @@ -163,7 +189,9 @@ export function repairOpenClawStateDatabaseSchema(options: OpenClawStateDatabase } // Current-schema doctor repair may normalize recognized columns or // table options, but it must never recreate a missing table empty. - assertSqliteSchemaTablesPresent(db, pathname, OPENCLAW_STATE_SCHEMA_SQL); + assertSqliteSchemaTablesPresent(db, pathname, OPENCLAW_STATE_SCHEMA_SQL, { + allowedMissingTables: LAZY_ADDITIVE_STATE_TABLES, + }); } if (rebuiltIndexNames.size === 0) { assertSqliteIntegrity(db, pathname); @@ -189,7 +217,9 @@ export function repairOpenClawStateDatabaseSchema(options: OpenClawStateDatabase assertCanonicalStateSchemaShape(db, pathname); if (tableExists(db, "audit_events")) { ensureAdditiveStateColumns(db); - db.exec(OPENCLAW_STATE_SCHEMA_SQL); + executeCanonicalStateSchema(db, { + includeLazyAdditiveTables: previousVersion !== OPENCLAW_STATE_SCHEMA_VERSION, + }); if (previousVersion < OPENCLAW_STATE_STRICT_SCHEMA_VERSION) { repairLegacyGatewayRestartHandoffsForStrictMigration(db); } @@ -279,7 +309,9 @@ function ensureSchema(db: DatabaseSync, pathname: string): void { ensureAdditiveStateColumns(db); sessionWatchMigration.migrateSessionWatchCursorProvenance(db); assertCanonicalStateSchemaShape(db, pathname); - db.exec(OPENCLAW_STATE_SCHEMA_SQL); + executeCanonicalStateSchema(db, { + includeLazyAdditiveTables: previousVersion !== OPENCLAW_STATE_SCHEMA_VERSION, + }); migrateLegacyCronRunLogsToTaskRuns(db); if (previousVersion < OPENCLAW_STATE_STRICT_SCHEMA_VERSION) { repairLegacyGatewayRestartHandoffsForStrictMigration(db);