fix(state): prevent gateway crash after model catalog upgrade (#113875)

* fix(state): preserve lazy additive tables on upgrade

* chore: keep release notes in PR

src/infra/sqlite-schema-contract.test.ts

@@ -145,6 +145,26 @@ describe("assertSqliteSchemaContains", () => {
     }
   });
 
+  it("accepts only allowlisted missing lazy-additive tables", () => {
+    const migratedSchema = CANONICAL_SCHEMA.replace(
+      / {2}CREATE TABLE events \([\s\S]*?\n {2}\);\n/u,
+      "",
+    );
+    const database = createDatabase(migratedSchema);
+    try {
+      expect(() => assertSqliteSchemaContains(database, "test database", CANONICAL_SCHEMA)).toThrow(
+        "missing table events",
+      );
+      expect(() =>
+        assertSqliteSchemaContains(database, "test database", CANONICAL_SCHEMA, {
+          allowedMissingTables: ["events"],
+        }),
+      ).not.toThrow();
+    } finally {
+      database.close();
+    }
+  });
+
   it("accepts equivalent foreign keys declared in migration order", () => {
     const migratedSchema = CANONICAL_SCHEMA.replace(
       `    FOREIGN KEY (parent_id) REFERENCES parents(id) ON DELETE CASCADE,

src/infra/sqlite-schema-contract.ts

@@ -67,6 +67,12 @@ export type CanonicalSqliteNamedIndexContract = {
 };
 
 export type SqliteSchemaCompatibility = {
+  /**
+   * Canonical additive tables that may be absent until their owning feature
+   * performs its one-time lazy ensure. Present tables still require the exact
+   * canonical shape.
+   */
+  allowedMissingTables?: readonly string[];
   /**
    * Exact definitions produced by supported additive migrations when SQLite
    * requires a temporary default that the clean schema does not retain.
@@ -99,11 +105,15 @@ export function assertSqliteSchemaContains(
   compatibility: SqliteSchemaCompatibility = {},
 ): void {
   const expected = getSqliteSchemaContract(schemaSql);
+  const allowedMissingTables = new Set(compatibility.allowedMissingTables ?? []);
 
   const mismatches: string[] = [];
   for (const [tableName, expectedTable] of expected) {
     const actualTable = collectSqliteTableContract(database, tableName);
     if (!actualTable) {
+      if (allowedMissingTables.has(tableName)) {
+        continue;
+      }
       mismatches.push(`missing table ${tableName}`);
       continue;
     }

src/model-catalog/remote-store.test.ts

@@ -2,7 +2,11 @@ import fs from "node:fs";
 import os from "node:os";
 import path from "node:path";
 import { afterEach, describe, expect, it } from "vitest";
-import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js";
+import { requireNodeSqlite } from "../infra/node-sqlite.js";
+import {
+  closeOpenClawStateDatabaseForTest,
+  openOpenClawStateDatabase,
+} from "../state/openclaw-state-db.js";
 import {
   markRemoteModelCatalogChecked,
   readRemoteModelCatalog,
@@ -18,6 +22,42 @@ afterEach(() => {
 });
 
 describe("remote model catalog store", () => {
+  it("lazily adds the cache table to an existing current-schema database", () => {
+    const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-catalog-")));
+    roots.push(root);
+    const options = { path: path.join(root, "state.sqlite") };
+    openOpenClawStateDatabase(options);
+    closeOpenClawStateDatabaseForTest();
+
+    const { DatabaseSync } = requireNodeSqlite();
+    const preCatalog = new DatabaseSync(options.path);
+    preCatalog.exec("DROP TABLE model_catalog_remote;");
+    preCatalog.exec("DROP INDEX idx_task_runs_status;");
+    preCatalog.close();
+
+    const reopened = openOpenClawStateDatabase(options);
+    expect(
+      reopened.db
+        .prepare("SELECT name FROM sqlite_schema WHERE type = 'table' AND name = ?")
+        .get("model_catalog_remote"),
+    ).toBeUndefined();
+    expect(
+      reopened.db
+        .prepare("SELECT name FROM sqlite_schema WHERE type = 'index' AND name = ?")
+        .get("idx_task_runs_status"),
+    ).toEqual({ name: "idx_task_runs_status" });
+    closeOpenClawStateDatabaseForTest();
+
+    expect(readRemoteModelCatalog(options)).toBeUndefined();
+    const upgraded = new DatabaseSync(options.path, { readOnly: true });
+    expect(
+      upgraded
+        .prepare("SELECT name FROM sqlite_schema WHERE type = 'table' AND name = ?")
+        .get("model_catalog_remote"),
+    ).toEqual({ name: "model_catalog_remote" });
+    upgraded.close();
+  });
+
   it("lazily ensures twice and upserts the single slot", () => {
     const root = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-catalog-")));
     roots.push(root);

src/state/openclaw-state-db-contract.ts

@@ -5,6 +5,9 @@ import type { SqliteWalMaintenance } from "../infra/sqlite-wal.js";
 // v5 records durable cloud-worker result refs on pending workspace fences.
 export const OPENCLAW_STATE_SCHEMA_VERSION = 6;
 export const OPENCLAW_STATE_STRICT_SCHEMA_VERSION = 3;
+// Added after v6 shipped. The cache stays optional until its feature-local
+// lazy ensure runs; fold it into the next natural schema-version bump.
+export const LAZY_ADDITIVE_STATE_TABLES = ["model_catalog_remote"] as const;
 /** Maximum time one synchronous SQLite call may wait for a lock. */
 export const OPENCLAW_SQLITE_BUSY_TIMEOUT_MS = 5_000;
 /** User-facing guide for schema refusals; lives here so error sites avoid import cycles. */

src/state/openclaw-state-db-maintenance.ts

@@ -11,6 +11,7 @@ import {
 } from "../infra/sqlite-user-version.js";
 import {
   OPENCLAW_DATABASE_SCHEMA_DOCS_URL,
+  LAZY_ADDITIVE_STATE_TABLES,
   OPENCLAW_STATE_SCHEMA_VERSION,
   type OpenClawStateDatabaseOptions,
 } from "./openclaw-state-db-contract.js";
@@ -18,6 +19,7 @@ import { resolveOpenClawStateSqlitePath } from "./openclaw-state-db.paths.js";
 import { OPENCLAW_STATE_SCHEMA_SQL } from "./openclaw-state-schema.generated.js";
 
 const OPENCLAW_STATE_MAINTENANCE_SCHEMA_COMPATIBILITY = {
+  allowedMissingTables: LAZY_ADDITIVE_STATE_TABLES,
   allowedColumnDefinitions: {
     "diagnostic_events.sequence": ["sequence INTEGER NOT NULL DEFAULT 0"],
     "commitments.attempts": ["attempts INTEGER NOT NULL DEFAULT 0"],

src/state/openclaw-state-db.ts

@@ -40,6 +40,7 @@ import {
 import { repairAuditEventsSchema } from "./openclaw-state-db-audit-migration.js";
 import {
   OPENCLAW_DATABASE_SCHEMA_DOCS_URL,
+  LAZY_ADDITIVE_STATE_TABLES,
   OPENCLAW_SQLITE_BUSY_TIMEOUT_MS,
   OPENCLAW_STATE_SCHEMA_VERSION,
   OPENCLAW_STATE_STRICT_SCHEMA_VERSION,
@@ -135,6 +136,31 @@ export function clearOpenClawStateDatabaseOpenFailure(pathname: string): void {
 type OpenClawStateMetadataDatabase = Pick<OpenClawStateKyselyDatabase, "schema_meta">;
 const stateDbLog = createSubsystemLogger("state/db");
 
+function executeCanonicalStateSchema(
+  database: DatabaseSync,
+  options: { includeLazyAdditiveTables: boolean },
+): void {
+  if (options.includeLazyAdditiveTables) {
+    database.exec(OPENCLAW_STATE_SCHEMA_SQL);
+    return;
+  }
+
+  // Current-version databases may lack lazy cache tables, but the remaining
+  // canonical DDL must still run so doctor can restore indexes and triggers.
+  let eagerSchema = OPENCLAW_STATE_SCHEMA_SQL;
+  for (const tableName of LAZY_ADDITIVE_STATE_TABLES) {
+    const startMarker = `CREATE TABLE IF NOT EXISTS ${tableName} (`;
+    const start = eagerSchema.indexOf(startMarker);
+    const endMarker = "\n) STRICT;";
+    const end = start >= 0 ? eagerSchema.indexOf(endMarker, start) : -1;
+    if (start < 0 || end < 0) {
+      throw new Error(`lazy additive state schema block is missing for ${tableName}`);
+    }
+    eagerSchema = `${eagerSchema.slice(0, start)}${eagerSchema.slice(end + endMarker.length)}`;
+  }
+  database.exec(eagerSchema);
+}
+
 export function repairOpenClawStateDatabaseSchema(options: OpenClawStateDatabaseOptions = {}): {
   changes: string[];
   warnings: string[];
@@ -163,7 +189,9 @@ export function repairOpenClawStateDatabaseSchema(options: OpenClawStateDatabase
           }
           // Current-schema doctor repair may normalize recognized columns or
           // table options, but it must never recreate a missing table empty.
-          assertSqliteSchemaTablesPresent(db, pathname, OPENCLAW_STATE_SCHEMA_SQL);
+          assertSqliteSchemaTablesPresent(db, pathname, OPENCLAW_STATE_SCHEMA_SQL, {
+            allowedMissingTables: LAZY_ADDITIVE_STATE_TABLES,
+          });
         }
         if (rebuiltIndexNames.size === 0) {
           assertSqliteIntegrity(db, pathname);
@@ -189,7 +217,9 @@ export function repairOpenClawStateDatabaseSchema(options: OpenClawStateDatabase
         assertCanonicalStateSchemaShape(db, pathname);
         if (tableExists(db, "audit_events")) {
           ensureAdditiveStateColumns(db);
-          db.exec(OPENCLAW_STATE_SCHEMA_SQL);
+          executeCanonicalStateSchema(db, {
+            includeLazyAdditiveTables: previousVersion !== OPENCLAW_STATE_SCHEMA_VERSION,
+          });
           if (previousVersion < OPENCLAW_STATE_STRICT_SCHEMA_VERSION) {
             repairLegacyGatewayRestartHandoffsForStrictMigration(db);
           }
@@ -279,7 +309,9 @@ function ensureSchema(db: DatabaseSync, pathname: string): void {
         ensureAdditiveStateColumns(db);
         sessionWatchMigration.migrateSessionWatchCursorProvenance(db);
         assertCanonicalStateSchemaShape(db, pathname);
-        db.exec(OPENCLAW_STATE_SCHEMA_SQL);
+        executeCanonicalStateSchema(db, {
+          includeLazyAdditiveTables: previousVersion !== OPENCLAW_STATE_SCHEMA_VERSION,
+        });
         migrateLegacyCronRunLogsToTaskRuns(db);
         if (previousVersion < OPENCLAW_STATE_STRICT_SCHEMA_VERSION) {
           repairLegacyGatewayRestartHandoffsForStrictMigration(db);