fix: speed up doctor on large SQLite state (#115005)

* fix(sqlite): avoid repeated integrity scans

* chore: leave release notes to release flow

* docs(sqlite): clarify post-repair integrity guard

src/infra/sqlite-index-schema.test.ts

@@ -1,6 +1,9 @@
 import { DatabaseSync } from "node:sqlite";
 import { describe, expect, it } from "vitest";
-import { repairCanonicalSqliteIndexes } from "./sqlite-index-schema.js";
+import {
+  repairCanonicalSqliteIndexes,
+  verifyAndRepairCanonicalSqliteIndexes,
+} from "./sqlite-index-schema.js";
 
 const CANONICAL_SCHEMA = `
   CREATE TABLE records (
@@ -25,13 +28,50 @@ function createDatabase(): DatabaseSync {
   return db;
 }
 
+function tracePreparedSql(database: DatabaseSync): {
+  database: DatabaseSync;
+  statements: string[];
+} {
+  const statements: string[] = [];
+  return {
+    database: new Proxy(database, {
+      get(target, property) {
+        if (property === "prepare") {
+          return (sql: string) => {
+            statements.push(sql);
+            return target.prepare(sql);
+          };
+        }
+        const value = Reflect.get(target, property, target) as unknown;
+        return typeof value === "function" ? value.bind(target) : value;
+      },
+    }) as DatabaseSync,
+    statements,
+  };
+}
+
 describe("repairCanonicalSqliteIndexes", () => {
+  it("runs one whole-file integrity check for healthy indexes", () => {
+    const db = createDatabase();
+    try {
+      const traced = tracePreparedSql(db);
+
+      verifyAndRepairCanonicalSqliteIndexes(traced.database, "test database", CANONICAL_SCHEMA);
+
+      expect(traced.statements.filter((sql) => sql.startsWith("PRAGMA integrity_check"))).toEqual([
+        "PRAGMA integrity_check;",
+      ]);
+    } finally {
+      db.close();
+    }
+  });
+
   it("does not rewrite an already canonical index", () => {
     const db = createDatabase();
     try {
       const before = db.prepare("PRAGMA schema_version").get();
 
-      repairCanonicalSqliteIndexes(db, "test database", CANONICAL_SCHEMA);
+      verifyAndRepairCanonicalSqliteIndexes(db, "test database", CANONICAL_SCHEMA);
 
       expect(db.prepare("PRAGMA schema_version").get()).toEqual(before);
     } finally {
@@ -159,7 +199,7 @@ describe("repairCanonicalSqliteIndexes", () => {
           .all(),
       ).toEqual([]);
 
-      repairCanonicalSqliteIndexes(db, "test database", CANONICAL_SCHEMA);
+      verifyAndRepairCanonicalSqliteIndexes(db, "test database", CANONICAL_SCHEMA);
 
       expect(db.prepare("PRAGMA integrity_check").get()).toEqual({ integrity_check: "ok" });
       expect(

src/infra/sqlite-index-schema.ts

@@ -19,6 +19,49 @@ type SqliteIndexListRow = {
   unique: number;
 };
 
+type RepairCanonicalSqliteIndexesOptions = {
+  /**
+   * A recognized schema migration may add a column before recreating its
+   * canonical index. No other repair failure is deferred.
+   */
+  allowMissingColumns?: boolean;
+  /** Keep index repair atomic with the caller's whole-schema validation. */
+  validateAfterRepair?: () => void;
+  verifyPhysicalIntegrity?: boolean;
+};
+
+/**
+ * Verify the whole file once, then use table scans only to locate repairable
+ * index damage. Healthy opens must not multiply integrity work by table count.
+ */
+export function verifyAndRepairCanonicalSqliteIndexes(
+  db: DatabaseSync,
+  databaseLabel: string,
+  schemaSql: string,
+  options: Omit<RepairCanonicalSqliteIndexesOptions, "verifyPhysicalIntegrity"> = {},
+): string[] {
+  let integrityFailure: Error | undefined;
+  try {
+    assertSqliteIntegrity(db, databaseLabel);
+  } catch (error) {
+    if (!(error instanceof Error) || !isTerminalSqliteIntegrityError(error)) {
+      throw error;
+    }
+    integrityFailure = error;
+  }
+
+  const repairedIndexes = repairCanonicalSqliteIndexes(db, databaseLabel, schemaSql, {
+    ...options,
+    verifyPhysicalIntegrity: integrityFailure !== undefined,
+  });
+  // A non-empty repair result already passed table and whole-file integrity
+  // checks inside the repair savepoint, so it supersedes the initial failure.
+  if (integrityFailure && repairedIndexes.length === 0) {
+    throw integrityFailure;
+  }
+  return repairedIndexes;
+}
+
 /**
  * Restore every named index when SQLite's IF NOT EXISTS semantics preserve a
  * same-name definition or b-tree that no longer matches the committed schema.
@@ -27,16 +70,7 @@ export function repairCanonicalSqliteIndexes(
   db: DatabaseSync,
   databaseLabel: string,
   schemaSql: string,
-  options: {
-    /**
-     * A recognized schema migration may add a column before recreating its
-     * canonical index. No other repair failure is deferred.
-     */
-    allowMissingColumns?: boolean;
-    /** Keep index repair atomic with the caller's whole-schema validation. */
-    validateAfterRepair?: () => void;
-    verifyPhysicalIntegrity?: boolean;
-  } = {},
+  options: RepairCanonicalSqliteIndexesOptions = {},
 ): string[] {
   const indexes = getCanonicalSqliteNamedIndexContracts(schemaSql);
   const indexesByTable = new Map<string, CanonicalSqliteNamedIndexContract[]>();

src/state/openclaw-agent-db-schema.ts

@@ -1,7 +1,10 @@
 import type { DatabaseSync } from "node:sqlite";
 import { migrateMemoryIndexSourcesIdentity } from "../../packages/memory-host-sdk/src/host/memory-schema.js";
 import { executeSqliteQuerySync, getNodeSqliteKysely } from "../infra/kysely-sync.js";
-import { repairCanonicalSqliteIndexes } from "../infra/sqlite-index-schema.js";
+import {
+  repairCanonicalSqliteIndexes,
+  verifyAndRepairCanonicalSqliteIndexes,
+} from "../infra/sqlite-index-schema.js";
 import { assertSqliteIntegrity } from "../infra/sqlite-integrity.js";
 import { migrateSqliteSchemaToStrictInTransaction } from "../infra/sqlite-strict.js";
 import { runSqliteImmediateTransactionSync } from "../infra/sqlite-transaction.js";
@@ -470,17 +473,13 @@ export function assertAgentDatabaseIntegrityBeforeMutation(
       toVersion: OPENCLAW_AGENT_SCHEMA_VERSION,
     });
   }
-  // Named indexes are repairable; the full schema assertion below must run
-  // after this repair while still rejecting table and constraint drift.
-  const rebuiltIndexes =
-    userVersion === OPENCLAW_AGENT_SCHEMA_VERSION
-      ? repairCanonicalSqliteIndexes(database, pathname, OPENCLAW_AGENT_SCHEMA_SQL, {
-          allowMissingColumns: true,
-          validateAfterRepair: () =>
-            assertOpenClawAgentCurrentRuntimeSchema(database, { agentId, pathname }),
-        })
-      : [];
-  if (rebuiltIndexes.length === 0) {
+  if (userVersion === OPENCLAW_AGENT_SCHEMA_VERSION) {
+    verifyAndRepairCanonicalSqliteIndexes(database, pathname, OPENCLAW_AGENT_SCHEMA_SQL, {
+      allowMissingColumns: true,
+      validateAfterRepair: () =>
+        assertOpenClawAgentCurrentRuntimeSchema(database, { agentId, pathname }),
+    });
+  } else {
     // Every physical open proves the full file before schema mutation or exposure.
     assertSqliteIntegrity(database, pathname);
   }

src/state/openclaw-state-db.ts

@@ -10,7 +10,10 @@ import {
 } from "../infra/kysely-sync.js";
 import { openNodeSqliteDatabase } from "../infra/node-sqlite.js";
 import type { SqliteFileGeneration } from "../infra/sqlite-file-generation.js";
-import { repairCanonicalSqliteIndexes } from "../infra/sqlite-index-schema.js";
+import {
+  repairCanonicalSqliteIndexes,
+  verifyAndRepairCanonicalSqliteIndexes,
+} from "../infra/sqlite-index-schema.js";
 import {
   assertSqliteIntegrity,
   confirmSqliteFileIntegrity,
@@ -463,14 +466,12 @@ function assertStateDatabaseIntegrityBeforeMutation(
       toVersion: OPENCLAW_STATE_SCHEMA_VERSION,
     });
   }
-  const rebuiltIndexes =
-    userVersion === OPENCLAW_STATE_SCHEMA_VERSION
-      ? repairCanonicalSqliteIndexes(database, pathname, OPENCLAW_STATE_SCHEMA_SQL, {
-          allowMissingColumns: true,
-          validateAfterRepair: () => assertCurrentStateRuntimeSchema(database, pathname),
-        })
-      : [];
-  if (rebuiltIndexes.length === 0) {
+  if (userVersion === OPENCLAW_STATE_SCHEMA_VERSION) {
+    verifyAndRepairCanonicalSqliteIndexes(database, pathname, OPENCLAW_STATE_SCHEMA_SQL, {
+      allowMissingColumns: true,
+      validateAfterRepair: () => assertCurrentStateRuntimeSchema(database, pathname),
+    });
+  } else {
     // Every physical open proves the full file before schema mutation or exposure.
     assertSqliteIntegrity(database, pathname);
   }