test(sqlite): harden compaction and restore reliability proof (#105811)

* test(sqlite): chain compaction and restore reliability proof

* test(sqlite): bound reliability stress storage

* test(sqlite): monitor transient WAL growth

scripts/bench-sqlite-reliability.ts

@@ -81,7 +81,7 @@ Usage:
 Options:
   --profile <smoke|default|large>  Stress profile (default: default)
   --agent <id>                     Stress one per-agent database (default: global)
-  --state-dir <path>               Reuse a state directory instead of a temp dir
+  --state-dir <path>               Reuse a state directory and retain proof artifacts
   --repository <path>              Snapshot repository path
   --output <path>                  Write machine-readable JSON report
   --help                           Show this text
@@ -95,6 +95,9 @@ function printProofLines(report: ReliabilityReport): void {
   console.log(`SQLITE_RELIABILITY_ARCH=${report.arch}`);
   console.log(`SQLITE_RELIABILITY_ITERATIONS=${report.iterations}`);
   console.log(`SQLITE_RELIABILITY_RETAINED_BATCHES=${report.retainedBatches}`);
+  console.log(
+    `SQLITE_RELIABILITY_CONCURRENT_RESTORES_VERIFIED=${report.concurrentRestoresVerified}`,
+  );
   console.log(`SQLITE_RELIABILITY_RESTORES_VERIFIED=${report.restoresVerified}`);
   console.log(`SQLITE_RELIABILITY_WRITER_ROWS=${report.writer.rowsCommitted}`);
   console.log(
@@ -104,6 +107,18 @@ function printProofLines(report: ReliabilityReport): void {
   console.log(`SQLITE_RELIABILITY_SNAPSHOT_P95_MS=${report.timingsMs.snapshotP95.toFixed(3)}`);
   console.log(`SQLITE_RELIABILITY_RESTORE_P95_MS=${report.timingsMs.restoreP95.toFixed(3)}`);
   console.log(`SQLITE_RELIABILITY_SNAPSHOT_BYTES_MAX=${report.snapshotBytes.max}`);
+  console.log(
+    `SQLITE_RELIABILITY_COMPACT_RECLAIMED_BYTES=${report.maintenanceProof.compaction.reclaimedBytes}`,
+  );
+  console.log(
+    `SQLITE_RELIABILITY_POST_COMPACT_RESTORE=${report.maintenanceProof.postCompact.restoreVerified ? "verified" : "missing"}`,
+  );
+  console.log(`SQLITE_RELIABILITY_FINAL_ROWS=${report.maintenanceProof.postCompact.state.rows}`);
+  console.log(
+    `SQLITE_RELIABILITY_FINAL_STATE_SHA256=${report.maintenanceProof.postCompact.state.sha256}`,
+  );
+  console.log(`SQLITE_RELIABILITY_WAL_PEAK_BYTES=${report.walBytes.peak}`);
+  console.log(`SQLITE_RELIABILITY_WAL_LIMIT_BYTES=${report.walBytes.limit}`);
 }
 
 async function main(argv: string[]): Promise<void> {

scripts/lib/sqlite-reliability-contract.ts

@@ -2,9 +2,11 @@ export type ProfileId = "smoke" | "default" | "large";
 
 export type ProfileConfig = {
   iterations: number;
+  maxWalBytes: number;
   payloadBytes: number;
   retainedBatches: number;
   rowsPerBatch: number;
+  walAutoCheckpointPages: number;
   writerPauseMs: number;
 };
 
@@ -16,9 +18,45 @@ export type CliOptions = {
   stateDir: string | null;
 };
 
+export type ReliabilityStateProof = {
+  batches: number;
+  rows: number;
+  sha256: string;
+};
+
 export type ReliabilityReport = {
   arch: string;
+  concurrentRestoresVerified: number;
   iterations: number;
+  maintenanceProof: {
+    bloatBytes: number;
+    compaction: {
+      autoVacuum: {
+        after: 2;
+        before: number;
+      };
+      databaseBytes: {
+        after: number;
+        before: number;
+      };
+      freelistPages: {
+        after: 0;
+        before: number;
+      };
+      reclaimedBytes: number;
+      walBytes: {
+        after: 0;
+        before: number;
+      };
+    };
+    postCompact: {
+      restoreMs: number;
+      restoreVerified: true;
+      snapshotBytes: number;
+      snapshotMs: number;
+      state: ReliabilityStateProof;
+    };
+  };
   node: string;
   paths: {
     repository: string;
@@ -52,6 +90,8 @@ export type ReliabilityReport = {
   walBytes: {
     after: number;
     before: number;
+    limit: number;
+    peak: number;
   };
   writer: {
     batchesCommitted: number;
@@ -62,23 +102,29 @@ export type ReliabilityReport = {
 export const PROFILES: Record<ProfileId, ProfileConfig> = {
   smoke: {
     iterations: 4,
+    maxWalBytes: 64 * 1024 * 1024,
     payloadBytes: 512,
     retainedBatches: 32,
     rowsPerBatch: 8,
+    walAutoCheckpointPages: 256,
     writerPauseMs: 5,
   },
   default: {
     iterations: 25,
+    maxWalBytes: 512 * 1024 * 1024,
     payloadBytes: 4 * 1024,
     retainedBatches: 128,
     rowsPerBatch: 32,
+    walAutoCheckpointPages: 4 * 1024,
     writerPauseMs: 5,
   },
   large: {
     iterations: 100,
+    maxWalBytes: 8 * 1024 * 1024 * 1024,
     payloadBytes: 8 * 1024,
     retainedBatches: 256,
     rowsPerBatch: 64,
+    walAutoCheckpointPages: 16 * 1024,
     writerPauseMs: 1,
   },
 };

scripts/lib/sqlite-reliability-runner.ts

@@ -1,8 +1,11 @@
+import { createHash } from "node:crypto";
 import { randomUUID } from "node:crypto";
 import fs from "node:fs";
 import os from "node:os";
 import path from "node:path";
 import type { DatabaseSync } from "node:sqlite";
+import { compactDoctorSessionSqliteTarget } from "../../src/commands/doctor-session-sqlite-compact.js";
+import { runDoctorStateSqliteCompact } from "../../src/commands/doctor-state-sqlite-compact.js";
 import { requireNodeSqlite } from "../../src/infra/node-sqlite.js";
 import { createLocalSqliteSnapshotProvider } from "../../src/snapshot/local-repository.js";
 import type { SnapshotDatabaseIdentity } from "../../src/snapshot/snapshot-provider.js";
@@ -22,7 +25,9 @@ import {
   STRESS_TABLE_SQL,
   type CliOptions,
   type ReliabilityReport,
+  type ReliabilityStateProof,
 } from "./sqlite-reliability-contract.js";
+import { monitorSqliteWalDuring } from "./sqlite-reliability-wal-monitor.js";
 import {
   startWriter,
   stopWriter,
@@ -43,6 +48,11 @@ type IterationMetric = {
   snapshotMs: number;
 };
 
+type CompactionProof = ReliabilityReport["maintenanceProof"]["compaction"];
+
+const COMPACTION_BLOAT_ROWS = 512;
+const COMPACTION_BLOAT_PAYLOAD_BYTES = 16 * 1024;
+
 function nowMs(): number {
   return Number(process.hrtime.bigint()) / 1e6;
 }
@@ -93,6 +103,7 @@ function setupStressTable(databasePath: string): void {
     database.exec("PRAGMA journal_mode = WAL;");
     database.exec("PRAGMA busy_timeout = 30000;");
     database.exec(STRESS_TABLE_SQL);
+    database.exec("DROP TABLE IF EXISTS openclaw_reliability_compaction_bloat;");
     database.prepare("DELETE FROM openclaw_reliability_entries").run();
     database.prepare("DELETE FROM openclaw_reliability_sentinel").run();
   } finally {
@@ -119,12 +130,81 @@ function assertPragmaOk(database: DatabaseSync, pragma: "integrity_check" | "qui
   }
 }
 
+function sqliteSafeInteger(value: unknown, label: string): number {
+  const numberValue = typeof value === "bigint" ? Number(value) : value;
+  if (typeof numberValue !== "number" || !Number.isSafeInteger(numberValue) || numberValue < 0) {
+    throw new Error(`${label} is not a non-negative safe integer: ${String(value)}`);
+  }
+  return numberValue;
+}
+
+function readReliabilityState(database: DatabaseSync, rowsPerBatch: number): ReliabilityStateProof {
+  const partial = database
+    .prepare(
+      `SELECT batch, COUNT(*) AS row_count
+         FROM openclaw_reliability_entries
+        GROUP BY batch
+       HAVING COUNT(*) <> ?
+        LIMIT 1`,
+    )
+    .get(rowsPerBatch) as { batch?: unknown; row_count?: unknown } | undefined;
+  if (partial) {
+    throw new Error(
+      `partial transaction visible: batch=${String(partial.batch)} rows=${String(partial.row_count)}`,
+    );
+  }
+
+  const hash = createHash("sha256");
+  const batches = new Set<number>();
+  let rows = 0;
+  const entries = database
+    .prepare(
+      `SELECT batch, ordinal, payload
+         FROM openclaw_reliability_entries
+        ORDER BY batch, ordinal`,
+    )
+    .iterate() as Iterable<{ batch?: unknown; ordinal?: unknown; payload?: unknown }>;
+  for (const entry of entries) {
+    const batch = sqliteSafeInteger(entry.batch, "reliability batch");
+    const ordinal = sqliteSafeInteger(entry.ordinal, "reliability ordinal");
+    if (typeof entry.payload !== "string") {
+      throw new Error(`reliability payload is not text for batch=${batch} ordinal=${ordinal}`);
+    }
+    hash.update(JSON.stringify([batch, ordinal, entry.payload]));
+    hash.update("\n");
+    batches.add(batch);
+    rows += 1;
+  }
+  return {
+    batches: batches.size,
+    rows,
+    sha256: hash.digest("hex"),
+  };
+}
+
+function assertSameReliabilityState(
+  actual: ReliabilityStateProof,
+  expected: ReliabilityStateProof,
+  label: string,
+): void {
+  if (
+    actual.batches !== expected.batches ||
+    actual.rows !== expected.rows ||
+    actual.sha256 !== expected.sha256
+  ) {
+    throw new Error(
+      `${label} changed reliability state: expected batches=${expected.batches} rows=${expected.rows} sha256=${expected.sha256}, got batches=${actual.batches} rows=${actual.rows} sha256=${actual.sha256}`,
+    );
+  }
+}
+
 function verifyRestoredDatabase(params: {
+  expectedState?: ReliabilityStateProof;
   identity: SnapshotDatabaseIdentity;
   path: string;
   rowsPerBatch: number;
   uncommittedBatch: number | null;
-}): void {
+}): ReliabilityStateProof {
   const { DatabaseSync } = requireNodeSqlite();
   const database = new DatabaseSync(params.path, { readOnly: true });
   try {
@@ -149,20 +229,7 @@ function verifyRestoredDatabase(params: {
     if (sentinel?.payload !== COMMITTED_WAL_SENTINEL) {
       throw new Error("committed WAL sentinel is missing after restore");
     }
-    const partial = database
-      .prepare(
-        `SELECT batch, COUNT(*) AS row_count
-           FROM openclaw_reliability_entries
-          GROUP BY batch
-         HAVING COUNT(*) <> ?
-          LIMIT 1`,
-      )
-      .get(params.rowsPerBatch) as { batch?: unknown; row_count?: unknown } | undefined;
-    if (partial) {
-      throw new Error(
-        `partial transaction visible after restore: batch=${String(partial.batch)} rows=${String(partial.row_count)}`,
-      );
-    }
+    const state = readReliabilityState(database, params.rowsPerBatch);
     if (params.uncommittedBatch !== null) {
       const held = database
         .prepare("SELECT COUNT(*) AS rows FROM openclaw_reliability_entries WHERE batch = ?")
@@ -173,12 +240,216 @@ function verifyRestoredDatabase(params: {
         );
       }
     }
+    if (params.expectedState) {
+      assertSameReliabilityState(state, params.expectedState, params.path);
+    }
+    return state;
   } finally {
     database.close();
   }
 }
 
+function createCompactionBloat(databasePath: string): number {
+  const { DatabaseSync } = requireNodeSqlite();
+  const database = new DatabaseSync(databasePath);
+  const payload = "b".repeat(COMPACTION_BLOAT_PAYLOAD_BYTES);
+  try {
+    database.exec("PRAGMA journal_mode = WAL;");
+    database.exec("PRAGMA wal_autocheckpoint = 0;");
+    database.exec("PRAGMA busy_timeout = 30000;");
+    database.exec(`
+      DROP TABLE IF EXISTS openclaw_reliability_compaction_bloat;
+      CREATE TABLE openclaw_reliability_compaction_bloat (
+        id INTEGER PRIMARY KEY,
+        payload TEXT NOT NULL
+      );
+      BEGIN IMMEDIATE;
+    `);
+    const insert = database.prepare(
+      "INSERT INTO openclaw_reliability_compaction_bloat (id, payload) VALUES (?, ?)",
+    );
+    try {
+      for (let id = 1; id <= COMPACTION_BLOAT_ROWS; id += 1) {
+        insert.run(id, payload);
+      }
+      database.exec("COMMIT;");
+    } catch (error) {
+      database.exec("ROLLBACK;");
+      throw error;
+    }
+    database.exec("DELETE FROM openclaw_reliability_compaction_bloat;");
+    return COMPACTION_BLOAT_ROWS * COMPACTION_BLOAT_PAYLOAD_BYTES;
+  } finally {
+    database.close();
+  }
+}
+
+function readAutoVacuum(databasePath: string): number {
+  const { DatabaseSync } = requireNodeSqlite();
+  const database = new DatabaseSync(databasePath, { readOnly: true });
+  try {
+    const row = database.prepare("PRAGMA auto_vacuum;").get() as
+      | Record<string, unknown>
+      | undefined;
+    return sqliteSafeInteger(
+      row?.auto_vacuum ?? (row ? Object.values(row)[0] : undefined),
+      "auto_vacuum",
+    );
+  } finally {
+    database.close();
+  }
+}
+
+function assertCompactionProof(proof: {
+  autoVacuumAfter: number;
+  autoVacuumBefore: number;
+  databaseBytesAfter: number;
+  databaseBytesBefore: number;
+  freelistPagesAfter: number;
+  freelistPagesBefore: number;
+  reclaimedBytes: number;
+  walBytesAfter: number;
+  walBytesBefore: number;
+}): CompactionProof {
+  if (proof.autoVacuumAfter !== 2) {
+    throw new Error(`compaction did not enable incremental auto_vacuum: ${proof.autoVacuumAfter}`);
+  }
+  if (proof.freelistPagesBefore <= 0 || proof.freelistPagesAfter !== 0) {
+    throw new Error(
+      `compaction did not clear the freelist: before=${proof.freelistPagesBefore} after=${proof.freelistPagesAfter}`,
+    );
+  }
+  if (proof.walBytesAfter !== 0) {
+    throw new Error(`compaction left a non-empty WAL: ${proof.walBytesAfter} bytes`);
+  }
+  if (proof.reclaimedBytes <= 0 || proof.databaseBytesAfter >= proof.databaseBytesBefore) {
+    throw new Error(
+      `compaction did not reclaim file bytes: before=${proof.databaseBytesBefore} after=${proof.databaseBytesAfter} reclaimed=${proof.reclaimedBytes}`,
+    );
+  }
+  return {
+    autoVacuum: {
+      after: 2,
+      before: proof.autoVacuumBefore,
+    },
+    databaseBytes: {
+      after: proof.databaseBytesAfter,
+      before: proof.databaseBytesBefore,
+    },
+    freelistPages: {
+      after: 0,
+      before: proof.freelistPagesBefore,
+    },
+    reclaimedBytes: proof.reclaimedBytes,
+    walBytes: {
+      after: 0,
+      before: proof.walBytesBefore,
+    },
+  };
+}
+
+function compactTargetDatabase(target: TargetDatabase, env: NodeJS.ProcessEnv): CompactionProof {
+  if (target.identity.role === "global") {
+    const report = runDoctorStateSqliteCompact({ env });
+    if (report.skipped) {
+      throw new Error(`global compaction unexpectedly skipped ${target.path}`);
+    }
+    return assertCompactionProof({
+      autoVacuumAfter: report.after.autoVacuum,
+      autoVacuumBefore: report.before.autoVacuum,
+      databaseBytesAfter: report.after.dbSizeBytes,
+      databaseBytesBefore: report.before.dbSizeBytes,
+      freelistPagesAfter: report.after.freelistPages,
+      freelistPagesBefore: report.before.freelistPages,
+      reclaimedBytes: report.reclaimedBytes,
+      walBytesAfter: report.after.walSizeBytes,
+      walBytesBefore: report.before.walSizeBytes,
+    });
+  }
+  if (target.identity.role !== "agent") {
+    throw new Error(`unsupported reliability target role: ${target.identity.role}`);
+  }
+  const autoVacuumBefore = readAutoVacuum(target.path);
+  const report = compactDoctorSessionSqliteTarget({
+    agentId: target.identity.agentId,
+    storePath: target.path,
+  });
+  if (report.skipped) {
+    throw new Error(`agent compaction unexpectedly skipped ${target.path}`);
+  }
+  return assertCompactionProof({
+    autoVacuumAfter: readAutoVacuum(target.path),
+    autoVacuumBefore,
+    databaseBytesAfter: report.dbSizeAfterBytes,
+    databaseBytesBefore: report.dbSizeBeforeBytes,
+    freelistPagesAfter: report.freelistAfterPages,
+    freelistPagesBefore: report.freelistBeforePages,
+    reclaimedBytes: report.reclaimedBytes,
+    walBytesAfter: report.walSizeAfterBytes,
+    walBytesBefore: report.walSizeBeforeBytes,
+  });
+}
+
+async function runMaintenanceRoundTrip(params: {
+  env: NodeJS.ProcessEnv;
+  repositoryProvider: ReturnType<typeof createLocalSqliteSnapshotProvider>;
+  restoreRoot: string;
+  rowsPerBatch: number;
+  syncedProvider: ReturnType<typeof createLocalSqliteSnapshotProvider>;
+  syncedRepository: string;
+  target: TargetDatabase;
+}): Promise<ReliabilityReport["maintenanceProof"]> {
+  const bloatBytes = createCompactionBloat(params.target.path);
+  const expectedState = verifyRestoredDatabase({
+    identity: params.target.identity,
+    path: params.target.path,
+    rowsPerBatch: params.rowsPerBatch,
+    uncommittedBatch: null,
+  });
+  const compaction = compactTargetDatabase(params.target, params.env);
+  verifyRestoredDatabase({
+    expectedState,
+    identity: params.target.identity,
+    path: params.target.path,
+    rowsPerBatch: params.rowsPerBatch,
+    uncommittedBatch: null,
+  });
+
+  const snapshotStarted = nowMs();
+  const snapshot = await params.repositoryProvider.create({
+    identity: params.target.identity,
+    path: params.target.path,
+  });
+  const snapshotMs = nowMs() - snapshotStarted;
+  const copiedPath = copySnapshotDirectory(snapshot.ref.path, params.syncedRepository);
+  const copiedRef = { path: copiedPath };
+  await params.syncedProvider.verify(copiedRef);
+  const restorePath = path.join(params.restoreRoot, "post-compact.sqlite");
+  const restoreStarted = nowMs();
+  await params.syncedProvider.restoreFresh(copiedRef, restorePath);
+  const restoreMs = nowMs() - restoreStarted;
+  const state = verifyRestoredDatabase({
+    expectedState,
+    identity: params.target.identity,
+    path: restorePath,
+    rowsPerBatch: params.rowsPerBatch,
+    uncommittedBatch: null,
+  });
+  return {
+    bloatBytes,
+    compaction,
+    postCompact: {
+      restoreMs: Number(restoreMs.toFixed(3)),
+      restoreVerified: true,
+      snapshotBytes: snapshot.manifest.artifact.sizeBytes,
+      snapshotMs: Number(snapshotMs.toFixed(3)),
+      state,
+    },
+  };
+}
+
 async function runSnapshotIteration(params: {
+  cleanupArtifacts: boolean;
   iteration: number;
   repositoryProvider: ReturnType<typeof createLocalSqliteSnapshotProvider>;
   restoreRoot: string;
@@ -207,6 +478,11 @@ async function runSnapshotIteration(params: {
     rowsPerBatch: params.rowsPerBatch,
     uncommittedBatch: params.uncommittedBatch,
   });
+  if (params.cleanupArtifacts) {
+    fs.rmSync(snapshot.ref.path, { force: true, recursive: true });
+    fs.rmSync(copiedPath, { force: true, recursive: true });
+    fs.rmSync(restorePath, { force: true });
+  }
   return {
     restoreMs: Number(restoreMs.toFixed(3)),
     snapshotBytes: snapshot.manifest.artifact.sizeBytes,
@@ -217,6 +493,7 @@ async function runSnapshotIteration(params: {
 export async function runReliabilityStress(options: CliOptions): Promise<ReliabilityReport> {
   const profile = PROFILES[options.profile];
   const ownsStateDir = options.stateDir === null;
+  const cleanupIterationArtifacts = ownsStateDir && options.repository === null;
   const stateDir =
     options.stateDir ?? fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-sqlite-reliability-"));
   const repository = options.repository ?? path.join(stateDir, "snapshots");
@@ -242,23 +519,37 @@ export async function runReliabilityStress(options: CliOptions): Promise<Reliabi
     writer = startWriter(target.path, profile);
     await waitForWriterMessage(writer, "ready");
     const walBytesBefore = fileSize(`${target.path}-wal`);
+    let peakWalBytes = walBytesBefore;
     const partial = await waitForWriterMessage(writer, "partial", () => {
       writer?.child.send?.({ kind: "hold-partial" });
     });
     const metrics: IterationMetric[] = [];
     for (let iteration = 0; iteration < profile.iterations; iteration += 1) {
-      metrics.push(
-        await runSnapshotIteration({
-          iteration,
-          repositoryProvider,
-          restoreRoot,
-          rowsPerBatch: profile.rowsPerBatch,
-          syncedProvider,
-          syncedRepository,
-          target,
-          uncommittedBatch: iteration === 0 ? partial.batch : null,
-        }),
-      );
+      const iterationProof = await monitorSqliteWalDuring({
+        maxWalBytes: profile.maxWalBytes,
+        onLimitExceeded: () => {
+          try {
+            writer?.child.send?.({ kind: "stop" }, () => undefined);
+          } catch {
+            // The operation still fails on the recorded peak if the writer exited first.
+          }
+        },
+        operation: async () =>
+          await runSnapshotIteration({
+            cleanupArtifacts: cleanupIterationArtifacts,
+            iteration,
+            repositoryProvider,
+            restoreRoot,
+            rowsPerBatch: profile.rowsPerBatch,
+            syncedProvider,
+            syncedRepository,
+            target,
+            uncommittedBatch: iteration === 0 ? partial.batch : null,
+          }),
+        walPath: `${target.path}-wal`,
+      });
+      metrics.push(iterationProof.result);
+      peakWalBytes = Math.max(peakWalBytes, iterationProof.peakWalBytes);
       if (iteration === 0) {
         await waitForWriterMessage(writer, "released", () => {
           writer?.child.send?.({ action: "rollback", kind: "release-partial" });
@@ -266,10 +557,21 @@ export async function runReliabilityStress(options: CliOptions): Promise<Reliabi
       }
     }
     const writerResult = await stopWriter(writer);
+    const maintenanceProof = await runMaintenanceRoundTrip({
+      env,
+      repositoryProvider,
+      restoreRoot,
+      rowsPerBatch: profile.rowsPerBatch,
+      syncedProvider,
+      syncedRepository,
+      target,
+    });
     const snapshotBytes = metrics.map((metric) => metric.snapshotBytes);
     return {
       arch: process.arch,
+      concurrentRestoresVerified: metrics.length,
       iterations: profile.iterations,
+      maintenanceProof,
       node: process.version,
       paths: {
         repository,
@@ -280,7 +582,7 @@ export async function runReliabilityStress(options: CliOptions): Promise<Reliabi
       platform: process.platform,
       profile: options.profile,
       retainedBatches: profile.retainedBatches,
-      restoresVerified: metrics.length,
+      restoresVerified: metrics.length + 1,
       rowsPerBatch: profile.rowsPerBatch,
       snapshotBytes: {
         max: Math.max(...snapshotBytes),
@@ -315,6 +617,8 @@ export async function runReliabilityStress(options: CliOptions): Promise<Reliabi
       walBytes: {
         after: fileSize(`${target.path}-wal`),
         before: walBytesBefore,
+        limit: profile.maxWalBytes,
+        peak: peakWalBytes,
       },
       writer: {
         batchesCommitted: writerResult.batchesCommitted,

scripts/lib/sqlite-reliability-wal-monitor.ts

@@ -0,0 +1,43 @@
+import fs from "node:fs";
+
+const DEFAULT_POLL_INTERVAL_MS = 25;
+
+function fileSize(pathname: string): number {
+  try {
+    return fs.statSync(pathname).size;
+  } catch {
+    return 0;
+  }
+}
+
+export async function monitorSqliteWalDuring<T>(params: {
+  maxWalBytes: number;
+  onLimitExceeded: () => void;
+  operation: () => Promise<T>;
+  pollIntervalMs?: number;
+  walPath: string;
+}): Promise<{ peakWalBytes: number; result: T }> {
+  let peakWalBytes = fileSize(params.walPath);
+  let limitExceeded = false;
+  const sample = () => {
+    const currentWalBytes = fileSize(params.walPath);
+    peakWalBytes = Math.max(peakWalBytes, currentWalBytes);
+    if (!limitExceeded && currentWalBytes > params.maxWalBytes) {
+      limitExceeded = true;
+      params.onLimitExceeded();
+    }
+  };
+  const timer = setInterval(sample, params.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS);
+  try {
+    const result = await params.operation();
+    sample();
+    if (limitExceeded) {
+      throw new Error(
+        `SQLite reliability WAL exceeded the ${params.maxWalBytes}-byte profile limit: ${peakWalBytes} bytes`,
+      );
+    }
+    return { peakWalBytes, result };
+  } finally {
+    clearInterval(timer);
+  }
+}

scripts/lib/sqlite-reliability-writer.ts

@@ -57,6 +57,8 @@ export function startWriter(databasePath: string, profile: ProfileConfig): Write
       String(profile.rowsPerBatch),
       String(profile.payloadBytes),
       String(profile.retainedBatches),
+      String(profile.walAutoCheckpointPages),
+      String(profile.maxWalBytes),
       String(profile.writerPauseMs),
     ],
     {
@@ -181,12 +183,25 @@ function parseWriterChildArgs(argv: string[]): {
   payloadBytes: number;
   retainedBatches: number;
   rowsPerBatch: number;
+  walAutoCheckpointPages: number;
+  walSizeLimitBytes: number;
   writerPauseMs: number;
 } {
-  const [databasePath, rowsRaw, payloadRaw, retainedRaw, pauseRaw, ...extra] = argv;
+  const [
+    databasePath,
+    rowsRaw,
+    payloadRaw,
+    retainedRaw,
+    checkpointRaw,
+    walSizeLimitRaw,
+    pauseRaw,
+    ...extra
+  ] = argv;
   const rowsPerBatch = Number(rowsRaw);
   const payloadBytes = Number(payloadRaw);
   const retainedBatches = Number(retainedRaw);
+  const walAutoCheckpointPages = Number(checkpointRaw);
+  const walSizeLimitBytes = Number(walSizeLimitRaw);
   const writerPauseMs = Number(pauseRaw);
   if (
     !databasePath ||
@@ -197,12 +212,24 @@ function parseWriterChildArgs(argv: string[]): {
     payloadBytes < 1 ||
     !Number.isSafeInteger(retainedBatches) ||
     retainedBatches < 1 ||
+    !Number.isSafeInteger(walAutoCheckpointPages) ||
+    walAutoCheckpointPages < 1 ||
+    !Number.isSafeInteger(walSizeLimitBytes) ||
+    walSizeLimitBytes < 1 ||
     !Number.isSafeInteger(writerPauseMs) ||
     writerPauseMs < 0
   ) {
     throw new Error("invalid SQLite reliability writer arguments");
   }
-  return { databasePath, payloadBytes, retainedBatches, rowsPerBatch, writerPauseMs };
+  return {
+    databasePath,
+    payloadBytes,
+    retainedBatches,
+    rowsPerBatch,
+    walAutoCheckpointPages,
+    walSizeLimitBytes,
+    writerPauseMs,
+  };
 }
 
 async function runWriterChild(argv: string[]): Promise<void> {
@@ -227,7 +254,8 @@ async function runWriterChild(argv: string[]): Promise<void> {
   };
   try {
     database.exec("PRAGMA journal_mode = WAL;");
-    database.exec("PRAGMA wal_autocheckpoint = 0;");
+    database.exec(`PRAGMA wal_autocheckpoint = ${options.walAutoCheckpointPages};`);
+    database.exec(`PRAGMA journal_size_limit = ${options.walSizeLimitBytes};`);
     database.exec("PRAGMA busy_timeout = 30000;");
     database.exec(STRESS_TABLE_SQL);
     const next = database

test/scripts/bench-sqlite-reliability.test.ts

@@ -5,6 +5,7 @@ import os from "node:os";
 import path from "node:path";
 import { DatabaseSync } from "node:sqlite";
 import { afterEach, describe, expect, it } from "vitest";
+import { monitorSqliteWalDuring } from "../../scripts/lib/sqlite-reliability-wal-monitor.js";
 
 const tempDirs: string[] = [];
 
@@ -96,6 +97,29 @@ afterEach(() => {
 });
 
 describe("scripts/bench-sqlite-reliability", () => {
+  it("detects a transient WAL overrun before the file shrinks", async () => {
+    const walPath = path.join(makeTempDir(), "database.sqlite-wal");
+    let stopRequests = 0;
+
+    await expect(
+      monitorSqliteWalDuring({
+        maxWalBytes: 1024,
+        onLimitExceeded: () => {
+          stopRequests += 1;
+        },
+        operation: async () => {
+          fs.writeFileSync(walPath, Buffer.alloc(2048));
+          await new Promise((resolve) => setTimeout(resolve, 25));
+          fs.truncateSync(walPath, 0);
+          return "complete";
+        },
+        pollIntervalMs: 5,
+        walPath,
+      }),
+    ).rejects.toThrow("SQLite reliability WAL exceeded the 1024-byte profile limit: 2048 bytes");
+    expect(stopRequests).toBe(1);
+  });
+
   it("rejects malformed arguments before creating state", () => {
     const unknown = runProof(["--wat"]);
     expect(unknown.status).toBe(2);
@@ -130,8 +154,27 @@ describe("scripts/bench-sqlite-reliability", () => {
     expect(firstResult.status, firstResult.stderr).toBe(0);
     expect(firstResult.stderr).toBe("");
     expect(firstResult.stdout).toContain("SQLITE_RELIABILITY_TARGET=global");
-    expect(firstResult.stdout).toContain("SQLITE_RELIABILITY_RESTORES_VERIFIED=4");
+    expect(firstResult.stdout).toContain("SQLITE_RELIABILITY_RESTORES_VERIFIED=5");
+    expect(firstResult.stdout).toContain("SQLITE_RELIABILITY_POST_COMPACT_RESTORE=verified");
     const firstReport = JSON.parse(fs.readFileSync(firstOutput, "utf8")) as {
+      concurrentRestoresVerified: number;
+      maintenanceProof: {
+        bloatBytes: number;
+        compaction: {
+          autoVacuum: { after: number };
+          freelistPages: { after: number; before: number };
+          reclaimedBytes: number;
+          walBytes: { after: number };
+        };
+        postCompact: {
+          restoreVerified: boolean;
+          state: {
+            batches: number;
+            rows: number;
+            sha256: string;
+          };
+        };
+      };
       paths: {
         sourceDatabase: string;
         syncedRepository: string;
@@ -142,15 +185,33 @@ describe("scripts/bench-sqlite-reliability", () => {
         heldRows: number;
         visibleAfterRestore: boolean;
       };
+      walBytes: {
+        limit: number;
+        peak: number;
+      };
       writer: {
         rowsCommitted: number;
       };
     };
-    expect(firstReport.restoresVerified).toBe(4);
+    expect(firstReport.concurrentRestoresVerified).toBe(4);
+    expect(firstReport.restoresVerified).toBe(5);
     expect(firstReport.transactionProof.committedWalSentinel).toBe(true);
     expect(firstReport.transactionProof.heldRows).toBeGreaterThan(0);
     expect(firstReport.transactionProof.visibleAfterRestore).toBe(false);
     expect(firstReport.writer.rowsCommitted).toBeGreaterThan(0);
+    expect(firstReport.maintenanceProof.bloatBytes).toBeGreaterThan(0);
+    expect(firstReport.maintenanceProof.compaction.autoVacuum.after).toBe(2);
+    expect(firstReport.maintenanceProof.compaction.freelistPages.before).toBeGreaterThan(0);
+    expect(firstReport.maintenanceProof.compaction.freelistPages.after).toBe(0);
+    expect(firstReport.maintenanceProof.compaction.reclaimedBytes).toBeGreaterThan(0);
+    expect(firstReport.maintenanceProof.compaction.walBytes.after).toBe(0);
+    expect(firstReport.maintenanceProof.postCompact.restoreVerified).toBe(true);
+    expect(firstReport.maintenanceProof.postCompact.state.batches).toBeGreaterThan(0);
+    expect(firstReport.maintenanceProof.postCompact.state.rows).toBeGreaterThan(0);
+    expect(firstReport.maintenanceProof.postCompact.state.sha256).toMatch(/^[a-f0-9]{64}$/);
+    expect(firstReport.walBytes.limit).toBeGreaterThan(0);
+    expect(firstReport.walBytes.peak).toBeGreaterThan(0);
+    expect(firstReport.walBytes.peak).toBeLessThanOrEqual(firstReport.walBytes.limit);
 
     const database = new DatabaseSync(firstReport.paths.sourceDatabase);
     try {
@@ -177,7 +238,7 @@ describe("scripts/bench-sqlite-reliability", () => {
       paths: { syncedRepository: string };
       restoresVerified: number;
     };
-    expect(secondReport.restoresVerified).toBe(4);
+    expect(secondReport.restoresVerified).toBe(5);
     expect(secondReport.paths.syncedRepository).not.toBe(firstReport.paths.syncedRepository);
   });
 
@@ -185,7 +246,7 @@ describe("scripts/bench-sqlite-reliability", () => {
     const databasePath = path.join(makeTempDir(), "writer.sqlite");
     const child = fork(
       path.resolve("scripts/lib/sqlite-reliability-writer.ts"),
-      [databasePath, "8", "64", "4", "1"],
+      [databasePath, "8", "64", "4", "256", String(64 * 1024 * 1024), "1"],
       {
         cwd: process.cwd(),
         execArgv: ["--import", "tsx"],