From 4afcbea5c580c73ec6b51f3e6349340b678f9421 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Mon, 13 Jul 2026 09:09:21 +0800 Subject: [PATCH] test(sqlite): harden compaction and restore reliability proof (#105811) * test(sqlite): chain compaction and restore reliability proof * test(sqlite): bound reliability stress storage * test(sqlite): monitor transient WAL growth --- scripts/bench-sqlite-reliability.ts | 17 +- scripts/lib/sqlite-reliability-contract.ts | 46 +++ scripts/lib/sqlite-reliability-runner.ts | 360 ++++++++++++++++-- scripts/lib/sqlite-reliability-wal-monitor.ts | 43 +++ scripts/lib/sqlite-reliability-writer.ts | 34 +- test/scripts/bench-sqlite-reliability.test.ts | 69 +++- 6 files changed, 533 insertions(+), 36 deletions(-) create mode 100644 scripts/lib/sqlite-reliability-wal-monitor.ts diff --git a/scripts/bench-sqlite-reliability.ts b/scripts/bench-sqlite-reliability.ts index 0e094beb4342..963a2f2c686b 100644 --- a/scripts/bench-sqlite-reliability.ts +++ b/scripts/bench-sqlite-reliability.ts @@ -81,7 +81,7 @@ Usage: Options: --profile Stress profile (default: default) --agent Stress one per-agent database (default: global) - --state-dir Reuse a state directory instead of a temp dir + --state-dir Reuse a state directory and retain proof artifacts --repository Snapshot repository path --output Write machine-readable JSON report --help Show this text @@ -95,6 +95,9 @@ function printProofLines(report: ReliabilityReport): void { console.log(`SQLITE_RELIABILITY_ARCH=${report.arch}`); console.log(`SQLITE_RELIABILITY_ITERATIONS=${report.iterations}`); console.log(`SQLITE_RELIABILITY_RETAINED_BATCHES=${report.retainedBatches}`); + console.log( + `SQLITE_RELIABILITY_CONCURRENT_RESTORES_VERIFIED=${report.concurrentRestoresVerified}`, + ); console.log(`SQLITE_RELIABILITY_RESTORES_VERIFIED=${report.restoresVerified}`); console.log(`SQLITE_RELIABILITY_WRITER_ROWS=${report.writer.rowsCommitted}`); console.log( @@ -104,6 +107,18 @@ function printProofLines(report: ReliabilityReport): void { console.log(`SQLITE_RELIABILITY_SNAPSHOT_P95_MS=${report.timingsMs.snapshotP95.toFixed(3)}`); console.log(`SQLITE_RELIABILITY_RESTORE_P95_MS=${report.timingsMs.restoreP95.toFixed(3)}`); console.log(`SQLITE_RELIABILITY_SNAPSHOT_BYTES_MAX=${report.snapshotBytes.max}`); + console.log( + `SQLITE_RELIABILITY_COMPACT_RECLAIMED_BYTES=${report.maintenanceProof.compaction.reclaimedBytes}`, + ); + console.log( + `SQLITE_RELIABILITY_POST_COMPACT_RESTORE=${report.maintenanceProof.postCompact.restoreVerified ? "verified" : "missing"}`, + ); + console.log(`SQLITE_RELIABILITY_FINAL_ROWS=${report.maintenanceProof.postCompact.state.rows}`); + console.log( + `SQLITE_RELIABILITY_FINAL_STATE_SHA256=${report.maintenanceProof.postCompact.state.sha256}`, + ); + console.log(`SQLITE_RELIABILITY_WAL_PEAK_BYTES=${report.walBytes.peak}`); + console.log(`SQLITE_RELIABILITY_WAL_LIMIT_BYTES=${report.walBytes.limit}`); } async function main(argv: string[]): Promise { diff --git a/scripts/lib/sqlite-reliability-contract.ts b/scripts/lib/sqlite-reliability-contract.ts index 10c0529cbed4..ec224330c457 100644 --- a/scripts/lib/sqlite-reliability-contract.ts +++ b/scripts/lib/sqlite-reliability-contract.ts @@ -2,9 +2,11 @@ export type ProfileId = "smoke" | "default" | "large"; export type ProfileConfig = { iterations: number; + maxWalBytes: number; payloadBytes: number; retainedBatches: number; rowsPerBatch: number; + walAutoCheckpointPages: number; writerPauseMs: number; }; @@ -16,9 +18,45 @@ export type CliOptions = { stateDir: string | null; }; +export type ReliabilityStateProof = { + batches: number; + rows: number; + sha256: string; +}; + export type ReliabilityReport = { arch: string; + concurrentRestoresVerified: number; iterations: number; + maintenanceProof: { + bloatBytes: number; + compaction: { + autoVacuum: { + after: 2; + before: number; + }; + databaseBytes: { + after: number; + before: number; + }; + freelistPages: { + after: 0; + before: number; + }; + reclaimedBytes: number; + walBytes: { + after: 0; + before: number; + }; + }; + postCompact: { + restoreMs: number; + restoreVerified: true; + snapshotBytes: number; + snapshotMs: number; + state: ReliabilityStateProof; + }; + }; node: string; paths: { repository: string; @@ -52,6 +90,8 @@ export type ReliabilityReport = { walBytes: { after: number; before: number; + limit: number; + peak: number; }; writer: { batchesCommitted: number; @@ -62,23 +102,29 @@ export type ReliabilityReport = { export const PROFILES: Record = { smoke: { iterations: 4, + maxWalBytes: 64 * 1024 * 1024, payloadBytes: 512, retainedBatches: 32, rowsPerBatch: 8, + walAutoCheckpointPages: 256, writerPauseMs: 5, }, default: { iterations: 25, + maxWalBytes: 512 * 1024 * 1024, payloadBytes: 4 * 1024, retainedBatches: 128, rowsPerBatch: 32, + walAutoCheckpointPages: 4 * 1024, writerPauseMs: 5, }, large: { iterations: 100, + maxWalBytes: 8 * 1024 * 1024 * 1024, payloadBytes: 8 * 1024, retainedBatches: 256, rowsPerBatch: 64, + walAutoCheckpointPages: 16 * 1024, writerPauseMs: 1, }, }; diff --git a/scripts/lib/sqlite-reliability-runner.ts b/scripts/lib/sqlite-reliability-runner.ts index 1619948f1610..bef78d0dcedb 100644 --- a/scripts/lib/sqlite-reliability-runner.ts +++ b/scripts/lib/sqlite-reliability-runner.ts @@ -1,8 +1,11 @@ +import { createHash } from "node:crypto"; import { randomUUID } from "node:crypto"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import type { DatabaseSync } from "node:sqlite"; +import { compactDoctorSessionSqliteTarget } from "../../src/commands/doctor-session-sqlite-compact.js"; +import { runDoctorStateSqliteCompact } from "../../src/commands/doctor-state-sqlite-compact.js"; import { requireNodeSqlite } from "../../src/infra/node-sqlite.js"; import { createLocalSqliteSnapshotProvider } from "../../src/snapshot/local-repository.js"; import type { SnapshotDatabaseIdentity } from "../../src/snapshot/snapshot-provider.js"; @@ -22,7 +25,9 @@ import { STRESS_TABLE_SQL, type CliOptions, type ReliabilityReport, + type ReliabilityStateProof, } from "./sqlite-reliability-contract.js"; +import { monitorSqliteWalDuring } from "./sqlite-reliability-wal-monitor.js"; import { startWriter, stopWriter, @@ -43,6 +48,11 @@ type IterationMetric = { snapshotMs: number; }; +type CompactionProof = ReliabilityReport["maintenanceProof"]["compaction"]; + +const COMPACTION_BLOAT_ROWS = 512; +const COMPACTION_BLOAT_PAYLOAD_BYTES = 16 * 1024; + function nowMs(): number { return Number(process.hrtime.bigint()) / 1e6; } @@ -93,6 +103,7 @@ function setupStressTable(databasePath: string): void { database.exec("PRAGMA journal_mode = WAL;"); database.exec("PRAGMA busy_timeout = 30000;"); database.exec(STRESS_TABLE_SQL); + database.exec("DROP TABLE IF EXISTS openclaw_reliability_compaction_bloat;"); database.prepare("DELETE FROM openclaw_reliability_entries").run(); database.prepare("DELETE FROM openclaw_reliability_sentinel").run(); } finally { @@ -119,12 +130,81 @@ function assertPragmaOk(database: DatabaseSync, pragma: "integrity_check" | "qui } } +function sqliteSafeInteger(value: unknown, label: string): number { + const numberValue = typeof value === "bigint" ? Number(value) : value; + if (typeof numberValue !== "number" || !Number.isSafeInteger(numberValue) || numberValue < 0) { + throw new Error(`${label} is not a non-negative safe integer: ${String(value)}`); + } + return numberValue; +} + +function readReliabilityState(database: DatabaseSync, rowsPerBatch: number): ReliabilityStateProof { + const partial = database + .prepare( + `SELECT batch, COUNT(*) AS row_count + FROM openclaw_reliability_entries + GROUP BY batch + HAVING COUNT(*) <> ? + LIMIT 1`, + ) + .get(rowsPerBatch) as { batch?: unknown; row_count?: unknown } | undefined; + if (partial) { + throw new Error( + `partial transaction visible: batch=${String(partial.batch)} rows=${String(partial.row_count)}`, + ); + } + + const hash = createHash("sha256"); + const batches = new Set(); + let rows = 0; + const entries = database + .prepare( + `SELECT batch, ordinal, payload + FROM openclaw_reliability_entries + ORDER BY batch, ordinal`, + ) + .iterate() as Iterable<{ batch?: unknown; ordinal?: unknown; payload?: unknown }>; + for (const entry of entries) { + const batch = sqliteSafeInteger(entry.batch, "reliability batch"); + const ordinal = sqliteSafeInteger(entry.ordinal, "reliability ordinal"); + if (typeof entry.payload !== "string") { + throw new Error(`reliability payload is not text for batch=${batch} ordinal=${ordinal}`); + } + hash.update(JSON.stringify([batch, ordinal, entry.payload])); + hash.update("\n"); + batches.add(batch); + rows += 1; + } + return { + batches: batches.size, + rows, + sha256: hash.digest("hex"), + }; +} + +function assertSameReliabilityState( + actual: ReliabilityStateProof, + expected: ReliabilityStateProof, + label: string, +): void { + if ( + actual.batches !== expected.batches || + actual.rows !== expected.rows || + actual.sha256 !== expected.sha256 + ) { + throw new Error( + `${label} changed reliability state: expected batches=${expected.batches} rows=${expected.rows} sha256=${expected.sha256}, got batches=${actual.batches} rows=${actual.rows} sha256=${actual.sha256}`, + ); + } +} + function verifyRestoredDatabase(params: { + expectedState?: ReliabilityStateProof; identity: SnapshotDatabaseIdentity; path: string; rowsPerBatch: number; uncommittedBatch: number | null; -}): void { +}): ReliabilityStateProof { const { DatabaseSync } = requireNodeSqlite(); const database = new DatabaseSync(params.path, { readOnly: true }); try { @@ -149,20 +229,7 @@ function verifyRestoredDatabase(params: { if (sentinel?.payload !== COMMITTED_WAL_SENTINEL) { throw new Error("committed WAL sentinel is missing after restore"); } - const partial = database - .prepare( - `SELECT batch, COUNT(*) AS row_count - FROM openclaw_reliability_entries - GROUP BY batch - HAVING COUNT(*) <> ? - LIMIT 1`, - ) - .get(params.rowsPerBatch) as { batch?: unknown; row_count?: unknown } | undefined; - if (partial) { - throw new Error( - `partial transaction visible after restore: batch=${String(partial.batch)} rows=${String(partial.row_count)}`, - ); - } + const state = readReliabilityState(database, params.rowsPerBatch); if (params.uncommittedBatch !== null) { const held = database .prepare("SELECT COUNT(*) AS rows FROM openclaw_reliability_entries WHERE batch = ?") @@ -173,12 +240,216 @@ function verifyRestoredDatabase(params: { ); } } + if (params.expectedState) { + assertSameReliabilityState(state, params.expectedState, params.path); + } + return state; } finally { database.close(); } } +function createCompactionBloat(databasePath: string): number { + const { DatabaseSync } = requireNodeSqlite(); + const database = new DatabaseSync(databasePath); + const payload = "b".repeat(COMPACTION_BLOAT_PAYLOAD_BYTES); + try { + database.exec("PRAGMA journal_mode = WAL;"); + database.exec("PRAGMA wal_autocheckpoint = 0;"); + database.exec("PRAGMA busy_timeout = 30000;"); + database.exec(` + DROP TABLE IF EXISTS openclaw_reliability_compaction_bloat; + CREATE TABLE openclaw_reliability_compaction_bloat ( + id INTEGER PRIMARY KEY, + payload TEXT NOT NULL + ); + BEGIN IMMEDIATE; + `); + const insert = database.prepare( + "INSERT INTO openclaw_reliability_compaction_bloat (id, payload) VALUES (?, ?)", + ); + try { + for (let id = 1; id <= COMPACTION_BLOAT_ROWS; id += 1) { + insert.run(id, payload); + } + database.exec("COMMIT;"); + } catch (error) { + database.exec("ROLLBACK;"); + throw error; + } + database.exec("DELETE FROM openclaw_reliability_compaction_bloat;"); + return COMPACTION_BLOAT_ROWS * COMPACTION_BLOAT_PAYLOAD_BYTES; + } finally { + database.close(); + } +} + +function readAutoVacuum(databasePath: string): number { + const { DatabaseSync } = requireNodeSqlite(); + const database = new DatabaseSync(databasePath, { readOnly: true }); + try { + const row = database.prepare("PRAGMA auto_vacuum;").get() as + | Record + | undefined; + return sqliteSafeInteger( + row?.auto_vacuum ?? (row ? Object.values(row)[0] : undefined), + "auto_vacuum", + ); + } finally { + database.close(); + } +} + +function assertCompactionProof(proof: { + autoVacuumAfter: number; + autoVacuumBefore: number; + databaseBytesAfter: number; + databaseBytesBefore: number; + freelistPagesAfter: number; + freelistPagesBefore: number; + reclaimedBytes: number; + walBytesAfter: number; + walBytesBefore: number; +}): CompactionProof { + if (proof.autoVacuumAfter !== 2) { + throw new Error(`compaction did not enable incremental auto_vacuum: ${proof.autoVacuumAfter}`); + } + if (proof.freelistPagesBefore <= 0 || proof.freelistPagesAfter !== 0) { + throw new Error( + `compaction did not clear the freelist: before=${proof.freelistPagesBefore} after=${proof.freelistPagesAfter}`, + ); + } + if (proof.walBytesAfter !== 0) { + throw new Error(`compaction left a non-empty WAL: ${proof.walBytesAfter} bytes`); + } + if (proof.reclaimedBytes <= 0 || proof.databaseBytesAfter >= proof.databaseBytesBefore) { + throw new Error( + `compaction did not reclaim file bytes: before=${proof.databaseBytesBefore} after=${proof.databaseBytesAfter} reclaimed=${proof.reclaimedBytes}`, + ); + } + return { + autoVacuum: { + after: 2, + before: proof.autoVacuumBefore, + }, + databaseBytes: { + after: proof.databaseBytesAfter, + before: proof.databaseBytesBefore, + }, + freelistPages: { + after: 0, + before: proof.freelistPagesBefore, + }, + reclaimedBytes: proof.reclaimedBytes, + walBytes: { + after: 0, + before: proof.walBytesBefore, + }, + }; +} + +function compactTargetDatabase(target: TargetDatabase, env: NodeJS.ProcessEnv): CompactionProof { + if (target.identity.role === "global") { + const report = runDoctorStateSqliteCompact({ env }); + if (report.skipped) { + throw new Error(`global compaction unexpectedly skipped ${target.path}`); + } + return assertCompactionProof({ + autoVacuumAfter: report.after.autoVacuum, + autoVacuumBefore: report.before.autoVacuum, + databaseBytesAfter: report.after.dbSizeBytes, + databaseBytesBefore: report.before.dbSizeBytes, + freelistPagesAfter: report.after.freelistPages, + freelistPagesBefore: report.before.freelistPages, + reclaimedBytes: report.reclaimedBytes, + walBytesAfter: report.after.walSizeBytes, + walBytesBefore: report.before.walSizeBytes, + }); + } + if (target.identity.role !== "agent") { + throw new Error(`unsupported reliability target role: ${target.identity.role}`); + } + const autoVacuumBefore = readAutoVacuum(target.path); + const report = compactDoctorSessionSqliteTarget({ + agentId: target.identity.agentId, + storePath: target.path, + }); + if (report.skipped) { + throw new Error(`agent compaction unexpectedly skipped ${target.path}`); + } + return assertCompactionProof({ + autoVacuumAfter: readAutoVacuum(target.path), + autoVacuumBefore, + databaseBytesAfter: report.dbSizeAfterBytes, + databaseBytesBefore: report.dbSizeBeforeBytes, + freelistPagesAfter: report.freelistAfterPages, + freelistPagesBefore: report.freelistBeforePages, + reclaimedBytes: report.reclaimedBytes, + walBytesAfter: report.walSizeAfterBytes, + walBytesBefore: report.walSizeBeforeBytes, + }); +} + +async function runMaintenanceRoundTrip(params: { + env: NodeJS.ProcessEnv; + repositoryProvider: ReturnType; + restoreRoot: string; + rowsPerBatch: number; + syncedProvider: ReturnType; + syncedRepository: string; + target: TargetDatabase; +}): Promise { + const bloatBytes = createCompactionBloat(params.target.path); + const expectedState = verifyRestoredDatabase({ + identity: params.target.identity, + path: params.target.path, + rowsPerBatch: params.rowsPerBatch, + uncommittedBatch: null, + }); + const compaction = compactTargetDatabase(params.target, params.env); + verifyRestoredDatabase({ + expectedState, + identity: params.target.identity, + path: params.target.path, + rowsPerBatch: params.rowsPerBatch, + uncommittedBatch: null, + }); + + const snapshotStarted = nowMs(); + const snapshot = await params.repositoryProvider.create({ + identity: params.target.identity, + path: params.target.path, + }); + const snapshotMs = nowMs() - snapshotStarted; + const copiedPath = copySnapshotDirectory(snapshot.ref.path, params.syncedRepository); + const copiedRef = { path: copiedPath }; + await params.syncedProvider.verify(copiedRef); + const restorePath = path.join(params.restoreRoot, "post-compact.sqlite"); + const restoreStarted = nowMs(); + await params.syncedProvider.restoreFresh(copiedRef, restorePath); + const restoreMs = nowMs() - restoreStarted; + const state = verifyRestoredDatabase({ + expectedState, + identity: params.target.identity, + path: restorePath, + rowsPerBatch: params.rowsPerBatch, + uncommittedBatch: null, + }); + return { + bloatBytes, + compaction, + postCompact: { + restoreMs: Number(restoreMs.toFixed(3)), + restoreVerified: true, + snapshotBytes: snapshot.manifest.artifact.sizeBytes, + snapshotMs: Number(snapshotMs.toFixed(3)), + state, + }, + }; +} + async function runSnapshotIteration(params: { + cleanupArtifacts: boolean; iteration: number; repositoryProvider: ReturnType; restoreRoot: string; @@ -207,6 +478,11 @@ async function runSnapshotIteration(params: { rowsPerBatch: params.rowsPerBatch, uncommittedBatch: params.uncommittedBatch, }); + if (params.cleanupArtifacts) { + fs.rmSync(snapshot.ref.path, { force: true, recursive: true }); + fs.rmSync(copiedPath, { force: true, recursive: true }); + fs.rmSync(restorePath, { force: true }); + } return { restoreMs: Number(restoreMs.toFixed(3)), snapshotBytes: snapshot.manifest.artifact.sizeBytes, @@ -217,6 +493,7 @@ async function runSnapshotIteration(params: { export async function runReliabilityStress(options: CliOptions): Promise { const profile = PROFILES[options.profile]; const ownsStateDir = options.stateDir === null; + const cleanupIterationArtifacts = ownsStateDir && options.repository === null; const stateDir = options.stateDir ?? fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-sqlite-reliability-")); const repository = options.repository ?? path.join(stateDir, "snapshots"); @@ -242,23 +519,37 @@ export async function runReliabilityStress(options: CliOptions): Promise { writer?.child.send?.({ kind: "hold-partial" }); }); const metrics: IterationMetric[] = []; for (let iteration = 0; iteration < profile.iterations; iteration += 1) { - metrics.push( - await runSnapshotIteration({ - iteration, - repositoryProvider, - restoreRoot, - rowsPerBatch: profile.rowsPerBatch, - syncedProvider, - syncedRepository, - target, - uncommittedBatch: iteration === 0 ? partial.batch : null, - }), - ); + const iterationProof = await monitorSqliteWalDuring({ + maxWalBytes: profile.maxWalBytes, + onLimitExceeded: () => { + try { + writer?.child.send?.({ kind: "stop" }, () => undefined); + } catch { + // The operation still fails on the recorded peak if the writer exited first. + } + }, + operation: async () => + await runSnapshotIteration({ + cleanupArtifacts: cleanupIterationArtifacts, + iteration, + repositoryProvider, + restoreRoot, + rowsPerBatch: profile.rowsPerBatch, + syncedProvider, + syncedRepository, + target, + uncommittedBatch: iteration === 0 ? partial.batch : null, + }), + walPath: `${target.path}-wal`, + }); + metrics.push(iterationProof.result); + peakWalBytes = Math.max(peakWalBytes, iterationProof.peakWalBytes); if (iteration === 0) { await waitForWriterMessage(writer, "released", () => { writer?.child.send?.({ action: "rollback", kind: "release-partial" }); @@ -266,10 +557,21 @@ export async function runReliabilityStress(options: CliOptions): Promise metric.snapshotBytes); return { arch: process.arch, + concurrentRestoresVerified: metrics.length, iterations: profile.iterations, + maintenanceProof, node: process.version, paths: { repository, @@ -280,7 +582,7 @@ export async function runReliabilityStress(options: CliOptions): Promise(params: { + maxWalBytes: number; + onLimitExceeded: () => void; + operation: () => Promise; + pollIntervalMs?: number; + walPath: string; +}): Promise<{ peakWalBytes: number; result: T }> { + let peakWalBytes = fileSize(params.walPath); + let limitExceeded = false; + const sample = () => { + const currentWalBytes = fileSize(params.walPath); + peakWalBytes = Math.max(peakWalBytes, currentWalBytes); + if (!limitExceeded && currentWalBytes > params.maxWalBytes) { + limitExceeded = true; + params.onLimitExceeded(); + } + }; + const timer = setInterval(sample, params.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS); + try { + const result = await params.operation(); + sample(); + if (limitExceeded) { + throw new Error( + `SQLite reliability WAL exceeded the ${params.maxWalBytes}-byte profile limit: ${peakWalBytes} bytes`, + ); + } + return { peakWalBytes, result }; + } finally { + clearInterval(timer); + } +} diff --git a/scripts/lib/sqlite-reliability-writer.ts b/scripts/lib/sqlite-reliability-writer.ts index db82bea29eb5..a2ab9b840596 100644 --- a/scripts/lib/sqlite-reliability-writer.ts +++ b/scripts/lib/sqlite-reliability-writer.ts @@ -57,6 +57,8 @@ export function startWriter(databasePath: string, profile: ProfileConfig): Write String(profile.rowsPerBatch), String(profile.payloadBytes), String(profile.retainedBatches), + String(profile.walAutoCheckpointPages), + String(profile.maxWalBytes), String(profile.writerPauseMs), ], { @@ -181,12 +183,25 @@ function parseWriterChildArgs(argv: string[]): { payloadBytes: number; retainedBatches: number; rowsPerBatch: number; + walAutoCheckpointPages: number; + walSizeLimitBytes: number; writerPauseMs: number; } { - const [databasePath, rowsRaw, payloadRaw, retainedRaw, pauseRaw, ...extra] = argv; + const [ + databasePath, + rowsRaw, + payloadRaw, + retainedRaw, + checkpointRaw, + walSizeLimitRaw, + pauseRaw, + ...extra + ] = argv; const rowsPerBatch = Number(rowsRaw); const payloadBytes = Number(payloadRaw); const retainedBatches = Number(retainedRaw); + const walAutoCheckpointPages = Number(checkpointRaw); + const walSizeLimitBytes = Number(walSizeLimitRaw); const writerPauseMs = Number(pauseRaw); if ( !databasePath || @@ -197,12 +212,24 @@ function parseWriterChildArgs(argv: string[]): { payloadBytes < 1 || !Number.isSafeInteger(retainedBatches) || retainedBatches < 1 || + !Number.isSafeInteger(walAutoCheckpointPages) || + walAutoCheckpointPages < 1 || + !Number.isSafeInteger(walSizeLimitBytes) || + walSizeLimitBytes < 1 || !Number.isSafeInteger(writerPauseMs) || writerPauseMs < 0 ) { throw new Error("invalid SQLite reliability writer arguments"); } - return { databasePath, payloadBytes, retainedBatches, rowsPerBatch, writerPauseMs }; + return { + databasePath, + payloadBytes, + retainedBatches, + rowsPerBatch, + walAutoCheckpointPages, + walSizeLimitBytes, + writerPauseMs, + }; } async function runWriterChild(argv: string[]): Promise { @@ -227,7 +254,8 @@ async function runWriterChild(argv: string[]): Promise { }; try { database.exec("PRAGMA journal_mode = WAL;"); - database.exec("PRAGMA wal_autocheckpoint = 0;"); + database.exec(`PRAGMA wal_autocheckpoint = ${options.walAutoCheckpointPages};`); + database.exec(`PRAGMA journal_size_limit = ${options.walSizeLimitBytes};`); database.exec("PRAGMA busy_timeout = 30000;"); database.exec(STRESS_TABLE_SQL); const next = database diff --git a/test/scripts/bench-sqlite-reliability.test.ts b/test/scripts/bench-sqlite-reliability.test.ts index 5e77112d8d31..ee078525248c 100644 --- a/test/scripts/bench-sqlite-reliability.test.ts +++ b/test/scripts/bench-sqlite-reliability.test.ts @@ -5,6 +5,7 @@ import os from "node:os"; import path from "node:path"; import { DatabaseSync } from "node:sqlite"; import { afterEach, describe, expect, it } from "vitest"; +import { monitorSqliteWalDuring } from "../../scripts/lib/sqlite-reliability-wal-monitor.js"; const tempDirs: string[] = []; @@ -96,6 +97,29 @@ afterEach(() => { }); describe("scripts/bench-sqlite-reliability", () => { + it("detects a transient WAL overrun before the file shrinks", async () => { + const walPath = path.join(makeTempDir(), "database.sqlite-wal"); + let stopRequests = 0; + + await expect( + monitorSqliteWalDuring({ + maxWalBytes: 1024, + onLimitExceeded: () => { + stopRequests += 1; + }, + operation: async () => { + fs.writeFileSync(walPath, Buffer.alloc(2048)); + await new Promise((resolve) => setTimeout(resolve, 25)); + fs.truncateSync(walPath, 0); + return "complete"; + }, + pollIntervalMs: 5, + walPath, + }), + ).rejects.toThrow("SQLite reliability WAL exceeded the 1024-byte profile limit: 2048 bytes"); + expect(stopRequests).toBe(1); + }); + it("rejects malformed arguments before creating state", () => { const unknown = runProof(["--wat"]); expect(unknown.status).toBe(2); @@ -130,8 +154,27 @@ describe("scripts/bench-sqlite-reliability", () => { expect(firstResult.status, firstResult.stderr).toBe(0); expect(firstResult.stderr).toBe(""); expect(firstResult.stdout).toContain("SQLITE_RELIABILITY_TARGET=global"); - expect(firstResult.stdout).toContain("SQLITE_RELIABILITY_RESTORES_VERIFIED=4"); + expect(firstResult.stdout).toContain("SQLITE_RELIABILITY_RESTORES_VERIFIED=5"); + expect(firstResult.stdout).toContain("SQLITE_RELIABILITY_POST_COMPACT_RESTORE=verified"); const firstReport = JSON.parse(fs.readFileSync(firstOutput, "utf8")) as { + concurrentRestoresVerified: number; + maintenanceProof: { + bloatBytes: number; + compaction: { + autoVacuum: { after: number }; + freelistPages: { after: number; before: number }; + reclaimedBytes: number; + walBytes: { after: number }; + }; + postCompact: { + restoreVerified: boolean; + state: { + batches: number; + rows: number; + sha256: string; + }; + }; + }; paths: { sourceDatabase: string; syncedRepository: string; @@ -142,15 +185,33 @@ describe("scripts/bench-sqlite-reliability", () => { heldRows: number; visibleAfterRestore: boolean; }; + walBytes: { + limit: number; + peak: number; + }; writer: { rowsCommitted: number; }; }; - expect(firstReport.restoresVerified).toBe(4); + expect(firstReport.concurrentRestoresVerified).toBe(4); + expect(firstReport.restoresVerified).toBe(5); expect(firstReport.transactionProof.committedWalSentinel).toBe(true); expect(firstReport.transactionProof.heldRows).toBeGreaterThan(0); expect(firstReport.transactionProof.visibleAfterRestore).toBe(false); expect(firstReport.writer.rowsCommitted).toBeGreaterThan(0); + expect(firstReport.maintenanceProof.bloatBytes).toBeGreaterThan(0); + expect(firstReport.maintenanceProof.compaction.autoVacuum.after).toBe(2); + expect(firstReport.maintenanceProof.compaction.freelistPages.before).toBeGreaterThan(0); + expect(firstReport.maintenanceProof.compaction.freelistPages.after).toBe(0); + expect(firstReport.maintenanceProof.compaction.reclaimedBytes).toBeGreaterThan(0); + expect(firstReport.maintenanceProof.compaction.walBytes.after).toBe(0); + expect(firstReport.maintenanceProof.postCompact.restoreVerified).toBe(true); + expect(firstReport.maintenanceProof.postCompact.state.batches).toBeGreaterThan(0); + expect(firstReport.maintenanceProof.postCompact.state.rows).toBeGreaterThan(0); + expect(firstReport.maintenanceProof.postCompact.state.sha256).toMatch(/^[a-f0-9]{64}$/); + expect(firstReport.walBytes.limit).toBeGreaterThan(0); + expect(firstReport.walBytes.peak).toBeGreaterThan(0); + expect(firstReport.walBytes.peak).toBeLessThanOrEqual(firstReport.walBytes.limit); const database = new DatabaseSync(firstReport.paths.sourceDatabase); try { @@ -177,7 +238,7 @@ describe("scripts/bench-sqlite-reliability", () => { paths: { syncedRepository: string }; restoresVerified: number; }; - expect(secondReport.restoresVerified).toBe(4); + expect(secondReport.restoresVerified).toBe(5); expect(secondReport.paths.syncedRepository).not.toBe(firstReport.paths.syncedRepository); }); @@ -185,7 +246,7 @@ describe("scripts/bench-sqlite-reliability", () => { const databasePath = path.join(makeTempDir(), "writer.sqlite"); const child = fork( path.resolve("scripts/lib/sqlite-reliability-writer.ts"), - [databasePath, "8", "64", "4", "1"], + [databasePath, "8", "64", "4", "256", String(64 * 1024 * 1024), "1"], { cwd: process.cwd(), execArgv: ["--import", "tsx"],