From c76c6cd8a79b20c97919fec99fc4eb8fce617235 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Thu, 13 Aug 2026 00:16:44 +0800 Subject: [PATCH] fix(ci): stabilize sqlite vacuum interruption proof (#122700) --- scripts/lib/sqlite-reliability-runner.ts | 108 ++++++++++++------ test/scripts/bench-sqlite-reliability.test.ts | 23 +++- 2 files changed, 92 insertions(+), 39 deletions(-) diff --git a/scripts/lib/sqlite-reliability-runner.ts b/scripts/lib/sqlite-reliability-runner.ts index 1413ec2705d9..b7b5a4f2a5d0 100644 --- a/scripts/lib/sqlite-reliability-runner.ts +++ b/scripts/lib/sqlite-reliability-runner.ts @@ -61,6 +61,7 @@ type CompactionProof = ReliabilityReport["maintenanceProof"]["compaction"]; // an arbitrarily large payload through every repository and restore crash phase. const COMPACTION_BLOAT_ROWS = 12; const COMPACTION_BLOAT_PAYLOAD_BYTES = 256 * 1024; +const VACUUM_BLOAT_ROWS = 64; function nowMs(): number { return Number(process.hrtime.bigint()) / 1e6; @@ -275,26 +276,33 @@ function verifyRestoredDatabase(params: { } } -function createCompactionBloat(databasePath: string): number { +function writeCompactionBloatRange( + databasePath: string, + firstId: number, + lastId: number, + reset = false, +): void { const database = openNodeSqliteDatabase(databasePath); const payload = "b".repeat(COMPACTION_BLOAT_PAYLOAD_BYTES); try { database.exec("PRAGMA journal_mode = WAL;"); database.exec("PRAGMA wal_autocheckpoint = 0;"); database.exec("PRAGMA busy_timeout = 30000;"); - database.exec(` - DROP TABLE IF EXISTS openclaw_reliability_compaction_bloat; - CREATE TABLE openclaw_reliability_compaction_bloat ( - id INTEGER PRIMARY KEY, - payload TEXT NOT NULL - ); - BEGIN IMMEDIATE; - `); + if (reset) { + database.exec(` + DROP TABLE IF EXISTS openclaw_reliability_compaction_bloat; + CREATE TABLE openclaw_reliability_compaction_bloat ( + id INTEGER PRIMARY KEY, + payload TEXT NOT NULL + ); + `); + } + database.exec("BEGIN IMMEDIATE;"); const insert = database.prepare( "INSERT INTO openclaw_reliability_compaction_bloat (id, payload) VALUES (?, ?)", ); try { - for (let id = 1; id <= COMPACTION_BLOAT_ROWS; id += 1) { + for (let id = firstId; id <= lastId; id += 1) { insert.run(id, payload); } database.exec("COMMIT;"); @@ -303,7 +311,6 @@ function createCompactionBloat(databasePath: string): number { throw error; } database.exec("PRAGMA wal_checkpoint(TRUNCATE);"); - return COMPACTION_BLOAT_ROWS * COMPACTION_BLOAT_PAYLOAD_BYTES; } finally { database.close(); } @@ -335,13 +342,17 @@ function readCompactionPayload(databasePath: string): { } } -function deleteCompactionBloat(databasePath: string): void { +function deleteCompactionBloat(databasePath: string, retainThroughId?: number): void { const database = openNodeSqliteDatabase(databasePath); try { - database.exec(` - DELETE FROM openclaw_reliability_compaction_bloat; - PRAGMA wal_checkpoint(TRUNCATE); - `); + if (retainThroughId === undefined) { + database.exec("DELETE FROM openclaw_reliability_compaction_bloat;"); + } else { + database + .prepare("DELETE FROM openclaw_reliability_compaction_bloat WHERE id > ?") + .run(retainThroughId); + } + database.exec("PRAGMA wal_checkpoint(TRUNCATE);"); } finally { database.close(); } @@ -488,7 +499,7 @@ async function runMaintenanceRoundTrip(params: { validationRoot: string; }): Promise { const autoVacuumBeforeKill = prepareVacuumRollbackSentinel(params.target.path); - const bloatBytes = createCompactionBloat(params.target.path); + writeCompactionBloatRange(params.target.path, 1, COMPACTION_BLOAT_ROWS, true); const expectedState = verifyRestoredDatabase({ identity: params.target.identity, path: params.target.path, @@ -496,7 +507,10 @@ async function runMaintenanceRoundTrip(params: { uncommittedBatch: null, }); const expectedPayload = readCompactionPayload(params.target.path); - if (expectedPayload.rows !== COMPACTION_BLOAT_ROWS || expectedPayload.bytes !== bloatBytes) { + if ( + expectedPayload.rows !== COMPACTION_BLOAT_ROWS || + expectedPayload.bytes !== COMPACTION_BLOAT_ROWS * COMPACTION_BLOAT_PAYLOAD_BYTES + ) { throw new Error( `compaction payload setup failed: rows=${expectedPayload.rows} bytes=${expectedPayload.bytes}`, ); @@ -546,24 +560,44 @@ async function runMaintenanceRoundTrip(params: { }), }), ); - const vacuumInterruption = await runVacuumInterruptionProof({ - env: params.env, - expectedAutoVacuum: autoVacuumBeforeKill, - expectedPayload, - expectedState, - readAutoVacuum: () => readAutoVacuum(params.target.path), - readPayload: () => readCompactionPayload(params.target.path), - recoverAndVerifyDatabase: () => - verifyRestoredDatabase({ - expectedState, - identity: params.target.identity, - path: params.target.path, - readOnly: false, - rowsPerBatch: params.rowsPerBatch, - uncommittedBatch: null, - }), - target: params.target, - }); + let vacuumInterruption: ReliabilityReport["maintenanceProof"]["vacuumInterruption"]; + try { + writeCompactionBloatRange(params.target.path, COMPACTION_BLOAT_ROWS + 1, VACUUM_BLOAT_ROWS); + const vacuumExpectedPayload = readCompactionPayload(params.target.path); + if ( + vacuumExpectedPayload.rows !== VACUUM_BLOAT_ROWS || + vacuumExpectedPayload.bytes !== VACUUM_BLOAT_ROWS * COMPACTION_BLOAT_PAYLOAD_BYTES + ) { + throw new Error( + `vacuum payload setup failed: rows=${vacuumExpectedPayload.rows} bytes=${vacuumExpectedPayload.bytes}`, + ); + } + vacuumInterruption = await runVacuumInterruptionProof({ + env: params.env, + expectedAutoVacuum: autoVacuumBeforeKill, + expectedPayload: vacuumExpectedPayload, + expectedState, + readAutoVacuum: () => readAutoVacuum(params.target.path), + readPayload: () => readCompactionPayload(params.target.path), + recoverAndVerifyDatabase: () => + verifyRestoredDatabase({ + expectedState, + identity: params.target.identity, + path: params.target.path, + readOnly: false, + rowsPerBatch: params.rowsPerBatch, + uncommittedBatch: null, + }), + target: params.target, + }); + } catch (error) { + try { + deleteCompactionBloat(params.target.path, COMPACTION_BLOAT_ROWS); + } catch { + // Preserve the proof failure; it is the actionable root cause. + } + throw error; + } deleteCompactionBloat(params.target.path); const compaction = await compactTargetDatabase(params.target, params.env); verifyRestoredDatabase({ @@ -595,7 +629,7 @@ async function runMaintenanceRoundTrip(params: { uncommittedBatch: null, }); return { - bloatBytes, + bloatBytes: vacuumInterruption.payloadBeforeKill.bytes, compaction, postCompact: { restoreMs: Number(restoreMs.toFixed(3)), diff --git a/test/scripts/bench-sqlite-reliability.test.ts b/test/scripts/bench-sqlite-reliability.test.ts index 6838ae1ed9a8..c30f50052ae7 100644 --- a/test/scripts/bench-sqlite-reliability.test.ts +++ b/test/scripts/bench-sqlite-reliability.test.ts @@ -25,6 +25,9 @@ const tempDirs = useAutoCleanupTempDirTracker(afterEach); const RELIABILITY_PROOF_TIMEOUT_MS = process.platform === "win32" ? 480_000 : 240_000; const RELIABILITY_SMOKE_TEST_TIMEOUT_MS = process.platform === "win32" ? 1_200_000 : 300_000; const MIN_MULTICHUNK_RESTORE_BYTES = 2 * 1024 * 1024; +const COMPACTION_FIXTURE_ROWS = 12; +const COMPACTION_PAYLOAD_BYTES = 256 * 1024; +const VACUUM_PROOF_ROWS = 64; function reliabilitySmokeTest(name: string, test: () => void): void { it(name, test, RELIABILITY_SMOKE_TEST_TIMEOUT_MS); @@ -256,7 +259,9 @@ describe("scripts/bench-sqlite-reliability", () => { expect(firstReport.transactionProof.heldRows).toBeGreaterThan(0); expect(firstReport.transactionProof.visibleAfterRestore).toBe(false); expect(firstReport.writer.rowsCommitted).toBeGreaterThan(0); - expect(firstReport.maintenanceProof.bloatBytes).toBeGreaterThan(0); + expect(firstReport.maintenanceProof.bloatBytes).toBe( + VACUUM_PROOF_ROWS * COMPACTION_PAYLOAD_BYTES, + ); expect(firstReport.maintenanceProof.compaction.autoVacuum.after).toBe(2); expect(firstReport.maintenanceProof.compaction.freelistPages.before).toBeGreaterThan(0); expect(firstReport.maintenanceProof.compaction.freelistPages.after).toBe(0); @@ -276,6 +281,11 @@ describe("scripts/bench-sqlite-reliability", () => { expect(firstReport.maintenanceProof.vacuumInterruption.payloadAfterRecovery).toEqual( firstReport.maintenanceProof.vacuumInterruption.payloadBeforeKill, ); + expect(firstReport.maintenanceProof.vacuumInterruption.payloadBeforeKill).toEqual({ + bytes: VACUUM_PROOF_ROWS * COMPACTION_PAYLOAD_BYTES, + idSum: (VACUUM_PROOF_ROWS * (VACUUM_PROOF_ROWS + 1)) / 2, + rows: VACUUM_PROOF_ROWS, + }); expect(firstReport.maintenanceProof.vacuumInterruption.stateAfterRecovery).toEqual( firstReport.maintenanceProof.vacuumInterruption.stateBeforeKill, ); @@ -342,6 +352,11 @@ describe("scripts/bench-sqlite-reliability", () => { expect(firstReport.maintenanceProof.repositoryInterruption.pending.payload).toEqual( firstReport.maintenanceProof.repositoryInterruption.afterCommit.payload, ); + expect(firstReport.maintenanceProof.repositoryInterruption.beforePending.payload).toEqual({ + bytes: COMPACTION_FIXTURE_ROWS * COMPACTION_PAYLOAD_BYTES, + idSum: (COMPACTION_FIXTURE_ROWS * (COMPACTION_FIXTURE_ROWS + 1)) / 2, + rows: COMPACTION_FIXTURE_ROWS, + }); expect(firstReport.maintenanceProof.restoreInterruption.snapshotBytes).toBeGreaterThan( MIN_MULTICHUNK_RESTORE_BYTES, ); @@ -365,7 +380,11 @@ describe("scripts/bench-sqlite-reliability", () => { ).toEqual(firstReport.maintenanceProof.postCompact.state); expect( firstReport.maintenanceProof.restoreInterruption.beforePublish.payloadAfterRecovery, - ).toEqual(firstReport.maintenanceProof.vacuumInterruption.payloadBeforeKill); + ).toEqual({ + bytes: COMPACTION_FIXTURE_ROWS * COMPACTION_PAYLOAD_BYTES, + idSum: (COMPACTION_FIXTURE_ROWS * (COMPACTION_FIXTURE_ROWS + 1)) / 2, + rows: COMPACTION_FIXTURE_ROWS, + }); expect(firstReport.maintenanceProof.restoreInterruption.afterPublish).toMatchObject({ existingTargetPreserved: true, recoveryVerified: true,