From e8b6a04ef5fbe21ae18590d06e36ee2e98910189 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 25 Aug 2026 13:56:54 -0700 Subject: [PATCH] refactor(scripts): share SQLite payload proof contract (#129499) Amp-Thread-ID: https://ampcode.com/threads/T-01a037b7-6475-7130-8be6-39147410914b Co-authored-by: Amp --- scripts/lib/sqlite-reliability-compaction.ts | 35 ++++------- scripts/lib/sqlite-reliability-contract.ts | 64 +++++++++----------- scripts/lib/sqlite-reliability-repository.ts | 51 +++++++--------- scripts/lib/sqlite-reliability-restore.ts | 41 +++++-------- 4 files changed, 79 insertions(+), 112 deletions(-) diff --git a/scripts/lib/sqlite-reliability-compaction.ts b/scripts/lib/sqlite-reliability-compaction.ts index a6270ff9442d..0857bf30b86b 100644 --- a/scripts/lib/sqlite-reliability-compaction.ts +++ b/scripts/lib/sqlite-reliability-compaction.ts @@ -3,13 +3,12 @@ import fs from "node:fs"; import { setTimeout as delay } from "node:timers/promises"; import { fileURLToPath } from "node:url"; import type { SnapshotDatabaseIdentity } from "../../src/snapshot/snapshot-provider.js"; -import type { ReliabilityReport, ReliabilityStateProof } from "./sqlite-reliability-contract.js"; - -type CompactionPayloadProof = { - bytes: number; - idSum: number; - rows: number; -}; +import { + assertSameCompactionPayload, + type CompactionPayloadProof, + type ReliabilityReport, + type ReliabilityStateProof, +} from "./sqlite-reliability-contract.js"; type CompactionTarget = { identity: SnapshotDatabaseIdentity; @@ -51,22 +50,6 @@ function assertSameState( } } -function assertSamePayload( - actual: CompactionPayloadProof, - expected: CompactionPayloadProof, - label: string, -): void { - if ( - actual.bytes !== expected.bytes || - actual.idSum !== expected.idSum || - actual.rows !== expected.rows - ) { - throw new Error( - `${label} changed compaction payload: expected rows=${expected.rows} bytes=${expected.bytes} idSum=${expected.idSum}, got rows=${actual.rows} bytes=${actual.bytes} idSum=${actual.idSum}`, - ); - } -} - function workerArgs(target: CompactionTarget): string[] { if (target.identity.role === "global") { return ["global", target.path, ""]; @@ -241,7 +224,11 @@ export async function runVacuumInterruptionProof(params: { ); } const payloadAfterRecovery = params.readPayload(); - assertSamePayload(payloadAfterRecovery, params.expectedPayload, "vacuum crash recovery"); + assertSameCompactionPayload( + payloadAfterRecovery, + params.expectedPayload, + "vacuum crash recovery", + ); const journalBytesAfterRecovery = fileSize(`${params.target.path}-journal`); const walBytesAfterRecovery = fileSize(`${params.target.path}-wal`); if (journalBytesAfterRecovery !== 0 || walBytesAfterRecovery !== 0) { diff --git a/scripts/lib/sqlite-reliability-contract.ts b/scripts/lib/sqlite-reliability-contract.ts index 7db420caea4f..c9fc0d48692c 100644 --- a/scripts/lib/sqlite-reliability-contract.ts +++ b/scripts/lib/sqlite-reliability-contract.ts @@ -31,12 +31,34 @@ export type CliOptions = { stateDir: string | null; }; +export type CompactionPayloadProof = { + bytes: number; + idSum: number; + rows: number; +}; + export type ReliabilityStateProof = { batches: number; rows: number; sha256: string; }; +export function assertSameCompactionPayload( + actual: CompactionPayloadProof, + expected: CompactionPayloadProof, + label: string, +): void { + if ( + actual.bytes !== expected.bytes || + actual.idSum !== expected.idSum || + actual.rows !== expected.rows + ) { + throw new Error( + `${label} changed compaction payload: expected rows=${expected.rows} bytes=${expected.bytes} idSum=${expected.idSum}, got rows=${actual.rows} bytes=${actual.bytes} idSum=${actual.idSum}`, + ); + } +} + export type ReliabilityReport = { arch: string; concurrentRestoresVerified: number; @@ -106,16 +128,8 @@ export type ReliabilityReport = { signal: NodeJS.Signals | null; }; journalBytesObserved: number; - payloadAfterRecovery: { - bytes: number; - idSum: number; - rows: number; - }; - payloadBeforeKill: { - bytes: number; - idSum: number; - rows: number; - }; + payloadAfterRecovery: CompactionPayloadProof; + payloadBeforeKill: CompactionPayloadProof; recoveryVerified: true; stateAfterRecovery: ReliabilityStateProof; stateBeforeKill: ReliabilityStateProof; @@ -137,11 +151,7 @@ export type ReliabilityReport = { signal: NodeJS.Signals | null; }; incompleteEntries: 0; - payload: { - bytes: number; - idSum: number; - rows: number; - }; + payload: CompactionPayloadProof; repositoryVerified: true; retryCreated: true; sourcePayloadPreserved: true; @@ -158,11 +168,7 @@ export type ReliabilityReport = { signal: NodeJS.Signals | null; }; incompleteEntries: 1; - payload: { - bytes: number; - idSum: number; - rows: number; - }; + payload: CompactionPayloadProof; repositoryVerified: true; retryCreated: true; sourcePayloadPreserved: true; @@ -179,11 +185,7 @@ export type ReliabilityReport = { signal: NodeJS.Signals | null; }; incompleteEntries: 0; - payload: { - bytes: number; - idSum: number; - rows: number; - }; + payload: CompactionPayloadProof; repositoryVerified: true; retryCreated: true; sourcePayloadPreserved: true; @@ -200,11 +202,7 @@ export type ReliabilityReport = { code: number | null; signal: NodeJS.Signals | null; }; - payloadAfterRecovery: { - bytes: number; - idSum: number; - rows: number; - }; + payloadAfterRecovery: CompactionPayloadProof; recoveryVerified: true; repositoryVerified: true; retryRestored: false; @@ -219,11 +217,7 @@ export type ReliabilityReport = { code: number | null; signal: NodeJS.Signals | null; }; - payloadAfterRecovery: { - bytes: number; - idSum: number; - rows: number; - }; + payloadAfterRecovery: CompactionPayloadProof; recoveryVerified: true; repositoryVerified: true; retryRestored: true; diff --git a/scripts/lib/sqlite-reliability-repository.ts b/scripts/lib/sqlite-reliability-repository.ts index 7aaa827d577c..48558bd1b557 100644 --- a/scripts/lib/sqlite-reliability-repository.ts +++ b/scripts/lib/sqlite-reliability-repository.ts @@ -8,19 +8,22 @@ import { type SnapshotDatabaseIdentity, type SnapshotSummary, } from "../../src/snapshot/snapshot-provider.js"; -import type { ReliabilityReport, ReliabilityStateProof } from "./sqlite-reliability-contract.js"; +import { + assertSameCompactionPayload, + type CompactionPayloadProof, + type ReliabilityReport, + type ReliabilityStateProof, +} from "./sqlite-reliability-contract.js"; type RepositoryCrashPoint = "after-commit" | "before-pending" | "pending"; type RepositoryExit = ReliabilityReport["maintenanceProof"]["repositoryInterruption"]["beforePending"]["exit"]; -type RepositoryPayload = - ReliabilityReport["maintenanceProof"]["repositoryInterruption"]["beforePending"]["payload"]; type CrashPointResult = { crashSnapshotVerifiedAfterCrash: boolean; crashSnapshotVisibleAfterCrash: boolean; exit: RepositoryExit; incompleteEntries: number; - payload: RepositoryPayload; + payload: CompactionPayloadProof; stagingEntries: number; state: ReliabilityStateProof; visibleSnapshotsAfterCrash: number; @@ -47,22 +50,6 @@ function assertSameState( } } -function assertSamePayload( - actual: RepositoryPayload, - expected: RepositoryPayload, - label: string, -): void { - if ( - actual.bytes !== expected.bytes || - actual.idSum !== expected.idSum || - actual.rows !== expected.rows - ) { - throw new Error( - `${label} changed compaction payload: expected rows=${expected.rows} bytes=${expected.bytes} idSum=${expected.idSum}, got rows=${actual.rows} bytes=${actual.bytes} idSum=${actual.idSum}`, - ); - } -} - function formatWorkerStderr(stderr: string): string { const text = stderr.trim(); return text ? ` stderr=${JSON.stringify(text)}` : ""; @@ -158,17 +145,21 @@ function assertForcedExit(exit: RepositoryExit): void { } async function verifySnapshot(params: { - expectedPayload: RepositoryPayload; + expectedPayload: CompactionPayloadProof; expectedState: ReliabilityStateProof; provider: ReturnType; snapshot: SnapshotSummary; - verifyPayload: (databasePath: string) => RepositoryPayload; + verifyPayload: (databasePath: string) => CompactionPayloadProof; verifyState: (databasePath: string) => ReliabilityStateProof; }): Promise { await params.provider.verify(params.snapshot.ref); const artifactPath = path.join(params.snapshot.ref.path, SNAPSHOT_SQLITE_FILENAME); assertSameState(params.verifyState(artifactPath), params.expectedState, artifactPath); - assertSamePayload(params.verifyPayload(artifactPath), params.expectedPayload, artifactPath); + assertSameCompactionPayload( + params.verifyPayload(artifactPath), + params.expectedPayload, + artifactPath, + ); } function listRepositoryEntries(repositoryPath: string): string[] { @@ -177,14 +168,14 @@ function listRepositoryEntries(repositoryPath: string): string[] { async function runCrashPoint(params: { crashPoint: RepositoryCrashPoint; - expectedPayload: RepositoryPayload; + expectedPayload: CompactionPayloadProof; expectedState: ReliabilityStateProof; identity: SnapshotDatabaseIdentity; provider: ReturnType; repositoryPath: string; sourcePath: string; validationRootPath: string; - verifyPayload: (databasePath: string) => RepositoryPayload; + verifyPayload: (databasePath: string) => CompactionPayloadProof; verifyState: (databasePath: string) => ReliabilityStateProof; }): Promise { const visibleBefore = await params.provider.list(); @@ -248,7 +239,11 @@ async function runCrashPoint(params: { const sourceState = params.verifyState(params.sourcePath); assertSameState(sourceState, params.expectedState, `${params.crashPoint} source`); const sourcePayload = params.verifyPayload(params.sourcePath); - assertSamePayload(sourcePayload, params.expectedPayload, `${params.crashPoint} source`); + assertSameCompactionPayload( + sourcePayload, + params.expectedPayload, + `${params.crashPoint} source`, + ); const crashSnapshotNames = new Set( crashSnapshots.map((snapshot) => path.basename(snapshot.ref.path)), @@ -302,13 +297,13 @@ async function runCrashPoint(params: { } export async function runRepositoryInterruptionProof(params: { - expectedPayload: RepositoryPayload; + expectedPayload: CompactionPayloadProof; expectedState: ReliabilityStateProof; identity: SnapshotDatabaseIdentity; repositoryPath: string; sourcePath: string; validationRootPath: string; - verifyPayload: (databasePath: string) => RepositoryPayload; + verifyPayload: (databasePath: string) => CompactionPayloadProof; verifyState: (databasePath: string) => ReliabilityStateProof; }): Promise { const provider = createLocalSqliteSnapshotProvider({ diff --git a/scripts/lib/sqlite-reliability-restore.ts b/scripts/lib/sqlite-reliability-restore.ts index 7496e028135f..e4ebede703b6 100644 --- a/scripts/lib/sqlite-reliability-restore.ts +++ b/scripts/lib/sqlite-reliability-restore.ts @@ -4,17 +4,20 @@ import fs from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { createLocalSqliteSnapshotProvider } from "../../src/snapshot/local-repository.js"; -import type { ReliabilityReport, ReliabilityStateProof } from "./sqlite-reliability-contract.js"; +import { + assertSameCompactionPayload, + type CompactionPayloadProof, + type ReliabilityReport, + type ReliabilityStateProof, +} from "./sqlite-reliability-contract.js"; type RestoreCrashPoint = "after-publish" | "before-publish"; type RestoreExit = ReliabilityReport["maintenanceProof"]["restoreInterruption"]["beforePublish"]["exit"]; -type RestorePayloadProof = - ReliabilityReport["maintenanceProof"]["restoreInterruption"]["beforePublish"]["payloadAfterRecovery"]; type RestoreCrashResult = { existingTargetPreserved: boolean; exit: RestoreExit; - payloadAfterRecovery: RestorePayloadProof; + payloadAfterRecovery: CompactionPayloadProof; recoveryVerified: true; repositoryVerified: true; retryRestored: boolean; @@ -50,22 +53,6 @@ function assertSameState( } } -function assertSamePayload( - actual: RestorePayloadProof, - expected: RestorePayloadProof, - label: string, -): void { - if ( - actual.bytes !== expected.bytes || - actual.idSum !== expected.idSum || - actual.rows !== expected.rows - ) { - throw new Error( - `${label} changed compaction payload: expected rows=${expected.rows} bytes=${expected.bytes} idSum=${expected.idSum}, got rows=${actual.rows} bytes=${actual.bytes} idSum=${actual.idSum}`, - ); - } -} - function assertNoSqliteSidecars(targetPath: string): void { for (const suffix of ["-journal", "-shm", "-wal"]) { if (fs.existsSync(`${targetPath}${suffix}`)) { @@ -267,7 +254,7 @@ async function assertRepositorySnapshotAvailable(params: { async function runCrashPoint(params: { crashPoint: RestoreCrashPoint; - expectedPayload: RestorePayloadProof; + expectedPayload: CompactionPayloadProof; expectedSnapshotBytes: number; expectedState: ReliabilityStateProof; provider: ReturnType; @@ -275,7 +262,7 @@ async function runCrashPoint(params: { scratchPath: string; snapshotPath: string; validationRootPath: string; - verifyPayload: (databasePath: string) => RestorePayloadProof; + verifyPayload: (databasePath: string) => CompactionPayloadProof; verifyState: (databasePath: string) => ReliabilityStateProof; }): Promise { const targetPath = path.join(params.scratchPath, `${params.crashPoint}.sqlite`); @@ -370,7 +357,11 @@ async function runCrashPoint(params: { const stateAfterRecovery = params.verifyState(targetPath); assertSameState(stateAfterRecovery, params.expectedState, `${params.crashPoint} restore`); const payloadAfterRecovery = params.verifyPayload(targetPath); - assertSamePayload(payloadAfterRecovery, params.expectedPayload, `${params.crashPoint} restore`); + assertSameCompactionPayload( + payloadAfterRecovery, + params.expectedPayload, + `${params.crashPoint} restore`, + ); await assertRepositorySnapshotAvailable({ expectedSnapshotBytes: params.expectedSnapshotBytes, provider: params.provider, @@ -407,14 +398,14 @@ async function runCrashPoint(params: { } export async function runRestoreInterruptionProof(params: { - expectedPayload: RestorePayloadProof; + expectedPayload: CompactionPayloadProof; expectedSnapshotBytes: number; expectedState: ReliabilityStateProof; repositoryPath: string; scratchPath: string; snapshotPath: string; validationRootPath: string; - verifyPayload: (databasePath: string) => RestorePayloadProof; + verifyPayload: (databasePath: string) => CompactionPayloadProof; verifyState: (databasePath: string) => ReliabilityStateProof; }): Promise { if (params.expectedSnapshotBytes < MIN_STAGED_RESTORE_BYTES * 2) {