refactor(scripts): share SQLite payload proof contract (#129499)

Amp-Thread-ID: https://ampcode.com/threads/T-01a037b7-6475-7130-8be6-39147410914b

Co-authored-by: Amp <amp@ampcode.com>

scripts/lib/sqlite-reliability-compaction.ts

@@ -3,13 +3,12 @@ import fs from "node:fs";
 import { setTimeout as delay } from "node:timers/promises";
 import { fileURLToPath } from "node:url";
 import type { SnapshotDatabaseIdentity } from "../../src/snapshot/snapshot-provider.js";
-import type { ReliabilityReport, ReliabilityStateProof } from "./sqlite-reliability-contract.js";
-
-type CompactionPayloadProof = {
-  bytes: number;
-  idSum: number;
-  rows: number;
-};
+import {
+  assertSameCompactionPayload,
+  type CompactionPayloadProof,
+  type ReliabilityReport,
+  type ReliabilityStateProof,
+} from "./sqlite-reliability-contract.js";
 
 type CompactionTarget = {
   identity: SnapshotDatabaseIdentity;
@@ -51,22 +50,6 @@ function assertSameState(
   }
 }
 
-function assertSamePayload(
-  actual: CompactionPayloadProof,
-  expected: CompactionPayloadProof,
-  label: string,
-): void {
-  if (
-    actual.bytes !== expected.bytes ||
-    actual.idSum !== expected.idSum ||
-    actual.rows !== expected.rows
-  ) {
-    throw new Error(
-      `${label} changed compaction payload: expected rows=${expected.rows} bytes=${expected.bytes} idSum=${expected.idSum}, got rows=${actual.rows} bytes=${actual.bytes} idSum=${actual.idSum}`,
-    );
-  }
-}
-
 function workerArgs(target: CompactionTarget): string[] {
   if (target.identity.role === "global") {
     return ["global", target.path, ""];
@@ -241,7 +224,11 @@ export async function runVacuumInterruptionProof(params: {
       );
     }
     const payloadAfterRecovery = params.readPayload();
-    assertSamePayload(payloadAfterRecovery, params.expectedPayload, "vacuum crash recovery");
+    assertSameCompactionPayload(
+      payloadAfterRecovery,
+      params.expectedPayload,
+      "vacuum crash recovery",
+    );
     const journalBytesAfterRecovery = fileSize(`${params.target.path}-journal`);
     const walBytesAfterRecovery = fileSize(`${params.target.path}-wal`);
     if (journalBytesAfterRecovery !== 0 || walBytesAfterRecovery !== 0) {

scripts/lib/sqlite-reliability-contract.ts

@@ -31,12 +31,34 @@ export type CliOptions = {
   stateDir: string | null;
 };
 
+export type CompactionPayloadProof = {
+  bytes: number;
+  idSum: number;
+  rows: number;
+};
+
 export type ReliabilityStateProof = {
   batches: number;
   rows: number;
   sha256: string;
 };
 
+export function assertSameCompactionPayload(
+  actual: CompactionPayloadProof,
+  expected: CompactionPayloadProof,
+  label: string,
+): void {
+  if (
+    actual.bytes !== expected.bytes ||
+    actual.idSum !== expected.idSum ||
+    actual.rows !== expected.rows
+  ) {
+    throw new Error(
+      `${label} changed compaction payload: expected rows=${expected.rows} bytes=${expected.bytes} idSum=${expected.idSum}, got rows=${actual.rows} bytes=${actual.bytes} idSum=${actual.idSum}`,
+    );
+  }
+}
+
 export type ReliabilityReport = {
   arch: string;
   concurrentRestoresVerified: number;
@@ -106,16 +128,8 @@ export type ReliabilityReport = {
         signal: NodeJS.Signals | null;
       };
       journalBytesObserved: number;
-      payloadAfterRecovery: {
-        bytes: number;
-        idSum: number;
-        rows: number;
-      };
-      payloadBeforeKill: {
-        bytes: number;
-        idSum: number;
-        rows: number;
-      };
+      payloadAfterRecovery: CompactionPayloadProof;
+      payloadBeforeKill: CompactionPayloadProof;
       recoveryVerified: true;
       stateAfterRecovery: ReliabilityStateProof;
       stateBeforeKill: ReliabilityStateProof;
@@ -137,11 +151,7 @@ export type ReliabilityReport = {
           signal: NodeJS.Signals | null;
         };
         incompleteEntries: 0;
-        payload: {
-          bytes: number;
-          idSum: number;
-          rows: number;
-        };
+        payload: CompactionPayloadProof;
         repositoryVerified: true;
         retryCreated: true;
         sourcePayloadPreserved: true;
@@ -158,11 +168,7 @@ export type ReliabilityReport = {
           signal: NodeJS.Signals | null;
         };
         incompleteEntries: 1;
-        payload: {
-          bytes: number;
-          idSum: number;
-          rows: number;
-        };
+        payload: CompactionPayloadProof;
         repositoryVerified: true;
         retryCreated: true;
         sourcePayloadPreserved: true;
@@ -179,11 +185,7 @@ export type ReliabilityReport = {
           signal: NodeJS.Signals | null;
         };
         incompleteEntries: 0;
-        payload: {
-          bytes: number;
-          idSum: number;
-          rows: number;
-        };
+        payload: CompactionPayloadProof;
         repositoryVerified: true;
         retryCreated: true;
         sourcePayloadPreserved: true;
@@ -200,11 +202,7 @@ export type ReliabilityReport = {
           code: number | null;
           signal: NodeJS.Signals | null;
         };
-        payloadAfterRecovery: {
-          bytes: number;
-          idSum: number;
-          rows: number;
-        };
+        payloadAfterRecovery: CompactionPayloadProof;
         recoveryVerified: true;
         repositoryVerified: true;
         retryRestored: false;
@@ -219,11 +217,7 @@ export type ReliabilityReport = {
           code: number | null;
           signal: NodeJS.Signals | null;
         };
-        payloadAfterRecovery: {
-          bytes: number;
-          idSum: number;
-          rows: number;
-        };
+        payloadAfterRecovery: CompactionPayloadProof;
         recoveryVerified: true;
         repositoryVerified: true;
         retryRestored: true;

scripts/lib/sqlite-reliability-repository.ts

@@ -8,19 +8,22 @@ import {
   type SnapshotDatabaseIdentity,
   type SnapshotSummary,
 } from "../../src/snapshot/snapshot-provider.js";
-import type { ReliabilityReport, ReliabilityStateProof } from "./sqlite-reliability-contract.js";
+import {
+  assertSameCompactionPayload,
+  type CompactionPayloadProof,
+  type ReliabilityReport,
+  type ReliabilityStateProof,
+} from "./sqlite-reliability-contract.js";
 
 type RepositoryCrashPoint = "after-commit" | "before-pending" | "pending";
 type RepositoryExit =
   ReliabilityReport["maintenanceProof"]["repositoryInterruption"]["beforePending"]["exit"];
-type RepositoryPayload =
-  ReliabilityReport["maintenanceProof"]["repositoryInterruption"]["beforePending"]["payload"];
 type CrashPointResult = {
   crashSnapshotVerifiedAfterCrash: boolean;
   crashSnapshotVisibleAfterCrash: boolean;
   exit: RepositoryExit;
   incompleteEntries: number;
-  payload: RepositoryPayload;
+  payload: CompactionPayloadProof;
   stagingEntries: number;
   state: ReliabilityStateProof;
   visibleSnapshotsAfterCrash: number;
@@ -47,22 +50,6 @@ function assertSameState(
   }
 }
 
-function assertSamePayload(
-  actual: RepositoryPayload,
-  expected: RepositoryPayload,
-  label: string,
-): void {
-  if (
-    actual.bytes !== expected.bytes ||
-    actual.idSum !== expected.idSum ||
-    actual.rows !== expected.rows
-  ) {
-    throw new Error(
-      `${label} changed compaction payload: expected rows=${expected.rows} bytes=${expected.bytes} idSum=${expected.idSum}, got rows=${actual.rows} bytes=${actual.bytes} idSum=${actual.idSum}`,
-    );
-  }
-}
-
 function formatWorkerStderr(stderr: string): string {
   const text = stderr.trim();
   return text ? ` stderr=${JSON.stringify(text)}` : "";
@@ -158,17 +145,21 @@ function assertForcedExit(exit: RepositoryExit): void {
 }
 
 async function verifySnapshot(params: {
-  expectedPayload: RepositoryPayload;
+  expectedPayload: CompactionPayloadProof;
   expectedState: ReliabilityStateProof;
   provider: ReturnType<typeof createLocalSqliteSnapshotProvider>;
   snapshot: SnapshotSummary;
-  verifyPayload: (databasePath: string) => RepositoryPayload;
+  verifyPayload: (databasePath: string) => CompactionPayloadProof;
   verifyState: (databasePath: string) => ReliabilityStateProof;
 }): Promise<void> {
   await params.provider.verify(params.snapshot.ref);
   const artifactPath = path.join(params.snapshot.ref.path, SNAPSHOT_SQLITE_FILENAME);
   assertSameState(params.verifyState(artifactPath), params.expectedState, artifactPath);
-  assertSamePayload(params.verifyPayload(artifactPath), params.expectedPayload, artifactPath);
+  assertSameCompactionPayload(
+    params.verifyPayload(artifactPath),
+    params.expectedPayload,
+    artifactPath,
+  );
 }
 
 function listRepositoryEntries(repositoryPath: string): string[] {
@@ -177,14 +168,14 @@ function listRepositoryEntries(repositoryPath: string): string[] {
 
 async function runCrashPoint(params: {
   crashPoint: RepositoryCrashPoint;
-  expectedPayload: RepositoryPayload;
+  expectedPayload: CompactionPayloadProof;
   expectedState: ReliabilityStateProof;
   identity: SnapshotDatabaseIdentity;
   provider: ReturnType<typeof createLocalSqliteSnapshotProvider>;
   repositoryPath: string;
   sourcePath: string;
   validationRootPath: string;
-  verifyPayload: (databasePath: string) => RepositoryPayload;
+  verifyPayload: (databasePath: string) => CompactionPayloadProof;
   verifyState: (databasePath: string) => ReliabilityStateProof;
 }): Promise<CrashPointResult> {
   const visibleBefore = await params.provider.list();
@@ -248,7 +239,11 @@ async function runCrashPoint(params: {
     const sourceState = params.verifyState(params.sourcePath);
     assertSameState(sourceState, params.expectedState, `${params.crashPoint} source`);
     const sourcePayload = params.verifyPayload(params.sourcePath);
-    assertSamePayload(sourcePayload, params.expectedPayload, `${params.crashPoint} source`);
+    assertSameCompactionPayload(
+      sourcePayload,
+      params.expectedPayload,
+      `${params.crashPoint} source`,
+    );
 
     const crashSnapshotNames = new Set(
       crashSnapshots.map((snapshot) => path.basename(snapshot.ref.path)),
@@ -302,13 +297,13 @@ async function runCrashPoint(params: {
 }
 
 export async function runRepositoryInterruptionProof(params: {
-  expectedPayload: RepositoryPayload;
+  expectedPayload: CompactionPayloadProof;
   expectedState: ReliabilityStateProof;
   identity: SnapshotDatabaseIdentity;
   repositoryPath: string;
   sourcePath: string;
   validationRootPath: string;
-  verifyPayload: (databasePath: string) => RepositoryPayload;
+  verifyPayload: (databasePath: string) => CompactionPayloadProof;
   verifyState: (databasePath: string) => ReliabilityStateProof;
 }): Promise<ReliabilityReport["maintenanceProof"]["repositoryInterruption"]> {
   const provider = createLocalSqliteSnapshotProvider({

scripts/lib/sqlite-reliability-restore.ts

@@ -4,17 +4,20 @@ import fs from "node:fs";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
 import { createLocalSqliteSnapshotProvider } from "../../src/snapshot/local-repository.js";
-import type { ReliabilityReport, ReliabilityStateProof } from "./sqlite-reliability-contract.js";
+import {
+  assertSameCompactionPayload,
+  type CompactionPayloadProof,
+  type ReliabilityReport,
+  type ReliabilityStateProof,
+} from "./sqlite-reliability-contract.js";
 
 type RestoreCrashPoint = "after-publish" | "before-publish";
 type RestoreExit =
   ReliabilityReport["maintenanceProof"]["restoreInterruption"]["beforePublish"]["exit"];
-type RestorePayloadProof =
-  ReliabilityReport["maintenanceProof"]["restoreInterruption"]["beforePublish"]["payloadAfterRecovery"];
 type RestoreCrashResult = {
   existingTargetPreserved: boolean;
   exit: RestoreExit;
-  payloadAfterRecovery: RestorePayloadProof;
+  payloadAfterRecovery: CompactionPayloadProof;
   recoveryVerified: true;
   repositoryVerified: true;
   retryRestored: boolean;
@@ -50,22 +53,6 @@ function assertSameState(
   }
 }
 
-function assertSamePayload(
-  actual: RestorePayloadProof,
-  expected: RestorePayloadProof,
-  label: string,
-): void {
-  if (
-    actual.bytes !== expected.bytes ||
-    actual.idSum !== expected.idSum ||
-    actual.rows !== expected.rows
-  ) {
-    throw new Error(
-      `${label} changed compaction payload: expected rows=${expected.rows} bytes=${expected.bytes} idSum=${expected.idSum}, got rows=${actual.rows} bytes=${actual.bytes} idSum=${actual.idSum}`,
-    );
-  }
-}
-
 function assertNoSqliteSidecars(targetPath: string): void {
   for (const suffix of ["-journal", "-shm", "-wal"]) {
     if (fs.existsSync(`${targetPath}${suffix}`)) {
@@ -267,7 +254,7 @@ async function assertRepositorySnapshotAvailable(params: {
 
 async function runCrashPoint(params: {
   crashPoint: RestoreCrashPoint;
-  expectedPayload: RestorePayloadProof;
+  expectedPayload: CompactionPayloadProof;
   expectedSnapshotBytes: number;
   expectedState: ReliabilityStateProof;
   provider: ReturnType<typeof createLocalSqliteSnapshotProvider>;
@@ -275,7 +262,7 @@ async function runCrashPoint(params: {
   scratchPath: string;
   snapshotPath: string;
   validationRootPath: string;
-  verifyPayload: (databasePath: string) => RestorePayloadProof;
+  verifyPayload: (databasePath: string) => CompactionPayloadProof;
   verifyState: (databasePath: string) => ReliabilityStateProof;
 }): Promise<RestoreCrashResult> {
   const targetPath = path.join(params.scratchPath, `${params.crashPoint}.sqlite`);
@@ -370,7 +357,11 @@ async function runCrashPoint(params: {
     const stateAfterRecovery = params.verifyState(targetPath);
     assertSameState(stateAfterRecovery, params.expectedState, `${params.crashPoint} restore`);
     const payloadAfterRecovery = params.verifyPayload(targetPath);
-    assertSamePayload(payloadAfterRecovery, params.expectedPayload, `${params.crashPoint} restore`);
+    assertSameCompactionPayload(
+      payloadAfterRecovery,
+      params.expectedPayload,
+      `${params.crashPoint} restore`,
+    );
     await assertRepositorySnapshotAvailable({
       expectedSnapshotBytes: params.expectedSnapshotBytes,
       provider: params.provider,
@@ -407,14 +398,14 @@ async function runCrashPoint(params: {
 }
 
 export async function runRestoreInterruptionProof(params: {
-  expectedPayload: RestorePayloadProof;
+  expectedPayload: CompactionPayloadProof;
   expectedSnapshotBytes: number;
   expectedState: ReliabilityStateProof;
   repositoryPath: string;
   scratchPath: string;
   snapshotPath: string;
   validationRootPath: string;
-  verifyPayload: (databasePath: string) => RestorePayloadProof;
+  verifyPayload: (databasePath: string) => CompactionPayloadProof;
   verifyState: (databasePath: string) => ReliabilityStateProof;
 }): Promise<ReliabilityReport["maintenanceProof"]["restoreInterruption"]> {
   if (params.expectedSnapshotBytes < MIN_STAGED_RESTORE_BYTES * 2) {