diff --git a/scripts/lib/ci-node-test-plan.mts b/scripts/lib/ci-node-test-plan.mts index d116708863ef..f8b085ce9000 100644 --- a/scripts/lib/ci-node-test-plan.mts +++ b/scripts/lib/ci-node-test-plan.mts @@ -321,7 +321,8 @@ const STRIPE_FILE_SECONDS_HINTS = new Map([ ["src/auto-reply/reply/commands-status.test.ts", 12], ["src/auto-reply/reply/commands-system-prompt.test.ts", 8], ["src/scripts/test-projects.test.ts", 21], - ["test/scripts/bench-sqlite-reliability.test.ts", 9], + // Focused cold proof is ~34s after right-sizing and concurrent crash phases. + ["test/scripts/bench-sqlite-reliability.test.ts", 34], ["test/scripts/bundled-plugin-install-uninstall-probe.test.ts", 4], ["test/scripts/changed-lanes.test.ts", 5], ["test/scripts/ci-workflow-guards.test.ts", 12], diff --git a/scripts/lib/sqlite-reliability-contract.ts b/scripts/lib/sqlite-reliability-contract.ts index 924e0c205eba..93ad693887be 100644 --- a/scripts/lib/sqlite-reliability-contract.ts +++ b/scripts/lib/sqlite-reliability-contract.ts @@ -305,7 +305,9 @@ export type ReliabilityReport = { export const PROFILES: Record = { smoke: { - iterations: 4, + // One snapshot before the forced writer crash and one after restart prove + // both distinct smoke paths; larger profiles retain repeated stress loops. + iterations: 2, maxWalBytes: 64 * 1024 * 1024, payloadBytes: 512, retainedBatches: 32, diff --git a/scripts/lib/sqlite-reliability-runner.ts b/scripts/lib/sqlite-reliability-runner.ts index 499108446db0..1413ec2705d9 100644 --- a/scripts/lib/sqlite-reliability-runner.ts +++ b/scripts/lib/sqlite-reliability-runner.ts @@ -57,15 +57,31 @@ type IterationMetric = { type CompactionProof = ReliabilityReport["maintenanceProof"]["compaction"]; -// Exceed the 1 MiB interruption thresholds without copying an arbitrary 64 MiB -// through every repository and restore crash phase. -const COMPACTION_BLOAT_ROWS = 64; +// Keep 50% headroom above the 2 MiB staged-restore threshold without copying +// an arbitrarily large payload through every repository and restore crash phase. +const COMPACTION_BLOAT_ROWS = 12; const COMPACTION_BLOAT_PAYLOAD_BYTES = 256 * 1024; function nowMs(): number { return Number(process.hrtime.bigint()) / 1e6; } +async function runProofsConcurrently( + first: Promise, + second: Promise, +): Promise<[First, Second]> { + // Wait for both proofs to release their child processes before outer scratch + // cleanup starts, even when one proof fails. + const [firstResult, secondResult] = await Promise.allSettled([first, second]); + if (firstResult.status === "rejected") { + throw firstResult.reason; + } + if (secondResult.status === "rejected") { + throw secondResult.reason; + } + return [firstResult.value, secondResult.value]; +} + function percentile(values: number[], pct: number): number { if (values.length === 0) { return 0; @@ -485,23 +501,6 @@ async function runMaintenanceRoundTrip(params: { `compaction payload setup failed: rows=${expectedPayload.rows} bytes=${expectedPayload.bytes}`, ); } - const repositoryInterruption = await runRepositoryInterruptionProof({ - expectedPayload, - expectedState, - identity: params.target.identity, - repositoryPath: path.join(params.restoreRoot, "repository-interruptions"), - sourcePath: params.target.path, - validationRootPath: params.validationRoot, - verifyPayload: readCompactionPayload, - verifyState: (databasePath) => - verifyRestoredDatabase({ - expectedState, - identity: params.target.identity, - path: databasePath, - rowsPerBatch: params.rowsPerBatch, - uncommittedBatch: null, - }), - }); const interruptedSnapshot = await params.repositoryProvider.create({ identity: params.target.identity, path: params.target.path, @@ -510,24 +509,43 @@ async function runMaintenanceRoundTrip(params: { interruptedSnapshot.ref.path, params.syncedRepository, ); - const restoreInterruption = await runRestoreInterruptionProof({ - expectedPayload, - expectedSnapshotBytes: interruptedSnapshot.manifest.artifact.sizeBytes, - expectedState, - repositoryPath: params.syncedRepository, - scratchPath: path.join(params.restoreRoot, "interrupted"), - snapshotPath: interruptedCopiedPath, - validationRootPath: params.validationRoot, - verifyPayload: readCompactionPayload, - verifyState: (databasePath) => - verifyRestoredDatabase({ - expectedState, - identity: params.target.identity, - path: databasePath, - rowsPerBatch: params.rowsPerBatch, - uncommittedBatch: null, - }), - }); + const [repositoryInterruption, restoreInterruption] = await runProofsConcurrently( + runRepositoryInterruptionProof({ + expectedPayload, + expectedState, + identity: params.target.identity, + repositoryPath: path.join(params.restoreRoot, "repository-interruptions"), + sourcePath: params.target.path, + validationRootPath: params.validationRoot, + verifyPayload: readCompactionPayload, + verifyState: (databasePath) => + verifyRestoredDatabase({ + expectedState, + identity: params.target.identity, + path: databasePath, + rowsPerBatch: params.rowsPerBatch, + uncommittedBatch: null, + }), + }), + runRestoreInterruptionProof({ + expectedPayload, + expectedSnapshotBytes: interruptedSnapshot.manifest.artifact.sizeBytes, + expectedState, + repositoryPath: params.syncedRepository, + scratchPath: path.join(params.restoreRoot, "interrupted"), + snapshotPath: interruptedCopiedPath, + validationRootPath: params.validationRoot, + verifyPayload: readCompactionPayload, + verifyState: (databasePath) => + verifyRestoredDatabase({ + expectedState, + identity: params.target.identity, + path: databasePath, + rowsPerBatch: params.rowsPerBatch, + uncommittedBatch: null, + }), + }), + ); const vacuumInterruption = await runVacuumInterruptionProof({ env: params.env, expectedAutoVacuum: autoVacuumBeforeKill, @@ -725,22 +743,23 @@ export async function runReliabilityStress(options: CliOptions): Promise - verifyRestoredDatabase({ + const [publicationInterruptionProof, indexRepairInterruptionProof] = + await runProofsConcurrently( + runPublicationInterruptionProof({ expectedState: stableState, - identity: target.identity, - path: databasePath, - rowsPerBatch: profile.rowsPerBatch, - uncommittedBatch: null, + scratchPath: path.join(runScratch, "publication-interruptions"), + sourcePath: target.path, + verifyDatabase: (databasePath) => + verifyRestoredDatabase({ + expectedState: stableState, + identity: target.identity, + path: databasePath, + rowsPerBatch: profile.rowsPerBatch, + uncommittedBatch: null, + }), }), - }); - const indexRepairInterruptionProof = await runIndexRepairInterruptionProof( - path.join(runScratch, "index-repair-interruptions"), - ); + runIndexRepairInterruptionProof(path.join(runScratch, "index-repair-interruptions")), + ); const maintenanceProof = await runMaintenanceRoundTrip({ env, repositoryProvider, diff --git a/test/scripts/bench-sqlite-reliability.test.ts b/test/scripts/bench-sqlite-reliability.test.ts index ce90865617ad..6838ae1ed9a8 100644 --- a/test/scripts/bench-sqlite-reliability.test.ts +++ b/test/scripts/bench-sqlite-reliability.test.ts @@ -187,7 +187,7 @@ describe("scripts/bench-sqlite-reliability", () => { expect(result.status, result.stderr).toBe(0); expect(result.stderr).toBe(""); expect(result.stdout).toContain("SQLITE_RELIABILITY_TARGET=global"); - expect(result.stdout).toContain("SQLITE_RELIABILITY_RESTORES_VERIFIED=7"); + expect(result.stdout).toContain("SQLITE_RELIABILITY_RESTORES_VERIFIED=5"); expect(result.stdout).toContain("SQLITE_RELIABILITY_CRASH_RECOVERY=verified"); expect(result.stdout).toContain("SQLITE_RELIABILITY_PUBLICATION_INTERRUPTION=verified"); expect(result.stdout).toContain("SQLITE_RELIABILITY_RESTORE_INTERRUPTION=verified"); @@ -197,8 +197,8 @@ describe("scripts/bench-sqlite-reliability", () => { expect(result.stdout).toContain("SQLITE_RELIABILITY_POST_COMPACT_RESTORE=verified"); expect(result.stdout).not.toContain("=missing"); const firstReport = JSON.parse(fs.readFileSync(output, "utf8")) as ReliabilityReport; - expect(firstReport.concurrentRestoresVerified).toBe(4); - expect(firstReport.restoresVerified).toBe(7); + expect(firstReport.concurrentRestoresVerified).toBe(2); + expect(firstReport.restoresVerified).toBe(5); expect( firstReport.crashRecoveryProof.exit.code !== null || firstReport.crashRecoveryProof.exit.signal !== null,