diff --git a/src/infra/sqlite-file-generation.ts b/src/infra/sqlite-file-generation.ts new file mode 100644 index 000000000000..2da4f92e2d5b --- /dev/null +++ b/src/infra/sqlite-file-generation.ts @@ -0,0 +1,212 @@ +import { createHash } from "node:crypto"; +import fs, { type BigIntStats } from "node:fs"; +import { sameFileIdentity } from "./fs-safe-advanced.js"; + +const SQLITE_GENERATION_HASH_BUFFER_BYTES = 1024 * 1024; + +type SqliteFileFingerprint = { + birthtimeNs: bigint; + ctimeNs: bigint; + dev: bigint; + ino: bigint; + mtimeNs: bigint; + sha256: string; + size: bigint; +}; + +type SerializedSqliteFileFingerprint = { + birthtimeNs: string; + ctimeNs: string; + dev: string; + ino: string; + mtimeNs: string; + sha256: string; + size: string; +}; + +export type SqliteFileGeneration = { + database: SqliteFileFingerprint; + journal?: SqliteFileFingerprint; + wal?: SqliteFileFingerprint; +}; + +function assertRegularFile(stat: BigIntStats): void { + if (!stat.isFile()) { + throw new Error("SQLite generation target must be a regular file"); + } +} + +function sameFileState(left: BigIntStats, right: BigIntStats): boolean { + return ( + sameFileIdentity(left, right) && + left.birthtimeNs === right.birthtimeNs && + left.ctimeNs === right.ctimeNs && + left.mtimeNs === right.mtimeNs && + left.size === right.size + ); +} + +function hashFileDescriptor(fd: number): string { + const hash = createHash("sha256"); + const buffer = Buffer.allocUnsafe(SQLITE_GENERATION_HASH_BUFFER_BYTES); + let position = 0; + while (true) { + const bytesRead = fs.readSync(fd, buffer, 0, buffer.length, position); + if (bytesRead === 0) { + break; + } + hash.update(buffer.subarray(0, bytesRead)); + position += bytesRead; + } + return hash.digest("hex"); +} + +function fingerprintFile(pathname: string): SqliteFileFingerprint { + const fd = fs.openSync(pathname, "r"); + try { + const before = fs.fstatSync(fd, { bigint: true }); + assertRegularFile(before); + const sha256 = hashFileDescriptor(fd); + const after = fs.fstatSync(fd, { bigint: true }); + const current = fs.statSync(pathname, { bigint: true }); + if (!sameFileState(before, after) || !sameFileState(after, current)) { + throw new Error(`SQLite generation target changed while hashing: ${pathname}`); + } + return { + birthtimeNs: after.birthtimeNs, + ctimeNs: after.ctimeNs, + dev: after.dev, + ino: after.ino, + mtimeNs: after.mtimeNs, + sha256, + size: after.size, + }; + } finally { + fs.closeSync(fd); + } +} + +function readOptionalFile(pathname: string): SqliteFileFingerprint | undefined { + try { + return fingerprintFile(pathname); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return undefined; + } + throw error; + } +} + +function readGeneration(pathname: string): SqliteFileGeneration { + const database = fingerprintFile(pathname); + const journal = readOptionalFile(`${pathname}-journal`); + const wal = readOptionalFile(`${pathname}-wal`); + return { + database, + ...(journal ? { journal } : {}), + ...(wal ? { wal } : {}), + }; +} + +export function readStableSqliteFileGeneration(pathname: string): SqliteFileGeneration { + const first = readGeneration(pathname); + const second = readGeneration(pathname); + if (!sameSqliteFileGeneration(first, second)) { + throw new Error(`SQLite file generation changed while reading: ${pathname}`); + } + return second; +} + +function sameFileFingerprint(left: SqliteFileFingerprint, right: SqliteFileFingerprint): boolean { + return ( + left.dev === right.dev && + left.ino === right.ino && + left.birthtimeNs === right.birthtimeNs && + left.ctimeNs === right.ctimeNs && + left.mtimeNs === right.mtimeNs && + left.sha256 === right.sha256 && + left.size === right.size + ); +} + +function sameOptionalFileFingerprint( + left: SqliteFileFingerprint | undefined, + right: SqliteFileFingerprint | undefined, +): boolean { + return left === undefined + ? right === undefined + : right !== undefined && sameFileFingerprint(left, right); +} + +export function sameSqliteFileGeneration( + left: SqliteFileGeneration, + right: SqliteFileGeneration, +): boolean { + return ( + sameFileFingerprint(left.database, right.database) && + sameOptionalFileFingerprint(left.journal, right.journal) && + sameOptionalFileFingerprint(left.wal, right.wal) + ); +} + +function serializeFileFingerprint( + fingerprint: SqliteFileFingerprint, +): SerializedSqliteFileFingerprint { + return { + birthtimeNs: fingerprint.birthtimeNs.toString(), + ctimeNs: fingerprint.ctimeNs.toString(), + dev: fingerprint.dev.toString(), + ino: fingerprint.ino.toString(), + mtimeNs: fingerprint.mtimeNs.toString(), + sha256: fingerprint.sha256, + size: fingerprint.size.toString(), + }; +} + +export function serializeSqliteFileGeneration(generation: SqliteFileGeneration): string { + return JSON.stringify({ + database: serializeFileFingerprint(generation.database), + ...(generation.journal ? { journal: serializeFileFingerprint(generation.journal) } : {}), + ...(generation.wal ? { wal: serializeFileFingerprint(generation.wal) } : {}), + }); +} + +function parseFileFingerprint(value: unknown): SqliteFileFingerprint { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("SQLite file fingerprint must be an object"); + } + const fingerprint = value as Record; + const fields = ["birthtimeNs", "ctimeNs", "dev", "ino", "mtimeNs", "size"] as const; + for (const field of fields) { + if (typeof fingerprint[field] !== "string" || !/^-?\d+$/u.test(fingerprint[field])) { + throw new Error(`SQLite file fingerprint ${field} is invalid`); + } + } + if (typeof fingerprint.sha256 !== "string" || !/^[a-f0-9]{64}$/u.test(fingerprint.sha256)) { + throw new Error("SQLite file fingerprint sha256 is invalid"); + } + return { + birthtimeNs: BigInt(fingerprint.birthtimeNs as string), + ctimeNs: BigInt(fingerprint.ctimeNs as string), + dev: BigInt(fingerprint.dev as string), + ino: BigInt(fingerprint.ino as string), + mtimeNs: BigInt(fingerprint.mtimeNs as string), + sha256: fingerprint.sha256, + size: BigInt(fingerprint.size as string), + }; +} + +export function parseSqliteFileGeneration(serialized: string): SqliteFileGeneration { + const value = JSON.parse(serialized) as unknown; + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("SQLite file generation must be an object"); + } + const generation = value as Record; + return { + database: parseFileFingerprint(generation.database), + ...(generation.journal === undefined + ? {} + : { journal: parseFileFingerprint(generation.journal) }), + ...(generation.wal === undefined ? {} : { wal: parseFileFingerprint(generation.wal) }), + }; +} diff --git a/src/infra/sqlite-integrity.test.ts b/src/infra/sqlite-integrity.test.ts index 8021e6828577..9d38addaf085 100644 --- a/src/infra/sqlite-integrity.test.ts +++ b/src/infra/sqlite-integrity.test.ts @@ -1,7 +1,17 @@ +import fs from "node:fs"; +import path from "node:path"; import type { DatabaseSync } from "node:sqlite"; -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import * as nodeSqlite from "./node-sqlite.js"; import { requireNodeSqlite } from "./node-sqlite.js"; -import { assertSqliteIntegrity, isTerminalSqliteIntegrityError } from "./sqlite-integrity.js"; +import { + assertSqliteIntegrity, + confirmSqliteFileIntegrity, + isTerminalSqliteIntegrityError, +} from "./sqlite-integrity.js"; + +const tempDirs = useAutoCleanupTempDirTracker(afterEach); describe("assertSqliteIntegrity", () => { it("accepts structurally and referentially consistent databases", () => { @@ -201,3 +211,24 @@ describe("isTerminalSqliteIntegrityError", () => { expect(isTerminalSqliteIntegrityError(corruptIndex)).toBe(true); }); }); + +describe("confirmSqliteFileIntegrity", () => { + it("leaves SQLite open failures unbound because the failed file identity is unknown", () => { + const databasePath = path.join(tempDirs.make("sqlite-open-integrity-"), "database.sqlite"); + fs.writeFileSync(databasePath, "not a sqlite database"); + const openError = Object.assign(new Error("file is not a database"), { errcode: 26 }); + const open = vi.spyOn(nodeSqlite, "openNodeSqliteDatabase").mockImplementationOnce(() => { + throw openError; + }); + + try { + expect(confirmSqliteFileIntegrity(databasePath, "test database")).toEqual({ + status: "failed", + error: openError, + terminal: false, + }); + } finally { + open.mockRestore(); + } + }); +}); diff --git a/src/infra/sqlite-integrity.ts b/src/infra/sqlite-integrity.ts index 51af2606b9b1..5830b6ff721b 100644 --- a/src/infra/sqlite-integrity.ts +++ b/src/infra/sqlite-integrity.ts @@ -1,9 +1,24 @@ import type { DatabaseSync } from "node:sqlite"; +import { openNodeSqliteDatabase } from "./node-sqlite.js"; +import { + readStableSqliteFileGeneration, + sameSqliteFileGeneration, + type SqliteFileGeneration, +} from "./sqlite-file-generation.js"; type SqliteIntegrityChecks = { integrityCheck: "ok"; }; +type UnboundSqliteIntegrityConfirmation = + | { status: "failed"; error: Error; terminal: boolean } + | { status: "healthy" }; + +export type SqliteIntegrityConfirmation = + | { status: "failed"; error: Error; terminal: false } + | { status: "failed"; error: Error; generation: SqliteFileGeneration; terminal: true } + | { status: "healthy"; generation: SqliteFileGeneration }; + type SqliteCheckPragma = "integrity_check"; type SqliteForeignKeyViolation = { fkid: bigint; @@ -45,6 +60,117 @@ export function assertSqliteIntegrity( return { integrityCheck }; } +/** Run integrity checks and preserve whether a failure proves persistent damage. */ +function confirmSqliteIntegrity( + database: DatabaseSync, + databaseLabel: string, +): UnboundSqliteIntegrityConfirmation { + try { + assertSqliteIntegrity(database, databaseLabel); + return { status: "healthy" }; + } catch (error) { + return failedSqliteIntegrityConfirmation(error); + } +} + +/** Reconfirm an advisory failure against the database currently at a closed path. */ +export function confirmSqliteFileIntegrity( + pathname: string, + databaseLabel: string, +): SqliteIntegrityConfirmation { + for (let attempt = 0; attempt < 3; attempt += 1) { + let initial: SqliteFileGeneration; + try { + initial = readStableSqliteFileGeneration(pathname); + } catch (error) { + return unboundSqliteIntegrityFailure(error); + } + + let database: DatabaseSync; + try { + database = openNodeSqliteDatabase(pathname, { readOnly: true }); + } catch (error) { + // A failed SQLite open exposes no descriptor identity. Path snapshots + // cannot bind the error safely across an A -> B -> A file rotation. + return unboundSqliteIntegrityFailure(error); + } + + let opened: SqliteFileGeneration; + try { + opened = readStableSqliteFileGeneration(pathname); + } catch { + const closeError = closeSqliteDatabase(database); + if (closeError) { + return unboundSqliteIntegrityFailure(closeError); + } + continue; + } + if (!sameSqliteFileGeneration(initial, opened)) { + const closeError = closeSqliteDatabase(database); + if (closeError) { + return unboundSqliteIntegrityFailure(closeError); + } + continue; + } + + let confirmation = confirmSqliteIntegrity(database, databaseLabel); + const closeError = closeSqliteDatabase(database); + if (closeError && confirmation.status === "healthy") { + confirmation = failedSqliteIntegrityConfirmation(closeError); + } + + let final: SqliteFileGeneration; + try { + final = readStableSqliteFileGeneration(pathname); + } catch { + continue; + } + if (!sameSqliteFileGeneration(opened, final)) { + continue; + } + return bindSqliteIntegrityConfirmation(confirmation, final); + } + return unboundSqliteIntegrityFailure( + new Error(`SQLite file generation did not stabilize during confirmation: ${pathname}`), + ); +} + +function bindSqliteIntegrityConfirmation( + confirmation: UnboundSqliteIntegrityConfirmation, + generation: SqliteFileGeneration, +): SqliteIntegrityConfirmation { + if (confirmation.status === "healthy") { + return { status: "healthy", generation }; + } + if (confirmation.terminal) { + return { ...confirmation, generation, terminal: true }; + } + return { ...confirmation, terminal: false }; +} + +function failedSqliteIntegrityConfirmation(error: unknown): UnboundSqliteIntegrityConfirmation { + const normalized = error instanceof Error ? error : new Error(String(error)); + return { + status: "failed", + error: normalized, + terminal: isTerminalSqliteIntegrityError(normalized), + }; +} + +function unboundSqliteIntegrityFailure(error: unknown): SqliteIntegrityConfirmation { + const normalized = error instanceof Error ? error : new Error(String(error)); + return { status: "failed", error: normalized, terminal: false }; +} + +function closeSqliteDatabase(database: DatabaseSync): Error | undefined { + try { + database.close(); + return undefined; + } catch (error) { + return error instanceof Error ? error : new Error(String(error)); + } +} + /** Require table and associated index consistency before trusting indexed reads. */ export function assertSqliteTableIntegrity( database: DatabaseSync, diff --git a/src/infra/sqlite-terminal-open-latch.ts b/src/infra/sqlite-terminal-open-latch.ts index d5496c06759f..772a5df67af2 100644 --- a/src/infra/sqlite-terminal-open-latch.ts +++ b/src/infra/sqlite-terminal-open-latch.ts @@ -1,4 +1,22 @@ import path from "node:path"; +import { + readStableSqliteFileGeneration, + sameSqliteFileGeneration, + type SqliteFileGeneration, +} from "./sqlite-file-generation.js"; + +type TerminalOpenFailure = { + error: Error; + generation?: SqliteFileGeneration; +}; + +function generationMatchesPath(pathname: string, expected: SqliteFileGeneration): boolean { + try { + return sameSqliteFileGeneration(expected, readStableSqliteFileGeneration(pathname)); + } catch { + return false; + } +} /** * Per-path latch for terminal database-open failures (newer schema, proven @@ -8,15 +26,34 @@ import path from "node:path"; export function createSqliteTerminalOpenLatch(options: { closeByPath: (pathname: string) => void; }) { - const failures = new Map(); + const failures = new Map(); return { - get: (pathname: string): Error | undefined => failures.get(path.resolve(pathname)), - record: (pathname: string, error: Error): void => { + get: (pathname: string): Error | undefined => { const resolvedPath = path.resolve(pathname); - failures.set(resolvedPath, error); + const failure = failures.get(resolvedPath); + if (!failure) { + return undefined; + } + if (failure.generation && !generationMatchesPath(resolvedPath, failure.generation)) { + failures.delete(resolvedPath); + return undefined; + } + return failure.error; + }, + record: (pathname: string, error: Error, generation?: SqliteFileGeneration): boolean => { + const resolvedPath = path.resolve(pathname); + if (generation && !generationMatchesPath(resolvedPath, generation)) { + return false; + } + failures.set(resolvedPath, { error, ...(generation ? { generation } : {}) }); // Latch first. Close hooks may reenter. options.closeByPath(resolvedPath); + if (generation && !generationMatchesPath(resolvedPath, generation)) { + failures.delete(resolvedPath); + return false; + } + return true; }, clear: (pathname: string): void => { failures.delete(path.resolve(pathname)); diff --git a/src/state/openclaw-agent-db.ts b/src/state/openclaw-agent-db.ts index 2cdd34a01e6e..161f4a73711c 100644 --- a/src/state/openclaw-agent-db.ts +++ b/src/state/openclaw-agent-db.ts @@ -4,7 +4,12 @@ import path from "node:path"; import type { DatabaseSync } from "node:sqlite"; import { clearNodeSqliteKyselyCacheForDatabase } from "../infra/kysely-sync.js"; import { openNodeSqliteDatabase } from "../infra/node-sqlite.js"; -import { isTerminalSqliteIntegrityError } from "../infra/sqlite-integrity.js"; +import type { SqliteFileGeneration } from "../infra/sqlite-file-generation.js"; +import { + confirmSqliteFileIntegrity, + isTerminalSqliteIntegrityError, + type SqliteIntegrityConfirmation, +} from "../infra/sqlite-integrity.js"; import { createSqliteTerminalOpenLatch } from "../infra/sqlite-terminal-open-latch.js"; import { runSqliteImmediateTransactionSync, @@ -113,11 +118,30 @@ const terminalOpenLatch = createSqliteTerminalOpenLatch({ closeByPath: closeOpenClawAgentDatabaseByPath, }); +/** Reconfirm an advisory worker failure on the live owner connection. */ +export function confirmOpenClawAgentDatabaseIntegrity( + pathname: string, +): SqliteIntegrityConfirmation { + const resolvedPath = path.resolve(pathname); + closeOpenClawAgentDatabaseByPath(resolvedPath); + // Closing breaks process ownership of the pathname. A replacement must + // revalidate and claim its schema before the path can become trusted again. + validatedAgentDatabasePaths.delete(resolvedPath); + return confirmSqliteFileIntegrity(resolvedPath, resolvedPath); +} + /** Latch background verification damage so later opens fail without rescanning. */ -export function recordOpenClawAgentDatabaseOpenFailure(pathname: string, error: Error): void { - // Quarantine revokes this process's trust because doctor may replace the file. - validatedAgentDatabasePaths.delete(path.resolve(pathname)); - terminalOpenLatch.record(pathname, error); +export function recordOpenClawAgentDatabaseOpenFailure( + pathname: string, + error: Error, + generation?: SqliteFileGeneration, +): boolean { + const recorded = terminalOpenLatch.record(pathname, error, generation); + if (recorded) { + // Quarantine revokes this process's trust because doctor may replace the file. + validatedAgentDatabasePaths.delete(path.resolve(pathname)); + } + return recorded; } /** diff --git a/src/state/openclaw-database-verify.impl.ts b/src/state/openclaw-database-verify.impl.ts index fda3a1497f14..da885e53e8f0 100644 --- a/src/state/openclaw-database-verify.impl.ts +++ b/src/state/openclaw-database-verify.impl.ts @@ -4,6 +4,7 @@ import { fileURLToPath, pathToFileURL } from "node:url"; import { Worker } from "node:worker_threads"; import { createSubsystemLogger } from "../logging/subsystem.js"; import { + confirmOpenClawAgentDatabaseIntegrity, listOpenClawRegisteredAgentDatabases, recordOpenClawAgentDatabaseOpenFailure, } from "./openclaw-agent-db.js"; @@ -12,7 +13,10 @@ import type { OpenClawDatabaseVerifyTarget, } from "./openclaw-database-verify.worker.js"; import { recordOpenClawDatabaseQuarantine } from "./openclaw-quarantine-store.js"; -import { recordOpenClawStateDatabaseOpenFailure } from "./openclaw-state-db.js"; +import { + confirmOpenClawStateDatabaseIntegrity, + recordOpenClawStateDatabaseOpenFailure, +} from "./openclaw-state-db.js"; import { resolveOpenClawStateSqlitePath } from "./openclaw-state-db.paths.js"; export const OPENCLAW_DATABASE_VERIFY_INITIAL_DELAY_MS = 5 * 60_000; @@ -126,15 +130,7 @@ export function collectOpenClawDatabaseVerifyTargets(options: { return [...targets.values()]; } -function createVerificationFailure(result: OpenClawDatabaseVerifyResult): Error { - const error = new Error( - result.error ?? `SQLite integrity verification failed for ${result.path}`, - ); - error.name = "SqliteIntegrityError"; - return error; -} - -/** Quarantine terminal failures and log the worker batch. */ +/** Reconfirm worker failures on live owners before quarantine and latching. */ export function applyOpenClawDatabaseVerificationResults(options: { env: NodeJS.ProcessEnv; results: readonly OpenClawDatabaseVerifyResult[]; @@ -164,11 +160,53 @@ export function applyOpenClawDatabaseVerificationResults(options: { }); continue; } + const confirmation = + target.kind === "state" + ? confirmOpenClawStateDatabaseIntegrity(result.path) + : confirmOpenClawAgentDatabaseIntegrity(result.path); + if (confirmation.status === "healthy") { + log.info("discarding stale database integrity verification result", { + kind: target.kind, + label: target.label, + path: result.path, + }); + continue; + } + if (!confirmation.terminal) { + log.warn("database integrity verification was inconclusive", { + kind: target.kind, + label: target.label, + path: result.path, + error: confirmation.error.message, + }); + continue; + } + const latched = + target.kind === "state" + ? recordOpenClawStateDatabaseOpenFailure( + result.path, + confirmation.error, + confirmation.generation, + ) + : recordOpenClawAgentDatabaseOpenFailure( + result.path, + confirmation.error, + confirmation.generation, + ); + if (!latched) { + log.info("discarding database integrity result after database generation changed", { + kind: target.kind, + label: target.label, + path: result.path, + }); + continue; + } const recorded = recordOpenClawDatabaseQuarantine({ env: options.env, + generation: confirmation.generation, kind: target.kind, path: result.path, - reason: result.error ?? `SQLite integrity verification failed for ${result.path}`, + reason: confirmation.error.message, }); if (!recorded) { // Store unavailable. Daily verification retries persistence. @@ -177,17 +215,11 @@ export function applyOpenClawDatabaseVerificationResults(options: { path: result.path, }); } - const error = createVerificationFailure(result); - if (target.kind === "state") { - recordOpenClawStateDatabaseOpenFailure(result.path, error); - } else { - recordOpenClawAgentDatabaseOpenFailure(result.path, error); - } log.error("database integrity verification failed", { kind: target.kind, label: target.label, path: result.path, - error: error.message, + error: confirmation.error.message, }); } } diff --git a/src/state/openclaw-database-verify.test.ts b/src/state/openclaw-database-verify.test.ts index 0e7539faa66e..23ad5942fbcb 100644 --- a/src/state/openclaw-database-verify.test.ts +++ b/src/state/openclaw-database-verify.test.ts @@ -4,11 +4,14 @@ import path from "node:path"; import { afterAll, afterEach, describe, expect, it } from "vitest"; import { cleanupTempDirs, makeTempDir } from "../../test/helpers/temp-dir.js"; import { requireNodeSqlite } from "../infra/node-sqlite.js"; +import { readStableSqliteFileGeneration } from "../infra/sqlite-file-generation.js"; import { readSqliteNumberPragma } from "../infra/sqlite-pragma.test-support.js"; import { clearOpenClawAgentDatabaseOpenFailure, + closeOpenClawAgentDatabaseByPath, closeOpenClawAgentDatabasesForTest, openOpenClawAgentDatabase, + recordOpenClawAgentDatabaseOpenFailure, } from "./openclaw-agent-db.js"; import { applyOpenClawDatabaseVerificationResults, @@ -26,6 +29,7 @@ import { import { closeOpenClawStateDatabaseForTest, openOpenClawStateDatabase, + recordOpenClawStateDatabaseOpenFailure, repairOpenClawStateDatabaseSchema, } from "./openclaw-state-db.js"; @@ -66,6 +70,34 @@ function createUnsafeIndexDrift(databasePath: string): void { } } +function repairUnsafeIndexDrift(databasePath: string): void { + const { DatabaseSync } = requireNodeSqlite(); + const database = new DatabaseSync(databasePath); + try { + database.enableDefensive?.(false); + database.exec("PRAGMA writable_schema = ON;"); + database + .prepare( + "UPDATE sqlite_schema SET sql = 'CREATE INDEX unsafe_index_records_value ON unsafe_index_records(indexed_value)' WHERE name = 'unsafe_index_records_value'", + ) + .run(); + const schemaVersion = readSqliteNumberPragma(database, "schema_version"); + database.exec(`PRAGMA writable_schema = OFF; PRAGMA schema_version = ${schemaVersion + 1};`); + } finally { + database.close(); + } +} + +async function copyHealthyDatabase(sourcePath: string, targetPath: string): Promise { + const sqlite = requireNodeSqlite(); + const source = new sqlite.DatabaseSync(sourcePath, { readOnly: true }); + try { + await sqlite.backup(source, targetPath); + } finally { + source.close(); + } +} + function quarantineStorePath(stateDir: string): string { return path.join(stateDir, "state", "openclaw-quarantine.sqlite"); } @@ -75,14 +107,12 @@ describe("OpenClaw database integrity verifier", () => { const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-"); const env = { OPENCLAW_STATE_DIR: stateDir }; const agentPath = openOpenClawAgentDatabase({ agentId: "worker-1", env }).path; - closeOpenClawAgentDatabasesForTest(); - closeOpenClawStateDatabaseForTest(); createUnsafeIndexDrift(agentPath); const targets: OpenClawDatabaseVerifyTarget[] = [ { kind: "agent", label: "OpenClaw agent database worker-1", path: agentPath }, ]; - const directResults = verifyOpenClawDatabases(targets); + const directResults = await verifyOpenClawDatabases(targets); expect(directResults).toEqual([ { path: agentPath, @@ -93,21 +123,16 @@ describe("OpenClaw database integrity verifier", () => { ]); await expect(runDatabaseVerifyWorker(targets)).resolves.toEqual(directResults); - // The drift is not a committed canonical index, so open must fail closed - // instead of guessing a replacement definition. - expect(() => openOpenClawAgentDatabase({ agentId: "worker-1", env })).toThrow( - expect.objectContaining({ name: "SqliteIntegrityError" }), - ); - applyOpenClawDatabaseVerificationResults({ env, results: directResults, targets, }); - expect(readOpenClawDatabaseQuarantine(agentPath, { env })).toEqual({ + const quarantine = readOpenClawDatabaseQuarantine(agentPath, { env }); + expect(quarantine).toEqual({ kind: "agent", quarantinedAt: expect.any(Number), - reason: directResults[0]?.error, + reason: expect.stringMatching(/missing from index unsafe_index_records_value/iu), }); expect(() => openOpenClawAgentDatabase({ agentId: "worker-1", env })).toThrow( @@ -119,7 +144,7 @@ describe("OpenClaw database integrity verifier", () => { expect(() => openOpenClawAgentDatabase({ agentId: "worker-1", env })).toThrow( expect.objectContaining({ name: "SqliteIntegrityError", - message: expect.stringContaining(directResults[0]?.error ?? ""), + message: expect.stringContaining(quarantine?.reason ?? ""), }), ); clearOpenClawAgentDatabaseOpenFailure(agentPath, { env }); @@ -128,6 +153,215 @@ describe("OpenClaw database integrity verifier", () => { ); }); + it("does not quarantine a healthy database that replaced the verified file", async () => { + const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-replacement-"); + const env = { OPENCLAW_STATE_DIR: stateDir }; + const agentPath = openOpenClawAgentDatabase({ agentId: "worker-1", env }).path; + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + + const healthyReplacementPath = `${agentPath}.healthy`; + const corruptArchivePath = `${agentPath}.corrupt`; + fs.copyFileSync(agentPath, healthyReplacementPath); + createUnsafeIndexDrift(agentPath); + const targets: OpenClawDatabaseVerifyTarget[] = [ + { kind: "agent", label: "OpenClaw agent database worker-1", path: agentPath }, + ]; + const results = await verifyOpenClawDatabases(targets); + expect(results).toEqual([ + expect.objectContaining({ + path: agentPath, + ok: false, + terminal: true, + }), + ]); + + fs.renameSync(agentPath, corruptArchivePath); + fs.renameSync(healthyReplacementPath, agentPath); + applyOpenClawDatabaseVerificationResults({ env, results, targets }); + + expect(readOpenClawDatabaseQuarantine(agentPath, { env })).toBeUndefined(); + expect(openOpenClawAgentDatabase({ agentId: "worker-1", env }).db.isOpen).toBe(true); + }); + + it.skipIf(process.platform === "win32")( + "does not quarantine a healthy replacement while the corrupt agent inode is cached", + async () => { + const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-live-agent-replace-"); + const env = { OPENCLAW_STATE_DIR: stateDir }; + const agent = openOpenClawAgentDatabase({ agentId: "worker-1", env }); + const healthyReplacementPath = `${agent.path}.healthy`; + const corruptArchivePath = `${agent.path}.corrupt`; + await copyHealthyDatabase(agent.path, healthyReplacementPath); + createUnsafeIndexDrift(agent.path); + const targets: OpenClawDatabaseVerifyTarget[] = [ + { kind: "agent", label: "OpenClaw agent database worker-1", path: agent.path }, + ]; + const results = await verifyOpenClawDatabases(targets); + + fs.renameSync(agent.path, corruptArchivePath); + fs.renameSync(healthyReplacementPath, agent.path); + applyOpenClawDatabaseVerificationResults({ env, results, targets }); + + expect(agent.db.isOpen).toBe(false); + expect(readOpenClawDatabaseQuarantine(agent.path, { env })).toBeUndefined(); + expect(openOpenClawAgentDatabase({ agentId: "worker-1", env }).db.isOpen).toBe(true); + }, + ); + + it.skipIf(process.platform === "win32")( + "does not quarantine a healthy replacement while the corrupt state inode is cached", + async () => { + const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-live-state-replace-"); + const env = { OPENCLAW_STATE_DIR: stateDir }; + const state = openOpenClawStateDatabase({ env }); + const healthyReplacementPath = `${state.path}.healthy`; + const corruptArchivePath = `${state.path}.corrupt`; + await copyHealthyDatabase(state.path, healthyReplacementPath); + createUnsafeIndexDrift(state.path); + const targets: OpenClawDatabaseVerifyTarget[] = [ + { kind: "state", label: "OpenClaw state database", path: state.path }, + ]; + const results = await verifyOpenClawDatabases(targets); + + fs.renameSync(state.path, corruptArchivePath); + fs.renameSync(healthyReplacementPath, state.path); + applyOpenClawDatabaseVerificationResults({ env, results, targets }); + + expect(state.db.isOpen).toBe(false); + expect(readOpenClawDatabaseQuarantine(state.path, { env })).toBeUndefined(); + expect(openOpenClawStateDatabase({ env }).db.isOpen).toBe(true); + }, + ); + + it("reconfirms and quarantines a corrupt closed database", async () => { + const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-closed-"); + const env = { OPENCLAW_STATE_DIR: stateDir }; + const agentPath = openOpenClawAgentDatabase({ agentId: "worker-1", env }).path; + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + createUnsafeIndexDrift(agentPath); + const targets: OpenClawDatabaseVerifyTarget[] = [ + { kind: "agent", label: "OpenClaw agent database worker-1", path: agentPath }, + ]; + const results = await verifyOpenClawDatabases(targets); + + applyOpenClawDatabaseVerificationResults({ env, results, targets }); + + expect(readOpenClawDatabaseQuarantine(agentPath, { env })?.reason).toMatch( + /missing from index unsafe_index_records_value/iu, + ); + expect(() => openOpenClawAgentDatabase({ agentId: "worker-1", env })).toThrow( + expect.objectContaining({ name: "SqliteIntegrityError" }), + ); + }); + + it("does not quarantine a repaired database after same-inode mutation", async () => { + const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-repair-race-"); + const env = { OPENCLAW_STATE_DIR: stateDir }; + const agentPath = openOpenClawAgentDatabase({ agentId: "worker-1", env }).path; + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + createUnsafeIndexDrift(agentPath); + const targets: OpenClawDatabaseVerifyTarget[] = [ + { kind: "agent", label: "OpenClaw agent database worker-1", path: agentPath }, + ]; + const results = await verifyOpenClawDatabases(targets); + expect(results[0]).toEqual(expect.objectContaining({ ok: false, terminal: true })); + + repairUnsafeIndexDrift(agentPath); + await expect(verifyOpenClawDatabases(targets)).resolves.toEqual([ + expect.objectContaining({ ok: true }), + ]); + applyOpenClawDatabaseVerificationResults({ env, results, targets }); + + expect(readOpenClawDatabaseQuarantine(agentPath, { env })).toBeUndefined(); + expect(openOpenClawAgentDatabase({ agentId: "worker-1", env }).db.isOpen).toBe(true); + }); + + it("rejects stale terminal results after draining healthy owners", () => { + const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-live-stale-"); + const env = { OPENCLAW_STATE_DIR: stateDir }; + const state = openOpenClawStateDatabase({ env }); + const agent = openOpenClawAgentDatabase({ agentId: "worker-1", env }); + const targets: OpenClawDatabaseVerifyTarget[] = [ + { kind: "state", label: "OpenClaw state database", path: state.path }, + { kind: "agent", label: "OpenClaw agent database worker-1", path: agent.path }, + ]; + + applyOpenClawDatabaseVerificationResults({ + env, + results: targets.map((target) => ({ + path: target.path, + ok: false, + error: "stale terminal result", + terminal: true, + })), + targets, + }); + + expect(readOpenClawDatabaseQuarantine(state.path, { env })).toBeUndefined(); + expect(readOpenClawDatabaseQuarantine(agent.path, { env })).toBeUndefined(); + expect(state.db.isOpen).toBe(false); + expect(agent.db.isOpen).toBe(false); + expect(openOpenClawStateDatabase({ env }).db.isOpen).toBe(true); + expect(openOpenClawAgentDatabase({ agentId: "worker-1", env }).db.isOpen).toBe(true); + }); + + it("revalidates agent schema ownership after confirmation drains a pathname", () => { + const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-agent-revalidate-"); + const env = { OPENCLAW_STATE_DIR: stateDir }; + const agent = openOpenClawAgentDatabase({ agentId: "worker-1", env }); + const replacementPath = `${agent.path}.replacement`; + const { DatabaseSync } = requireNodeSqlite(); + const replacement = new DatabaseSync(replacementPath); + try { + replacement.exec("CREATE TABLE replacement_marker (id INTEGER PRIMARY KEY) STRICT;"); + } finally { + replacement.close(); + } + expect(closeOpenClawAgentDatabaseByPath(agent.path)).toBe(true); + fs.rmSync(agent.path); + fs.renameSync(replacementPath, agent.path); + const targets: OpenClawDatabaseVerifyTarget[] = [ + { kind: "agent", label: "OpenClaw agent database worker-1", path: agent.path }, + ]; + + applyOpenClawDatabaseVerificationResults({ + env, + results: [{ path: agent.path, ok: false, error: "stale terminal result", terminal: true }], + targets, + }); + + const reopened = openOpenClawAgentDatabase({ agentId: "worker-1", env }); + expect( + reopened.db + .prepare("SELECT role, agent_id FROM schema_meta WHERE meta_key = 'primary'") + .get(), + ).toEqual({ role: "agent", agent_id: "worker-1" }); + }); + + it("expires a generation-bound process latch after the database changes", () => { + const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-latch-generation-"); + const env = { OPENCLAW_STATE_DIR: stateDir }; + const agentPath = openOpenClawAgentDatabase({ agentId: "worker-1", env }).path; + closeOpenClawAgentDatabasesForTest(); + const generation = readStableSqliteFileGeneration(agentPath); + const error = new Error("verified corrupt generation"); + error.name = "SqliteIntegrityError"; + + expect(recordOpenClawAgentDatabaseOpenFailure(agentPath, error, generation)).toBe(true); + const { DatabaseSync } = requireNodeSqlite(); + const changed = new DatabaseSync(agentPath); + try { + changed.exec("CREATE TABLE generation_change (id INTEGER PRIMARY KEY) STRICT;"); + } finally { + changed.close(); + } + + expect(openOpenClawAgentDatabase({ agentId: "worker-1", env }).db.isOpen).toBe(true); + }); + it("reports an uncleared quarantine row instead of claiming repair success", () => { const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-clear-failure-"); const env = { OPENCLAW_STATE_DIR: stateDir }; @@ -135,11 +369,17 @@ describe("OpenClaw database integrity verifier", () => { openOpenClawStateDatabase({ env }); closeOpenClawAgentDatabasesForTest(); closeOpenClawStateDatabaseForTest(); - applyOpenClawDatabaseVerificationResults({ - env, - results: [{ path: agentPath, ok: false, error: "corrupt index", terminal: true }], - targets: [{ kind: "agent", label: "OpenClaw agent database worker-1", path: agentPath }], - }); + expect( + recordOpenClawDatabaseQuarantine({ + env, + kind: "agent", + path: agentPath, + reason: "corrupt index", + }), + ).toBe(true); + const error = new Error("corrupt index"); + error.name = "SqliteIntegrityError"; + recordOpenClawAgentDatabaseOpenFailure(agentPath, error); closeOpenClawStateDatabaseForTest(); const storePath = quarantineStorePath(stateDir); @@ -199,7 +439,7 @@ describe("OpenClaw database integrity verifier", () => { try { expect(raw.prepare("PRAGMA journal_mode").get()).toEqual({ journal_mode: "delete" }); expect(readSqliteNumberPragma(raw, "synchronous")).toBe(2); - expect(readSqliteNumberPragma(raw, "user_version")).toBe(1); + expect(readSqliteNumberPragma(raw, "user_version")).toBe(2); } finally { raw.close(); } @@ -212,6 +452,96 @@ describe("OpenClaw database integrity verifier", () => { expect(readOpenClawDatabaseQuarantine(databasePath, { env })).toBeUndefined(); }); + it("expires a persisted quarantine when the verified database generation changes", () => { + const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-store-generation-"); + const env = { OPENCLAW_STATE_DIR: stateDir }; + const databasePath = path.join(stateDir, "agent.sqlite"); + const { DatabaseSync } = requireNodeSqlite(); + const database = new DatabaseSync(databasePath); + try { + database.exec("CREATE TABLE records (id INTEGER PRIMARY KEY) STRICT;"); + } finally { + database.close(); + } + const generation = readStableSqliteFileGeneration(databasePath); + + expect( + recordOpenClawDatabaseQuarantine({ + env, + generation, + kind: "agent", + path: databasePath, + reason: "corrupt generation", + }), + ).toBe(true); + expect(readOpenClawDatabaseQuarantine(databasePath, { env })?.reason).toBe( + "corrupt generation", + ); + + const changed = new DatabaseSync(databasePath); + try { + changed.exec("INSERT INTO records DEFAULT VALUES;"); + } finally { + changed.close(); + } + expect(readOpenClawDatabaseQuarantine(databasePath, { env })).toBeUndefined(); + }); + + it("reads schema-v1 quarantine rows and migrates them on the next write", () => { + const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-store-v1-"); + const env = { OPENCLAW_STATE_DIR: stateDir }; + const databasePath = path.join(stateDir, "agent.sqlite"); + const storePath = quarantineStorePath(stateDir); + fs.mkdirSync(path.dirname(storePath), { recursive: true }); + const { DatabaseSync } = requireNodeSqlite(); + const legacy = new DatabaseSync(storePath); + try { + legacy.exec(` + CREATE TABLE quarantined_databases ( + path TEXT NOT NULL PRIMARY KEY, + kind TEXT NOT NULL, + reason TEXT NOT NULL, + quarantined_at INTEGER NOT NULL, + writer_app_version TEXT + ) STRICT; + PRAGMA user_version = 1; + `); + legacy + .prepare( + "INSERT INTO quarantined_databases (path, kind, reason, quarantined_at) VALUES (?, ?, ?, ?)", + ) + .run(path.resolve(databasePath), "agent", "legacy quarantine", 1); + } finally { + legacy.close(); + } + + expect(readOpenClawDatabaseQuarantine(databasePath, { env })).toEqual({ + kind: "agent", + quarantinedAt: 1, + reason: "legacy quarantine", + }); + expect( + recordOpenClawDatabaseQuarantine({ + env, + kind: "agent", + path: databasePath, + reason: "migrated quarantine", + }), + ).toBe(true); + + const migrated = new DatabaseSync(storePath, { readOnly: true }); + try { + expect(readSqliteNumberPragma(migrated, "user_version")).toBe(2); + expect( + migrated + .prepare("SELECT reason, verified_generation FROM quarantined_databases WHERE path = ?") + .get(path.resolve(databasePath)), + ).toEqual({ reason: "migrated quarantine", verified_generation: null }); + } finally { + migrated.close(); + } + }); + it("recovers an interrupted empty quarantine-store initialization", () => { const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-empty-store-"); const env = { OPENCLAW_STATE_DIR: stateDir }; @@ -297,15 +627,18 @@ describe("OpenClaw database integrity verifier", () => { const env = { OPENCLAW_STATE_DIR: stateDir }; const statePath = openOpenClawStateDatabase({ env }).path; closeOpenClawStateDatabaseForTest(); - const targets: OpenClawDatabaseVerifyTarget[] = [ - { kind: "state", label: "OpenClaw state database", path: statePath }, - ]; - applyOpenClawDatabaseVerificationResults({ - env, - results: [{ path: statePath, ok: false, error: "corrupt index", terminal: true }], - targets, - }); + expect( + recordOpenClawDatabaseQuarantine({ + env, + kind: "state", + path: statePath, + reason: "corrupt index", + }), + ).toBe(true); + const error = new Error("corrupt index"); + error.name = "SqliteIntegrityError"; + recordOpenClawStateDatabaseOpenFailure(statePath, error); const { DatabaseSync } = requireNodeSqlite(); const raw = new DatabaseSync(quarantineStorePath(stateDir), { readOnly: true }); diff --git a/src/state/openclaw-database-verify.worker.ts b/src/state/openclaw-database-verify.worker.ts index 7171995bf9f6..a5a2c336b78b 100644 --- a/src/state/openclaw-database-verify.worker.ts +++ b/src/state/openclaw-database-verify.worker.ts @@ -4,6 +4,7 @@ import { assertSqliteIntegrity, isTerminalSqliteIntegrityError, } from "../infra/sqlite-integrity.js"; +import { prepareSqliteReadOnlyLocation } from "../infra/sqlite-readonly-location.js"; import { OPENCLAW_SQLITE_BUSY_TIMEOUT_MS } from "./openclaw-state-db.js"; export type OpenClawDatabaseVerifyTarget = { @@ -31,30 +32,64 @@ function isVerifyTarget(value: unknown): value is OpenClawDatabaseVerifyTarget { ); } -/** Verify database files serially so large agent scans never compete for I/O. */ -export function verifyOpenClawDatabases( - targets: readonly OpenClawDatabaseVerifyTarget[], -): OpenClawDatabaseVerifyResult[] { - return targets.map((target) => { - let database: import("node:sqlite").DatabaseSync | undefined; +function formatVerifyError(error: unknown): string { + return error instanceof Error ? `${error.name}: ${error.message}` : String(error); +} + +async function verifyOpenClawDatabase( + target: OpenClawDatabaseVerifyTarget, +): Promise { + let cleanup: (() => boolean) | undefined; + let database: import("node:sqlite").DatabaseSync | undefined; + let result = await (async (): Promise => { try { - database = openNodeSqliteDatabase(target.path, { + const prepared = await prepareSqliteReadOnlyLocation(target.path); + cleanup = prepared.cleanup; + database = openNodeSqliteDatabase(prepared.location, { readOnly: true, }); database.exec(`PRAGMA busy_timeout = ${OPENCLAW_SQLITE_BUSY_TIMEOUT_MS};`); assertSqliteIntegrity(database, target.label); return { path: target.path, ok: true }; } catch (error) { - const detail = error instanceof Error ? `${error.name}: ${error.message}` : String(error); const terminal = error instanceof Error && isTerminalSqliteIntegrityError(error); - return { path: target.path, ok: false, error: detail, terminal }; - } finally { - database?.close(); + return { + path: target.path, + ok: false, + error: formatVerifyError(error), + terminal, + }; } - }); + })(); + try { + database?.close(); + } catch (error) { + if (result.ok) { + result = { + path: target.path, + ok: false, + error: formatVerifyError(error), + terminal: false, + }; + } + } finally { + cleanup?.(); + } + return result; +} + +/** Verify database files serially so large agent scans never compete for I/O. */ +export async function verifyOpenClawDatabases( + targets: readonly OpenClawDatabaseVerifyTarget[], +): Promise { + const results: OpenClawDatabaseVerifyResult[] = []; + for (const target of targets) { + results.push(await verifyOpenClawDatabase(target)); + } + return results; } if (parentPort) { const targets = Array.isArray(workerData) ? workerData.filter(isVerifyTarget) : []; - parentPort.postMessage(verifyOpenClawDatabases(targets), []); + parentPort.postMessage(await verifyOpenClawDatabases(targets), []); } diff --git a/src/state/openclaw-quarantine-store.ts b/src/state/openclaw-quarantine-store.ts index addf191456d9..9c4c6c83e60c 100644 --- a/src/state/openclaw-quarantine-store.ts +++ b/src/state/openclaw-quarantine-store.ts @@ -4,10 +4,17 @@ import path from "node:path"; import type { DatabaseSync } from "node:sqlite"; import { openNodeSqliteDatabase } from "../infra/node-sqlite.js"; import { applyPrivateModeSync } from "../infra/private-mode.js"; +import { + parseSqliteFileGeneration, + readStableSqliteFileGeneration, + sameSqliteFileGeneration, + serializeSqliteFileGeneration, + type SqliteFileGeneration, +} from "../infra/sqlite-file-generation.js"; import { VERSION } from "../version.js"; import { resolveOpenClawStateSqliteDir } from "./openclaw-state-db.paths.js"; -const OPENCLAW_QUARANTINE_SCHEMA_VERSION = 1; +const OPENCLAW_QUARANTINE_SCHEMA_VERSION = 2; const OPENCLAW_QUARANTINE_BUSY_TIMEOUT_MS = 5_000; const OPENCLAW_QUARANTINE_DIR_MODE = 0o700; const OPENCLAW_QUARANTINE_FILE_MODE = 0o600; @@ -45,6 +52,15 @@ function configureQuarantineWriter(database: DatabaseSync, storePath: string): v if (userVersion === OPENCLAW_QUARANTINE_SCHEMA_VERSION) { return; } + if (userVersion === 1) { + database.exec(` + BEGIN IMMEDIATE; + ALTER TABLE quarantined_databases ADD COLUMN verified_generation TEXT; + PRAGMA user_version = ${OPENCLAW_QUARANTINE_SCHEMA_VERSION}; + COMMIT; + `); + return; + } database.exec(` BEGIN IMMEDIATE; CREATE TABLE IF NOT EXISTS quarantined_databases ( @@ -52,7 +68,8 @@ function configureQuarantineWriter(database: DatabaseSync, storePath: string): v kind TEXT NOT NULL, reason TEXT NOT NULL, quarantined_at INTEGER NOT NULL, - writer_app_version TEXT + writer_app_version TEXT, + verified_generation TEXT ) STRICT; PRAGMA user_version = ${OPENCLAW_QUARANTINE_SCHEMA_VERSION}; COMMIT; @@ -109,17 +126,23 @@ export function readOpenClawDatabaseQuarantine( if (userVersion === 0) { return undefined; } - if (userVersion !== OPENCLAW_QUARANTINE_SCHEMA_VERSION) { + if (userVersion > OPENCLAW_QUARANTINE_SCHEMA_VERSION) { throw new Error( `OpenClaw quarantine store ${storePath} uses newer schema version ${userVersion}.`, ); } + const generationColumn = userVersion >= 2 ? ", verified_generation" : ""; const row = database .prepare( - "SELECT kind, reason, quarantined_at FROM quarantined_databases WHERE path = ? LIMIT 1", + `SELECT kind, reason, quarantined_at${generationColumn} FROM quarantined_databases WHERE path = ? LIMIT 1`, ) .get(path.resolve(pathname)) as - | { kind?: unknown; quarantined_at?: unknown; reason?: unknown } + | { + kind?: unknown; + quarantined_at?: unknown; + reason?: unknown; + verified_generation?: unknown; + } | undefined; if (!row) { return undefined; @@ -128,10 +151,29 @@ export function readOpenClawDatabaseQuarantine( (row.kind !== "agent" && row.kind !== "state") || typeof row.reason !== "string" || typeof row.quarantined_at !== "number" || - !Number.isInteger(row.quarantined_at) + !Number.isInteger(row.quarantined_at) || + (row.verified_generation !== undefined && + row.verified_generation !== null && + typeof row.verified_generation !== "string") ) { throw new Error(`OpenClaw quarantine store ${storePath} contains an invalid row.`); } + if (typeof row.verified_generation === "string") { + let verifiedGeneration: SqliteFileGeneration; + try { + verifiedGeneration = parseSqliteFileGeneration(row.verified_generation); + } catch { + throw new Error(`OpenClaw quarantine store ${storePath} contains an invalid row.`); + } + try { + const currentGeneration = readStableSqliteFileGeneration(path.resolve(pathname)); + if (!sameSqliteFileGeneration(verifiedGeneration, currentGeneration)) { + return undefined; + } + } catch { + return undefined; + } + } return { kind: row.kind, quarantinedAt: row.quarantined_at, reason: row.reason }; } finally { database.close(); @@ -141,10 +183,14 @@ export function readOpenClawDatabaseQuarantine( /** Persist one authoritative quarantine decision. */ export function recordOpenClawDatabaseQuarantine(options: { env?: NodeJS.ProcessEnv; + generation?: SqliteFileGeneration; kind: OpenClawDatabaseKind; path: string; reason: string; }): boolean { + const serializedGeneration = options.generation + ? serializeSqliteFileGeneration(options.generation) + : null; try { return withQuarantineWriter(options.env ?? process.env, (database) => { database.exec("BEGIN IMMEDIATE;"); @@ -153,16 +199,24 @@ export function recordOpenClawDatabaseQuarantine(options: { .prepare( ` INSERT INTO quarantined_databases ( - path, kind, reason, quarantined_at, writer_app_version - ) VALUES (?, ?, ?, ?, ?) + path, kind, reason, quarantined_at, writer_app_version, verified_generation + ) VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT(path) DO UPDATE SET kind = excluded.kind, reason = excluded.reason, quarantined_at = excluded.quarantined_at, - writer_app_version = excluded.writer_app_version + writer_app_version = excluded.writer_app_version, + verified_generation = excluded.verified_generation `, ) - .run(path.resolve(options.path), options.kind, options.reason, Date.now(), VERSION); + .run( + path.resolve(options.path), + options.kind, + options.reason, + Date.now(), + VERSION, + serializedGeneration, + ); database.exec("COMMIT;"); return true; } catch (error) { diff --git a/src/state/openclaw-state-db.ts b/src/state/openclaw-state-db.ts index 8a3234a2f435..bb83f7189219 100644 --- a/src/state/openclaw-state-db.ts +++ b/src/state/openclaw-state-db.ts @@ -8,10 +8,13 @@ import { getNodeSqliteKysely, } from "../infra/kysely-sync.js"; import { openNodeSqliteDatabase } from "../infra/node-sqlite.js"; +import type { SqliteFileGeneration } from "../infra/sqlite-file-generation.js"; import { repairCanonicalSqliteIndexes } from "../infra/sqlite-index-schema.js"; import { assertSqliteIntegrity, + confirmSqliteFileIntegrity, isTerminalSqliteIntegrityError, + type SqliteIntegrityConfirmation, } from "../infra/sqlite-integrity.js"; import { prepareSqliteReadOnlyLocation } from "../infra/sqlite-readonly-location.js"; import { migrateSqliteSchemaToStrictInTransaction } from "../infra/sqlite-strict.js"; @@ -104,9 +107,22 @@ const terminalOpenLatch = createSqliteTerminalOpenLatch({ }, }); +/** Reconfirm an advisory worker failure on the live owner connection. */ +export function confirmOpenClawStateDatabaseIntegrity( + pathname: string, +): SqliteIntegrityConfirmation { + const resolvedPath = path.resolve(pathname); + closeOpenClawStateDatabaseByPath(resolvedPath); + return confirmSqliteFileIntegrity(resolvedPath, resolvedPath); +} + /** Latch background verification damage so later opens fail without rescanning. */ -export function recordOpenClawStateDatabaseOpenFailure(pathname: string, error: Error): void { - terminalOpenLatch.record(pathname, error); +export function recordOpenClawStateDatabaseOpenFailure( + pathname: string, + error: Error, + generation?: SqliteFileGeneration, +): boolean { + return terminalOpenLatch.record(pathname, error, generation); } /** Clear a terminal open failure after doctor rewrites the database file. */