diff --git a/src/infra/sqlite-files.ts b/src/infra/sqlite-files.ts index 8099cf38d4dc..53229f8ac792 100644 --- a/src/infra/sqlite-files.ts +++ b/src/infra/sqlite-files.ts @@ -1,7 +1,139 @@ +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { createSubsystemLogger } from "../logging/subsystem.js"; + /** SQLite main database plus every journal-mode sidecar that can contain database pages. */ const SQLITE_DATABASE_FILE_SUFFIXES = ["", "-wal", "-shm", "-journal"] as const; +// SQLite WAL format: https://sqlite.org/fileformat2.html#walformat defines a 32-byte header. +const SQLITE_WAL_HEADER_BYTES = 32; +const SQLITE_SIDECAR_HASH_BUFFER_BYTES = 1024 * 1024; +const sqliteFilesLog = createSubsystemLogger("state/sqlite"); + +class SqliteOrphanedSidecarsError extends Error { + constructor(pathname: string, sidecarPaths: string[], cause: unknown) { + super( + `SQLite database is missing at ${pathname}, and orphaned sidecars could not be copied: ${sidecarPaths.join(", ")}. ` + + "Refusing to open because SQLite could delete orphan WAL or journal state. Preserve the sidecar bytes, restore the main database, and pair it with the matching sidecar before retrying.", + { cause }, + ); + this.name = "SqliteOrphanedSidecarsError"; + } +} + +type CopiedSqliteSidecar = { + quarantinePath: string; + sourcePath: string; +}; /** Resolves the main database and all possible journal-mode sidecar paths. */ export function resolveSqliteDatabaseFilePaths(pathname: string): string[] { return SQLITE_DATABASE_FILE_SUFFIXES.map((suffix) => `${pathname}${suffix}`); } + +function sha256FileSync(pathname: string): string { + const descriptor = fs.openSync(pathname, "r"); + const digest = createHash("sha256"); + const buffer = Buffer.allocUnsafe(SQLITE_SIDECAR_HASH_BUFFER_BYTES); + try { + while (true) { + const bytesRead = fs.readSync(descriptor, buffer, 0, buffer.length, null); + if (bytesRead === 0) { + return digest.digest("hex"); + } + digest.update(buffer.subarray(0, bytesRead)); + } + } finally { + fs.closeSync(descriptor); + } +} + +function findMatchingOrphanedSidecarCopy( + sourcePath: string, + sourceSize: number, +): string | undefined { + const directory = path.dirname(sourcePath); + const prefix = `${path.basename(sourcePath)}.orphaned-`; + const candidates = fs + .readdirSync(directory, { withFileTypes: true }) + .filter((entry) => entry.isFile() && entry.name.startsWith(prefix)) + .map((entry) => path.join(directory, entry.name)) + .filter((candidate) => fs.statSync(candidate).size === sourceSize); + if (candidates.length === 0) { + return undefined; + } + const sourceHash = sha256FileSync(sourcePath); + for (const candidate of candidates) { + if (sha256FileSync(candidate) === sourceHash) { + return candidate; + } + } + return undefined; +} + +function copyOrphanedSidecar(sourcePath: string, epochMs: number): string { + const basePath = `${sourcePath}.orphaned-${epochMs}`; + for (let suffix = 0; ; suffix += 1) { + const candidate = suffix === 0 ? basePath : `${basePath}-${suffix}`; + try { + fs.copyFileSync(sourcePath, candidate, fs.constants.COPYFILE_EXCL); + return candidate; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") { + throw error; + } + } + } +} + +/** Preserve durable orphan sidecars before SQLite creates a replacement main database. */ +export function quarantineOrphanedSqliteSidecars(pathname: string): void { + if (fs.existsSync(pathname)) { + return; + } + const sidecars = [ + { path: `${pathname}-wal`, minimumBytes: SQLITE_WAL_HEADER_BYTES }, + { path: `${pathname}-journal`, minimumBytes: 0 }, + ].flatMap((sidecar) => { + const stat = fs.statSync(sidecar.path, { throwIfNoEntry: false }); + return stat?.isFile() === true && stat.size > sidecar.minimumBytes + ? [{ path: sidecar.path, size: stat.size }] + : []; + }); + if (sidecars.length === 0) { + return; + } + + const epochMs = Date.now(); + const copied: CopiedSqliteSidecar[] = []; + try { + for (const sidecar of sidecars) { + if (findMatchingOrphanedSidecarCopy(sidecar.path, sidecar.size)) { + continue; + } + const quarantinePath = copyOrphanedSidecar(sidecar.path, epochMs); + copied.push({ quarantinePath, sourcePath: sidecar.path }); + } + } catch (error) { + throw new SqliteOrphanedSidecarsError( + pathname, + sidecars.map((sidecar) => sidecar.path), + error, + ); + } + if (copied.length === 0) { + return; + } + + const copies = copied.map( + ({ sourcePath, quarantinePath }) => `${sourcePath} -> ${quarantinePath}`, + ); + sqliteFilesLog.warn( + `SQLite database is missing at ${pathname}; copied orphaned sidecars: ${copies.join(", ")}. ` + + "Committed frames could not be applied because the main database is missing. The bytes are preserved. Recovery requires restoring the main database and pairing it with the quarantined file.", + { + databasePath: pathname, + copiedSidecars: copied, + }, + ); +} diff --git a/src/state/openclaw-agent-db.ts b/src/state/openclaw-agent-db.ts index 33aa15a1b4a0..77704d84c406 100644 --- a/src/state/openclaw-agent-db.ts +++ b/src/state/openclaw-agent-db.ts @@ -5,6 +5,7 @@ import type { DatabaseSync } from "node:sqlite"; import { enableNodeSqliteKyselyStatementCache } from "../infra/kysely-sync.js"; import { openNodeSqliteDatabase } from "../infra/node-sqlite.js"; import type { SqliteFileGeneration } from "../infra/sqlite-file-generation.js"; +import { quarantineOrphanedSqliteSidecars } from "../infra/sqlite-files.js"; import { confirmSqliteFileIntegrity, isTerminalSqliteIntegrityError, @@ -262,6 +263,7 @@ export function openOpenClawAgentDatabase( cachedDatabases.set(pathname, database); return database; } + quarantineOrphanedSqliteSidecars(pathname); // Latched paths are quarantined; every fresh open fails fast here until // doctor repairs the file and clears the latch plus the persisted row. const terminalFailure = terminalOpenLatch.get(pathname); diff --git a/src/state/openclaw-database-orphan-sidecars.test.ts b/src/state/openclaw-database-orphan-sidecars.test.ts new file mode 100644 index 000000000000..0b9cb54bbbbe --- /dev/null +++ b/src/state/openclaw-database-orphan-sidecars.test.ts @@ -0,0 +1,264 @@ +// Orphan-sidecar tests prove fresh database opens preserve recoverable SQLite families. +import fs from "node:fs"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import { requireNodeSqlite } from "../infra/node-sqlite.js"; +import { quarantineOrphanedSqliteSidecars } from "../infra/sqlite-files.js"; +import { + closeOpenClawAgentDatabasesForTest, + openOpenClawAgentDatabase, +} from "./openclaw-agent-db.js"; +import { resolveOpenClawAgentSqlitePath } from "./openclaw-agent-db.paths.js"; +import { + closeOpenClawStateDatabaseForTest, + openOpenClawStateDatabase, +} from "./openclaw-state-db.js"; +import { resolveOpenClawStateSqlitePath } from "./openclaw-state-db.paths.js"; + +const tempStateDirs = useAutoCleanupTempDirTracker(afterEach); +const databaseKinds = ["state", "agent"] as const; +const rollbackJournalContents = Buffer.from("recoverable rollback journal content"); +const shmIndexContents = Buffer.alloc(32 * 1024, 0x53); +const emptySidecar = Buffer.alloc(0); +const loggerMocks = vi.hoisted(() => ({ warn: vi.fn() })); + +vi.mock("../logging/subsystem.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + createSubsystemLogger: (subsystem: string) => + subsystem === "state/sqlite" + ? { ...actual.createSubsystemLogger(subsystem), warn: loggerMocks.warn } + : actual.createSubsystemLogger(subsystem), + }; +}); + +function createWalFixture(): { header: Buffer; withFrames: Buffer } { + const fixtureDir = fs.realpathSync(tempStateDirs.make("openclaw-wal-header-")); + const databasePath = path.join(fixtureDir, "header.sqlite"); + const { DatabaseSync } = requireNodeSqlite(); + const database = new DatabaseSync(databasePath); + try { + database.exec("PRAGMA journal_mode = WAL; PRAGMA wal_autocheckpoint = 0;"); + database.exec("CREATE TABLE header_probe (value TEXT);"); + const wal = fs.readFileSync(`${databasePath}-wal`); + if (wal.length <= 32) { + throw new Error("SQLite did not write a WAL frame for the header fixture"); + } + return { + header: Buffer.from(wal.subarray(0, 32)), + withFrames: Buffer.from(wal), + }; + } finally { + database.close(); + } +} + +const walFixture = createWalFixture(); + +afterEach(() => { + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + loggerMocks.warn.mockReset(); + vi.restoreAllMocks(); +}); + +function prepareCase(kind: (typeof databaseKinds)[number]) { + const stateDir = fs.realpathSync(tempStateDirs.make("openclaw-orphan-sidecar-")); + const env = { OPENCLAW_STATE_DIR: stateDir }; + const databasePath = + kind === "state" + ? resolveOpenClawStateSqlitePath(env) + : resolveOpenClawAgentSqlitePath({ agentId: "main", env }); + fs.mkdirSync(path.dirname(databasePath), { recursive: true }); + return { databasePath, env }; +} + +function openDatabase(kind: (typeof databaseKinds)[number], env: NodeJS.ProcessEnv) { + return kind === "state" + ? openOpenClawStateDatabase({ env }) + : openOpenClawAgentDatabase({ agentId: "main", env }); +} + +function listQuarantinePaths(sourcePath: string): string[] { + const prefix = `${path.basename(sourcePath)}.orphaned-`; + return fs + .readdirSync(path.dirname(sourcePath)) + .filter((entry) => entry.startsWith(prefix)) + .map((entry) => path.join(path.dirname(sourcePath), entry)); +} + +describe("orphan SQLite sidecar admission", () => { + const recoverableCases = [ + { + label: "real WAL with committed frames", + suffix: "-wal", + contents: walFixture.withFrames, + benignSidecars: [ + { suffix: "-shm", contents: shmIndexContents }, + { suffix: "-journal", contents: emptySidecar }, + ], + }, + { + label: "non-empty rollback journal", + suffix: "-journal", + contents: rollbackJournalContents, + benignSidecars: [ + { suffix: "-wal", contents: walFixture.header }, + { suffix: "-shm", contents: shmIndexContents }, + ], + }, + ] as const; + + describe("recoverable sidecars", () => { + for (const testCase of recoverableCases) { + for (const kind of databaseKinds) { + const regression = + kind === "state" && testCase.suffix === "-wal" ? " (plugin lifecycle regression)" : ""; + it(`opens a missing ${kind} database after copying a ${testCase.label}${regression}`, () => { + const { databasePath, env } = prepareCase(kind); + const sidecars = [ + { suffix: testCase.suffix, contents: testCase.contents }, + ...testCase.benignSidecars, + ].map((sidecar) => ({ + contents: sidecar.contents, + path: `${databasePath}${sidecar.suffix}`, + suffix: sidecar.suffix, + })); + for (const sidecar of sidecars) { + fs.writeFileSync(sidecar.path, sidecar.contents); + } + + const database = openDatabase(kind, env); + const sourcePath = `${databasePath}${testCase.suffix}`; + const quarantinePaths = listQuarantinePaths(sourcePath); + + expect(database.db.isOpen).toBe(true); + expect(fs.existsSync(databasePath)).toBe(true); + expect(quarantinePaths).toHaveLength(1); + const quarantinePath = quarantinePaths.at(0); + if (!quarantinePath) { + throw new Error(`missing quarantine for ${sourcePath}`); + } + expect(fs.readFileSync(quarantinePath)).toEqual(testCase.contents); + if (fs.existsSync(sourcePath)) { + // A successful WAL-mode open may create a new sidecar at the canonical path. + expect(fs.readFileSync(sourcePath)).not.toEqual(testCase.contents); + } + expect(loggerMocks.warn).toHaveBeenCalledOnce(); + expect(loggerMocks.warn).toHaveBeenCalledWith( + expect.stringContaining(quarantinePath), + expect.objectContaining({ + databasePath, + copiedSidecars: [{ quarantinePath, sourcePath }], + }), + ); + expect(String(loggerMocks.warn.mock.calls[0]?.[0])).toContain( + "Committed frames could not be applied because the main database is missing", + ); + expect(String(loggerMocks.warn.mock.calls[0]?.[0])).toContain( + "Recovery requires restoring the main database and pairing it with the quarantined file", + ); + }); + } + } + }); + + const nonBlockingCases = [ + { label: "lone SHM index", suffix: "-shm", contents: shmIndexContents }, + { label: "zero-byte WAL", suffix: "-wal", contents: emptySidecar }, + { label: "32-byte header-only WAL", suffix: "-wal", contents: walFixture.header }, + { label: "zero-byte rollback journal", suffix: "-journal", contents: emptySidecar }, + ] as const; + + describe("non-blocking sidecars", () => { + for (const testCase of nonBlockingCases) { + for (const kind of databaseKinds) { + it(`creates a missing ${kind} database with a ${testCase.label}`, () => { + const { databasePath, env } = prepareCase(kind); + fs.writeFileSync(`${databasePath}${testCase.suffix}`, testCase.contents); + + const database = openDatabase(kind, env); + + expect(database.db.isOpen).toBe(true); + expect(fs.existsSync(databasePath)).toBe(true); + expect(database.db.prepare("PRAGMA integrity_check").get()).toEqual({ + integrity_check: "ok", + }); + expect(listQuarantinePaths(`${databasePath}${testCase.suffix}`)).toEqual([]); + expect(loggerMocks.warn).not.toHaveBeenCalled(); + }); + } + } + }); + + it("uses a unique suffix instead of overwriting an existing quarantine", () => { + const { databasePath, env } = prepareCase("state"); + const sourcePath = `${databasePath}-wal`; + const epochMs = 1_786_738_000_000; + const existingQuarantinePath = `${sourcePath}.orphaned-${epochMs}`; + const newQuarantinePath = `${existingQuarantinePath}-1`; + const existingContents = Buffer.from("previously quarantined WAL"); + fs.writeFileSync(sourcePath, walFixture.withFrames); + fs.writeFileSync(existingQuarantinePath, existingContents); + vi.spyOn(Date, "now").mockReturnValue(epochMs); + + const database = openDatabase("state", env); + + expect(database.db.isOpen).toBe(true); + expect(fs.readFileSync(existingQuarantinePath)).toEqual(existingContents); + expect(fs.readFileSync(newQuarantinePath)).toEqual(walFixture.withFrames); + expect(loggerMocks.warn).toHaveBeenCalledWith( + expect.stringContaining(newQuarantinePath), + expect.any(Object), + ); + }); + + it("leaves the live sidecar untouched and reuses an identical preserved copy", () => { + const { databasePath } = prepareCase("state"); + const sourcePath = `${databasePath}-wal`; + fs.writeFileSync(sourcePath, walFixture.withFrames); + + quarantineOrphanedSqliteSidecars(databasePath); + const quarantinePaths = listQuarantinePaths(sourcePath); + + expect(quarantinePaths).toHaveLength(1); + expect(fs.readFileSync(sourcePath)).toEqual(walFixture.withFrames); + expect(fs.readFileSync(quarantinePaths[0] ?? "")).toEqual(walFixture.withFrames); + loggerMocks.warn.mockClear(); + + quarantineOrphanedSqliteSidecars(databasePath); + + expect(listQuarantinePaths(sourcePath)).toEqual(quarantinePaths); + expect(fs.readFileSync(sourcePath)).toEqual(walFixture.withFrames); + expect(loggerMocks.warn).not.toHaveBeenCalled(); + }); + + it("fails closed with the typed error when quarantine copy fails", () => { + const { databasePath, env } = prepareCase("state"); + const sourcePath = `${databasePath}-wal`; + fs.writeFileSync(sourcePath, walFixture.withFrames); + const copyError = Object.assign(new Error("read-only volume"), { code: "EROFS" }); + vi.spyOn(fs, "copyFileSync").mockImplementationOnce(() => { + throw copyError; + }); + + let thrown: unknown; + try { + openDatabase("state", env); + } catch (error) { + thrown = error; + } + + expect(thrown).toBeInstanceOf(Error); + expect((thrown as Error).name).toBe("SqliteOrphanedSidecarsError"); + expect((thrown as Error).message).toContain(databasePath); + expect((thrown as Error).message).toContain(sourcePath); + expect((thrown as Error).message).toContain("restore the main database"); + expect(fs.existsSync(databasePath)).toBe(false); + expect(fs.readFileSync(sourcePath)).toEqual(walFixture.withFrames); + expect(listQuarantinePaths(sourcePath)).toEqual([]); + expect(loggerMocks.warn).not.toHaveBeenCalled(); + }); +}); diff --git a/src/state/openclaw-state-ownership.ts b/src/state/openclaw-state-ownership.ts index 7da010a751b2..fdb843c86a38 100644 --- a/src/state/openclaw-state-ownership.ts +++ b/src/state/openclaw-state-ownership.ts @@ -16,6 +16,7 @@ import { runWithSqliteCoordinator, SqliteCoordinatorError, } from "../infra/sqlite-coordinator.js"; +import { quarantineOrphanedSqliteSidecars } from "../infra/sqlite-files.js"; import { prepareSqliteReadOnlyLocation, prepareSqliteReadOnlyLocationSync, @@ -244,6 +245,7 @@ function acquireOpenClawStateWriteAccess(options: { const resolvedPath = path.resolve(options.databasePath); const access = acquireOpenClawStateOwnershipCoordinator(resolvedPath); try { + quarantineOrphanedSqliteSidecars(resolvedPath); assertOwnershipAllowsWrite( inspectOpenClawStateOwnershipAtPathWhileCoordinatorHeld(resolvedPath), resolvedPath, @@ -288,6 +290,7 @@ export async function assertOpenClawStateWriteAllowedAtPath(options: { env?: NodeJS.ProcessEnv; }): Promise { const databasePath = path.resolve(options.databasePath); + quarantineOrphanedSqliteSidecars(databasePath); if (!existsSync(databasePath)) { return; }