diff --git a/docs/reference/database-schemas.md b/docs/reference/database-schemas.md index 8db3ccbecb74..868d18c7d988 100644 --- a/docs/reference/database-schemas.md +++ b/docs/reference/database-schemas.md @@ -56,14 +56,15 @@ Version 3 was an unshipped development step folded into version 4. ## Integrity checks -| When | Check | -| ------------------------------------------- | -------------------------------------------------------------------------------------------------- | -| Every open | Validate the `schema_meta` table and primary metadata row | -| Before a pending migration | Run a full integrity, foreign-key, role, schema, and index scan | -| Gateway background verifier | Run the full scan about once daily and record results in the global `database_verifications` table | -| Doctor, backup verification, and compaction | Run the full scan before accepting or rewriting the database | +| When | Check | +| ------------------------------------------- | --------------------------------------------------------------- | +| Every open | Validate the `schema_meta` table and primary metadata row | +| Before a pending migration | Run a full integrity, foreign-key, role, schema, and index scan | +| Gateway background verifier | Run the full scan about once daily and record results | +| Doctor, backup verification, and compaction | Run the full scan before accepting or rewriting the database | The Gateway preflight reads schema headers only. The background verifier owns the slower full scan for databases that do not need migration. +Quarantine decisions live in a dedicated `openclaw-quarantine.sqlite` store, so they survive damage to the databases being quarantined. The global `database_verifications` table remains verification history. ## Downgrades are unsupported diff --git a/scripts/check-kysely-guardrails.mjs b/scripts/check-kysely-guardrails.mjs index 7c1254ab6137..eeb346273e74 100644 --- a/scripts/check-kysely-guardrails.mjs +++ b/scripts/check-kysely-guardrails.mjs @@ -57,6 +57,9 @@ const rawSqliteAllowPathGroups = { "src/state/openclaw-database-preflight.ts", "src/state/openclaw-database-verify.worker.ts", ], + "quarantine store must work when other databases are damaged": [ + "src/state/openclaw-quarantine-store.ts", + ], "read-only SQLite status probes": [ "src/commands/doctor-db-bloat.ts", "src/commands/status.scan.shared.ts", diff --git a/src/commands/doctor-session-sqlite-compact.ts b/src/commands/doctor-session-sqlite-compact.ts index 6aa5b0803e70..a386761b33de 100644 --- a/src/commands/doctor-session-sqlite-compact.ts +++ b/src/commands/doctor-session-sqlite-compact.ts @@ -15,7 +15,7 @@ import { compactDoctorSqliteFile } from "./doctor-sqlite-compact.js"; /** Reclaim free pages from one agent session SQLite database. */ export function compactDoctorSessionSqliteTarget( target: SessionStoreTarget, - options: { migrateOlderSchema?: boolean } = {}, + options: { env?: NodeJS.ProcessEnv; migrateOlderSchema?: boolean } = {}, ): DoctorSessionSqliteCompactReport { const sqlitePath = resolveTargetSqlitePath(target); const beforeFileSizes = readSqliteFileSizes(sqlitePath); @@ -42,7 +42,7 @@ export function compactDoctorSessionSqliteTarget( ); } const requireQuarantineCleared = () => { - if (!clearOpenClawAgentDatabaseOpenFailure(sqlitePath)) { + if (!clearOpenClawAgentDatabaseOpenFailure(sqlitePath, { env: options.env })) { throw new Error( `OpenClaw agent database ${sqlitePath} was repaired, but its persisted quarantine record could not be cleared. Rerun openclaw doctor --fix so the database is not refused again.`, ); diff --git a/src/commands/doctor-session-sqlite.test.ts b/src/commands/doctor-session-sqlite.test.ts index 471ef9e9a4a3..95b781e9c520 100644 --- a/src/commands/doctor-session-sqlite.test.ts +++ b/src/commands/doctor-session-sqlite.test.ts @@ -21,7 +21,14 @@ import { OPENCLAW_AGENT_SCHEMA_VERSION, resolveOpenClawAgentSqlitePath, } from "../state/openclaw-agent-db.js"; -import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; +import { + readOpenClawDatabaseQuarantine, + recordOpenClawDatabaseQuarantine, +} from "../state/openclaw-quarantine-store.js"; +import { + closeOpenClawStateDatabaseForTest, + openOpenClawStateDatabase, +} from "../state/openclaw-state-db.js"; import { assertSafeSessionSqliteMigrationMove, createSessionSqliteMigrationFailureIssue, @@ -611,6 +618,40 @@ describe("runDoctorSessionSqlite", () => { }, ); + it("clears agent quarantine and verification history after compaction", async () => { + const { sqlitePath, store } = await createImportedStoreForCompaction(); + const state = openOpenClawStateDatabase({ env: store.env }); + state.db + .prepare( + ` + INSERT INTO database_verifications (path, kind, verified_at, result, error) + VALUES (?, 'agent', 1, 'error', 'corrupt index') + `, + ) + .run(sqlitePath); + expect( + recordOpenClawDatabaseQuarantine({ + env: store.env, + kind: "agent", + path: sqlitePath, + reason: "corrupt index", + }), + ).toBe(true); + + const report = await runDoctorSessionSqlite({ + env: store.env, + mode: "compact", + store: store.storePath, + }); + + expect(report.totals.issues).toBe(0); + expect(readOpenClawDatabaseQuarantine(sqlitePath, { env: store.env })).toBeUndefined(); + expect( + state.db.prepare("SELECT 1 FROM database_verifications WHERE path = ?").get(sqlitePath), + ).toBeUndefined(); + expect(openOpenClawAgentDatabase({ agentId: "main", env: store.env }).db.isOpen).toBe(true); + }); + it.skipIf(process.platform === "win32")( "reapplies owner-only permissions after compaction", async () => { diff --git a/src/commands/doctor-session-sqlite.ts b/src/commands/doctor-session-sqlite.ts index ed93844853b2..6f024a89e124 100644 --- a/src/commands/doctor-session-sqlite.ts +++ b/src/commands/doctor-session-sqlite.ts @@ -259,7 +259,7 @@ async function inspectOrMigrateTarget(params: { return report; } if (params.mode === "compact") { - compactSqliteDatabase(params.target, report); + compactSqliteDatabase(params.target, report, { env: params.env }); report.sqliteEntries = readSqliteEntryCount(params.target); appendSqliteDbStats(params.target, report); return report; @@ -299,6 +299,7 @@ async function inspectOrMigrateTarget(params: { // databases and returns the pages the import churn freed. compactSqliteDatabase(params.target, report, { closeImportedHandle: true, + env: params.env, migrateOlderSchema: true, }); } @@ -988,15 +989,22 @@ function appendSqliteDbStats( function compactSqliteDatabase( target: SessionStoreTarget, report: DoctorSessionSqliteTargetReport, - options: { closeImportedHandle?: boolean; migrateOlderSchema?: boolean } = {}, + options: { + closeImportedHandle?: boolean; + env?: NodeJS.ProcessEnv; + migrateOlderSchema?: boolean; + } = {}, ): void { try { if (options.closeImportedHandle) { closeOpenClawAgentDatabaseByPath(resolveTargetSqlitePath(target)); } report.compact = options.migrateOlderSchema - ? compactDoctorSessionSqliteTarget(target, { migrateOlderSchema: true }) - : compactDoctorSessionSqliteTarget(target); + ? compactDoctorSessionSqliteTarget(target, { + env: options.env, + migrateOlderSchema: true, + }) + : compactDoctorSessionSqliteTarget(target, { env: options.env }); } catch (err) { report.issues.push({ code: "sqlite_compact_failed", diff --git a/src/commands/doctor-state-sqlite-compact.test.ts b/src/commands/doctor-state-sqlite-compact.test.ts index 90d9f59ec93e..2739c1d7f764 100644 --- a/src/commands/doctor-state-sqlite-compact.test.ts +++ b/src/commands/doctor-state-sqlite-compact.test.ts @@ -4,6 +4,10 @@ import type { DatabaseSync } from "node:sqlite"; import { afterEach, describe, expect, it } from "vitest"; import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; import { requireNodeSqlite } from "../infra/node-sqlite.js"; +import { + readOpenClawDatabaseQuarantine, + recordOpenClawDatabaseQuarantine, +} from "../state/openclaw-quarantine-store.js"; import { closeOpenClawStateDatabase, openOpenClawStateDatabase, @@ -166,6 +170,41 @@ describe("runDoctorStateSqliteCompact", () => { expect(report.integrityCheck).toBe("ok"); }); + it("clears authoritative quarantine and verification history after compaction", async () => { + const env = createStateEnv(); + const sqlitePath = seedStateDatabase({ env, withBloat: true }); + const sqlite = requireNodeSqlite(); + const history = new sqlite.DatabaseSync(sqlitePath); + try { + history + .prepare( + ` + INSERT INTO database_verifications (path, kind, verified_at, result, error) + VALUES (?, 'state', 1, 'error', 'corrupt index') + `, + ) + .run(sqlitePath); + } finally { + history.close(); + } + expect( + recordOpenClawDatabaseQuarantine({ + env, + kind: "state", + path: sqlitePath, + reason: "corrupt index", + }), + ).toBe(true); + + await runDoctorStateSqliteCompact({ env }); + + expect(readOpenClawDatabaseQuarantine(sqlitePath, { env })).toBeUndefined(); + const repaired = openOpenClawStateDatabase({ env }); + expect( + repaired.db.prepare("SELECT 1 FROM database_verifications WHERE path = ?").get(sqlitePath), + ).toBeUndefined(); + }); + it.skipIf(process.platform === "win32")("reapplies owner-only SQLite permissions", async () => { const env = createStateEnv(); const sqlitePath = seedStateDatabase({ env, withBloat: true }); diff --git a/src/commands/doctor-state-sqlite-compact.ts b/src/commands/doctor-state-sqlite-compact.ts index a030ec075ea8..4fe1df0122df 100644 --- a/src/commands/doctor-state-sqlite-compact.ts +++ b/src/commands/doctor-state-sqlite-compact.ts @@ -1,8 +1,9 @@ /** Explicit doctor maintenance for the canonical shared state SQLite database. */ import fs from "node:fs"; +import { clearOpenClawDatabaseQuarantine } from "../state/openclaw-quarantine-store.js"; import { assertOpenClawStateDatabaseForMaintenance, - clearOpenClawDatabaseVerification, + clearOpenClawDatabaseVerificationHistory, clearOpenClawStateDatabaseOpenFailure, ensureOpenClawStatePermissions, isOpenClawStateDatabaseOpen, @@ -72,11 +73,12 @@ export async function runDoctorStateSqliteCompact( const compact = compactDoctorSqliteFile({ afterMutation: () => { - if (!clearOpenClawDatabaseVerification(sqlitePath, { path: sqlitePath })) { + if (!clearOpenClawDatabaseQuarantine(sqlitePath, { env })) { throw new Error( `OpenClaw state database ${sqlitePath} was compacted, but its persisted quarantine record could not be cleared. Rerun openclaw doctor --fix so the database is not refused again.`, ); } + clearOpenClawDatabaseVerificationHistory(sqlitePath, { env }); clearOpenClawStateDatabaseOpenFailure(sqlitePath); ensureOpenClawStatePermissions(sqlitePath, env); }, diff --git a/src/state/openclaw-agent-db.ts b/src/state/openclaw-agent-db.ts index ce49d254b773..7b5be5b95573 100644 --- a/src/state/openclaw-agent-db.ts +++ b/src/state/openclaw-agent-db.ts @@ -59,14 +59,17 @@ import { import type { DB as OpenClawAgentKyselyDatabase } from "./openclaw-agent-db.generated.js"; import { resolveOpenClawAgentSqlitePath } from "./openclaw-agent-db.paths.js"; import { OPENCLAW_AGENT_SCHEMA_SQL } from "./openclaw-agent-schema.generated.js"; +import { + clearOpenClawDatabaseQuarantine, + readOpenClawDatabaseQuarantine, +} from "./openclaw-quarantine-store.js"; import type { DB as OpenClawStateKyselyDatabase } from "./openclaw-state-db.generated.js"; import { - clearOpenClawDatabaseVerification, + clearOpenClawDatabaseVerificationHistory, createOpenClawDatabaseVerificationError, detectOpenClawStateDatabaseSchemaMigrations, OPENCLAW_STATE_SCHEMA_VERSION, OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, - readOpenClawDatabaseVerification, runOpenClawStateWriteTransaction, type OpenClawStateDatabaseOptions, } from "./openclaw-state-db.js"; @@ -183,7 +186,8 @@ export function clearOpenClawAgentDatabaseOpenFailure( options: OpenClawStateDatabaseOptions = {}, ): boolean { const resolvedPath = path.resolve(pathname); - const cleared = clearOpenClawDatabaseVerification(resolvedPath, options); + const cleared = clearOpenClawDatabaseQuarantine(resolvedPath, { env: options.env }); + clearOpenClawDatabaseVerificationHistory(resolvedPath, options); terminalOpenFailures.delete(resolvedPath); return cleared; } @@ -1082,19 +1086,17 @@ export function openOpenClawAgentDatabase( } let persistedFailure: Error | undefined; try { - const verification = readOpenClawDatabaseVerification(pathname, { env: databaseOptions.env }); - if (verification?.result === "error") { + const quarantine = readOpenClawDatabaseQuarantine(pathname, { env: databaseOptions.env }); + if (quarantine) { persistedFailure = createOpenClawDatabaseVerificationError( "agent", pathname, - verification.error, + quarantine.reason, ); } } catch { - // Accepted tradeoff: a locked/unavailable state DB must not block agent - // opens, or every transient state hiccup takes all agents down. The - // in-process latch still covers this process; a missed cross-process - // quarantine is re-detected by the next daily verifier pass. + // A broken quarantine store must not brick every agent open. + // The process latch and daily verifier still cover known damage. } if (persistedFailure) { recordOpenClawAgentDatabaseOpenFailure(pathname, persistedFailure); diff --git a/src/state/openclaw-database-verify.impl.ts b/src/state/openclaw-database-verify.impl.ts index 21da5d44497c..5248b98bd27d 100644 --- a/src/state/openclaw-database-verify.impl.ts +++ b/src/state/openclaw-database-verify.impl.ts @@ -12,6 +12,7 @@ import type { OpenClawDatabaseVerifyResult, OpenClawDatabaseVerifyTarget, } from "./openclaw-database-verify.worker.js"; +import { recordOpenClawDatabaseQuarantine } from "./openclaw-quarantine-store.js"; import type { DB as OpenClawStateKyselyDatabase } from "./openclaw-state-db.generated.js"; import { recordOpenClawStateDatabaseOpenFailure, @@ -149,28 +150,36 @@ export function applyOpenClawDatabaseVerificationResults(options: { const targetByPath = new Map(options.targets.map((target) => [target.path, target])); const verifiedAt = options.verifiedAt ?? Date.now(); - // One best-effort row write into a possibly corrupt state file buys restart-durable quarantine. - // Doctor rebuilds the file during repair, so this bounded mutation is an accepted tradeoff. - const persistResults = (results: readonly OpenClawDatabaseVerifyResult[]) => { + for (const result of options.results) { + const target = targetByPath.get(result.path); + if (!target || result.ok || !result.terminal) { + continue; + } + const recorded = recordOpenClawDatabaseQuarantine({ + env: options.env, + kind: target.kind, + path: result.path, + reason: result.error ?? `SQLite integrity verification failed for ${result.path}`, + }); + if (!recorded) { + // Accepted residual: quarantine stays process-local when this tiny, + // independent store is unavailable. Daily verification retries it. + log.error("failed to persist database quarantine; quarantine is process-local", { + kind: target.kind, + path: result.path, + }); + } + } + + try { runOpenClawStateWriteTransaction( (database) => { const db = getNodeSqliteKysely(database.db); - for (const result of results) { + for (const result of options.results) { const target = targetByPath.get(result.path); if (!target) { continue; } - const existing = executeSqliteQuerySync( - database.db, - db - .selectFrom("database_verifications") - .select("result") - .where("path", "=", result.path) - .limit(1), - ).rows[0]; - if (existing?.result === "error") { - continue; - } executeSqliteQuerySync( database.db, db @@ -196,31 +205,8 @@ export function applyOpenClawDatabaseVerificationResults(options: { { env: options.env }, { operationLabel: "state.database-verifications.record" }, ); - }; - try { - persistResults(options.results); } catch (error) { - // Terminal rows are the restart-durable quarantine; retry those once so a - // transient state-DB lock does not silently downgrade them to process-local. - const terminalResults = options.results.filter((result) => !result.ok && result.terminal); - let retried = false; - if (terminalResults.length > 0) { - try { - persistResults(terminalResults); - retried = true; - } catch { - // fall through to the process-local warning below - } - } - if (!retried) { - // Accepted tradeoff: when this write fails the quarantine stays process-local - // until the next daily verifier pass re-detects and re-persists. A sidecar - // marker would break the SQLite-only storage contract, and startup full - // scans would reintroduce the multi-second opens this design removes. - log.error("failed to persist database verification results; quarantine is process-local", { - error: String(error), - }); - } + log.error("failed to persist database verification history", { error: String(error) }); } for (const result of options.results) { diff --git a/src/state/openclaw-database-verify.test.ts b/src/state/openclaw-database-verify.test.ts index 253f5144cd06..e060338a097d 100644 --- a/src/state/openclaw-database-verify.test.ts +++ b/src/state/openclaw-database-verify.test.ts @@ -1,4 +1,6 @@ +import { spawnSync } from "node:child_process"; import fs from "node:fs"; +import path from "node:path"; import { afterAll, afterEach, describe, expect, it } from "vitest"; import { cleanupTempDirs, makeTempDir } from "../../test/helpers/temp-dir.js"; import { requireNodeSqlite } from "../infra/node-sqlite.js"; @@ -16,6 +18,11 @@ import { type OpenClawDatabaseVerifyTarget, verifyOpenClawDatabases, } from "./openclaw-database-verify.worker.js"; +import { + clearOpenClawDatabaseQuarantine, + readOpenClawDatabaseQuarantine, + recordOpenClawDatabaseQuarantine, +} from "./openclaw-quarantine-store.js"; import { closeOpenClawStateDatabaseForTest, openOpenClawStateDatabase, @@ -59,6 +66,10 @@ function createUnsafeIndexDrift(databasePath: string): void { } } +function quarantineStorePath(stateDir: string): string { + return path.join(stateDir, "state", "openclaw-quarantine.sqlite"); +} + describe("OpenClaw database integrity verifier", () => { it("detects corruption off-thread, persists it, and latches later opens", async () => { const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-"); @@ -94,6 +105,11 @@ describe("OpenClaw database integrity verifier", () => { verifiedAt: 1234, }); expect(liveHandle.db.isOpen).toBe(false); + expect(readOpenClawDatabaseQuarantine(agentPath, { env })).toEqual({ + kind: "agent", + quarantinedAt: expect.any(Number), + reason: directResults[0]?.error, + }); expect( openOpenClawStateDatabase({ env }) @@ -118,7 +134,7 @@ describe("OpenClaw database integrity verifier", () => { openOpenClawStateDatabase({ env }) .db.prepare("SELECT verified_at, result, error FROM database_verifications WHERE path = ?") .get(agentPath), - ).toEqual({ verified_at: 1234, result: "error", error: directResults[0]?.error }); + ).toEqual({ verified_at: 1235, result: "inconclusive", error: "database busy" }); expect(() => openOpenClawAgentDatabase({ agentId: "worker-1", env })).toThrow( expect.objectContaining({ name: "SqliteIntegrityError" }), ); @@ -139,7 +155,7 @@ describe("OpenClaw database integrity verifier", () => { const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-clear-failure-"); const env = { OPENCLAW_STATE_DIR: stateDir }; const agentPath = openOpenClawAgentDatabase({ agentId: "worker-1", env }).path; - const statePath = openOpenClawStateDatabase({ env }).path; + openOpenClawStateDatabase({ env }); closeOpenClawAgentDatabasesForTest(); closeOpenClawStateDatabaseForTest(); applyOpenClawDatabaseVerificationResults({ @@ -150,23 +166,137 @@ describe("OpenClaw database integrity verifier", () => { }); closeOpenClawStateDatabaseForTest(); - // A read-only state DB cannot drop the quarantine row; the clear must say so + const storePath = quarantineStorePath(stateDir); + // A read-only quarantine store cannot drop the row; the clear must say so // instead of letting doctor report success while the next open still refuses. - fs.chmodSync(statePath, 0o444); + fs.chmodSync(storePath, 0o444); try { expect(clearOpenClawAgentDatabaseOpenFailure(agentPath, { env })).toBe(false); + expect( + recordOpenClawDatabaseQuarantine({ + env, + kind: "agent", + path: agentPath, + reason: "new reason", + }), + ).toBe(false); } finally { - // WAL sidecars minted during the read-only attempt inherit its mode. - for (const sidecar of [statePath, `${statePath}-wal`, `${statePath}-shm`]) { - if (fs.existsSync(sidecar)) { - fs.chmodSync(sidecar, 0o600); - } - } + fs.chmodSync(storePath, 0o600); } expect(clearOpenClawAgentDatabaseOpenFailure(agentPath, { env })).toBe(true); expect(openOpenClawAgentDatabase({ agentId: "worker-1", env }).db.isOpen).toBe(true); }); + it("keeps healthy opens on the missing-store fast path", () => { + const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-clean-"); + const env = { OPENCLAW_STATE_DIR: stateDir }; + + openOpenClawStateDatabase({ env }); + openOpenClawAgentDatabase({ agentId: "worker-1", env }); + + expect(fs.existsSync(quarantineStorePath(stateDir))).toBe(false); + }); + + it("records and clears dedicated quarantine rows with rollback journaling", () => { + const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-store-"); + const env = { OPENCLAW_STATE_DIR: stateDir }; + const databasePath = path.join(stateDir, "agent.sqlite"); + const storePath = quarantineStorePath(stateDir); + + expect(clearOpenClawDatabaseQuarantine(databasePath, { env })).toBe(true); + expect( + recordOpenClawDatabaseQuarantine({ + env, + kind: "agent", + path: databasePath, + reason: "corrupt index", + }), + ).toBe(true); + expect(readOpenClawDatabaseQuarantine(databasePath, { env })).toEqual({ + kind: "agent", + quarantinedAt: expect.any(Number), + reason: "corrupt index", + }); + + const { DatabaseSync } = requireNodeSqlite(); + const raw = new DatabaseSync(storePath, { readOnly: true }); + try { + expect(raw.prepare("PRAGMA journal_mode").get()).toEqual({ journal_mode: "delete" }); + expect(readSqliteNumberPragma(raw, "synchronous")).toBe(2); + expect(readSqliteNumberPragma(raw, "user_version")).toBe(1); + } finally { + raw.close(); + } + if (process.platform !== "win32") { + expect(fs.statSync(path.dirname(storePath)).mode & 0o777).toBe(0o700); + expect(fs.statSync(storePath).mode & 0o777).toBe(0o600); + } + expect(clearOpenClawDatabaseQuarantine(databasePath, { env })).toBe(true); + expect(clearOpenClawDatabaseQuarantine(databasePath, { env })).toBe(true); + expect(readOpenClawDatabaseQuarantine(databasePath, { env })).toBeUndefined(); + }); + + it("recovers an interrupted empty quarantine-store initialization", () => { + const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-empty-store-"); + const env = { OPENCLAW_STATE_DIR: stateDir }; + const databasePath = path.join(stateDir, "agent.sqlite"); + const storePath = quarantineStorePath(stateDir); + fs.mkdirSync(path.dirname(storePath), { recursive: true }); + fs.writeFileSync(storePath, "", { mode: 0o600 }); + + expect(readOpenClawDatabaseQuarantine(databasePath, { env })).toBeUndefined(); + expect( + recordOpenClawDatabaseQuarantine({ + env, + kind: "agent", + path: databasePath, + reason: "corrupt index", + }), + ).toBe(true); + expect(readOpenClawDatabaseQuarantine(databasePath, { env })?.reason).toBe("corrupt index"); + }); + + it.skipIf(process.platform === "win32")( + "recovers a hot rollback journal before reading quarantine", + () => { + const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-hot-journal-"); + const env = { OPENCLAW_STATE_DIR: stateDir }; + const databasePath = path.join(stateDir, "agent.sqlite"); + const storePath = quarantineStorePath(stateDir); + expect( + recordOpenClawDatabaseQuarantine({ + env, + kind: "agent", + path: databasePath, + reason: "committed reason", + }), + ).toBe(true); + + const crashed = spawnSync( + process.execPath, + [ + "--no-warnings", + "--input-type=module", + "-e", + ` + import { DatabaseSync } from "node:sqlite"; + const database = new DatabaseSync(process.env.OPENCLAW_QUARANTINE_TEST_PATH); + database.exec("PRAGMA journal_mode = DELETE; PRAGMA synchronous = FULL; BEGIN IMMEDIATE;"); + database.prepare("UPDATE quarantined_databases SET reason = 'uncommitted reason'").run(); + process.kill(process.pid, "SIGKILL"); + `, + ], + { env: { ...process.env, OPENCLAW_QUARANTINE_TEST_PATH: storePath } }, + ); + expect(crashed.signal).toBe("SIGKILL"); + expect(fs.existsSync(`${storePath}-journal`)).toBe(true); + + expect(readOpenClawDatabaseQuarantine(databasePath, { env })?.reason).toBe( + "committed reason", + ); + }, + ); + it("persists transient verifier errors as inconclusive without latching", () => { const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-transient-"); const env = { OPENCLAW_STATE_DIR: stateDir }; @@ -209,13 +339,11 @@ describe("OpenClaw database integrity verifier", () => { }); const { DatabaseSync } = requireNodeSqlite(); - const raw = new DatabaseSync(statePath, { readOnly: true }); + const raw = new DatabaseSync(quarantineStorePath(stateDir), { readOnly: true }); try { expect( - raw - .prepare("SELECT result, error FROM database_verifications WHERE path = ?") - .get(statePath), - ).toEqual({ result: "error", error: "corrupt index" }); + raw.prepare("SELECT kind, reason FROM quarantined_databases WHERE path = ?").get(statePath), + ).toEqual({ kind: "state", reason: "corrupt index" }); } finally { raw.close(); } diff --git a/src/state/openclaw-quarantine-store.ts b/src/state/openclaw-quarantine-store.ts new file mode 100644 index 000000000000..5f19baafb867 --- /dev/null +++ b/src/state/openclaw-quarantine-store.ts @@ -0,0 +1,206 @@ +// Dedicated quarantine decisions stay available when primary databases fail. +import { existsSync, mkdirSync } from "node:fs"; +import path from "node:path"; +import type { DatabaseSync } from "node:sqlite"; +import { requireNodeSqlite } from "../infra/node-sqlite.js"; +import { applyPrivateModeSync } from "../infra/private-mode.js"; +import { VERSION } from "../version.js"; +import { resolveOpenClawStateSqliteDir } from "./openclaw-state-db.paths.js"; + +const OPENCLAW_QUARANTINE_SCHEMA_VERSION = 1; +const OPENCLAW_QUARANTINE_BUSY_TIMEOUT_MS = 5_000; +const OPENCLAW_QUARANTINE_DIR_MODE = 0o700; +const OPENCLAW_QUARANTINE_FILE_MODE = 0o600; + +type OpenClawDatabaseKind = "agent" | "state"; + +type OpenClawDatabaseQuarantine = { + kind: OpenClawDatabaseKind; + quarantinedAt: number; + reason: string; +}; + +function resolveQuarantineStorePath(env: NodeJS.ProcessEnv): string { + return path.join(resolveOpenClawStateSqliteDir(env), "openclaw-quarantine.sqlite"); +} + +function ensureQuarantineStoreDirectory(storePath: string): void { + const dir = path.dirname(storePath); + mkdirSync(dir, { recursive: true, mode: OPENCLAW_QUARANTINE_DIR_MODE }); + applyPrivateModeSync(dir, OPENCLAW_QUARANTINE_DIR_MODE); +} + +function configureQuarantineWriter(database: DatabaseSync, storePath: string): void { + database.exec(` + PRAGMA busy_timeout = ${OPENCLAW_QUARANTINE_BUSY_TIMEOUT_MS}; + PRAGMA journal_mode = DELETE; + PRAGMA synchronous = FULL; + `); + const userVersion = readQuarantineSchemaVersion(database, storePath); + if (userVersion > OPENCLAW_QUARANTINE_SCHEMA_VERSION) { + throw new Error( + `OpenClaw quarantine store ${storePath} uses newer schema version ${userVersion}.`, + ); + } + if (userVersion === OPENCLAW_QUARANTINE_SCHEMA_VERSION) { + return; + } + database.exec(` + BEGIN IMMEDIATE; + CREATE TABLE IF NOT EXISTS quarantined_databases ( + path TEXT NOT NULL PRIMARY KEY, + kind TEXT NOT NULL, + reason TEXT NOT NULL, + quarantined_at INTEGER NOT NULL, + writer_app_version TEXT + ) STRICT; + PRAGMA user_version = ${OPENCLAW_QUARANTINE_SCHEMA_VERSION}; + COMMIT; + `); +} + +function readQuarantineSchemaVersion(database: DatabaseSync, storePath: string): number { + const row = database.prepare("PRAGMA user_version").get() as + | { user_version?: unknown } + | undefined; + const userVersion = row?.user_version; + if (typeof userVersion !== "number" || !Number.isInteger(userVersion)) { + throw new Error(`OpenClaw quarantine store ${storePath} has an invalid schema version.`); + } + return userVersion; +} + +function withQuarantineWriter(env: NodeJS.ProcessEnv, operation: (db: DatabaseSync) => T): T { + const storePath = resolveQuarantineStorePath(env); + const existed = existsSync(storePath); + ensureQuarantineStoreDirectory(storePath); + const sqlite = requireNodeSqlite(); + const database = new sqlite.DatabaseSync(storePath); + let completed = false; + try { + if (!existed) { + applyPrivateModeSync(storePath, OPENCLAW_QUARANTINE_FILE_MODE); + } + configureQuarantineWriter(database, storePath); + const result = operation(database); + completed = true; + return result; + } finally { + database.close(); + if (completed || !existed) { + applyPrivateModeSync(storePath, OPENCLAW_QUARANTINE_FILE_MODE); + } + } +} + +/** Read one authoritative quarantine decision without creating the store. */ +export function readOpenClawDatabaseQuarantine( + pathname: string, + options: { env?: NodeJS.ProcessEnv } = {}, +): OpenClawDatabaseQuarantine | undefined { + const storePath = resolveQuarantineStorePath(options.env ?? process.env); + // Clean installs pay one existence check. No directory or SQLite work. + if (!existsSync(storePath)) { + return undefined; + } + const sqlite = requireNodeSqlite(); + const database = new sqlite.DatabaseSync(storePath); + try { + database.exec(`PRAGMA busy_timeout = ${OPENCLAW_QUARANTINE_BUSY_TIMEOUT_MS};`); + const userVersion = readQuarantineSchemaVersion(database, storePath); + if (userVersion === 0) { + return undefined; + } + if (userVersion !== OPENCLAW_QUARANTINE_SCHEMA_VERSION) { + throw new Error( + `OpenClaw quarantine store ${storePath} uses newer schema version ${userVersion}.`, + ); + } + const row = database + .prepare( + "SELECT kind, reason, quarantined_at FROM quarantined_databases WHERE path = ? LIMIT 1", + ) + .get(path.resolve(pathname)) as + | { kind?: unknown; quarantined_at?: unknown; reason?: unknown } + | undefined; + if (!row) { + return undefined; + } + if ( + (row.kind !== "agent" && row.kind !== "state") || + typeof row.reason !== "string" || + typeof row.quarantined_at !== "number" || + !Number.isInteger(row.quarantined_at) + ) { + throw new Error(`OpenClaw quarantine store ${storePath} contains an invalid row.`); + } + return { kind: row.kind, quarantinedAt: row.quarantined_at, reason: row.reason }; + } finally { + database.close(); + } +} + +/** Persist one authoritative quarantine decision. */ +export function recordOpenClawDatabaseQuarantine(options: { + env?: NodeJS.ProcessEnv; + kind: OpenClawDatabaseKind; + path: string; + reason: string; +}): boolean { + try { + return withQuarantineWriter(options.env ?? process.env, (database) => { + database.exec("BEGIN IMMEDIATE;"); + try { + database + .prepare( + ` + INSERT INTO quarantined_databases ( + path, kind, reason, quarantined_at, writer_app_version + ) VALUES (?, ?, ?, ?, ?) + ON CONFLICT(path) DO UPDATE SET + kind = excluded.kind, + reason = excluded.reason, + quarantined_at = excluded.quarantined_at, + writer_app_version = excluded.writer_app_version + `, + ) + .run(path.resolve(options.path), options.kind, options.reason, Date.now(), VERSION); + database.exec("COMMIT;"); + return true; + } catch (error) { + database.exec("ROLLBACK;"); + throw error; + } + }); + } catch { + return false; + } +} + +/** Clear one authoritative quarantine decision. */ +export function clearOpenClawDatabaseQuarantine( + pathname: string, + options: { env?: NodeJS.ProcessEnv } = {}, +): boolean { + const env = options.env ?? process.env; + if (!existsSync(resolveQuarantineStorePath(env))) { + return true; + } + try { + return withQuarantineWriter(env, (database) => { + database.exec("BEGIN IMMEDIATE;"); + try { + database + .prepare("DELETE FROM quarantined_databases WHERE path = ?") + .run(path.resolve(pathname)); + database.exec("COMMIT;"); + return true; + } catch (error) { + database.exec("ROLLBACK;"); + throw error; + } + }); + } catch { + return false; + } +} diff --git a/src/state/openclaw-state-db.ts b/src/state/openclaw-state-db.ts index 0af4fa8e10e3..97a24c4946d5 100644 --- a/src/state/openclaw-state-db.ts +++ b/src/state/openclaw-state-db.ts @@ -42,6 +42,10 @@ import { import { migrateLegacyCronRunLogsToTaskRuns } from "../infra/state-migrations.cron-run-logs.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; import { VERSION } from "../version.js"; +import { + clearOpenClawDatabaseQuarantine, + readOpenClawDatabaseQuarantine, +} from "./openclaw-quarantine-store.js"; import * as operatorApprovalMigration from "./openclaw-state-db-operator-approval-migration.js"; import { ensureColumn, @@ -165,29 +169,6 @@ type OpenClawDatabaseVerificationDatabase = Pick< OpenClawStateKyselyDatabase, "database_verifications" >; -type OpenClawDatabaseVerification = { - error: string | null; - result: string; -}; - -function readDatabaseVerificationRow( - database: DatabaseSync, - pathname: string, -): OpenClawDatabaseVerification | undefined { - if (!tableExists(database, "database_verifications")) { - return undefined; - } - const db = getNodeSqliteKysely(database); - return executeSqliteQuerySync( - database, - db - .selectFrom("database_verifications") - .select(["result", "error"]) - .where("path", "=", path.resolve(pathname)) - .limit(1), - ).rows[0]; -} - function clearDatabaseVerificationRow(database: DatabaseSync, pathname: string): void { if (!tableExists(database, "database_verifications")) { return; @@ -199,27 +180,8 @@ function clearDatabaseVerificationRow(database: DatabaseSync, pathname: string): ); } -/** Read one durable verification row through the cached shared state database. */ -export function readOpenClawDatabaseVerification( - pathname: string, - options: OpenClawStateDatabaseOptions = {}, -): OpenClawDatabaseVerification | undefined { - const statePath = path.resolve( - options.path ?? resolveOpenClawStateSqlitePath(options.env ?? process.env), - ); - if (!existsSync(statePath)) { - return undefined; - } - return readDatabaseVerificationRow(openOpenClawStateDatabase(options).db, pathname); -} - -/** - * Remove a persisted quarantine row after doctor repairs a database. - * Returns false when the row could not be cleared: callers must surface that, - * or the next open re-quarantines a healthy repaired file while doctor - * reported success. - */ -export function clearOpenClawDatabaseVerification( +/** Best-effort deletion of one verification-history row after repair. */ +export function clearOpenClawDatabaseVerificationHistory( pathname: string, options: OpenClawStateDatabaseOptions = {}, ): boolean { @@ -1005,7 +967,16 @@ export function repairOpenClawStateDatabaseSchema(options: OpenClawStateDatabase operationLabel: "state.schema.repair", }, ); - const quarantineCleared = clearOpenClawDatabaseVerification(pathname, { path: pathname }); + try { + runSqliteImmediateTransactionSync(db, () => clearDatabaseVerificationRow(db, pathname), { + busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, + databaseLabel: pathname, + operationLabel: "state.database-verifications.clear-history", + }); + } catch { + // History cleanup must not override authoritative quarantine repair. + } + const quarantineCleared = clearOpenClawDatabaseQuarantine(pathname, { env }); clearOpenClawStateDatabaseOpenFailure(pathname); return { changes, @@ -1862,6 +1833,23 @@ export function openOpenClawStateDatabase( clearNodeSqliteKyselyCacheForDatabase(cached.db); cachedDatabases.delete(pathname); } + let quarantineFailure: Error | undefined; + try { + const quarantine = readOpenClawDatabaseQuarantine(pathname, { env }); + if (quarantine) { + quarantineFailure = createOpenClawDatabaseVerificationError( + "state", + pathname, + quarantine.reason, + ); + } + } catch { + // A broken quarantine store must not brick every state open. + // The process latch and daily verifier still cover known damage. + } + if (quarantineFailure) { + throw quarantineFailure; + } ensureOpenClawStatePermissions(pathname, env); const sqlite = requireNodeSqlite(); const db = new sqlite.DatabaseSync(pathname); @@ -1871,10 +1859,6 @@ export function openOpenClawStateDatabase( db.exec(`PRAGMA busy_timeout = ${OPENCLAW_SQLITE_BUSY_TIMEOUT_MS};`); assertSupportedSchemaVersion(db, pathname); assertStateDatabaseIntegrityBeforeMutation(db, pathname); - const verification = readDatabaseVerificationRow(db, pathname); - if (verification?.result === "error") { - throw createOpenClawDatabaseVerificationError("state", pathname, verification.error); - } configureSqlitePreSchemaPragmas(db, { busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, });