feat(sqlite): move quarantine decisions to a dedicated store that survives primary database damage (#110453)

This commit is contained in:
Peter Steinberger
2026-07-18 07:13:43 +01:00
committed by GitHub
parent 713b6151d1
commit deddeb3dac
12 changed files with 528 additions and 128 deletions
+7 -6
View File
@@ -56,14 +56,15 @@ Version 3 was an unshipped development step folded into version 4.
## Integrity checks
| When | Check |
| ------------------------------------------- | -------------------------------------------------------------------------------------------------- |
| Every open | Validate the `schema_meta` table and primary metadata row |
| Before a pending migration | Run a full integrity, foreign-key, role, schema, and index scan |
| Gateway background verifier | Run the full scan about once daily and record results in the global `database_verifications` table |
| Doctor, backup verification, and compaction | Run the full scan before accepting or rewriting the database |
| When | Check |
| ------------------------------------------- | --------------------------------------------------------------- |
| Every open | Validate the `schema_meta` table and primary metadata row |
| Before a pending migration | Run a full integrity, foreign-key, role, schema, and index scan |
| Gateway background verifier | Run the full scan about once daily and record results |
| Doctor, backup verification, and compaction | Run the full scan before accepting or rewriting the database |
The Gateway preflight reads schema headers only. The background verifier owns the slower full scan for databases that do not need migration.
Quarantine decisions live in a dedicated `openclaw-quarantine.sqlite` store, so they survive damage to the databases being quarantined. The global `database_verifications` table remains verification history.
## Downgrades are unsupported
+3
View File
@@ -57,6 +57,9 @@ const rawSqliteAllowPathGroups = {
"src/state/openclaw-database-preflight.ts",
"src/state/openclaw-database-verify.worker.ts",
],
"quarantine store must work when other databases are damaged": [
"src/state/openclaw-quarantine-store.ts",
],
"read-only SQLite status probes": [
"src/commands/doctor-db-bloat.ts",
"src/commands/status.scan.shared.ts",
@@ -15,7 +15,7 @@ import { compactDoctorSqliteFile } from "./doctor-sqlite-compact.js";
/** Reclaim free pages from one agent session SQLite database. */
export function compactDoctorSessionSqliteTarget(
target: SessionStoreTarget,
options: { migrateOlderSchema?: boolean } = {},
options: { env?: NodeJS.ProcessEnv; migrateOlderSchema?: boolean } = {},
): DoctorSessionSqliteCompactReport {
const sqlitePath = resolveTargetSqlitePath(target);
const beforeFileSizes = readSqliteFileSizes(sqlitePath);
@@ -42,7 +42,7 @@ export function compactDoctorSessionSqliteTarget(
);
}
const requireQuarantineCleared = () => {
if (!clearOpenClawAgentDatabaseOpenFailure(sqlitePath)) {
if (!clearOpenClawAgentDatabaseOpenFailure(sqlitePath, { env: options.env })) {
throw new Error(
`OpenClaw agent database ${sqlitePath} was repaired, but its persisted quarantine record could not be cleared. Rerun openclaw doctor --fix so the database is not refused again.`,
);
+42 -1
View File
@@ -21,7 +21,14 @@ import {
OPENCLAW_AGENT_SCHEMA_VERSION,
resolveOpenClawAgentSqlitePath,
} from "../state/openclaw-agent-db.js";
import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js";
import {
readOpenClawDatabaseQuarantine,
recordOpenClawDatabaseQuarantine,
} from "../state/openclaw-quarantine-store.js";
import {
closeOpenClawStateDatabaseForTest,
openOpenClawStateDatabase,
} from "../state/openclaw-state-db.js";
import {
assertSafeSessionSqliteMigrationMove,
createSessionSqliteMigrationFailureIssue,
@@ -611,6 +618,40 @@ describe("runDoctorSessionSqlite", () => {
},
);
it("clears agent quarantine and verification history after compaction", async () => {
const { sqlitePath, store } = await createImportedStoreForCompaction();
const state = openOpenClawStateDatabase({ env: store.env });
state.db
.prepare(
`
INSERT INTO database_verifications (path, kind, verified_at, result, error)
VALUES (?, 'agent', 1, 'error', 'corrupt index')
`,
)
.run(sqlitePath);
expect(
recordOpenClawDatabaseQuarantine({
env: store.env,
kind: "agent",
path: sqlitePath,
reason: "corrupt index",
}),
).toBe(true);
const report = await runDoctorSessionSqlite({
env: store.env,
mode: "compact",
store: store.storePath,
});
expect(report.totals.issues).toBe(0);
expect(readOpenClawDatabaseQuarantine(sqlitePath, { env: store.env })).toBeUndefined();
expect(
state.db.prepare("SELECT 1 FROM database_verifications WHERE path = ?").get(sqlitePath),
).toBeUndefined();
expect(openOpenClawAgentDatabase({ agentId: "main", env: store.env }).db.isOpen).toBe(true);
});
it.skipIf(process.platform === "win32")(
"reapplies owner-only permissions after compaction",
async () => {
+12 -4
View File
@@ -259,7 +259,7 @@ async function inspectOrMigrateTarget(params: {
return report;
}
if (params.mode === "compact") {
compactSqliteDatabase(params.target, report);
compactSqliteDatabase(params.target, report, { env: params.env });
report.sqliteEntries = readSqliteEntryCount(params.target);
appendSqliteDbStats(params.target, report);
return report;
@@ -299,6 +299,7 @@ async function inspectOrMigrateTarget(params: {
// databases and returns the pages the import churn freed.
compactSqliteDatabase(params.target, report, {
closeImportedHandle: true,
env: params.env,
migrateOlderSchema: true,
});
}
@@ -988,15 +989,22 @@ function appendSqliteDbStats(
function compactSqliteDatabase(
target: SessionStoreTarget,
report: DoctorSessionSqliteTargetReport,
options: { closeImportedHandle?: boolean; migrateOlderSchema?: boolean } = {},
options: {
closeImportedHandle?: boolean;
env?: NodeJS.ProcessEnv;
migrateOlderSchema?: boolean;
} = {},
): void {
try {
if (options.closeImportedHandle) {
closeOpenClawAgentDatabaseByPath(resolveTargetSqlitePath(target));
}
report.compact = options.migrateOlderSchema
? compactDoctorSessionSqliteTarget(target, { migrateOlderSchema: true })
: compactDoctorSessionSqliteTarget(target);
? compactDoctorSessionSqliteTarget(target, {
env: options.env,
migrateOlderSchema: true,
})
: compactDoctorSessionSqliteTarget(target, { env: options.env });
} catch (err) {
report.issues.push({
code: "sqlite_compact_failed",
@@ -4,6 +4,10 @@ import type { DatabaseSync } from "node:sqlite";
import { afterEach, describe, expect, it } from "vitest";
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
import { requireNodeSqlite } from "../infra/node-sqlite.js";
import {
readOpenClawDatabaseQuarantine,
recordOpenClawDatabaseQuarantine,
} from "../state/openclaw-quarantine-store.js";
import {
closeOpenClawStateDatabase,
openOpenClawStateDatabase,
@@ -166,6 +170,41 @@ describe("runDoctorStateSqliteCompact", () => {
expect(report.integrityCheck).toBe("ok");
});
it("clears authoritative quarantine and verification history after compaction", async () => {
const env = createStateEnv();
const sqlitePath = seedStateDatabase({ env, withBloat: true });
const sqlite = requireNodeSqlite();
const history = new sqlite.DatabaseSync(sqlitePath);
try {
history
.prepare(
`
INSERT INTO database_verifications (path, kind, verified_at, result, error)
VALUES (?, 'state', 1, 'error', 'corrupt index')
`,
)
.run(sqlitePath);
} finally {
history.close();
}
expect(
recordOpenClawDatabaseQuarantine({
env,
kind: "state",
path: sqlitePath,
reason: "corrupt index",
}),
).toBe(true);
await runDoctorStateSqliteCompact({ env });
expect(readOpenClawDatabaseQuarantine(sqlitePath, { env })).toBeUndefined();
const repaired = openOpenClawStateDatabase({ env });
expect(
repaired.db.prepare("SELECT 1 FROM database_verifications WHERE path = ?").get(sqlitePath),
).toBeUndefined();
});
it.skipIf(process.platform === "win32")("reapplies owner-only SQLite permissions", async () => {
const env = createStateEnv();
const sqlitePath = seedStateDatabase({ env, withBloat: true });
+4 -2
View File
@@ -1,8 +1,9 @@
/** Explicit doctor maintenance for the canonical shared state SQLite database. */
import fs from "node:fs";
import { clearOpenClawDatabaseQuarantine } from "../state/openclaw-quarantine-store.js";
import {
assertOpenClawStateDatabaseForMaintenance,
clearOpenClawDatabaseVerification,
clearOpenClawDatabaseVerificationHistory,
clearOpenClawStateDatabaseOpenFailure,
ensureOpenClawStatePermissions,
isOpenClawStateDatabaseOpen,
@@ -72,11 +73,12 @@ export async function runDoctorStateSqliteCompact(
const compact = compactDoctorSqliteFile({
afterMutation: () => {
if (!clearOpenClawDatabaseVerification(sqlitePath, { path: sqlitePath })) {
if (!clearOpenClawDatabaseQuarantine(sqlitePath, { env })) {
throw new Error(
`OpenClaw state database ${sqlitePath} was compacted, but its persisted quarantine record could not be cleared. Rerun openclaw doctor --fix so the database is not refused again.`,
);
}
clearOpenClawDatabaseVerificationHistory(sqlitePath, { env });
clearOpenClawStateDatabaseOpenFailure(sqlitePath);
ensureOpenClawStatePermissions(sqlitePath, env);
},
+12 -10
View File
@@ -59,14 +59,17 @@ import {
import type { DB as OpenClawAgentKyselyDatabase } from "./openclaw-agent-db.generated.js";
import { resolveOpenClawAgentSqlitePath } from "./openclaw-agent-db.paths.js";
import { OPENCLAW_AGENT_SCHEMA_SQL } from "./openclaw-agent-schema.generated.js";
import {
clearOpenClawDatabaseQuarantine,
readOpenClawDatabaseQuarantine,
} from "./openclaw-quarantine-store.js";
import type { DB as OpenClawStateKyselyDatabase } from "./openclaw-state-db.generated.js";
import {
clearOpenClawDatabaseVerification,
clearOpenClawDatabaseVerificationHistory,
createOpenClawDatabaseVerificationError,
detectOpenClawStateDatabaseSchemaMigrations,
OPENCLAW_STATE_SCHEMA_VERSION,
OPENCLAW_SQLITE_BUSY_TIMEOUT_MS,
readOpenClawDatabaseVerification,
runOpenClawStateWriteTransaction,
type OpenClawStateDatabaseOptions,
} from "./openclaw-state-db.js";
@@ -183,7 +186,8 @@ export function clearOpenClawAgentDatabaseOpenFailure(
options: OpenClawStateDatabaseOptions = {},
): boolean {
const resolvedPath = path.resolve(pathname);
const cleared = clearOpenClawDatabaseVerification(resolvedPath, options);
const cleared = clearOpenClawDatabaseQuarantine(resolvedPath, { env: options.env });
clearOpenClawDatabaseVerificationHistory(resolvedPath, options);
terminalOpenFailures.delete(resolvedPath);
return cleared;
}
@@ -1082,19 +1086,17 @@ export function openOpenClawAgentDatabase(
}
let persistedFailure: Error | undefined;
try {
const verification = readOpenClawDatabaseVerification(pathname, { env: databaseOptions.env });
if (verification?.result === "error") {
const quarantine = readOpenClawDatabaseQuarantine(pathname, { env: databaseOptions.env });
if (quarantine) {
persistedFailure = createOpenClawDatabaseVerificationError(
"agent",
pathname,
verification.error,
quarantine.reason,
);
}
} catch {
// Accepted tradeoff: a locked/unavailable state DB must not block agent
// opens, or every transient state hiccup takes all agents down. The
// in-process latch still covers this process; a missed cross-process
// quarantine is re-detected by the next daily verifier pass.
// A broken quarantine store must not brick every agent open.
// The process latch and daily verifier still cover known damage.
}
if (persistedFailure) {
recordOpenClawAgentDatabaseOpenFailure(pathname, persistedFailure);
+25 -39
View File
@@ -12,6 +12,7 @@ import type {
OpenClawDatabaseVerifyResult,
OpenClawDatabaseVerifyTarget,
} from "./openclaw-database-verify.worker.js";
import { recordOpenClawDatabaseQuarantine } from "./openclaw-quarantine-store.js";
import type { DB as OpenClawStateKyselyDatabase } from "./openclaw-state-db.generated.js";
import {
recordOpenClawStateDatabaseOpenFailure,
@@ -149,28 +150,36 @@ export function applyOpenClawDatabaseVerificationResults(options: {
const targetByPath = new Map(options.targets.map((target) => [target.path, target]));
const verifiedAt = options.verifiedAt ?? Date.now();
// One best-effort row write into a possibly corrupt state file buys restart-durable quarantine.
// Doctor rebuilds the file during repair, so this bounded mutation is an accepted tradeoff.
const persistResults = (results: readonly OpenClawDatabaseVerifyResult[]) => {
for (const result of options.results) {
const target = targetByPath.get(result.path);
if (!target || result.ok || !result.terminal) {
continue;
}
const recorded = recordOpenClawDatabaseQuarantine({
env: options.env,
kind: target.kind,
path: result.path,
reason: result.error ?? `SQLite integrity verification failed for ${result.path}`,
});
if (!recorded) {
// Accepted residual: quarantine stays process-local when this tiny,
// independent store is unavailable. Daily verification retries it.
log.error("failed to persist database quarantine; quarantine is process-local", {
kind: target.kind,
path: result.path,
});
}
}
try {
runOpenClawStateWriteTransaction(
(database) => {
const db = getNodeSqliteKysely<VerificationDatabase>(database.db);
for (const result of results) {
for (const result of options.results) {
const target = targetByPath.get(result.path);
if (!target) {
continue;
}
const existing = executeSqliteQuerySync(
database.db,
db
.selectFrom("database_verifications")
.select("result")
.where("path", "=", result.path)
.limit(1),
).rows[0];
if (existing?.result === "error") {
continue;
}
executeSqliteQuerySync(
database.db,
db
@@ -196,31 +205,8 @@ export function applyOpenClawDatabaseVerificationResults(options: {
{ env: options.env },
{ operationLabel: "state.database-verifications.record" },
);
};
try {
persistResults(options.results);
} catch (error) {
// Terminal rows are the restart-durable quarantine; retry those once so a
// transient state-DB lock does not silently downgrade them to process-local.
const terminalResults = options.results.filter((result) => !result.ok && result.terminal);
let retried = false;
if (terminalResults.length > 0) {
try {
persistResults(terminalResults);
retried = true;
} catch {
// fall through to the process-local warning below
}
}
if (!retried) {
// Accepted tradeoff: when this write fails the quarantine stays process-local
// until the next daily verifier pass re-detects and re-persists. A sidecar
// marker would break the SQLite-only storage contract, and startup full
// scans would reintroduce the multi-second opens this design removes.
log.error("failed to persist database verification results; quarantine is process-local", {
error: String(error),
});
}
log.error("failed to persist database verification history", { error: String(error) });
}
for (const result of options.results) {
+143 -15
View File
@@ -1,4 +1,6 @@
import { spawnSync } from "node:child_process";
import fs from "node:fs";
import path from "node:path";
import { afterAll, afterEach, describe, expect, it } from "vitest";
import { cleanupTempDirs, makeTempDir } from "../../test/helpers/temp-dir.js";
import { requireNodeSqlite } from "../infra/node-sqlite.js";
@@ -16,6 +18,11 @@ import {
type OpenClawDatabaseVerifyTarget,
verifyOpenClawDatabases,
} from "./openclaw-database-verify.worker.js";
import {
clearOpenClawDatabaseQuarantine,
readOpenClawDatabaseQuarantine,
recordOpenClawDatabaseQuarantine,
} from "./openclaw-quarantine-store.js";
import {
closeOpenClawStateDatabaseForTest,
openOpenClawStateDatabase,
@@ -59,6 +66,10 @@ function createUnsafeIndexDrift(databasePath: string): void {
}
}
function quarantineStorePath(stateDir: string): string {
return path.join(stateDir, "state", "openclaw-quarantine.sqlite");
}
describe("OpenClaw database integrity verifier", () => {
it("detects corruption off-thread, persists it, and latches later opens", async () => {
const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-");
@@ -94,6 +105,11 @@ describe("OpenClaw database integrity verifier", () => {
verifiedAt: 1234,
});
expect(liveHandle.db.isOpen).toBe(false);
expect(readOpenClawDatabaseQuarantine(agentPath, { env })).toEqual({
kind: "agent",
quarantinedAt: expect.any(Number),
reason: directResults[0]?.error,
});
expect(
openOpenClawStateDatabase({ env })
@@ -118,7 +134,7 @@ describe("OpenClaw database integrity verifier", () => {
openOpenClawStateDatabase({ env })
.db.prepare("SELECT verified_at, result, error FROM database_verifications WHERE path = ?")
.get(agentPath),
).toEqual({ verified_at: 1234, result: "error", error: directResults[0]?.error });
).toEqual({ verified_at: 1235, result: "inconclusive", error: "database busy" });
expect(() => openOpenClawAgentDatabase({ agentId: "worker-1", env })).toThrow(
expect.objectContaining({ name: "SqliteIntegrityError" }),
);
@@ -139,7 +155,7 @@ describe("OpenClaw database integrity verifier", () => {
const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-clear-failure-");
const env = { OPENCLAW_STATE_DIR: stateDir };
const agentPath = openOpenClawAgentDatabase({ agentId: "worker-1", env }).path;
const statePath = openOpenClawStateDatabase({ env }).path;
openOpenClawStateDatabase({ env });
closeOpenClawAgentDatabasesForTest();
closeOpenClawStateDatabaseForTest();
applyOpenClawDatabaseVerificationResults({
@@ -150,23 +166,137 @@ describe("OpenClaw database integrity verifier", () => {
});
closeOpenClawStateDatabaseForTest();
// A read-only state DB cannot drop the quarantine row; the clear must say so
const storePath = quarantineStorePath(stateDir);
// A read-only quarantine store cannot drop the row; the clear must say so
// instead of letting doctor report success while the next open still refuses.
fs.chmodSync(statePath, 0o444);
fs.chmodSync(storePath, 0o444);
try {
expect(clearOpenClawAgentDatabaseOpenFailure(agentPath, { env })).toBe(false);
expect(
recordOpenClawDatabaseQuarantine({
env,
kind: "agent",
path: agentPath,
reason: "new reason",
}),
).toBe(false);
} finally {
// WAL sidecars minted during the read-only attempt inherit its mode.
for (const sidecar of [statePath, `${statePath}-wal`, `${statePath}-shm`]) {
if (fs.existsSync(sidecar)) {
fs.chmodSync(sidecar, 0o600);
}
}
fs.chmodSync(storePath, 0o600);
}
expect(clearOpenClawAgentDatabaseOpenFailure(agentPath, { env })).toBe(true);
expect(openOpenClawAgentDatabase({ agentId: "worker-1", env }).db.isOpen).toBe(true);
});
it("keeps healthy opens on the missing-store fast path", () => {
const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-clean-");
const env = { OPENCLAW_STATE_DIR: stateDir };
openOpenClawStateDatabase({ env });
openOpenClawAgentDatabase({ agentId: "worker-1", env });
expect(fs.existsSync(quarantineStorePath(stateDir))).toBe(false);
});
it("records and clears dedicated quarantine rows with rollback journaling", () => {
const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-store-");
const env = { OPENCLAW_STATE_DIR: stateDir };
const databasePath = path.join(stateDir, "agent.sqlite");
const storePath = quarantineStorePath(stateDir);
expect(clearOpenClawDatabaseQuarantine(databasePath, { env })).toBe(true);
expect(
recordOpenClawDatabaseQuarantine({
env,
kind: "agent",
path: databasePath,
reason: "corrupt index",
}),
).toBe(true);
expect(readOpenClawDatabaseQuarantine(databasePath, { env })).toEqual({
kind: "agent",
quarantinedAt: expect.any(Number),
reason: "corrupt index",
});
const { DatabaseSync } = requireNodeSqlite();
const raw = new DatabaseSync(storePath, { readOnly: true });
try {
expect(raw.prepare("PRAGMA journal_mode").get()).toEqual({ journal_mode: "delete" });
expect(readSqliteNumberPragma(raw, "synchronous")).toBe(2);
expect(readSqliteNumberPragma(raw, "user_version")).toBe(1);
} finally {
raw.close();
}
if (process.platform !== "win32") {
expect(fs.statSync(path.dirname(storePath)).mode & 0o777).toBe(0o700);
expect(fs.statSync(storePath).mode & 0o777).toBe(0o600);
}
expect(clearOpenClawDatabaseQuarantine(databasePath, { env })).toBe(true);
expect(clearOpenClawDatabaseQuarantine(databasePath, { env })).toBe(true);
expect(readOpenClawDatabaseQuarantine(databasePath, { env })).toBeUndefined();
});
it("recovers an interrupted empty quarantine-store initialization", () => {
const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-empty-store-");
const env = { OPENCLAW_STATE_DIR: stateDir };
const databasePath = path.join(stateDir, "agent.sqlite");
const storePath = quarantineStorePath(stateDir);
fs.mkdirSync(path.dirname(storePath), { recursive: true });
fs.writeFileSync(storePath, "", { mode: 0o600 });
expect(readOpenClawDatabaseQuarantine(databasePath, { env })).toBeUndefined();
expect(
recordOpenClawDatabaseQuarantine({
env,
kind: "agent",
path: databasePath,
reason: "corrupt index",
}),
).toBe(true);
expect(readOpenClawDatabaseQuarantine(databasePath, { env })?.reason).toBe("corrupt index");
});
it.skipIf(process.platform === "win32")(
"recovers a hot rollback journal before reading quarantine",
() => {
const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-hot-journal-");
const env = { OPENCLAW_STATE_DIR: stateDir };
const databasePath = path.join(stateDir, "agent.sqlite");
const storePath = quarantineStorePath(stateDir);
expect(
recordOpenClawDatabaseQuarantine({
env,
kind: "agent",
path: databasePath,
reason: "committed reason",
}),
).toBe(true);
const crashed = spawnSync(
process.execPath,
[
"--no-warnings",
"--input-type=module",
"-e",
`
import { DatabaseSync } from "node:sqlite";
const database = new DatabaseSync(process.env.OPENCLAW_QUARANTINE_TEST_PATH);
database.exec("PRAGMA journal_mode = DELETE; PRAGMA synchronous = FULL; BEGIN IMMEDIATE;");
database.prepare("UPDATE quarantined_databases SET reason = 'uncommitted reason'").run();
process.kill(process.pid, "SIGKILL");
`,
],
{ env: { ...process.env, OPENCLAW_QUARANTINE_TEST_PATH: storePath } },
);
expect(crashed.signal).toBe("SIGKILL");
expect(fs.existsSync(`${storePath}-journal`)).toBe(true);
expect(readOpenClawDatabaseQuarantine(databasePath, { env })?.reason).toBe(
"committed reason",
);
},
);
it("persists transient verifier errors as inconclusive without latching", () => {
const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-transient-");
const env = { OPENCLAW_STATE_DIR: stateDir };
@@ -209,13 +339,11 @@ describe("OpenClaw database integrity verifier", () => {
});
const { DatabaseSync } = requireNodeSqlite();
const raw = new DatabaseSync(statePath, { readOnly: true });
const raw = new DatabaseSync(quarantineStorePath(stateDir), { readOnly: true });
try {
expect(
raw
.prepare("SELECT result, error FROM database_verifications WHERE path = ?")
.get(statePath),
).toEqual({ result: "error", error: "corrupt index" });
raw.prepare("SELECT kind, reason FROM quarantined_databases WHERE path = ?").get(statePath),
).toEqual({ kind: "state", reason: "corrupt index" });
} finally {
raw.close();
}
+206
View File
@@ -0,0 +1,206 @@
// Dedicated quarantine decisions stay available when primary databases fail.
import { existsSync, mkdirSync } from "node:fs";
import path from "node:path";
import type { DatabaseSync } from "node:sqlite";
import { requireNodeSqlite } from "../infra/node-sqlite.js";
import { applyPrivateModeSync } from "../infra/private-mode.js";
import { VERSION } from "../version.js";
import { resolveOpenClawStateSqliteDir } from "./openclaw-state-db.paths.js";
const OPENCLAW_QUARANTINE_SCHEMA_VERSION = 1;
const OPENCLAW_QUARANTINE_BUSY_TIMEOUT_MS = 5_000;
const OPENCLAW_QUARANTINE_DIR_MODE = 0o700;
const OPENCLAW_QUARANTINE_FILE_MODE = 0o600;
type OpenClawDatabaseKind = "agent" | "state";
type OpenClawDatabaseQuarantine = {
kind: OpenClawDatabaseKind;
quarantinedAt: number;
reason: string;
};
function resolveQuarantineStorePath(env: NodeJS.ProcessEnv): string {
return path.join(resolveOpenClawStateSqliteDir(env), "openclaw-quarantine.sqlite");
}
function ensureQuarantineStoreDirectory(storePath: string): void {
const dir = path.dirname(storePath);
mkdirSync(dir, { recursive: true, mode: OPENCLAW_QUARANTINE_DIR_MODE });
applyPrivateModeSync(dir, OPENCLAW_QUARANTINE_DIR_MODE);
}
function configureQuarantineWriter(database: DatabaseSync, storePath: string): void {
database.exec(`
PRAGMA busy_timeout = ${OPENCLAW_QUARANTINE_BUSY_TIMEOUT_MS};
PRAGMA journal_mode = DELETE;
PRAGMA synchronous = FULL;
`);
const userVersion = readQuarantineSchemaVersion(database, storePath);
if (userVersion > OPENCLAW_QUARANTINE_SCHEMA_VERSION) {
throw new Error(
`OpenClaw quarantine store ${storePath} uses newer schema version ${userVersion}.`,
);
}
if (userVersion === OPENCLAW_QUARANTINE_SCHEMA_VERSION) {
return;
}
database.exec(`
BEGIN IMMEDIATE;
CREATE TABLE IF NOT EXISTS quarantined_databases (
path TEXT NOT NULL PRIMARY KEY,
kind TEXT NOT NULL,
reason TEXT NOT NULL,
quarantined_at INTEGER NOT NULL,
writer_app_version TEXT
) STRICT;
PRAGMA user_version = ${OPENCLAW_QUARANTINE_SCHEMA_VERSION};
COMMIT;
`);
}
function readQuarantineSchemaVersion(database: DatabaseSync, storePath: string): number {
const row = database.prepare("PRAGMA user_version").get() as
| { user_version?: unknown }
| undefined;
const userVersion = row?.user_version;
if (typeof userVersion !== "number" || !Number.isInteger(userVersion)) {
throw new Error(`OpenClaw quarantine store ${storePath} has an invalid schema version.`);
}
return userVersion;
}
function withQuarantineWriter<T>(env: NodeJS.ProcessEnv, operation: (db: DatabaseSync) => T): T {
const storePath = resolveQuarantineStorePath(env);
const existed = existsSync(storePath);
ensureQuarantineStoreDirectory(storePath);
const sqlite = requireNodeSqlite();
const database = new sqlite.DatabaseSync(storePath);
let completed = false;
try {
if (!existed) {
applyPrivateModeSync(storePath, OPENCLAW_QUARANTINE_FILE_MODE);
}
configureQuarantineWriter(database, storePath);
const result = operation(database);
completed = true;
return result;
} finally {
database.close();
if (completed || !existed) {
applyPrivateModeSync(storePath, OPENCLAW_QUARANTINE_FILE_MODE);
}
}
}
/** Read one authoritative quarantine decision without creating the store. */
export function readOpenClawDatabaseQuarantine(
pathname: string,
options: { env?: NodeJS.ProcessEnv } = {},
): OpenClawDatabaseQuarantine | undefined {
const storePath = resolveQuarantineStorePath(options.env ?? process.env);
// Clean installs pay one existence check. No directory or SQLite work.
if (!existsSync(storePath)) {
return undefined;
}
const sqlite = requireNodeSqlite();
const database = new sqlite.DatabaseSync(storePath);
try {
database.exec(`PRAGMA busy_timeout = ${OPENCLAW_QUARANTINE_BUSY_TIMEOUT_MS};`);
const userVersion = readQuarantineSchemaVersion(database, storePath);
if (userVersion === 0) {
return undefined;
}
if (userVersion !== OPENCLAW_QUARANTINE_SCHEMA_VERSION) {
throw new Error(
`OpenClaw quarantine store ${storePath} uses newer schema version ${userVersion}.`,
);
}
const row = database
.prepare(
"SELECT kind, reason, quarantined_at FROM quarantined_databases WHERE path = ? LIMIT 1",
)
.get(path.resolve(pathname)) as
| { kind?: unknown; quarantined_at?: unknown; reason?: unknown }
| undefined;
if (!row) {
return undefined;
}
if (
(row.kind !== "agent" && row.kind !== "state") ||
typeof row.reason !== "string" ||
typeof row.quarantined_at !== "number" ||
!Number.isInteger(row.quarantined_at)
) {
throw new Error(`OpenClaw quarantine store ${storePath} contains an invalid row.`);
}
return { kind: row.kind, quarantinedAt: row.quarantined_at, reason: row.reason };
} finally {
database.close();
}
}
/** Persist one authoritative quarantine decision. */
export function recordOpenClawDatabaseQuarantine(options: {
env?: NodeJS.ProcessEnv;
kind: OpenClawDatabaseKind;
path: string;
reason: string;
}): boolean {
try {
return withQuarantineWriter(options.env ?? process.env, (database) => {
database.exec("BEGIN IMMEDIATE;");
try {
database
.prepare(
`
INSERT INTO quarantined_databases (
path, kind, reason, quarantined_at, writer_app_version
) VALUES (?, ?, ?, ?, ?)
ON CONFLICT(path) DO UPDATE SET
kind = excluded.kind,
reason = excluded.reason,
quarantined_at = excluded.quarantined_at,
writer_app_version = excluded.writer_app_version
`,
)
.run(path.resolve(options.path), options.kind, options.reason, Date.now(), VERSION);
database.exec("COMMIT;");
return true;
} catch (error) {
database.exec("ROLLBACK;");
throw error;
}
});
} catch {
return false;
}
}
/** Clear one authoritative quarantine decision. */
export function clearOpenClawDatabaseQuarantine(
pathname: string,
options: { env?: NodeJS.ProcessEnv } = {},
): boolean {
const env = options.env ?? process.env;
if (!existsSync(resolveQuarantineStorePath(env))) {
return true;
}
try {
return withQuarantineWriter(env, (database) => {
database.exec("BEGIN IMMEDIATE;");
try {
database
.prepare("DELETE FROM quarantined_databases WHERE path = ?")
.run(path.resolve(pathname));
database.exec("COMMIT;");
return true;
} catch (error) {
database.exec("ROLLBACK;");
throw error;
}
});
} catch {
return false;
}
}
+33 -49
View File
@@ -42,6 +42,10 @@ import {
import { migrateLegacyCronRunLogsToTaskRuns } from "../infra/state-migrations.cron-run-logs.js";
import { createSubsystemLogger } from "../logging/subsystem.js";
import { VERSION } from "../version.js";
import {
clearOpenClawDatabaseQuarantine,
readOpenClawDatabaseQuarantine,
} from "./openclaw-quarantine-store.js";
import * as operatorApprovalMigration from "./openclaw-state-db-operator-approval-migration.js";
import {
ensureColumn,
@@ -165,29 +169,6 @@ type OpenClawDatabaseVerificationDatabase = Pick<
OpenClawStateKyselyDatabase,
"database_verifications"
>;
type OpenClawDatabaseVerification = {
error: string | null;
result: string;
};
function readDatabaseVerificationRow(
database: DatabaseSync,
pathname: string,
): OpenClawDatabaseVerification | undefined {
if (!tableExists(database, "database_verifications")) {
return undefined;
}
const db = getNodeSqliteKysely<OpenClawDatabaseVerificationDatabase>(database);
return executeSqliteQuerySync(
database,
db
.selectFrom("database_verifications")
.select(["result", "error"])
.where("path", "=", path.resolve(pathname))
.limit(1),
).rows[0];
}
function clearDatabaseVerificationRow(database: DatabaseSync, pathname: string): void {
if (!tableExists(database, "database_verifications")) {
return;
@@ -199,27 +180,8 @@ function clearDatabaseVerificationRow(database: DatabaseSync, pathname: string):
);
}
/** Read one durable verification row through the cached shared state database. */
export function readOpenClawDatabaseVerification(
pathname: string,
options: OpenClawStateDatabaseOptions = {},
): OpenClawDatabaseVerification | undefined {
const statePath = path.resolve(
options.path ?? resolveOpenClawStateSqlitePath(options.env ?? process.env),
);
if (!existsSync(statePath)) {
return undefined;
}
return readDatabaseVerificationRow(openOpenClawStateDatabase(options).db, pathname);
}
/**
* Remove a persisted quarantine row after doctor repairs a database.
* Returns false when the row could not be cleared: callers must surface that,
* or the next open re-quarantines a healthy repaired file while doctor
* reported success.
*/
export function clearOpenClawDatabaseVerification(
/** Best-effort deletion of one verification-history row after repair. */
export function clearOpenClawDatabaseVerificationHistory(
pathname: string,
options: OpenClawStateDatabaseOptions = {},
): boolean {
@@ -1005,7 +967,16 @@ export function repairOpenClawStateDatabaseSchema(options: OpenClawStateDatabase
operationLabel: "state.schema.repair",
},
);
const quarantineCleared = clearOpenClawDatabaseVerification(pathname, { path: pathname });
try {
runSqliteImmediateTransactionSync(db, () => clearDatabaseVerificationRow(db, pathname), {
busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS,
databaseLabel: pathname,
operationLabel: "state.database-verifications.clear-history",
});
} catch {
// History cleanup must not override authoritative quarantine repair.
}
const quarantineCleared = clearOpenClawDatabaseQuarantine(pathname, { env });
clearOpenClawStateDatabaseOpenFailure(pathname);
return {
changes,
@@ -1862,6 +1833,23 @@ export function openOpenClawStateDatabase(
clearNodeSqliteKyselyCacheForDatabase(cached.db);
cachedDatabases.delete(pathname);
}
let quarantineFailure: Error | undefined;
try {
const quarantine = readOpenClawDatabaseQuarantine(pathname, { env });
if (quarantine) {
quarantineFailure = createOpenClawDatabaseVerificationError(
"state",
pathname,
quarantine.reason,
);
}
} catch {
// A broken quarantine store must not brick every state open.
// The process latch and daily verifier still cover known damage.
}
if (quarantineFailure) {
throw quarantineFailure;
}
ensureOpenClawStatePermissions(pathname, env);
const sqlite = requireNodeSqlite();
const db = new sqlite.DatabaseSync(pathname);
@@ -1871,10 +1859,6 @@ export function openOpenClawStateDatabase(
db.exec(`PRAGMA busy_timeout = ${OPENCLAW_SQLITE_BUSY_TIMEOUT_MS};`);
assertSupportedSchemaVersion(db, pathname);
assertStateDatabaseIntegrityBeforeMutation(db, pathname);
const verification = readDatabaseVerificationRow(db, pathname);
if (verification?.result === "error") {
throw createOpenClawDatabaseVerificationError("state", pathname, verification.error);
}
configureSqlitePreSchemaPragmas(db, {
busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS,
});