mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-12 21:53:00 -06:00
feat(sqlite): move quarantine decisions to a dedicated store that survives primary database damage (#110453)
This commit is contained in:
committed by
GitHub
parent
713b6151d1
commit
deddeb3dac
@@ -56,14 +56,15 @@ Version 3 was an unshipped development step folded into version 4.
|
||||
|
||||
## Integrity checks
|
||||
|
||||
| When | Check |
|
||||
| ------------------------------------------- | -------------------------------------------------------------------------------------------------- |
|
||||
| Every open | Validate the `schema_meta` table and primary metadata row |
|
||||
| Before a pending migration | Run a full integrity, foreign-key, role, schema, and index scan |
|
||||
| Gateway background verifier | Run the full scan about once daily and record results in the global `database_verifications` table |
|
||||
| Doctor, backup verification, and compaction | Run the full scan before accepting or rewriting the database |
|
||||
| When | Check |
|
||||
| ------------------------------------------- | --------------------------------------------------------------- |
|
||||
| Every open | Validate the `schema_meta` table and primary metadata row |
|
||||
| Before a pending migration | Run a full integrity, foreign-key, role, schema, and index scan |
|
||||
| Gateway background verifier | Run the full scan about once daily and record results |
|
||||
| Doctor, backup verification, and compaction | Run the full scan before accepting or rewriting the database |
|
||||
|
||||
The Gateway preflight reads schema headers only. The background verifier owns the slower full scan for databases that do not need migration.
|
||||
Quarantine decisions live in a dedicated `openclaw-quarantine.sqlite` store, so they survive damage to the databases being quarantined. The global `database_verifications` table remains verification history.
|
||||
|
||||
## Downgrades are unsupported
|
||||
|
||||
|
||||
@@ -57,6 +57,9 @@ const rawSqliteAllowPathGroups = {
|
||||
"src/state/openclaw-database-preflight.ts",
|
||||
"src/state/openclaw-database-verify.worker.ts",
|
||||
],
|
||||
"quarantine store must work when other databases are damaged": [
|
||||
"src/state/openclaw-quarantine-store.ts",
|
||||
],
|
||||
"read-only SQLite status probes": [
|
||||
"src/commands/doctor-db-bloat.ts",
|
||||
"src/commands/status.scan.shared.ts",
|
||||
|
||||
@@ -15,7 +15,7 @@ import { compactDoctorSqliteFile } from "./doctor-sqlite-compact.js";
|
||||
/** Reclaim free pages from one agent session SQLite database. */
|
||||
export function compactDoctorSessionSqliteTarget(
|
||||
target: SessionStoreTarget,
|
||||
options: { migrateOlderSchema?: boolean } = {},
|
||||
options: { env?: NodeJS.ProcessEnv; migrateOlderSchema?: boolean } = {},
|
||||
): DoctorSessionSqliteCompactReport {
|
||||
const sqlitePath = resolveTargetSqlitePath(target);
|
||||
const beforeFileSizes = readSqliteFileSizes(sqlitePath);
|
||||
@@ -42,7 +42,7 @@ export function compactDoctorSessionSqliteTarget(
|
||||
);
|
||||
}
|
||||
const requireQuarantineCleared = () => {
|
||||
if (!clearOpenClawAgentDatabaseOpenFailure(sqlitePath)) {
|
||||
if (!clearOpenClawAgentDatabaseOpenFailure(sqlitePath, { env: options.env })) {
|
||||
throw new Error(
|
||||
`OpenClaw agent database ${sqlitePath} was repaired, but its persisted quarantine record could not be cleared. Rerun openclaw doctor --fix so the database is not refused again.`,
|
||||
);
|
||||
|
||||
@@ -21,7 +21,14 @@ import {
|
||||
OPENCLAW_AGENT_SCHEMA_VERSION,
|
||||
resolveOpenClawAgentSqlitePath,
|
||||
} from "../state/openclaw-agent-db.js";
|
||||
import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js";
|
||||
import {
|
||||
readOpenClawDatabaseQuarantine,
|
||||
recordOpenClawDatabaseQuarantine,
|
||||
} from "../state/openclaw-quarantine-store.js";
|
||||
import {
|
||||
closeOpenClawStateDatabaseForTest,
|
||||
openOpenClawStateDatabase,
|
||||
} from "../state/openclaw-state-db.js";
|
||||
import {
|
||||
assertSafeSessionSqliteMigrationMove,
|
||||
createSessionSqliteMigrationFailureIssue,
|
||||
@@ -611,6 +618,40 @@ describe("runDoctorSessionSqlite", () => {
|
||||
},
|
||||
);
|
||||
|
||||
it("clears agent quarantine and verification history after compaction", async () => {
|
||||
const { sqlitePath, store } = await createImportedStoreForCompaction();
|
||||
const state = openOpenClawStateDatabase({ env: store.env });
|
||||
state.db
|
||||
.prepare(
|
||||
`
|
||||
INSERT INTO database_verifications (path, kind, verified_at, result, error)
|
||||
VALUES (?, 'agent', 1, 'error', 'corrupt index')
|
||||
`,
|
||||
)
|
||||
.run(sqlitePath);
|
||||
expect(
|
||||
recordOpenClawDatabaseQuarantine({
|
||||
env: store.env,
|
||||
kind: "agent",
|
||||
path: sqlitePath,
|
||||
reason: "corrupt index",
|
||||
}),
|
||||
).toBe(true);
|
||||
|
||||
const report = await runDoctorSessionSqlite({
|
||||
env: store.env,
|
||||
mode: "compact",
|
||||
store: store.storePath,
|
||||
});
|
||||
|
||||
expect(report.totals.issues).toBe(0);
|
||||
expect(readOpenClawDatabaseQuarantine(sqlitePath, { env: store.env })).toBeUndefined();
|
||||
expect(
|
||||
state.db.prepare("SELECT 1 FROM database_verifications WHERE path = ?").get(sqlitePath),
|
||||
).toBeUndefined();
|
||||
expect(openOpenClawAgentDatabase({ agentId: "main", env: store.env }).db.isOpen).toBe(true);
|
||||
});
|
||||
|
||||
it.skipIf(process.platform === "win32")(
|
||||
"reapplies owner-only permissions after compaction",
|
||||
async () => {
|
||||
|
||||
@@ -259,7 +259,7 @@ async function inspectOrMigrateTarget(params: {
|
||||
return report;
|
||||
}
|
||||
if (params.mode === "compact") {
|
||||
compactSqliteDatabase(params.target, report);
|
||||
compactSqliteDatabase(params.target, report, { env: params.env });
|
||||
report.sqliteEntries = readSqliteEntryCount(params.target);
|
||||
appendSqliteDbStats(params.target, report);
|
||||
return report;
|
||||
@@ -299,6 +299,7 @@ async function inspectOrMigrateTarget(params: {
|
||||
// databases and returns the pages the import churn freed.
|
||||
compactSqliteDatabase(params.target, report, {
|
||||
closeImportedHandle: true,
|
||||
env: params.env,
|
||||
migrateOlderSchema: true,
|
||||
});
|
||||
}
|
||||
@@ -988,15 +989,22 @@ function appendSqliteDbStats(
|
||||
function compactSqliteDatabase(
|
||||
target: SessionStoreTarget,
|
||||
report: DoctorSessionSqliteTargetReport,
|
||||
options: { closeImportedHandle?: boolean; migrateOlderSchema?: boolean } = {},
|
||||
options: {
|
||||
closeImportedHandle?: boolean;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
migrateOlderSchema?: boolean;
|
||||
} = {},
|
||||
): void {
|
||||
try {
|
||||
if (options.closeImportedHandle) {
|
||||
closeOpenClawAgentDatabaseByPath(resolveTargetSqlitePath(target));
|
||||
}
|
||||
report.compact = options.migrateOlderSchema
|
||||
? compactDoctorSessionSqliteTarget(target, { migrateOlderSchema: true })
|
||||
: compactDoctorSessionSqliteTarget(target);
|
||||
? compactDoctorSessionSqliteTarget(target, {
|
||||
env: options.env,
|
||||
migrateOlderSchema: true,
|
||||
})
|
||||
: compactDoctorSessionSqliteTarget(target, { env: options.env });
|
||||
} catch (err) {
|
||||
report.issues.push({
|
||||
code: "sqlite_compact_failed",
|
||||
|
||||
@@ -4,6 +4,10 @@ import type { DatabaseSync } from "node:sqlite";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
|
||||
import { requireNodeSqlite } from "../infra/node-sqlite.js";
|
||||
import {
|
||||
readOpenClawDatabaseQuarantine,
|
||||
recordOpenClawDatabaseQuarantine,
|
||||
} from "../state/openclaw-quarantine-store.js";
|
||||
import {
|
||||
closeOpenClawStateDatabase,
|
||||
openOpenClawStateDatabase,
|
||||
@@ -166,6 +170,41 @@ describe("runDoctorStateSqliteCompact", () => {
|
||||
expect(report.integrityCheck).toBe("ok");
|
||||
});
|
||||
|
||||
it("clears authoritative quarantine and verification history after compaction", async () => {
|
||||
const env = createStateEnv();
|
||||
const sqlitePath = seedStateDatabase({ env, withBloat: true });
|
||||
const sqlite = requireNodeSqlite();
|
||||
const history = new sqlite.DatabaseSync(sqlitePath);
|
||||
try {
|
||||
history
|
||||
.prepare(
|
||||
`
|
||||
INSERT INTO database_verifications (path, kind, verified_at, result, error)
|
||||
VALUES (?, 'state', 1, 'error', 'corrupt index')
|
||||
`,
|
||||
)
|
||||
.run(sqlitePath);
|
||||
} finally {
|
||||
history.close();
|
||||
}
|
||||
expect(
|
||||
recordOpenClawDatabaseQuarantine({
|
||||
env,
|
||||
kind: "state",
|
||||
path: sqlitePath,
|
||||
reason: "corrupt index",
|
||||
}),
|
||||
).toBe(true);
|
||||
|
||||
await runDoctorStateSqliteCompact({ env });
|
||||
|
||||
expect(readOpenClawDatabaseQuarantine(sqlitePath, { env })).toBeUndefined();
|
||||
const repaired = openOpenClawStateDatabase({ env });
|
||||
expect(
|
||||
repaired.db.prepare("SELECT 1 FROM database_verifications WHERE path = ?").get(sqlitePath),
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it.skipIf(process.platform === "win32")("reapplies owner-only SQLite permissions", async () => {
|
||||
const env = createStateEnv();
|
||||
const sqlitePath = seedStateDatabase({ env, withBloat: true });
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
/** Explicit doctor maintenance for the canonical shared state SQLite database. */
|
||||
import fs from "node:fs";
|
||||
import { clearOpenClawDatabaseQuarantine } from "../state/openclaw-quarantine-store.js";
|
||||
import {
|
||||
assertOpenClawStateDatabaseForMaintenance,
|
||||
clearOpenClawDatabaseVerification,
|
||||
clearOpenClawDatabaseVerificationHistory,
|
||||
clearOpenClawStateDatabaseOpenFailure,
|
||||
ensureOpenClawStatePermissions,
|
||||
isOpenClawStateDatabaseOpen,
|
||||
@@ -72,11 +73,12 @@ export async function runDoctorStateSqliteCompact(
|
||||
|
||||
const compact = compactDoctorSqliteFile({
|
||||
afterMutation: () => {
|
||||
if (!clearOpenClawDatabaseVerification(sqlitePath, { path: sqlitePath })) {
|
||||
if (!clearOpenClawDatabaseQuarantine(sqlitePath, { env })) {
|
||||
throw new Error(
|
||||
`OpenClaw state database ${sqlitePath} was compacted, but its persisted quarantine record could not be cleared. Rerun openclaw doctor --fix so the database is not refused again.`,
|
||||
);
|
||||
}
|
||||
clearOpenClawDatabaseVerificationHistory(sqlitePath, { env });
|
||||
clearOpenClawStateDatabaseOpenFailure(sqlitePath);
|
||||
ensureOpenClawStatePermissions(sqlitePath, env);
|
||||
},
|
||||
|
||||
@@ -59,14 +59,17 @@ import {
|
||||
import type { DB as OpenClawAgentKyselyDatabase } from "./openclaw-agent-db.generated.js";
|
||||
import { resolveOpenClawAgentSqlitePath } from "./openclaw-agent-db.paths.js";
|
||||
import { OPENCLAW_AGENT_SCHEMA_SQL } from "./openclaw-agent-schema.generated.js";
|
||||
import {
|
||||
clearOpenClawDatabaseQuarantine,
|
||||
readOpenClawDatabaseQuarantine,
|
||||
} from "./openclaw-quarantine-store.js";
|
||||
import type { DB as OpenClawStateKyselyDatabase } from "./openclaw-state-db.generated.js";
|
||||
import {
|
||||
clearOpenClawDatabaseVerification,
|
||||
clearOpenClawDatabaseVerificationHistory,
|
||||
createOpenClawDatabaseVerificationError,
|
||||
detectOpenClawStateDatabaseSchemaMigrations,
|
||||
OPENCLAW_STATE_SCHEMA_VERSION,
|
||||
OPENCLAW_SQLITE_BUSY_TIMEOUT_MS,
|
||||
readOpenClawDatabaseVerification,
|
||||
runOpenClawStateWriteTransaction,
|
||||
type OpenClawStateDatabaseOptions,
|
||||
} from "./openclaw-state-db.js";
|
||||
@@ -183,7 +186,8 @@ export function clearOpenClawAgentDatabaseOpenFailure(
|
||||
options: OpenClawStateDatabaseOptions = {},
|
||||
): boolean {
|
||||
const resolvedPath = path.resolve(pathname);
|
||||
const cleared = clearOpenClawDatabaseVerification(resolvedPath, options);
|
||||
const cleared = clearOpenClawDatabaseQuarantine(resolvedPath, { env: options.env });
|
||||
clearOpenClawDatabaseVerificationHistory(resolvedPath, options);
|
||||
terminalOpenFailures.delete(resolvedPath);
|
||||
return cleared;
|
||||
}
|
||||
@@ -1082,19 +1086,17 @@ export function openOpenClawAgentDatabase(
|
||||
}
|
||||
let persistedFailure: Error | undefined;
|
||||
try {
|
||||
const verification = readOpenClawDatabaseVerification(pathname, { env: databaseOptions.env });
|
||||
if (verification?.result === "error") {
|
||||
const quarantine = readOpenClawDatabaseQuarantine(pathname, { env: databaseOptions.env });
|
||||
if (quarantine) {
|
||||
persistedFailure = createOpenClawDatabaseVerificationError(
|
||||
"agent",
|
||||
pathname,
|
||||
verification.error,
|
||||
quarantine.reason,
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
// Accepted tradeoff: a locked/unavailable state DB must not block agent
|
||||
// opens, or every transient state hiccup takes all agents down. The
|
||||
// in-process latch still covers this process; a missed cross-process
|
||||
// quarantine is re-detected by the next daily verifier pass.
|
||||
// A broken quarantine store must not brick every agent open.
|
||||
// The process latch and daily verifier still cover known damage.
|
||||
}
|
||||
if (persistedFailure) {
|
||||
recordOpenClawAgentDatabaseOpenFailure(pathname, persistedFailure);
|
||||
|
||||
@@ -12,6 +12,7 @@ import type {
|
||||
OpenClawDatabaseVerifyResult,
|
||||
OpenClawDatabaseVerifyTarget,
|
||||
} from "./openclaw-database-verify.worker.js";
|
||||
import { recordOpenClawDatabaseQuarantine } from "./openclaw-quarantine-store.js";
|
||||
import type { DB as OpenClawStateKyselyDatabase } from "./openclaw-state-db.generated.js";
|
||||
import {
|
||||
recordOpenClawStateDatabaseOpenFailure,
|
||||
@@ -149,28 +150,36 @@ export function applyOpenClawDatabaseVerificationResults(options: {
|
||||
const targetByPath = new Map(options.targets.map((target) => [target.path, target]));
|
||||
const verifiedAt = options.verifiedAt ?? Date.now();
|
||||
|
||||
// One best-effort row write into a possibly corrupt state file buys restart-durable quarantine.
|
||||
// Doctor rebuilds the file during repair, so this bounded mutation is an accepted tradeoff.
|
||||
const persistResults = (results: readonly OpenClawDatabaseVerifyResult[]) => {
|
||||
for (const result of options.results) {
|
||||
const target = targetByPath.get(result.path);
|
||||
if (!target || result.ok || !result.terminal) {
|
||||
continue;
|
||||
}
|
||||
const recorded = recordOpenClawDatabaseQuarantine({
|
||||
env: options.env,
|
||||
kind: target.kind,
|
||||
path: result.path,
|
||||
reason: result.error ?? `SQLite integrity verification failed for ${result.path}`,
|
||||
});
|
||||
if (!recorded) {
|
||||
// Accepted residual: quarantine stays process-local when this tiny,
|
||||
// independent store is unavailable. Daily verification retries it.
|
||||
log.error("failed to persist database quarantine; quarantine is process-local", {
|
||||
kind: target.kind,
|
||||
path: result.path,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
runOpenClawStateWriteTransaction(
|
||||
(database) => {
|
||||
const db = getNodeSqliteKysely<VerificationDatabase>(database.db);
|
||||
for (const result of results) {
|
||||
for (const result of options.results) {
|
||||
const target = targetByPath.get(result.path);
|
||||
if (!target) {
|
||||
continue;
|
||||
}
|
||||
const existing = executeSqliteQuerySync(
|
||||
database.db,
|
||||
db
|
||||
.selectFrom("database_verifications")
|
||||
.select("result")
|
||||
.where("path", "=", result.path)
|
||||
.limit(1),
|
||||
).rows[0];
|
||||
if (existing?.result === "error") {
|
||||
continue;
|
||||
}
|
||||
executeSqliteQuerySync(
|
||||
database.db,
|
||||
db
|
||||
@@ -196,31 +205,8 @@ export function applyOpenClawDatabaseVerificationResults(options: {
|
||||
{ env: options.env },
|
||||
{ operationLabel: "state.database-verifications.record" },
|
||||
);
|
||||
};
|
||||
try {
|
||||
persistResults(options.results);
|
||||
} catch (error) {
|
||||
// Terminal rows are the restart-durable quarantine; retry those once so a
|
||||
// transient state-DB lock does not silently downgrade them to process-local.
|
||||
const terminalResults = options.results.filter((result) => !result.ok && result.terminal);
|
||||
let retried = false;
|
||||
if (terminalResults.length > 0) {
|
||||
try {
|
||||
persistResults(terminalResults);
|
||||
retried = true;
|
||||
} catch {
|
||||
// fall through to the process-local warning below
|
||||
}
|
||||
}
|
||||
if (!retried) {
|
||||
// Accepted tradeoff: when this write fails the quarantine stays process-local
|
||||
// until the next daily verifier pass re-detects and re-persists. A sidecar
|
||||
// marker would break the SQLite-only storage contract, and startup full
|
||||
// scans would reintroduce the multi-second opens this design removes.
|
||||
log.error("failed to persist database verification results; quarantine is process-local", {
|
||||
error: String(error),
|
||||
});
|
||||
}
|
||||
log.error("failed to persist database verification history", { error: String(error) });
|
||||
}
|
||||
|
||||
for (const result of options.results) {
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { afterAll, afterEach, describe, expect, it } from "vitest";
|
||||
import { cleanupTempDirs, makeTempDir } from "../../test/helpers/temp-dir.js";
|
||||
import { requireNodeSqlite } from "../infra/node-sqlite.js";
|
||||
@@ -16,6 +18,11 @@ import {
|
||||
type OpenClawDatabaseVerifyTarget,
|
||||
verifyOpenClawDatabases,
|
||||
} from "./openclaw-database-verify.worker.js";
|
||||
import {
|
||||
clearOpenClawDatabaseQuarantine,
|
||||
readOpenClawDatabaseQuarantine,
|
||||
recordOpenClawDatabaseQuarantine,
|
||||
} from "./openclaw-quarantine-store.js";
|
||||
import {
|
||||
closeOpenClawStateDatabaseForTest,
|
||||
openOpenClawStateDatabase,
|
||||
@@ -59,6 +66,10 @@ function createUnsafeIndexDrift(databasePath: string): void {
|
||||
}
|
||||
}
|
||||
|
||||
function quarantineStorePath(stateDir: string): string {
|
||||
return path.join(stateDir, "state", "openclaw-quarantine.sqlite");
|
||||
}
|
||||
|
||||
describe("OpenClaw database integrity verifier", () => {
|
||||
it("detects corruption off-thread, persists it, and latches later opens", async () => {
|
||||
const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-");
|
||||
@@ -94,6 +105,11 @@ describe("OpenClaw database integrity verifier", () => {
|
||||
verifiedAt: 1234,
|
||||
});
|
||||
expect(liveHandle.db.isOpen).toBe(false);
|
||||
expect(readOpenClawDatabaseQuarantine(agentPath, { env })).toEqual({
|
||||
kind: "agent",
|
||||
quarantinedAt: expect.any(Number),
|
||||
reason: directResults[0]?.error,
|
||||
});
|
||||
|
||||
expect(
|
||||
openOpenClawStateDatabase({ env })
|
||||
@@ -118,7 +134,7 @@ describe("OpenClaw database integrity verifier", () => {
|
||||
openOpenClawStateDatabase({ env })
|
||||
.db.prepare("SELECT verified_at, result, error FROM database_verifications WHERE path = ?")
|
||||
.get(agentPath),
|
||||
).toEqual({ verified_at: 1234, result: "error", error: directResults[0]?.error });
|
||||
).toEqual({ verified_at: 1235, result: "inconclusive", error: "database busy" });
|
||||
expect(() => openOpenClawAgentDatabase({ agentId: "worker-1", env })).toThrow(
|
||||
expect.objectContaining({ name: "SqliteIntegrityError" }),
|
||||
);
|
||||
@@ -139,7 +155,7 @@ describe("OpenClaw database integrity verifier", () => {
|
||||
const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-clear-failure-");
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
const agentPath = openOpenClawAgentDatabase({ agentId: "worker-1", env }).path;
|
||||
const statePath = openOpenClawStateDatabase({ env }).path;
|
||||
openOpenClawStateDatabase({ env });
|
||||
closeOpenClawAgentDatabasesForTest();
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
applyOpenClawDatabaseVerificationResults({
|
||||
@@ -150,23 +166,137 @@ describe("OpenClaw database integrity verifier", () => {
|
||||
});
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
|
||||
// A read-only state DB cannot drop the quarantine row; the clear must say so
|
||||
const storePath = quarantineStorePath(stateDir);
|
||||
// A read-only quarantine store cannot drop the row; the clear must say so
|
||||
// instead of letting doctor report success while the next open still refuses.
|
||||
fs.chmodSync(statePath, 0o444);
|
||||
fs.chmodSync(storePath, 0o444);
|
||||
try {
|
||||
expect(clearOpenClawAgentDatabaseOpenFailure(agentPath, { env })).toBe(false);
|
||||
expect(
|
||||
recordOpenClawDatabaseQuarantine({
|
||||
env,
|
||||
kind: "agent",
|
||||
path: agentPath,
|
||||
reason: "new reason",
|
||||
}),
|
||||
).toBe(false);
|
||||
} finally {
|
||||
// WAL sidecars minted during the read-only attempt inherit its mode.
|
||||
for (const sidecar of [statePath, `${statePath}-wal`, `${statePath}-shm`]) {
|
||||
if (fs.existsSync(sidecar)) {
|
||||
fs.chmodSync(sidecar, 0o600);
|
||||
}
|
||||
}
|
||||
fs.chmodSync(storePath, 0o600);
|
||||
}
|
||||
expect(clearOpenClawAgentDatabaseOpenFailure(agentPath, { env })).toBe(true);
|
||||
expect(openOpenClawAgentDatabase({ agentId: "worker-1", env }).db.isOpen).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps healthy opens on the missing-store fast path", () => {
|
||||
const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-clean-");
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
|
||||
openOpenClawStateDatabase({ env });
|
||||
openOpenClawAgentDatabase({ agentId: "worker-1", env });
|
||||
|
||||
expect(fs.existsSync(quarantineStorePath(stateDir))).toBe(false);
|
||||
});
|
||||
|
||||
it("records and clears dedicated quarantine rows with rollback journaling", () => {
|
||||
const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-store-");
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
const databasePath = path.join(stateDir, "agent.sqlite");
|
||||
const storePath = quarantineStorePath(stateDir);
|
||||
|
||||
expect(clearOpenClawDatabaseQuarantine(databasePath, { env })).toBe(true);
|
||||
expect(
|
||||
recordOpenClawDatabaseQuarantine({
|
||||
env,
|
||||
kind: "agent",
|
||||
path: databasePath,
|
||||
reason: "corrupt index",
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(readOpenClawDatabaseQuarantine(databasePath, { env })).toEqual({
|
||||
kind: "agent",
|
||||
quarantinedAt: expect.any(Number),
|
||||
reason: "corrupt index",
|
||||
});
|
||||
|
||||
const { DatabaseSync } = requireNodeSqlite();
|
||||
const raw = new DatabaseSync(storePath, { readOnly: true });
|
||||
try {
|
||||
expect(raw.prepare("PRAGMA journal_mode").get()).toEqual({ journal_mode: "delete" });
|
||||
expect(readSqliteNumberPragma(raw, "synchronous")).toBe(2);
|
||||
expect(readSqliteNumberPragma(raw, "user_version")).toBe(1);
|
||||
} finally {
|
||||
raw.close();
|
||||
}
|
||||
if (process.platform !== "win32") {
|
||||
expect(fs.statSync(path.dirname(storePath)).mode & 0o777).toBe(0o700);
|
||||
expect(fs.statSync(storePath).mode & 0o777).toBe(0o600);
|
||||
}
|
||||
expect(clearOpenClawDatabaseQuarantine(databasePath, { env })).toBe(true);
|
||||
expect(clearOpenClawDatabaseQuarantine(databasePath, { env })).toBe(true);
|
||||
expect(readOpenClawDatabaseQuarantine(databasePath, { env })).toBeUndefined();
|
||||
});
|
||||
|
||||
it("recovers an interrupted empty quarantine-store initialization", () => {
|
||||
const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-empty-store-");
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
const databasePath = path.join(stateDir, "agent.sqlite");
|
||||
const storePath = quarantineStorePath(stateDir);
|
||||
fs.mkdirSync(path.dirname(storePath), { recursive: true });
|
||||
fs.writeFileSync(storePath, "", { mode: 0o600 });
|
||||
|
||||
expect(readOpenClawDatabaseQuarantine(databasePath, { env })).toBeUndefined();
|
||||
expect(
|
||||
recordOpenClawDatabaseQuarantine({
|
||||
env,
|
||||
kind: "agent",
|
||||
path: databasePath,
|
||||
reason: "corrupt index",
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(readOpenClawDatabaseQuarantine(databasePath, { env })?.reason).toBe("corrupt index");
|
||||
});
|
||||
|
||||
it.skipIf(process.platform === "win32")(
|
||||
"recovers a hot rollback journal before reading quarantine",
|
||||
() => {
|
||||
const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-hot-journal-");
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
const databasePath = path.join(stateDir, "agent.sqlite");
|
||||
const storePath = quarantineStorePath(stateDir);
|
||||
expect(
|
||||
recordOpenClawDatabaseQuarantine({
|
||||
env,
|
||||
kind: "agent",
|
||||
path: databasePath,
|
||||
reason: "committed reason",
|
||||
}),
|
||||
).toBe(true);
|
||||
|
||||
const crashed = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
"--no-warnings",
|
||||
"--input-type=module",
|
||||
"-e",
|
||||
`
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
const database = new DatabaseSync(process.env.OPENCLAW_QUARANTINE_TEST_PATH);
|
||||
database.exec("PRAGMA journal_mode = DELETE; PRAGMA synchronous = FULL; BEGIN IMMEDIATE;");
|
||||
database.prepare("UPDATE quarantined_databases SET reason = 'uncommitted reason'").run();
|
||||
process.kill(process.pid, "SIGKILL");
|
||||
`,
|
||||
],
|
||||
{ env: { ...process.env, OPENCLAW_QUARANTINE_TEST_PATH: storePath } },
|
||||
);
|
||||
expect(crashed.signal).toBe("SIGKILL");
|
||||
expect(fs.existsSync(`${storePath}-journal`)).toBe(true);
|
||||
|
||||
expect(readOpenClawDatabaseQuarantine(databasePath, { env })?.reason).toBe(
|
||||
"committed reason",
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("persists transient verifier errors as inconclusive without latching", () => {
|
||||
const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-transient-");
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
@@ -209,13 +339,11 @@ describe("OpenClaw database integrity verifier", () => {
|
||||
});
|
||||
|
||||
const { DatabaseSync } = requireNodeSqlite();
|
||||
const raw = new DatabaseSync(statePath, { readOnly: true });
|
||||
const raw = new DatabaseSync(quarantineStorePath(stateDir), { readOnly: true });
|
||||
try {
|
||||
expect(
|
||||
raw
|
||||
.prepare("SELECT result, error FROM database_verifications WHERE path = ?")
|
||||
.get(statePath),
|
||||
).toEqual({ result: "error", error: "corrupt index" });
|
||||
raw.prepare("SELECT kind, reason FROM quarantined_databases WHERE path = ?").get(statePath),
|
||||
).toEqual({ kind: "state", reason: "corrupt index" });
|
||||
} finally {
|
||||
raw.close();
|
||||
}
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
// Dedicated quarantine decisions stay available when primary databases fail.
|
||||
import { existsSync, mkdirSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import { requireNodeSqlite } from "../infra/node-sqlite.js";
|
||||
import { applyPrivateModeSync } from "../infra/private-mode.js";
|
||||
import { VERSION } from "../version.js";
|
||||
import { resolveOpenClawStateSqliteDir } from "./openclaw-state-db.paths.js";
|
||||
|
||||
const OPENCLAW_QUARANTINE_SCHEMA_VERSION = 1;
|
||||
const OPENCLAW_QUARANTINE_BUSY_TIMEOUT_MS = 5_000;
|
||||
const OPENCLAW_QUARANTINE_DIR_MODE = 0o700;
|
||||
const OPENCLAW_QUARANTINE_FILE_MODE = 0o600;
|
||||
|
||||
type OpenClawDatabaseKind = "agent" | "state";
|
||||
|
||||
type OpenClawDatabaseQuarantine = {
|
||||
kind: OpenClawDatabaseKind;
|
||||
quarantinedAt: number;
|
||||
reason: string;
|
||||
};
|
||||
|
||||
function resolveQuarantineStorePath(env: NodeJS.ProcessEnv): string {
|
||||
return path.join(resolveOpenClawStateSqliteDir(env), "openclaw-quarantine.sqlite");
|
||||
}
|
||||
|
||||
function ensureQuarantineStoreDirectory(storePath: string): void {
|
||||
const dir = path.dirname(storePath);
|
||||
mkdirSync(dir, { recursive: true, mode: OPENCLAW_QUARANTINE_DIR_MODE });
|
||||
applyPrivateModeSync(dir, OPENCLAW_QUARANTINE_DIR_MODE);
|
||||
}
|
||||
|
||||
function configureQuarantineWriter(database: DatabaseSync, storePath: string): void {
|
||||
database.exec(`
|
||||
PRAGMA busy_timeout = ${OPENCLAW_QUARANTINE_BUSY_TIMEOUT_MS};
|
||||
PRAGMA journal_mode = DELETE;
|
||||
PRAGMA synchronous = FULL;
|
||||
`);
|
||||
const userVersion = readQuarantineSchemaVersion(database, storePath);
|
||||
if (userVersion > OPENCLAW_QUARANTINE_SCHEMA_VERSION) {
|
||||
throw new Error(
|
||||
`OpenClaw quarantine store ${storePath} uses newer schema version ${userVersion}.`,
|
||||
);
|
||||
}
|
||||
if (userVersion === OPENCLAW_QUARANTINE_SCHEMA_VERSION) {
|
||||
return;
|
||||
}
|
||||
database.exec(`
|
||||
BEGIN IMMEDIATE;
|
||||
CREATE TABLE IF NOT EXISTS quarantined_databases (
|
||||
path TEXT NOT NULL PRIMARY KEY,
|
||||
kind TEXT NOT NULL,
|
||||
reason TEXT NOT NULL,
|
||||
quarantined_at INTEGER NOT NULL,
|
||||
writer_app_version TEXT
|
||||
) STRICT;
|
||||
PRAGMA user_version = ${OPENCLAW_QUARANTINE_SCHEMA_VERSION};
|
||||
COMMIT;
|
||||
`);
|
||||
}
|
||||
|
||||
function readQuarantineSchemaVersion(database: DatabaseSync, storePath: string): number {
|
||||
const row = database.prepare("PRAGMA user_version").get() as
|
||||
| { user_version?: unknown }
|
||||
| undefined;
|
||||
const userVersion = row?.user_version;
|
||||
if (typeof userVersion !== "number" || !Number.isInteger(userVersion)) {
|
||||
throw new Error(`OpenClaw quarantine store ${storePath} has an invalid schema version.`);
|
||||
}
|
||||
return userVersion;
|
||||
}
|
||||
|
||||
function withQuarantineWriter<T>(env: NodeJS.ProcessEnv, operation: (db: DatabaseSync) => T): T {
|
||||
const storePath = resolveQuarantineStorePath(env);
|
||||
const existed = existsSync(storePath);
|
||||
ensureQuarantineStoreDirectory(storePath);
|
||||
const sqlite = requireNodeSqlite();
|
||||
const database = new sqlite.DatabaseSync(storePath);
|
||||
let completed = false;
|
||||
try {
|
||||
if (!existed) {
|
||||
applyPrivateModeSync(storePath, OPENCLAW_QUARANTINE_FILE_MODE);
|
||||
}
|
||||
configureQuarantineWriter(database, storePath);
|
||||
const result = operation(database);
|
||||
completed = true;
|
||||
return result;
|
||||
} finally {
|
||||
database.close();
|
||||
if (completed || !existed) {
|
||||
applyPrivateModeSync(storePath, OPENCLAW_QUARANTINE_FILE_MODE);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Read one authoritative quarantine decision without creating the store. */
|
||||
export function readOpenClawDatabaseQuarantine(
|
||||
pathname: string,
|
||||
options: { env?: NodeJS.ProcessEnv } = {},
|
||||
): OpenClawDatabaseQuarantine | undefined {
|
||||
const storePath = resolveQuarantineStorePath(options.env ?? process.env);
|
||||
// Clean installs pay one existence check. No directory or SQLite work.
|
||||
if (!existsSync(storePath)) {
|
||||
return undefined;
|
||||
}
|
||||
const sqlite = requireNodeSqlite();
|
||||
const database = new sqlite.DatabaseSync(storePath);
|
||||
try {
|
||||
database.exec(`PRAGMA busy_timeout = ${OPENCLAW_QUARANTINE_BUSY_TIMEOUT_MS};`);
|
||||
const userVersion = readQuarantineSchemaVersion(database, storePath);
|
||||
if (userVersion === 0) {
|
||||
return undefined;
|
||||
}
|
||||
if (userVersion !== OPENCLAW_QUARANTINE_SCHEMA_VERSION) {
|
||||
throw new Error(
|
||||
`OpenClaw quarantine store ${storePath} uses newer schema version ${userVersion}.`,
|
||||
);
|
||||
}
|
||||
const row = database
|
||||
.prepare(
|
||||
"SELECT kind, reason, quarantined_at FROM quarantined_databases WHERE path = ? LIMIT 1",
|
||||
)
|
||||
.get(path.resolve(pathname)) as
|
||||
| { kind?: unknown; quarantined_at?: unknown; reason?: unknown }
|
||||
| undefined;
|
||||
if (!row) {
|
||||
return undefined;
|
||||
}
|
||||
if (
|
||||
(row.kind !== "agent" && row.kind !== "state") ||
|
||||
typeof row.reason !== "string" ||
|
||||
typeof row.quarantined_at !== "number" ||
|
||||
!Number.isInteger(row.quarantined_at)
|
||||
) {
|
||||
throw new Error(`OpenClaw quarantine store ${storePath} contains an invalid row.`);
|
||||
}
|
||||
return { kind: row.kind, quarantinedAt: row.quarantined_at, reason: row.reason };
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
}
|
||||
|
||||
/** Persist one authoritative quarantine decision. */
|
||||
export function recordOpenClawDatabaseQuarantine(options: {
|
||||
env?: NodeJS.ProcessEnv;
|
||||
kind: OpenClawDatabaseKind;
|
||||
path: string;
|
||||
reason: string;
|
||||
}): boolean {
|
||||
try {
|
||||
return withQuarantineWriter(options.env ?? process.env, (database) => {
|
||||
database.exec("BEGIN IMMEDIATE;");
|
||||
try {
|
||||
database
|
||||
.prepare(
|
||||
`
|
||||
INSERT INTO quarantined_databases (
|
||||
path, kind, reason, quarantined_at, writer_app_version
|
||||
) VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(path) DO UPDATE SET
|
||||
kind = excluded.kind,
|
||||
reason = excluded.reason,
|
||||
quarantined_at = excluded.quarantined_at,
|
||||
writer_app_version = excluded.writer_app_version
|
||||
`,
|
||||
)
|
||||
.run(path.resolve(options.path), options.kind, options.reason, Date.now(), VERSION);
|
||||
database.exec("COMMIT;");
|
||||
return true;
|
||||
} catch (error) {
|
||||
database.exec("ROLLBACK;");
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** Clear one authoritative quarantine decision. */
|
||||
export function clearOpenClawDatabaseQuarantine(
|
||||
pathname: string,
|
||||
options: { env?: NodeJS.ProcessEnv } = {},
|
||||
): boolean {
|
||||
const env = options.env ?? process.env;
|
||||
if (!existsSync(resolveQuarantineStorePath(env))) {
|
||||
return true;
|
||||
}
|
||||
try {
|
||||
return withQuarantineWriter(env, (database) => {
|
||||
database.exec("BEGIN IMMEDIATE;");
|
||||
try {
|
||||
database
|
||||
.prepare("DELETE FROM quarantined_databases WHERE path = ?")
|
||||
.run(path.resolve(pathname));
|
||||
database.exec("COMMIT;");
|
||||
return true;
|
||||
} catch (error) {
|
||||
database.exec("ROLLBACK;");
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -42,6 +42,10 @@ import {
|
||||
import { migrateLegacyCronRunLogsToTaskRuns } from "../infra/state-migrations.cron-run-logs.js";
|
||||
import { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
import { VERSION } from "../version.js";
|
||||
import {
|
||||
clearOpenClawDatabaseQuarantine,
|
||||
readOpenClawDatabaseQuarantine,
|
||||
} from "./openclaw-quarantine-store.js";
|
||||
import * as operatorApprovalMigration from "./openclaw-state-db-operator-approval-migration.js";
|
||||
import {
|
||||
ensureColumn,
|
||||
@@ -165,29 +169,6 @@ type OpenClawDatabaseVerificationDatabase = Pick<
|
||||
OpenClawStateKyselyDatabase,
|
||||
"database_verifications"
|
||||
>;
|
||||
type OpenClawDatabaseVerification = {
|
||||
error: string | null;
|
||||
result: string;
|
||||
};
|
||||
|
||||
function readDatabaseVerificationRow(
|
||||
database: DatabaseSync,
|
||||
pathname: string,
|
||||
): OpenClawDatabaseVerification | undefined {
|
||||
if (!tableExists(database, "database_verifications")) {
|
||||
return undefined;
|
||||
}
|
||||
const db = getNodeSqliteKysely<OpenClawDatabaseVerificationDatabase>(database);
|
||||
return executeSqliteQuerySync(
|
||||
database,
|
||||
db
|
||||
.selectFrom("database_verifications")
|
||||
.select(["result", "error"])
|
||||
.where("path", "=", path.resolve(pathname))
|
||||
.limit(1),
|
||||
).rows[0];
|
||||
}
|
||||
|
||||
function clearDatabaseVerificationRow(database: DatabaseSync, pathname: string): void {
|
||||
if (!tableExists(database, "database_verifications")) {
|
||||
return;
|
||||
@@ -199,27 +180,8 @@ function clearDatabaseVerificationRow(database: DatabaseSync, pathname: string):
|
||||
);
|
||||
}
|
||||
|
||||
/** Read one durable verification row through the cached shared state database. */
|
||||
export function readOpenClawDatabaseVerification(
|
||||
pathname: string,
|
||||
options: OpenClawStateDatabaseOptions = {},
|
||||
): OpenClawDatabaseVerification | undefined {
|
||||
const statePath = path.resolve(
|
||||
options.path ?? resolveOpenClawStateSqlitePath(options.env ?? process.env),
|
||||
);
|
||||
if (!existsSync(statePath)) {
|
||||
return undefined;
|
||||
}
|
||||
return readDatabaseVerificationRow(openOpenClawStateDatabase(options).db, pathname);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove a persisted quarantine row after doctor repairs a database.
|
||||
* Returns false when the row could not be cleared: callers must surface that,
|
||||
* or the next open re-quarantines a healthy repaired file while doctor
|
||||
* reported success.
|
||||
*/
|
||||
export function clearOpenClawDatabaseVerification(
|
||||
/** Best-effort deletion of one verification-history row after repair. */
|
||||
export function clearOpenClawDatabaseVerificationHistory(
|
||||
pathname: string,
|
||||
options: OpenClawStateDatabaseOptions = {},
|
||||
): boolean {
|
||||
@@ -1005,7 +967,16 @@ export function repairOpenClawStateDatabaseSchema(options: OpenClawStateDatabase
|
||||
operationLabel: "state.schema.repair",
|
||||
},
|
||||
);
|
||||
const quarantineCleared = clearOpenClawDatabaseVerification(pathname, { path: pathname });
|
||||
try {
|
||||
runSqliteImmediateTransactionSync(db, () => clearDatabaseVerificationRow(db, pathname), {
|
||||
busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS,
|
||||
databaseLabel: pathname,
|
||||
operationLabel: "state.database-verifications.clear-history",
|
||||
});
|
||||
} catch {
|
||||
// History cleanup must not override authoritative quarantine repair.
|
||||
}
|
||||
const quarantineCleared = clearOpenClawDatabaseQuarantine(pathname, { env });
|
||||
clearOpenClawStateDatabaseOpenFailure(pathname);
|
||||
return {
|
||||
changes,
|
||||
@@ -1862,6 +1833,23 @@ export function openOpenClawStateDatabase(
|
||||
clearNodeSqliteKyselyCacheForDatabase(cached.db);
|
||||
cachedDatabases.delete(pathname);
|
||||
}
|
||||
let quarantineFailure: Error | undefined;
|
||||
try {
|
||||
const quarantine = readOpenClawDatabaseQuarantine(pathname, { env });
|
||||
if (quarantine) {
|
||||
quarantineFailure = createOpenClawDatabaseVerificationError(
|
||||
"state",
|
||||
pathname,
|
||||
quarantine.reason,
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
// A broken quarantine store must not brick every state open.
|
||||
// The process latch and daily verifier still cover known damage.
|
||||
}
|
||||
if (quarantineFailure) {
|
||||
throw quarantineFailure;
|
||||
}
|
||||
ensureOpenClawStatePermissions(pathname, env);
|
||||
const sqlite = requireNodeSqlite();
|
||||
const db = new sqlite.DatabaseSync(pathname);
|
||||
@@ -1871,10 +1859,6 @@ export function openOpenClawStateDatabase(
|
||||
db.exec(`PRAGMA busy_timeout = ${OPENCLAW_SQLITE_BUSY_TIMEOUT_MS};`);
|
||||
assertSupportedSchemaVersion(db, pathname);
|
||||
assertStateDatabaseIntegrityBeforeMutation(db, pathname);
|
||||
const verification = readDatabaseVerificationRow(db, pathname);
|
||||
if (verification?.result === "error") {
|
||||
throw createOpenClawDatabaseVerificationError("state", pathname, verification.error);
|
||||
}
|
||||
configureSqlitePreSchemaPragmas(db, {
|
||||
busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS,
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user