mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-25 11:55:47 -06:00
fix(sqlite): bind verifier quarantine to database generation (#113459)
This commit is contained in:
@@ -0,0 +1,212 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import fs, { type BigIntStats } from "node:fs";
|
||||
import { sameFileIdentity } from "./fs-safe-advanced.js";
|
||||
|
||||
const SQLITE_GENERATION_HASH_BUFFER_BYTES = 1024 * 1024;
|
||||
|
||||
type SqliteFileFingerprint = {
|
||||
birthtimeNs: bigint;
|
||||
ctimeNs: bigint;
|
||||
dev: bigint;
|
||||
ino: bigint;
|
||||
mtimeNs: bigint;
|
||||
sha256: string;
|
||||
size: bigint;
|
||||
};
|
||||
|
||||
type SerializedSqliteFileFingerprint = {
|
||||
birthtimeNs: string;
|
||||
ctimeNs: string;
|
||||
dev: string;
|
||||
ino: string;
|
||||
mtimeNs: string;
|
||||
sha256: string;
|
||||
size: string;
|
||||
};
|
||||
|
||||
export type SqliteFileGeneration = {
|
||||
database: SqliteFileFingerprint;
|
||||
journal?: SqliteFileFingerprint;
|
||||
wal?: SqliteFileFingerprint;
|
||||
};
|
||||
|
||||
function assertRegularFile(stat: BigIntStats): void {
|
||||
if (!stat.isFile()) {
|
||||
throw new Error("SQLite generation target must be a regular file");
|
||||
}
|
||||
}
|
||||
|
||||
function sameFileState(left: BigIntStats, right: BigIntStats): boolean {
|
||||
return (
|
||||
sameFileIdentity(left, right) &&
|
||||
left.birthtimeNs === right.birthtimeNs &&
|
||||
left.ctimeNs === right.ctimeNs &&
|
||||
left.mtimeNs === right.mtimeNs &&
|
||||
left.size === right.size
|
||||
);
|
||||
}
|
||||
|
||||
function hashFileDescriptor(fd: number): string {
|
||||
const hash = createHash("sha256");
|
||||
const buffer = Buffer.allocUnsafe(SQLITE_GENERATION_HASH_BUFFER_BYTES);
|
||||
let position = 0;
|
||||
while (true) {
|
||||
const bytesRead = fs.readSync(fd, buffer, 0, buffer.length, position);
|
||||
if (bytesRead === 0) {
|
||||
break;
|
||||
}
|
||||
hash.update(buffer.subarray(0, bytesRead));
|
||||
position += bytesRead;
|
||||
}
|
||||
return hash.digest("hex");
|
||||
}
|
||||
|
||||
function fingerprintFile(pathname: string): SqliteFileFingerprint {
|
||||
const fd = fs.openSync(pathname, "r");
|
||||
try {
|
||||
const before = fs.fstatSync(fd, { bigint: true });
|
||||
assertRegularFile(before);
|
||||
const sha256 = hashFileDescriptor(fd);
|
||||
const after = fs.fstatSync(fd, { bigint: true });
|
||||
const current = fs.statSync(pathname, { bigint: true });
|
||||
if (!sameFileState(before, after) || !sameFileState(after, current)) {
|
||||
throw new Error(`SQLite generation target changed while hashing: ${pathname}`);
|
||||
}
|
||||
return {
|
||||
birthtimeNs: after.birthtimeNs,
|
||||
ctimeNs: after.ctimeNs,
|
||||
dev: after.dev,
|
||||
ino: after.ino,
|
||||
mtimeNs: after.mtimeNs,
|
||||
sha256,
|
||||
size: after.size,
|
||||
};
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
}
|
||||
|
||||
function readOptionalFile(pathname: string): SqliteFileFingerprint | undefined {
|
||||
try {
|
||||
return fingerprintFile(pathname);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
|
||||
return undefined;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function readGeneration(pathname: string): SqliteFileGeneration {
|
||||
const database = fingerprintFile(pathname);
|
||||
const journal = readOptionalFile(`${pathname}-journal`);
|
||||
const wal = readOptionalFile(`${pathname}-wal`);
|
||||
return {
|
||||
database,
|
||||
...(journal ? { journal } : {}),
|
||||
...(wal ? { wal } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function readStableSqliteFileGeneration(pathname: string): SqliteFileGeneration {
|
||||
const first = readGeneration(pathname);
|
||||
const second = readGeneration(pathname);
|
||||
if (!sameSqliteFileGeneration(first, second)) {
|
||||
throw new Error(`SQLite file generation changed while reading: ${pathname}`);
|
||||
}
|
||||
return second;
|
||||
}
|
||||
|
||||
function sameFileFingerprint(left: SqliteFileFingerprint, right: SqliteFileFingerprint): boolean {
|
||||
return (
|
||||
left.dev === right.dev &&
|
||||
left.ino === right.ino &&
|
||||
left.birthtimeNs === right.birthtimeNs &&
|
||||
left.ctimeNs === right.ctimeNs &&
|
||||
left.mtimeNs === right.mtimeNs &&
|
||||
left.sha256 === right.sha256 &&
|
||||
left.size === right.size
|
||||
);
|
||||
}
|
||||
|
||||
function sameOptionalFileFingerprint(
|
||||
left: SqliteFileFingerprint | undefined,
|
||||
right: SqliteFileFingerprint | undefined,
|
||||
): boolean {
|
||||
return left === undefined
|
||||
? right === undefined
|
||||
: right !== undefined && sameFileFingerprint(left, right);
|
||||
}
|
||||
|
||||
export function sameSqliteFileGeneration(
|
||||
left: SqliteFileGeneration,
|
||||
right: SqliteFileGeneration,
|
||||
): boolean {
|
||||
return (
|
||||
sameFileFingerprint(left.database, right.database) &&
|
||||
sameOptionalFileFingerprint(left.journal, right.journal) &&
|
||||
sameOptionalFileFingerprint(left.wal, right.wal)
|
||||
);
|
||||
}
|
||||
|
||||
function serializeFileFingerprint(
|
||||
fingerprint: SqliteFileFingerprint,
|
||||
): SerializedSqliteFileFingerprint {
|
||||
return {
|
||||
birthtimeNs: fingerprint.birthtimeNs.toString(),
|
||||
ctimeNs: fingerprint.ctimeNs.toString(),
|
||||
dev: fingerprint.dev.toString(),
|
||||
ino: fingerprint.ino.toString(),
|
||||
mtimeNs: fingerprint.mtimeNs.toString(),
|
||||
sha256: fingerprint.sha256,
|
||||
size: fingerprint.size.toString(),
|
||||
};
|
||||
}
|
||||
|
||||
export function serializeSqliteFileGeneration(generation: SqliteFileGeneration): string {
|
||||
return JSON.stringify({
|
||||
database: serializeFileFingerprint(generation.database),
|
||||
...(generation.journal ? { journal: serializeFileFingerprint(generation.journal) } : {}),
|
||||
...(generation.wal ? { wal: serializeFileFingerprint(generation.wal) } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
function parseFileFingerprint(value: unknown): SqliteFileFingerprint {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
throw new Error("SQLite file fingerprint must be an object");
|
||||
}
|
||||
const fingerprint = value as Record<string, unknown>;
|
||||
const fields = ["birthtimeNs", "ctimeNs", "dev", "ino", "mtimeNs", "size"] as const;
|
||||
for (const field of fields) {
|
||||
if (typeof fingerprint[field] !== "string" || !/^-?\d+$/u.test(fingerprint[field])) {
|
||||
throw new Error(`SQLite file fingerprint ${field} is invalid`);
|
||||
}
|
||||
}
|
||||
if (typeof fingerprint.sha256 !== "string" || !/^[a-f0-9]{64}$/u.test(fingerprint.sha256)) {
|
||||
throw new Error("SQLite file fingerprint sha256 is invalid");
|
||||
}
|
||||
return {
|
||||
birthtimeNs: BigInt(fingerprint.birthtimeNs as string),
|
||||
ctimeNs: BigInt(fingerprint.ctimeNs as string),
|
||||
dev: BigInt(fingerprint.dev as string),
|
||||
ino: BigInt(fingerprint.ino as string),
|
||||
mtimeNs: BigInt(fingerprint.mtimeNs as string),
|
||||
sha256: fingerprint.sha256,
|
||||
size: BigInt(fingerprint.size as string),
|
||||
};
|
||||
}
|
||||
|
||||
export function parseSqliteFileGeneration(serialized: string): SqliteFileGeneration {
|
||||
const value = JSON.parse(serialized) as unknown;
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
throw new Error("SQLite file generation must be an object");
|
||||
}
|
||||
const generation = value as Record<string, unknown>;
|
||||
return {
|
||||
database: parseFileFingerprint(generation.database),
|
||||
...(generation.journal === undefined
|
||||
? {}
|
||||
: { journal: parseFileFingerprint(generation.journal) }),
|
||||
...(generation.wal === undefined ? {} : { wal: parseFileFingerprint(generation.wal) }),
|
||||
};
|
||||
}
|
||||
@@ -1,7 +1,17 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
|
||||
import * as nodeSqlite from "./node-sqlite.js";
|
||||
import { requireNodeSqlite } from "./node-sqlite.js";
|
||||
import { assertSqliteIntegrity, isTerminalSqliteIntegrityError } from "./sqlite-integrity.js";
|
||||
import {
|
||||
assertSqliteIntegrity,
|
||||
confirmSqliteFileIntegrity,
|
||||
isTerminalSqliteIntegrityError,
|
||||
} from "./sqlite-integrity.js";
|
||||
|
||||
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
|
||||
describe("assertSqliteIntegrity", () => {
|
||||
it("accepts structurally and referentially consistent databases", () => {
|
||||
@@ -201,3 +211,24 @@ describe("isTerminalSqliteIntegrityError", () => {
|
||||
expect(isTerminalSqliteIntegrityError(corruptIndex)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("confirmSqliteFileIntegrity", () => {
|
||||
it("leaves SQLite open failures unbound because the failed file identity is unknown", () => {
|
||||
const databasePath = path.join(tempDirs.make("sqlite-open-integrity-"), "database.sqlite");
|
||||
fs.writeFileSync(databasePath, "not a sqlite database");
|
||||
const openError = Object.assign(new Error("file is not a database"), { errcode: 26 });
|
||||
const open = vi.spyOn(nodeSqlite, "openNodeSqliteDatabase").mockImplementationOnce(() => {
|
||||
throw openError;
|
||||
});
|
||||
|
||||
try {
|
||||
expect(confirmSqliteFileIntegrity(databasePath, "test database")).toEqual({
|
||||
status: "failed",
|
||||
error: openError,
|
||||
terminal: false,
|
||||
});
|
||||
} finally {
|
||||
open.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,9 +1,24 @@
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import { openNodeSqliteDatabase } from "./node-sqlite.js";
|
||||
import {
|
||||
readStableSqliteFileGeneration,
|
||||
sameSqliteFileGeneration,
|
||||
type SqliteFileGeneration,
|
||||
} from "./sqlite-file-generation.js";
|
||||
|
||||
type SqliteIntegrityChecks = {
|
||||
integrityCheck: "ok";
|
||||
};
|
||||
|
||||
type UnboundSqliteIntegrityConfirmation =
|
||||
| { status: "failed"; error: Error; terminal: boolean }
|
||||
| { status: "healthy" };
|
||||
|
||||
export type SqliteIntegrityConfirmation =
|
||||
| { status: "failed"; error: Error; terminal: false }
|
||||
| { status: "failed"; error: Error; generation: SqliteFileGeneration; terminal: true }
|
||||
| { status: "healthy"; generation: SqliteFileGeneration };
|
||||
|
||||
type SqliteCheckPragma = "integrity_check";
|
||||
type SqliteForeignKeyViolation = {
|
||||
fkid: bigint;
|
||||
@@ -45,6 +60,117 @@ export function assertSqliteIntegrity(
|
||||
return { integrityCheck };
|
||||
}
|
||||
|
||||
/** Run integrity checks and preserve whether a failure proves persistent damage. */
|
||||
function confirmSqliteIntegrity(
|
||||
database: DatabaseSync,
|
||||
databaseLabel: string,
|
||||
): UnboundSqliteIntegrityConfirmation {
|
||||
try {
|
||||
assertSqliteIntegrity(database, databaseLabel);
|
||||
return { status: "healthy" };
|
||||
} catch (error) {
|
||||
return failedSqliteIntegrityConfirmation(error);
|
||||
}
|
||||
}
|
||||
|
||||
/** Reconfirm an advisory failure against the database currently at a closed path. */
|
||||
export function confirmSqliteFileIntegrity(
|
||||
pathname: string,
|
||||
databaseLabel: string,
|
||||
): SqliteIntegrityConfirmation {
|
||||
for (let attempt = 0; attempt < 3; attempt += 1) {
|
||||
let initial: SqliteFileGeneration;
|
||||
try {
|
||||
initial = readStableSqliteFileGeneration(pathname);
|
||||
} catch (error) {
|
||||
return unboundSqliteIntegrityFailure(error);
|
||||
}
|
||||
|
||||
let database: DatabaseSync;
|
||||
try {
|
||||
database = openNodeSqliteDatabase(pathname, { readOnly: true });
|
||||
} catch (error) {
|
||||
// A failed SQLite open exposes no descriptor identity. Path snapshots
|
||||
// cannot bind the error safely across an A -> B -> A file rotation.
|
||||
return unboundSqliteIntegrityFailure(error);
|
||||
}
|
||||
|
||||
let opened: SqliteFileGeneration;
|
||||
try {
|
||||
opened = readStableSqliteFileGeneration(pathname);
|
||||
} catch {
|
||||
const closeError = closeSqliteDatabase(database);
|
||||
if (closeError) {
|
||||
return unboundSqliteIntegrityFailure(closeError);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (!sameSqliteFileGeneration(initial, opened)) {
|
||||
const closeError = closeSqliteDatabase(database);
|
||||
if (closeError) {
|
||||
return unboundSqliteIntegrityFailure(closeError);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
let confirmation = confirmSqliteIntegrity(database, databaseLabel);
|
||||
const closeError = closeSqliteDatabase(database);
|
||||
if (closeError && confirmation.status === "healthy") {
|
||||
confirmation = failedSqliteIntegrityConfirmation(closeError);
|
||||
}
|
||||
|
||||
let final: SqliteFileGeneration;
|
||||
try {
|
||||
final = readStableSqliteFileGeneration(pathname);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (!sameSqliteFileGeneration(opened, final)) {
|
||||
continue;
|
||||
}
|
||||
return bindSqliteIntegrityConfirmation(confirmation, final);
|
||||
}
|
||||
return unboundSqliteIntegrityFailure(
|
||||
new Error(`SQLite file generation did not stabilize during confirmation: ${pathname}`),
|
||||
);
|
||||
}
|
||||
|
||||
function bindSqliteIntegrityConfirmation(
|
||||
confirmation: UnboundSqliteIntegrityConfirmation,
|
||||
generation: SqliteFileGeneration,
|
||||
): SqliteIntegrityConfirmation {
|
||||
if (confirmation.status === "healthy") {
|
||||
return { status: "healthy", generation };
|
||||
}
|
||||
if (confirmation.terminal) {
|
||||
return { ...confirmation, generation, terminal: true };
|
||||
}
|
||||
return { ...confirmation, terminal: false };
|
||||
}
|
||||
|
||||
function failedSqliteIntegrityConfirmation(error: unknown): UnboundSqliteIntegrityConfirmation {
|
||||
const normalized = error instanceof Error ? error : new Error(String(error));
|
||||
return {
|
||||
status: "failed",
|
||||
error: normalized,
|
||||
terminal: isTerminalSqliteIntegrityError(normalized),
|
||||
};
|
||||
}
|
||||
|
||||
function unboundSqliteIntegrityFailure(error: unknown): SqliteIntegrityConfirmation {
|
||||
const normalized = error instanceof Error ? error : new Error(String(error));
|
||||
return { status: "failed", error: normalized, terminal: false };
|
||||
}
|
||||
|
||||
function closeSqliteDatabase(database: DatabaseSync): Error | undefined {
|
||||
try {
|
||||
database.close();
|
||||
return undefined;
|
||||
} catch (error) {
|
||||
return error instanceof Error ? error : new Error(String(error));
|
||||
}
|
||||
}
|
||||
|
||||
/** Require table and associated index consistency before trusting indexed reads. */
|
||||
export function assertSqliteTableIntegrity(
|
||||
database: DatabaseSync,
|
||||
|
||||
@@ -1,4 +1,22 @@
|
||||
import path from "node:path";
|
||||
import {
|
||||
readStableSqliteFileGeneration,
|
||||
sameSqliteFileGeneration,
|
||||
type SqliteFileGeneration,
|
||||
} from "./sqlite-file-generation.js";
|
||||
|
||||
type TerminalOpenFailure = {
|
||||
error: Error;
|
||||
generation?: SqliteFileGeneration;
|
||||
};
|
||||
|
||||
function generationMatchesPath(pathname: string, expected: SqliteFileGeneration): boolean {
|
||||
try {
|
||||
return sameSqliteFileGeneration(expected, readStableSqliteFileGeneration(pathname));
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-path latch for terminal database-open failures (newer schema, proven
|
||||
@@ -8,15 +26,34 @@ import path from "node:path";
|
||||
export function createSqliteTerminalOpenLatch(options: {
|
||||
closeByPath: (pathname: string) => void;
|
||||
}) {
|
||||
const failures = new Map<string, Error>();
|
||||
const failures = new Map<string, TerminalOpenFailure>();
|
||||
|
||||
return {
|
||||
get: (pathname: string): Error | undefined => failures.get(path.resolve(pathname)),
|
||||
record: (pathname: string, error: Error): void => {
|
||||
get: (pathname: string): Error | undefined => {
|
||||
const resolvedPath = path.resolve(pathname);
|
||||
failures.set(resolvedPath, error);
|
||||
const failure = failures.get(resolvedPath);
|
||||
if (!failure) {
|
||||
return undefined;
|
||||
}
|
||||
if (failure.generation && !generationMatchesPath(resolvedPath, failure.generation)) {
|
||||
failures.delete(resolvedPath);
|
||||
return undefined;
|
||||
}
|
||||
return failure.error;
|
||||
},
|
||||
record: (pathname: string, error: Error, generation?: SqliteFileGeneration): boolean => {
|
||||
const resolvedPath = path.resolve(pathname);
|
||||
if (generation && !generationMatchesPath(resolvedPath, generation)) {
|
||||
return false;
|
||||
}
|
||||
failures.set(resolvedPath, { error, ...(generation ? { generation } : {}) });
|
||||
// Latch first. Close hooks may reenter.
|
||||
options.closeByPath(resolvedPath);
|
||||
if (generation && !generationMatchesPath(resolvedPath, generation)) {
|
||||
failures.delete(resolvedPath);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
},
|
||||
clear: (pathname: string): void => {
|
||||
failures.delete(path.resolve(pathname));
|
||||
|
||||
@@ -4,7 +4,12 @@ import path from "node:path";
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import { clearNodeSqliteKyselyCacheForDatabase } from "../infra/kysely-sync.js";
|
||||
import { openNodeSqliteDatabase } from "../infra/node-sqlite.js";
|
||||
import { isTerminalSqliteIntegrityError } from "../infra/sqlite-integrity.js";
|
||||
import type { SqliteFileGeneration } from "../infra/sqlite-file-generation.js";
|
||||
import {
|
||||
confirmSqliteFileIntegrity,
|
||||
isTerminalSqliteIntegrityError,
|
||||
type SqliteIntegrityConfirmation,
|
||||
} from "../infra/sqlite-integrity.js";
|
||||
import { createSqliteTerminalOpenLatch } from "../infra/sqlite-terminal-open-latch.js";
|
||||
import {
|
||||
runSqliteImmediateTransactionSync,
|
||||
@@ -113,11 +118,30 @@ const terminalOpenLatch = createSqliteTerminalOpenLatch({
|
||||
closeByPath: closeOpenClawAgentDatabaseByPath,
|
||||
});
|
||||
|
||||
/** Reconfirm an advisory worker failure on the live owner connection. */
|
||||
export function confirmOpenClawAgentDatabaseIntegrity(
|
||||
pathname: string,
|
||||
): SqliteIntegrityConfirmation {
|
||||
const resolvedPath = path.resolve(pathname);
|
||||
closeOpenClawAgentDatabaseByPath(resolvedPath);
|
||||
// Closing breaks process ownership of the pathname. A replacement must
|
||||
// revalidate and claim its schema before the path can become trusted again.
|
||||
validatedAgentDatabasePaths.delete(resolvedPath);
|
||||
return confirmSqliteFileIntegrity(resolvedPath, resolvedPath);
|
||||
}
|
||||
|
||||
/** Latch background verification damage so later opens fail without rescanning. */
|
||||
export function recordOpenClawAgentDatabaseOpenFailure(pathname: string, error: Error): void {
|
||||
// Quarantine revokes this process's trust because doctor may replace the file.
|
||||
validatedAgentDatabasePaths.delete(path.resolve(pathname));
|
||||
terminalOpenLatch.record(pathname, error);
|
||||
export function recordOpenClawAgentDatabaseOpenFailure(
|
||||
pathname: string,
|
||||
error: Error,
|
||||
generation?: SqliteFileGeneration,
|
||||
): boolean {
|
||||
const recorded = terminalOpenLatch.record(pathname, error, generation);
|
||||
if (recorded) {
|
||||
// Quarantine revokes this process's trust because doctor may replace the file.
|
||||
validatedAgentDatabasePaths.delete(path.resolve(pathname));
|
||||
}
|
||||
return recorded;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -4,6 +4,7 @@ import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { Worker } from "node:worker_threads";
|
||||
import { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
import {
|
||||
confirmOpenClawAgentDatabaseIntegrity,
|
||||
listOpenClawRegisteredAgentDatabases,
|
||||
recordOpenClawAgentDatabaseOpenFailure,
|
||||
} from "./openclaw-agent-db.js";
|
||||
@@ -12,7 +13,10 @@ import type {
|
||||
OpenClawDatabaseVerifyTarget,
|
||||
} from "./openclaw-database-verify.worker.js";
|
||||
import { recordOpenClawDatabaseQuarantine } from "./openclaw-quarantine-store.js";
|
||||
import { recordOpenClawStateDatabaseOpenFailure } from "./openclaw-state-db.js";
|
||||
import {
|
||||
confirmOpenClawStateDatabaseIntegrity,
|
||||
recordOpenClawStateDatabaseOpenFailure,
|
||||
} from "./openclaw-state-db.js";
|
||||
import { resolveOpenClawStateSqlitePath } from "./openclaw-state-db.paths.js";
|
||||
|
||||
export const OPENCLAW_DATABASE_VERIFY_INITIAL_DELAY_MS = 5 * 60_000;
|
||||
@@ -126,15 +130,7 @@ export function collectOpenClawDatabaseVerifyTargets(options: {
|
||||
return [...targets.values()];
|
||||
}
|
||||
|
||||
function createVerificationFailure(result: OpenClawDatabaseVerifyResult): Error {
|
||||
const error = new Error(
|
||||
result.error ?? `SQLite integrity verification failed for ${result.path}`,
|
||||
);
|
||||
error.name = "SqliteIntegrityError";
|
||||
return error;
|
||||
}
|
||||
|
||||
/** Quarantine terminal failures and log the worker batch. */
|
||||
/** Reconfirm worker failures on live owners before quarantine and latching. */
|
||||
export function applyOpenClawDatabaseVerificationResults(options: {
|
||||
env: NodeJS.ProcessEnv;
|
||||
results: readonly OpenClawDatabaseVerifyResult[];
|
||||
@@ -164,11 +160,53 @@ export function applyOpenClawDatabaseVerificationResults(options: {
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const confirmation =
|
||||
target.kind === "state"
|
||||
? confirmOpenClawStateDatabaseIntegrity(result.path)
|
||||
: confirmOpenClawAgentDatabaseIntegrity(result.path);
|
||||
if (confirmation.status === "healthy") {
|
||||
log.info("discarding stale database integrity verification result", {
|
||||
kind: target.kind,
|
||||
label: target.label,
|
||||
path: result.path,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if (!confirmation.terminal) {
|
||||
log.warn("database integrity verification was inconclusive", {
|
||||
kind: target.kind,
|
||||
label: target.label,
|
||||
path: result.path,
|
||||
error: confirmation.error.message,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const latched =
|
||||
target.kind === "state"
|
||||
? recordOpenClawStateDatabaseOpenFailure(
|
||||
result.path,
|
||||
confirmation.error,
|
||||
confirmation.generation,
|
||||
)
|
||||
: recordOpenClawAgentDatabaseOpenFailure(
|
||||
result.path,
|
||||
confirmation.error,
|
||||
confirmation.generation,
|
||||
);
|
||||
if (!latched) {
|
||||
log.info("discarding database integrity result after database generation changed", {
|
||||
kind: target.kind,
|
||||
label: target.label,
|
||||
path: result.path,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const recorded = recordOpenClawDatabaseQuarantine({
|
||||
env: options.env,
|
||||
generation: confirmation.generation,
|
||||
kind: target.kind,
|
||||
path: result.path,
|
||||
reason: result.error ?? `SQLite integrity verification failed for ${result.path}`,
|
||||
reason: confirmation.error.message,
|
||||
});
|
||||
if (!recorded) {
|
||||
// Store unavailable. Daily verification retries persistence.
|
||||
@@ -177,17 +215,11 @@ export function applyOpenClawDatabaseVerificationResults(options: {
|
||||
path: result.path,
|
||||
});
|
||||
}
|
||||
const error = createVerificationFailure(result);
|
||||
if (target.kind === "state") {
|
||||
recordOpenClawStateDatabaseOpenFailure(result.path, error);
|
||||
} else {
|
||||
recordOpenClawAgentDatabaseOpenFailure(result.path, error);
|
||||
}
|
||||
log.error("database integrity verification failed", {
|
||||
kind: target.kind,
|
||||
label: target.label,
|
||||
path: result.path,
|
||||
error: error.message,
|
||||
error: confirmation.error.message,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,11 +4,14 @@ import path from "node:path";
|
||||
import { afterAll, afterEach, describe, expect, it } from "vitest";
|
||||
import { cleanupTempDirs, makeTempDir } from "../../test/helpers/temp-dir.js";
|
||||
import { requireNodeSqlite } from "../infra/node-sqlite.js";
|
||||
import { readStableSqliteFileGeneration } from "../infra/sqlite-file-generation.js";
|
||||
import { readSqliteNumberPragma } from "../infra/sqlite-pragma.test-support.js";
|
||||
import {
|
||||
clearOpenClawAgentDatabaseOpenFailure,
|
||||
closeOpenClawAgentDatabaseByPath,
|
||||
closeOpenClawAgentDatabasesForTest,
|
||||
openOpenClawAgentDatabase,
|
||||
recordOpenClawAgentDatabaseOpenFailure,
|
||||
} from "./openclaw-agent-db.js";
|
||||
import {
|
||||
applyOpenClawDatabaseVerificationResults,
|
||||
@@ -26,6 +29,7 @@ import {
|
||||
import {
|
||||
closeOpenClawStateDatabaseForTest,
|
||||
openOpenClawStateDatabase,
|
||||
recordOpenClawStateDatabaseOpenFailure,
|
||||
repairOpenClawStateDatabaseSchema,
|
||||
} from "./openclaw-state-db.js";
|
||||
|
||||
@@ -66,6 +70,34 @@ function createUnsafeIndexDrift(databasePath: string): void {
|
||||
}
|
||||
}
|
||||
|
||||
function repairUnsafeIndexDrift(databasePath: string): void {
|
||||
const { DatabaseSync } = requireNodeSqlite();
|
||||
const database = new DatabaseSync(databasePath);
|
||||
try {
|
||||
database.enableDefensive?.(false);
|
||||
database.exec("PRAGMA writable_schema = ON;");
|
||||
database
|
||||
.prepare(
|
||||
"UPDATE sqlite_schema SET sql = 'CREATE INDEX unsafe_index_records_value ON unsafe_index_records(indexed_value)' WHERE name = 'unsafe_index_records_value'",
|
||||
)
|
||||
.run();
|
||||
const schemaVersion = readSqliteNumberPragma(database, "schema_version");
|
||||
database.exec(`PRAGMA writable_schema = OFF; PRAGMA schema_version = ${schemaVersion + 1};`);
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function copyHealthyDatabase(sourcePath: string, targetPath: string): Promise<void> {
|
||||
const sqlite = requireNodeSqlite();
|
||||
const source = new sqlite.DatabaseSync(sourcePath, { readOnly: true });
|
||||
try {
|
||||
await sqlite.backup(source, targetPath);
|
||||
} finally {
|
||||
source.close();
|
||||
}
|
||||
}
|
||||
|
||||
function quarantineStorePath(stateDir: string): string {
|
||||
return path.join(stateDir, "state", "openclaw-quarantine.sqlite");
|
||||
}
|
||||
@@ -75,14 +107,12 @@ describe("OpenClaw database integrity verifier", () => {
|
||||
const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-");
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
const agentPath = openOpenClawAgentDatabase({ agentId: "worker-1", env }).path;
|
||||
closeOpenClawAgentDatabasesForTest();
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
createUnsafeIndexDrift(agentPath);
|
||||
const targets: OpenClawDatabaseVerifyTarget[] = [
|
||||
{ kind: "agent", label: "OpenClaw agent database worker-1", path: agentPath },
|
||||
];
|
||||
|
||||
const directResults = verifyOpenClawDatabases(targets);
|
||||
const directResults = await verifyOpenClawDatabases(targets);
|
||||
expect(directResults).toEqual([
|
||||
{
|
||||
path: agentPath,
|
||||
@@ -93,21 +123,16 @@ describe("OpenClaw database integrity verifier", () => {
|
||||
]);
|
||||
await expect(runDatabaseVerifyWorker(targets)).resolves.toEqual(directResults);
|
||||
|
||||
// The drift is not a committed canonical index, so open must fail closed
|
||||
// instead of guessing a replacement definition.
|
||||
expect(() => openOpenClawAgentDatabase({ agentId: "worker-1", env })).toThrow(
|
||||
expect.objectContaining({ name: "SqliteIntegrityError" }),
|
||||
);
|
||||
|
||||
applyOpenClawDatabaseVerificationResults({
|
||||
env,
|
||||
results: directResults,
|
||||
targets,
|
||||
});
|
||||
expect(readOpenClawDatabaseQuarantine(agentPath, { env })).toEqual({
|
||||
const quarantine = readOpenClawDatabaseQuarantine(agentPath, { env });
|
||||
expect(quarantine).toEqual({
|
||||
kind: "agent",
|
||||
quarantinedAt: expect.any(Number),
|
||||
reason: directResults[0]?.error,
|
||||
reason: expect.stringMatching(/missing from index unsafe_index_records_value/iu),
|
||||
});
|
||||
|
||||
expect(() => openOpenClawAgentDatabase({ agentId: "worker-1", env })).toThrow(
|
||||
@@ -119,7 +144,7 @@ describe("OpenClaw database integrity verifier", () => {
|
||||
expect(() => openOpenClawAgentDatabase({ agentId: "worker-1", env })).toThrow(
|
||||
expect.objectContaining({
|
||||
name: "SqliteIntegrityError",
|
||||
message: expect.stringContaining(directResults[0]?.error ?? ""),
|
||||
message: expect.stringContaining(quarantine?.reason ?? ""),
|
||||
}),
|
||||
);
|
||||
clearOpenClawAgentDatabaseOpenFailure(agentPath, { env });
|
||||
@@ -128,6 +153,215 @@ describe("OpenClaw database integrity verifier", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("does not quarantine a healthy database that replaced the verified file", async () => {
|
||||
const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-replacement-");
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
const agentPath = openOpenClawAgentDatabase({ agentId: "worker-1", env }).path;
|
||||
closeOpenClawAgentDatabasesForTest();
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
|
||||
const healthyReplacementPath = `${agentPath}.healthy`;
|
||||
const corruptArchivePath = `${agentPath}.corrupt`;
|
||||
fs.copyFileSync(agentPath, healthyReplacementPath);
|
||||
createUnsafeIndexDrift(agentPath);
|
||||
const targets: OpenClawDatabaseVerifyTarget[] = [
|
||||
{ kind: "agent", label: "OpenClaw agent database worker-1", path: agentPath },
|
||||
];
|
||||
const results = await verifyOpenClawDatabases(targets);
|
||||
expect(results).toEqual([
|
||||
expect.objectContaining({
|
||||
path: agentPath,
|
||||
ok: false,
|
||||
terminal: true,
|
||||
}),
|
||||
]);
|
||||
|
||||
fs.renameSync(agentPath, corruptArchivePath);
|
||||
fs.renameSync(healthyReplacementPath, agentPath);
|
||||
applyOpenClawDatabaseVerificationResults({ env, results, targets });
|
||||
|
||||
expect(readOpenClawDatabaseQuarantine(agentPath, { env })).toBeUndefined();
|
||||
expect(openOpenClawAgentDatabase({ agentId: "worker-1", env }).db.isOpen).toBe(true);
|
||||
});
|
||||
|
||||
it.skipIf(process.platform === "win32")(
|
||||
"does not quarantine a healthy replacement while the corrupt agent inode is cached",
|
||||
async () => {
|
||||
const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-live-agent-replace-");
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
const agent = openOpenClawAgentDatabase({ agentId: "worker-1", env });
|
||||
const healthyReplacementPath = `${agent.path}.healthy`;
|
||||
const corruptArchivePath = `${agent.path}.corrupt`;
|
||||
await copyHealthyDatabase(agent.path, healthyReplacementPath);
|
||||
createUnsafeIndexDrift(agent.path);
|
||||
const targets: OpenClawDatabaseVerifyTarget[] = [
|
||||
{ kind: "agent", label: "OpenClaw agent database worker-1", path: agent.path },
|
||||
];
|
||||
const results = await verifyOpenClawDatabases(targets);
|
||||
|
||||
fs.renameSync(agent.path, corruptArchivePath);
|
||||
fs.renameSync(healthyReplacementPath, agent.path);
|
||||
applyOpenClawDatabaseVerificationResults({ env, results, targets });
|
||||
|
||||
expect(agent.db.isOpen).toBe(false);
|
||||
expect(readOpenClawDatabaseQuarantine(agent.path, { env })).toBeUndefined();
|
||||
expect(openOpenClawAgentDatabase({ agentId: "worker-1", env }).db.isOpen).toBe(true);
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(process.platform === "win32")(
|
||||
"does not quarantine a healthy replacement while the corrupt state inode is cached",
|
||||
async () => {
|
||||
const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-live-state-replace-");
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
const state = openOpenClawStateDatabase({ env });
|
||||
const healthyReplacementPath = `${state.path}.healthy`;
|
||||
const corruptArchivePath = `${state.path}.corrupt`;
|
||||
await copyHealthyDatabase(state.path, healthyReplacementPath);
|
||||
createUnsafeIndexDrift(state.path);
|
||||
const targets: OpenClawDatabaseVerifyTarget[] = [
|
||||
{ kind: "state", label: "OpenClaw state database", path: state.path },
|
||||
];
|
||||
const results = await verifyOpenClawDatabases(targets);
|
||||
|
||||
fs.renameSync(state.path, corruptArchivePath);
|
||||
fs.renameSync(healthyReplacementPath, state.path);
|
||||
applyOpenClawDatabaseVerificationResults({ env, results, targets });
|
||||
|
||||
expect(state.db.isOpen).toBe(false);
|
||||
expect(readOpenClawDatabaseQuarantine(state.path, { env })).toBeUndefined();
|
||||
expect(openOpenClawStateDatabase({ env }).db.isOpen).toBe(true);
|
||||
},
|
||||
);
|
||||
|
||||
it("reconfirms and quarantines a corrupt closed database", async () => {
|
||||
const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-closed-");
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
const agentPath = openOpenClawAgentDatabase({ agentId: "worker-1", env }).path;
|
||||
closeOpenClawAgentDatabasesForTest();
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
createUnsafeIndexDrift(agentPath);
|
||||
const targets: OpenClawDatabaseVerifyTarget[] = [
|
||||
{ kind: "agent", label: "OpenClaw agent database worker-1", path: agentPath },
|
||||
];
|
||||
const results = await verifyOpenClawDatabases(targets);
|
||||
|
||||
applyOpenClawDatabaseVerificationResults({ env, results, targets });
|
||||
|
||||
expect(readOpenClawDatabaseQuarantine(agentPath, { env })?.reason).toMatch(
|
||||
/missing from index unsafe_index_records_value/iu,
|
||||
);
|
||||
expect(() => openOpenClawAgentDatabase({ agentId: "worker-1", env })).toThrow(
|
||||
expect.objectContaining({ name: "SqliteIntegrityError" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("does not quarantine a repaired database after same-inode mutation", async () => {
|
||||
const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-repair-race-");
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
const agentPath = openOpenClawAgentDatabase({ agentId: "worker-1", env }).path;
|
||||
closeOpenClawAgentDatabasesForTest();
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
createUnsafeIndexDrift(agentPath);
|
||||
const targets: OpenClawDatabaseVerifyTarget[] = [
|
||||
{ kind: "agent", label: "OpenClaw agent database worker-1", path: agentPath },
|
||||
];
|
||||
const results = await verifyOpenClawDatabases(targets);
|
||||
expect(results[0]).toEqual(expect.objectContaining({ ok: false, terminal: true }));
|
||||
|
||||
repairUnsafeIndexDrift(agentPath);
|
||||
await expect(verifyOpenClawDatabases(targets)).resolves.toEqual([
|
||||
expect.objectContaining({ ok: true }),
|
||||
]);
|
||||
applyOpenClawDatabaseVerificationResults({ env, results, targets });
|
||||
|
||||
expect(readOpenClawDatabaseQuarantine(agentPath, { env })).toBeUndefined();
|
||||
expect(openOpenClawAgentDatabase({ agentId: "worker-1", env }).db.isOpen).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects stale terminal results after draining healthy owners", () => {
|
||||
const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-live-stale-");
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
const state = openOpenClawStateDatabase({ env });
|
||||
const agent = openOpenClawAgentDatabase({ agentId: "worker-1", env });
|
||||
const targets: OpenClawDatabaseVerifyTarget[] = [
|
||||
{ kind: "state", label: "OpenClaw state database", path: state.path },
|
||||
{ kind: "agent", label: "OpenClaw agent database worker-1", path: agent.path },
|
||||
];
|
||||
|
||||
applyOpenClawDatabaseVerificationResults({
|
||||
env,
|
||||
results: targets.map((target) => ({
|
||||
path: target.path,
|
||||
ok: false,
|
||||
error: "stale terminal result",
|
||||
terminal: true,
|
||||
})),
|
||||
targets,
|
||||
});
|
||||
|
||||
expect(readOpenClawDatabaseQuarantine(state.path, { env })).toBeUndefined();
|
||||
expect(readOpenClawDatabaseQuarantine(agent.path, { env })).toBeUndefined();
|
||||
expect(state.db.isOpen).toBe(false);
|
||||
expect(agent.db.isOpen).toBe(false);
|
||||
expect(openOpenClawStateDatabase({ env }).db.isOpen).toBe(true);
|
||||
expect(openOpenClawAgentDatabase({ agentId: "worker-1", env }).db.isOpen).toBe(true);
|
||||
});
|
||||
|
||||
it("revalidates agent schema ownership after confirmation drains a pathname", () => {
|
||||
const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-agent-revalidate-");
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
const agent = openOpenClawAgentDatabase({ agentId: "worker-1", env });
|
||||
const replacementPath = `${agent.path}.replacement`;
|
||||
const { DatabaseSync } = requireNodeSqlite();
|
||||
const replacement = new DatabaseSync(replacementPath);
|
||||
try {
|
||||
replacement.exec("CREATE TABLE replacement_marker (id INTEGER PRIMARY KEY) STRICT;");
|
||||
} finally {
|
||||
replacement.close();
|
||||
}
|
||||
expect(closeOpenClawAgentDatabaseByPath(agent.path)).toBe(true);
|
||||
fs.rmSync(agent.path);
|
||||
fs.renameSync(replacementPath, agent.path);
|
||||
const targets: OpenClawDatabaseVerifyTarget[] = [
|
||||
{ kind: "agent", label: "OpenClaw agent database worker-1", path: agent.path },
|
||||
];
|
||||
|
||||
applyOpenClawDatabaseVerificationResults({
|
||||
env,
|
||||
results: [{ path: agent.path, ok: false, error: "stale terminal result", terminal: true }],
|
||||
targets,
|
||||
});
|
||||
|
||||
const reopened = openOpenClawAgentDatabase({ agentId: "worker-1", env });
|
||||
expect(
|
||||
reopened.db
|
||||
.prepare("SELECT role, agent_id FROM schema_meta WHERE meta_key = 'primary'")
|
||||
.get(),
|
||||
).toEqual({ role: "agent", agent_id: "worker-1" });
|
||||
});
|
||||
|
||||
it("expires a generation-bound process latch after the database changes", () => {
|
||||
const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-latch-generation-");
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
const agentPath = openOpenClawAgentDatabase({ agentId: "worker-1", env }).path;
|
||||
closeOpenClawAgentDatabasesForTest();
|
||||
const generation = readStableSqliteFileGeneration(agentPath);
|
||||
const error = new Error("verified corrupt generation");
|
||||
error.name = "SqliteIntegrityError";
|
||||
|
||||
expect(recordOpenClawAgentDatabaseOpenFailure(agentPath, error, generation)).toBe(true);
|
||||
const { DatabaseSync } = requireNodeSqlite();
|
||||
const changed = new DatabaseSync(agentPath);
|
||||
try {
|
||||
changed.exec("CREATE TABLE generation_change (id INTEGER PRIMARY KEY) STRICT;");
|
||||
} finally {
|
||||
changed.close();
|
||||
}
|
||||
|
||||
expect(openOpenClawAgentDatabase({ agentId: "worker-1", env }).db.isOpen).toBe(true);
|
||||
});
|
||||
|
||||
it("reports an uncleared quarantine row instead of claiming repair success", () => {
|
||||
const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-clear-failure-");
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
@@ -135,11 +369,17 @@ describe("OpenClaw database integrity verifier", () => {
|
||||
openOpenClawStateDatabase({ env });
|
||||
closeOpenClawAgentDatabasesForTest();
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
applyOpenClawDatabaseVerificationResults({
|
||||
env,
|
||||
results: [{ path: agentPath, ok: false, error: "corrupt index", terminal: true }],
|
||||
targets: [{ kind: "agent", label: "OpenClaw agent database worker-1", path: agentPath }],
|
||||
});
|
||||
expect(
|
||||
recordOpenClawDatabaseQuarantine({
|
||||
env,
|
||||
kind: "agent",
|
||||
path: agentPath,
|
||||
reason: "corrupt index",
|
||||
}),
|
||||
).toBe(true);
|
||||
const error = new Error("corrupt index");
|
||||
error.name = "SqliteIntegrityError";
|
||||
recordOpenClawAgentDatabaseOpenFailure(agentPath, error);
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
|
||||
const storePath = quarantineStorePath(stateDir);
|
||||
@@ -199,7 +439,7 @@ describe("OpenClaw database integrity verifier", () => {
|
||||
try {
|
||||
expect(raw.prepare("PRAGMA journal_mode").get()).toEqual({ journal_mode: "delete" });
|
||||
expect(readSqliteNumberPragma(raw, "synchronous")).toBe(2);
|
||||
expect(readSqliteNumberPragma(raw, "user_version")).toBe(1);
|
||||
expect(readSqliteNumberPragma(raw, "user_version")).toBe(2);
|
||||
} finally {
|
||||
raw.close();
|
||||
}
|
||||
@@ -212,6 +452,96 @@ describe("OpenClaw database integrity verifier", () => {
|
||||
expect(readOpenClawDatabaseQuarantine(databasePath, { env })).toBeUndefined();
|
||||
});
|
||||
|
||||
it("expires a persisted quarantine when the verified database generation changes", () => {
|
||||
const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-store-generation-");
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
const databasePath = path.join(stateDir, "agent.sqlite");
|
||||
const { DatabaseSync } = requireNodeSqlite();
|
||||
const database = new DatabaseSync(databasePath);
|
||||
try {
|
||||
database.exec("CREATE TABLE records (id INTEGER PRIMARY KEY) STRICT;");
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
const generation = readStableSqliteFileGeneration(databasePath);
|
||||
|
||||
expect(
|
||||
recordOpenClawDatabaseQuarantine({
|
||||
env,
|
||||
generation,
|
||||
kind: "agent",
|
||||
path: databasePath,
|
||||
reason: "corrupt generation",
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(readOpenClawDatabaseQuarantine(databasePath, { env })?.reason).toBe(
|
||||
"corrupt generation",
|
||||
);
|
||||
|
||||
const changed = new DatabaseSync(databasePath);
|
||||
try {
|
||||
changed.exec("INSERT INTO records DEFAULT VALUES;");
|
||||
} finally {
|
||||
changed.close();
|
||||
}
|
||||
expect(readOpenClawDatabaseQuarantine(databasePath, { env })).toBeUndefined();
|
||||
});
|
||||
|
||||
it("reads schema-v1 quarantine rows and migrates them on the next write", () => {
|
||||
const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-store-v1-");
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
const databasePath = path.join(stateDir, "agent.sqlite");
|
||||
const storePath = quarantineStorePath(stateDir);
|
||||
fs.mkdirSync(path.dirname(storePath), { recursive: true });
|
||||
const { DatabaseSync } = requireNodeSqlite();
|
||||
const legacy = new DatabaseSync(storePath);
|
||||
try {
|
||||
legacy.exec(`
|
||||
CREATE TABLE quarantined_databases (
|
||||
path TEXT NOT NULL PRIMARY KEY,
|
||||
kind TEXT NOT NULL,
|
||||
reason TEXT NOT NULL,
|
||||
quarantined_at INTEGER NOT NULL,
|
||||
writer_app_version TEXT
|
||||
) STRICT;
|
||||
PRAGMA user_version = 1;
|
||||
`);
|
||||
legacy
|
||||
.prepare(
|
||||
"INSERT INTO quarantined_databases (path, kind, reason, quarantined_at) VALUES (?, ?, ?, ?)",
|
||||
)
|
||||
.run(path.resolve(databasePath), "agent", "legacy quarantine", 1);
|
||||
} finally {
|
||||
legacy.close();
|
||||
}
|
||||
|
||||
expect(readOpenClawDatabaseQuarantine(databasePath, { env })).toEqual({
|
||||
kind: "agent",
|
||||
quarantinedAt: 1,
|
||||
reason: "legacy quarantine",
|
||||
});
|
||||
expect(
|
||||
recordOpenClawDatabaseQuarantine({
|
||||
env,
|
||||
kind: "agent",
|
||||
path: databasePath,
|
||||
reason: "migrated quarantine",
|
||||
}),
|
||||
).toBe(true);
|
||||
|
||||
const migrated = new DatabaseSync(storePath, { readOnly: true });
|
||||
try {
|
||||
expect(readSqliteNumberPragma(migrated, "user_version")).toBe(2);
|
||||
expect(
|
||||
migrated
|
||||
.prepare("SELECT reason, verified_generation FROM quarantined_databases WHERE path = ?")
|
||||
.get(path.resolve(databasePath)),
|
||||
).toEqual({ reason: "migrated quarantine", verified_generation: null });
|
||||
} finally {
|
||||
migrated.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("recovers an interrupted empty quarantine-store initialization", () => {
|
||||
const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-empty-store-");
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
@@ -297,15 +627,18 @@ describe("OpenClaw database integrity verifier", () => {
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
const statePath = openOpenClawStateDatabase({ env }).path;
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
const targets: OpenClawDatabaseVerifyTarget[] = [
|
||||
{ kind: "state", label: "OpenClaw state database", path: statePath },
|
||||
];
|
||||
|
||||
applyOpenClawDatabaseVerificationResults({
|
||||
env,
|
||||
results: [{ path: statePath, ok: false, error: "corrupt index", terminal: true }],
|
||||
targets,
|
||||
});
|
||||
expect(
|
||||
recordOpenClawDatabaseQuarantine({
|
||||
env,
|
||||
kind: "state",
|
||||
path: statePath,
|
||||
reason: "corrupt index",
|
||||
}),
|
||||
).toBe(true);
|
||||
const error = new Error("corrupt index");
|
||||
error.name = "SqliteIntegrityError";
|
||||
recordOpenClawStateDatabaseOpenFailure(statePath, error);
|
||||
|
||||
const { DatabaseSync } = requireNodeSqlite();
|
||||
const raw = new DatabaseSync(quarantineStorePath(stateDir), { readOnly: true });
|
||||
|
||||
@@ -4,6 +4,7 @@ import {
|
||||
assertSqliteIntegrity,
|
||||
isTerminalSqliteIntegrityError,
|
||||
} from "../infra/sqlite-integrity.js";
|
||||
import { prepareSqliteReadOnlyLocation } from "../infra/sqlite-readonly-location.js";
|
||||
import { OPENCLAW_SQLITE_BUSY_TIMEOUT_MS } from "./openclaw-state-db.js";
|
||||
|
||||
export type OpenClawDatabaseVerifyTarget = {
|
||||
@@ -31,30 +32,64 @@ function isVerifyTarget(value: unknown): value is OpenClawDatabaseVerifyTarget {
|
||||
);
|
||||
}
|
||||
|
||||
/** Verify database files serially so large agent scans never compete for I/O. */
|
||||
export function verifyOpenClawDatabases(
|
||||
targets: readonly OpenClawDatabaseVerifyTarget[],
|
||||
): OpenClawDatabaseVerifyResult[] {
|
||||
return targets.map((target) => {
|
||||
let database: import("node:sqlite").DatabaseSync | undefined;
|
||||
function formatVerifyError(error: unknown): string {
|
||||
return error instanceof Error ? `${error.name}: ${error.message}` : String(error);
|
||||
}
|
||||
|
||||
async function verifyOpenClawDatabase(
|
||||
target: OpenClawDatabaseVerifyTarget,
|
||||
): Promise<OpenClawDatabaseVerifyResult> {
|
||||
let cleanup: (() => boolean) | undefined;
|
||||
let database: import("node:sqlite").DatabaseSync | undefined;
|
||||
let result = await (async (): Promise<OpenClawDatabaseVerifyResult> => {
|
||||
try {
|
||||
database = openNodeSqliteDatabase(target.path, {
|
||||
const prepared = await prepareSqliteReadOnlyLocation(target.path);
|
||||
cleanup = prepared.cleanup;
|
||||
database = openNodeSqliteDatabase(prepared.location, {
|
||||
readOnly: true,
|
||||
});
|
||||
database.exec(`PRAGMA busy_timeout = ${OPENCLAW_SQLITE_BUSY_TIMEOUT_MS};`);
|
||||
assertSqliteIntegrity(database, target.label);
|
||||
return { path: target.path, ok: true };
|
||||
} catch (error) {
|
||||
const detail = error instanceof Error ? `${error.name}: ${error.message}` : String(error);
|
||||
const terminal = error instanceof Error && isTerminalSqliteIntegrityError(error);
|
||||
return { path: target.path, ok: false, error: detail, terminal };
|
||||
} finally {
|
||||
database?.close();
|
||||
return {
|
||||
path: target.path,
|
||||
ok: false,
|
||||
error: formatVerifyError(error),
|
||||
terminal,
|
||||
};
|
||||
}
|
||||
});
|
||||
})();
|
||||
try {
|
||||
database?.close();
|
||||
} catch (error) {
|
||||
if (result.ok) {
|
||||
result = {
|
||||
path: target.path,
|
||||
ok: false,
|
||||
error: formatVerifyError(error),
|
||||
terminal: false,
|
||||
};
|
||||
}
|
||||
} finally {
|
||||
cleanup?.();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Verify database files serially so large agent scans never compete for I/O. */
|
||||
export async function verifyOpenClawDatabases(
|
||||
targets: readonly OpenClawDatabaseVerifyTarget[],
|
||||
): Promise<OpenClawDatabaseVerifyResult[]> {
|
||||
const results: OpenClawDatabaseVerifyResult[] = [];
|
||||
for (const target of targets) {
|
||||
results.push(await verifyOpenClawDatabase(target));
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
if (parentPort) {
|
||||
const targets = Array.isArray(workerData) ? workerData.filter(isVerifyTarget) : [];
|
||||
parentPort.postMessage(verifyOpenClawDatabases(targets), []);
|
||||
parentPort.postMessage(await verifyOpenClawDatabases(targets), []);
|
||||
}
|
||||
|
||||
@@ -4,10 +4,17 @@ import path from "node:path";
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import { openNodeSqliteDatabase } from "../infra/node-sqlite.js";
|
||||
import { applyPrivateModeSync } from "../infra/private-mode.js";
|
||||
import {
|
||||
parseSqliteFileGeneration,
|
||||
readStableSqliteFileGeneration,
|
||||
sameSqliteFileGeneration,
|
||||
serializeSqliteFileGeneration,
|
||||
type SqliteFileGeneration,
|
||||
} from "../infra/sqlite-file-generation.js";
|
||||
import { VERSION } from "../version.js";
|
||||
import { resolveOpenClawStateSqliteDir } from "./openclaw-state-db.paths.js";
|
||||
|
||||
const OPENCLAW_QUARANTINE_SCHEMA_VERSION = 1;
|
||||
const OPENCLAW_QUARANTINE_SCHEMA_VERSION = 2;
|
||||
const OPENCLAW_QUARANTINE_BUSY_TIMEOUT_MS = 5_000;
|
||||
const OPENCLAW_QUARANTINE_DIR_MODE = 0o700;
|
||||
const OPENCLAW_QUARANTINE_FILE_MODE = 0o600;
|
||||
@@ -45,6 +52,15 @@ function configureQuarantineWriter(database: DatabaseSync, storePath: string): v
|
||||
if (userVersion === OPENCLAW_QUARANTINE_SCHEMA_VERSION) {
|
||||
return;
|
||||
}
|
||||
if (userVersion === 1) {
|
||||
database.exec(`
|
||||
BEGIN IMMEDIATE;
|
||||
ALTER TABLE quarantined_databases ADD COLUMN verified_generation TEXT;
|
||||
PRAGMA user_version = ${OPENCLAW_QUARANTINE_SCHEMA_VERSION};
|
||||
COMMIT;
|
||||
`);
|
||||
return;
|
||||
}
|
||||
database.exec(`
|
||||
BEGIN IMMEDIATE;
|
||||
CREATE TABLE IF NOT EXISTS quarantined_databases (
|
||||
@@ -52,7 +68,8 @@ function configureQuarantineWriter(database: DatabaseSync, storePath: string): v
|
||||
kind TEXT NOT NULL,
|
||||
reason TEXT NOT NULL,
|
||||
quarantined_at INTEGER NOT NULL,
|
||||
writer_app_version TEXT
|
||||
writer_app_version TEXT,
|
||||
verified_generation TEXT
|
||||
) STRICT;
|
||||
PRAGMA user_version = ${OPENCLAW_QUARANTINE_SCHEMA_VERSION};
|
||||
COMMIT;
|
||||
@@ -109,17 +126,23 @@ export function readOpenClawDatabaseQuarantine(
|
||||
if (userVersion === 0) {
|
||||
return undefined;
|
||||
}
|
||||
if (userVersion !== OPENCLAW_QUARANTINE_SCHEMA_VERSION) {
|
||||
if (userVersion > OPENCLAW_QUARANTINE_SCHEMA_VERSION) {
|
||||
throw new Error(
|
||||
`OpenClaw quarantine store ${storePath} uses newer schema version ${userVersion}.`,
|
||||
);
|
||||
}
|
||||
const generationColumn = userVersion >= 2 ? ", verified_generation" : "";
|
||||
const row = database
|
||||
.prepare(
|
||||
"SELECT kind, reason, quarantined_at FROM quarantined_databases WHERE path = ? LIMIT 1",
|
||||
`SELECT kind, reason, quarantined_at${generationColumn} FROM quarantined_databases WHERE path = ? LIMIT 1`,
|
||||
)
|
||||
.get(path.resolve(pathname)) as
|
||||
| { kind?: unknown; quarantined_at?: unknown; reason?: unknown }
|
||||
| {
|
||||
kind?: unknown;
|
||||
quarantined_at?: unknown;
|
||||
reason?: unknown;
|
||||
verified_generation?: unknown;
|
||||
}
|
||||
| undefined;
|
||||
if (!row) {
|
||||
return undefined;
|
||||
@@ -128,10 +151,29 @@ export function readOpenClawDatabaseQuarantine(
|
||||
(row.kind !== "agent" && row.kind !== "state") ||
|
||||
typeof row.reason !== "string" ||
|
||||
typeof row.quarantined_at !== "number" ||
|
||||
!Number.isInteger(row.quarantined_at)
|
||||
!Number.isInteger(row.quarantined_at) ||
|
||||
(row.verified_generation !== undefined &&
|
||||
row.verified_generation !== null &&
|
||||
typeof row.verified_generation !== "string")
|
||||
) {
|
||||
throw new Error(`OpenClaw quarantine store ${storePath} contains an invalid row.`);
|
||||
}
|
||||
if (typeof row.verified_generation === "string") {
|
||||
let verifiedGeneration: SqliteFileGeneration;
|
||||
try {
|
||||
verifiedGeneration = parseSqliteFileGeneration(row.verified_generation);
|
||||
} catch {
|
||||
throw new Error(`OpenClaw quarantine store ${storePath} contains an invalid row.`);
|
||||
}
|
||||
try {
|
||||
const currentGeneration = readStableSqliteFileGeneration(path.resolve(pathname));
|
||||
if (!sameSqliteFileGeneration(verifiedGeneration, currentGeneration)) {
|
||||
return undefined;
|
||||
}
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
return { kind: row.kind, quarantinedAt: row.quarantined_at, reason: row.reason };
|
||||
} finally {
|
||||
database.close();
|
||||
@@ -141,10 +183,14 @@ export function readOpenClawDatabaseQuarantine(
|
||||
/** Persist one authoritative quarantine decision. */
|
||||
export function recordOpenClawDatabaseQuarantine(options: {
|
||||
env?: NodeJS.ProcessEnv;
|
||||
generation?: SqliteFileGeneration;
|
||||
kind: OpenClawDatabaseKind;
|
||||
path: string;
|
||||
reason: string;
|
||||
}): boolean {
|
||||
const serializedGeneration = options.generation
|
||||
? serializeSqliteFileGeneration(options.generation)
|
||||
: null;
|
||||
try {
|
||||
return withQuarantineWriter(options.env ?? process.env, (database) => {
|
||||
database.exec("BEGIN IMMEDIATE;");
|
||||
@@ -153,16 +199,24 @@ export function recordOpenClawDatabaseQuarantine(options: {
|
||||
.prepare(
|
||||
`
|
||||
INSERT INTO quarantined_databases (
|
||||
path, kind, reason, quarantined_at, writer_app_version
|
||||
) VALUES (?, ?, ?, ?, ?)
|
||||
path, kind, reason, quarantined_at, writer_app_version, verified_generation
|
||||
) VALUES (?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(path) DO UPDATE SET
|
||||
kind = excluded.kind,
|
||||
reason = excluded.reason,
|
||||
quarantined_at = excluded.quarantined_at,
|
||||
writer_app_version = excluded.writer_app_version
|
||||
writer_app_version = excluded.writer_app_version,
|
||||
verified_generation = excluded.verified_generation
|
||||
`,
|
||||
)
|
||||
.run(path.resolve(options.path), options.kind, options.reason, Date.now(), VERSION);
|
||||
.run(
|
||||
path.resolve(options.path),
|
||||
options.kind,
|
||||
options.reason,
|
||||
Date.now(),
|
||||
VERSION,
|
||||
serializedGeneration,
|
||||
);
|
||||
database.exec("COMMIT;");
|
||||
return true;
|
||||
} catch (error) {
|
||||
|
||||
@@ -8,10 +8,13 @@ import {
|
||||
getNodeSqliteKysely,
|
||||
} from "../infra/kysely-sync.js";
|
||||
import { openNodeSqliteDatabase } from "../infra/node-sqlite.js";
|
||||
import type { SqliteFileGeneration } from "../infra/sqlite-file-generation.js";
|
||||
import { repairCanonicalSqliteIndexes } from "../infra/sqlite-index-schema.js";
|
||||
import {
|
||||
assertSqliteIntegrity,
|
||||
confirmSqliteFileIntegrity,
|
||||
isTerminalSqliteIntegrityError,
|
||||
type SqliteIntegrityConfirmation,
|
||||
} from "../infra/sqlite-integrity.js";
|
||||
import { prepareSqliteReadOnlyLocation } from "../infra/sqlite-readonly-location.js";
|
||||
import { migrateSqliteSchemaToStrictInTransaction } from "../infra/sqlite-strict.js";
|
||||
@@ -104,9 +107,22 @@ const terminalOpenLatch = createSqliteTerminalOpenLatch({
|
||||
},
|
||||
});
|
||||
|
||||
/** Reconfirm an advisory worker failure on the live owner connection. */
|
||||
export function confirmOpenClawStateDatabaseIntegrity(
|
||||
pathname: string,
|
||||
): SqliteIntegrityConfirmation {
|
||||
const resolvedPath = path.resolve(pathname);
|
||||
closeOpenClawStateDatabaseByPath(resolvedPath);
|
||||
return confirmSqliteFileIntegrity(resolvedPath, resolvedPath);
|
||||
}
|
||||
|
||||
/** Latch background verification damage so later opens fail without rescanning. */
|
||||
export function recordOpenClawStateDatabaseOpenFailure(pathname: string, error: Error): void {
|
||||
terminalOpenLatch.record(pathname, error);
|
||||
export function recordOpenClawStateDatabaseOpenFailure(
|
||||
pathname: string,
|
||||
error: Error,
|
||||
generation?: SqliteFileGeneration,
|
||||
): boolean {
|
||||
return terminalOpenLatch.record(pathname, error, generation);
|
||||
}
|
||||
|
||||
/** Clear a terminal open failure after doctor rewrites the database file. */
|
||||
|
||||
Reference in New Issue
Block a user