fix(state): preserve orphaned SQLite sidecars (#123680)

* fix(state): preserve orphaned SQLite sidecars

* fix(state): narrow orphan sidecar recovery guard

* test(state): satisfy orphan sidecar lint

* fix(state): quarantine orphaned SQLite sidecars

* test(state): narrow quarantine path assertion

* fix(state): copy orphaned SQLite sidecars

src/infra/sqlite-files.ts

@@ -1,7 +1,139 @@
+import { createHash } from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { createSubsystemLogger } from "../logging/subsystem.js";
+
 /** SQLite main database plus every journal-mode sidecar that can contain database pages. */
 const SQLITE_DATABASE_FILE_SUFFIXES = ["", "-wal", "-shm", "-journal"] as const;
+// SQLite WAL format: https://sqlite.org/fileformat2.html#walformat defines a 32-byte header.
+const SQLITE_WAL_HEADER_BYTES = 32;
+const SQLITE_SIDECAR_HASH_BUFFER_BYTES = 1024 * 1024;
+const sqliteFilesLog = createSubsystemLogger("state/sqlite");
+
+class SqliteOrphanedSidecarsError extends Error {
+  constructor(pathname: string, sidecarPaths: string[], cause: unknown) {
+    super(
+      `SQLite database is missing at ${pathname}, and orphaned sidecars could not be copied: ${sidecarPaths.join(", ")}. ` +
+        "Refusing to open because SQLite could delete orphan WAL or journal state. Preserve the sidecar bytes, restore the main database, and pair it with the matching sidecar before retrying.",
+      { cause },
+    );
+    this.name = "SqliteOrphanedSidecarsError";
+  }
+}
+
+type CopiedSqliteSidecar = {
+  quarantinePath: string;
+  sourcePath: string;
+};
 
 /** Resolves the main database and all possible journal-mode sidecar paths. */
 export function resolveSqliteDatabaseFilePaths(pathname: string): string[] {
   return SQLITE_DATABASE_FILE_SUFFIXES.map((suffix) => `${pathname}${suffix}`);
 }
+
+function sha256FileSync(pathname: string): string {
+  const descriptor = fs.openSync(pathname, "r");
+  const digest = createHash("sha256");
+  const buffer = Buffer.allocUnsafe(SQLITE_SIDECAR_HASH_BUFFER_BYTES);
+  try {
+    while (true) {
+      const bytesRead = fs.readSync(descriptor, buffer, 0, buffer.length, null);
+      if (bytesRead === 0) {
+        return digest.digest("hex");
+      }
+      digest.update(buffer.subarray(0, bytesRead));
+    }
+  } finally {
+    fs.closeSync(descriptor);
+  }
+}
+
+function findMatchingOrphanedSidecarCopy(
+  sourcePath: string,
+  sourceSize: number,
+): string | undefined {
+  const directory = path.dirname(sourcePath);
+  const prefix = `${path.basename(sourcePath)}.orphaned-`;
+  const candidates = fs
+    .readdirSync(directory, { withFileTypes: true })
+    .filter((entry) => entry.isFile() && entry.name.startsWith(prefix))
+    .map((entry) => path.join(directory, entry.name))
+    .filter((candidate) => fs.statSync(candidate).size === sourceSize);
+  if (candidates.length === 0) {
+    return undefined;
+  }
+  const sourceHash = sha256FileSync(sourcePath);
+  for (const candidate of candidates) {
+    if (sha256FileSync(candidate) === sourceHash) {
+      return candidate;
+    }
+  }
+  return undefined;
+}
+
+function copyOrphanedSidecar(sourcePath: string, epochMs: number): string {
+  const basePath = `${sourcePath}.orphaned-${epochMs}`;
+  for (let suffix = 0; ; suffix += 1) {
+    const candidate = suffix === 0 ? basePath : `${basePath}-${suffix}`;
+    try {
+      fs.copyFileSync(sourcePath, candidate, fs.constants.COPYFILE_EXCL);
+      return candidate;
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "EEXIST") {
+        throw error;
+      }
+    }
+  }
+}
+
+/** Preserve durable orphan sidecars before SQLite creates a replacement main database. */
+export function quarantineOrphanedSqliteSidecars(pathname: string): void {
+  if (fs.existsSync(pathname)) {
+    return;
+  }
+  const sidecars = [
+    { path: `${pathname}-wal`, minimumBytes: SQLITE_WAL_HEADER_BYTES },
+    { path: `${pathname}-journal`, minimumBytes: 0 },
+  ].flatMap((sidecar) => {
+    const stat = fs.statSync(sidecar.path, { throwIfNoEntry: false });
+    return stat?.isFile() === true && stat.size > sidecar.minimumBytes
+      ? [{ path: sidecar.path, size: stat.size }]
+      : [];
+  });
+  if (sidecars.length === 0) {
+    return;
+  }
+
+  const epochMs = Date.now();
+  const copied: CopiedSqliteSidecar[] = [];
+  try {
+    for (const sidecar of sidecars) {
+      if (findMatchingOrphanedSidecarCopy(sidecar.path, sidecar.size)) {
+        continue;
+      }
+      const quarantinePath = copyOrphanedSidecar(sidecar.path, epochMs);
+      copied.push({ quarantinePath, sourcePath: sidecar.path });
+    }
+  } catch (error) {
+    throw new SqliteOrphanedSidecarsError(
+      pathname,
+      sidecars.map((sidecar) => sidecar.path),
+      error,
+    );
+  }
+  if (copied.length === 0) {
+    return;
+  }
+
+  const copies = copied.map(
+    ({ sourcePath, quarantinePath }) => `${sourcePath} -> ${quarantinePath}`,
+  );
+  sqliteFilesLog.warn(
+    `SQLite database is missing at ${pathname}; copied orphaned sidecars: ${copies.join(", ")}. ` +
+      "Committed frames could not be applied because the main database is missing. The bytes are preserved. Recovery requires restoring the main database and pairing it with the quarantined file.",
+    {
+      databasePath: pathname,
+      copiedSidecars: copied,
+    },
+  );
+}

src/state/openclaw-agent-db.ts

@@ -5,6 +5,7 @@ import type { DatabaseSync } from "node:sqlite";
 import { enableNodeSqliteKyselyStatementCache } from "../infra/kysely-sync.js";
 import { openNodeSqliteDatabase } from "../infra/node-sqlite.js";
 import type { SqliteFileGeneration } from "../infra/sqlite-file-generation.js";
+import { quarantineOrphanedSqliteSidecars } from "../infra/sqlite-files.js";
 import {
   confirmSqliteFileIntegrity,
   isTerminalSqliteIntegrityError,
@@ -262,6 +263,7 @@ export function openOpenClawAgentDatabase(
     cachedDatabases.set(pathname, database);
     return database;
   }
+  quarantineOrphanedSqliteSidecars(pathname);
   // Latched paths are quarantined; every fresh open fails fast here until
   // doctor repairs the file and clears the latch plus the persisted row.
   const terminalFailure = terminalOpenLatch.get(pathname);

src/state/openclaw-database-orphan-sidecars.test.ts

@@ -0,0 +1,264 @@
+// Orphan-sidecar tests prove fresh database opens preserve recoverable SQLite families.
+import fs from "node:fs";
+import path from "node:path";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
+import { requireNodeSqlite } from "../infra/node-sqlite.js";
+import { quarantineOrphanedSqliteSidecars } from "../infra/sqlite-files.js";
+import {
+  closeOpenClawAgentDatabasesForTest,
+  openOpenClawAgentDatabase,
+} from "./openclaw-agent-db.js";
+import { resolveOpenClawAgentSqlitePath } from "./openclaw-agent-db.paths.js";
+import {
+  closeOpenClawStateDatabaseForTest,
+  openOpenClawStateDatabase,
+} from "./openclaw-state-db.js";
+import { resolveOpenClawStateSqlitePath } from "./openclaw-state-db.paths.js";
+
+const tempStateDirs = useAutoCleanupTempDirTracker(afterEach);
+const databaseKinds = ["state", "agent"] as const;
+const rollbackJournalContents = Buffer.from("recoverable rollback journal content");
+const shmIndexContents = Buffer.alloc(32 * 1024, 0x53);
+const emptySidecar = Buffer.alloc(0);
+const loggerMocks = vi.hoisted(() => ({ warn: vi.fn() }));
+
+vi.mock("../logging/subsystem.js", async (importOriginal) => {
+  const actual = await importOriginal<typeof import("../logging/subsystem.js")>();
+  return {
+    ...actual,
+    createSubsystemLogger: (subsystem: string) =>
+      subsystem === "state/sqlite"
+        ? { ...actual.createSubsystemLogger(subsystem), warn: loggerMocks.warn }
+        : actual.createSubsystemLogger(subsystem),
+  };
+});
+
+function createWalFixture(): { header: Buffer; withFrames: Buffer } {
+  const fixtureDir = fs.realpathSync(tempStateDirs.make("openclaw-wal-header-"));
+  const databasePath = path.join(fixtureDir, "header.sqlite");
+  const { DatabaseSync } = requireNodeSqlite();
+  const database = new DatabaseSync(databasePath);
+  try {
+    database.exec("PRAGMA journal_mode = WAL; PRAGMA wal_autocheckpoint = 0;");
+    database.exec("CREATE TABLE header_probe (value TEXT);");
+    const wal = fs.readFileSync(`${databasePath}-wal`);
+    if (wal.length <= 32) {
+      throw new Error("SQLite did not write a WAL frame for the header fixture");
+    }
+    return {
+      header: Buffer.from(wal.subarray(0, 32)),
+      withFrames: Buffer.from(wal),
+    };
+  } finally {
+    database.close();
+  }
+}
+
+const walFixture = createWalFixture();
+
+afterEach(() => {
+  closeOpenClawAgentDatabasesForTest();
+  closeOpenClawStateDatabaseForTest();
+  loggerMocks.warn.mockReset();
+  vi.restoreAllMocks();
+});
+
+function prepareCase(kind: (typeof databaseKinds)[number]) {
+  const stateDir = fs.realpathSync(tempStateDirs.make("openclaw-orphan-sidecar-"));
+  const env = { OPENCLAW_STATE_DIR: stateDir };
+  const databasePath =
+    kind === "state"
+      ? resolveOpenClawStateSqlitePath(env)
+      : resolveOpenClawAgentSqlitePath({ agentId: "main", env });
+  fs.mkdirSync(path.dirname(databasePath), { recursive: true });
+  return { databasePath, env };
+}
+
+function openDatabase(kind: (typeof databaseKinds)[number], env: NodeJS.ProcessEnv) {
+  return kind === "state"
+    ? openOpenClawStateDatabase({ env })
+    : openOpenClawAgentDatabase({ agentId: "main", env });
+}
+
+function listQuarantinePaths(sourcePath: string): string[] {
+  const prefix = `${path.basename(sourcePath)}.orphaned-`;
+  return fs
+    .readdirSync(path.dirname(sourcePath))
+    .filter((entry) => entry.startsWith(prefix))
+    .map((entry) => path.join(path.dirname(sourcePath), entry));
+}
+
+describe("orphan SQLite sidecar admission", () => {
+  const recoverableCases = [
+    {
+      label: "real WAL with committed frames",
+      suffix: "-wal",
+      contents: walFixture.withFrames,
+      benignSidecars: [
+        { suffix: "-shm", contents: shmIndexContents },
+        { suffix: "-journal", contents: emptySidecar },
+      ],
+    },
+    {
+      label: "non-empty rollback journal",
+      suffix: "-journal",
+      contents: rollbackJournalContents,
+      benignSidecars: [
+        { suffix: "-wal", contents: walFixture.header },
+        { suffix: "-shm", contents: shmIndexContents },
+      ],
+    },
+  ] as const;
+
+  describe("recoverable sidecars", () => {
+    for (const testCase of recoverableCases) {
+      for (const kind of databaseKinds) {
+        const regression =
+          kind === "state" && testCase.suffix === "-wal" ? " (plugin lifecycle regression)" : "";
+        it(`opens a missing ${kind} database after copying a ${testCase.label}${regression}`, () => {
+          const { databasePath, env } = prepareCase(kind);
+          const sidecars = [
+            { suffix: testCase.suffix, contents: testCase.contents },
+            ...testCase.benignSidecars,
+          ].map((sidecar) => ({
+            contents: sidecar.contents,
+            path: `${databasePath}${sidecar.suffix}`,
+            suffix: sidecar.suffix,
+          }));
+          for (const sidecar of sidecars) {
+            fs.writeFileSync(sidecar.path, sidecar.contents);
+          }
+
+          const database = openDatabase(kind, env);
+          const sourcePath = `${databasePath}${testCase.suffix}`;
+          const quarantinePaths = listQuarantinePaths(sourcePath);
+
+          expect(database.db.isOpen).toBe(true);
+          expect(fs.existsSync(databasePath)).toBe(true);
+          expect(quarantinePaths).toHaveLength(1);
+          const quarantinePath = quarantinePaths.at(0);
+          if (!quarantinePath) {
+            throw new Error(`missing quarantine for ${sourcePath}`);
+          }
+          expect(fs.readFileSync(quarantinePath)).toEqual(testCase.contents);
+          if (fs.existsSync(sourcePath)) {
+            // A successful WAL-mode open may create a new sidecar at the canonical path.
+            expect(fs.readFileSync(sourcePath)).not.toEqual(testCase.contents);
+          }
+          expect(loggerMocks.warn).toHaveBeenCalledOnce();
+          expect(loggerMocks.warn).toHaveBeenCalledWith(
+            expect.stringContaining(quarantinePath),
+            expect.objectContaining({
+              databasePath,
+              copiedSidecars: [{ quarantinePath, sourcePath }],
+            }),
+          );
+          expect(String(loggerMocks.warn.mock.calls[0]?.[0])).toContain(
+            "Committed frames could not be applied because the main database is missing",
+          );
+          expect(String(loggerMocks.warn.mock.calls[0]?.[0])).toContain(
+            "Recovery requires restoring the main database and pairing it with the quarantined file",
+          );
+        });
+      }
+    }
+  });
+
+  const nonBlockingCases = [
+    { label: "lone SHM index", suffix: "-shm", contents: shmIndexContents },
+    { label: "zero-byte WAL", suffix: "-wal", contents: emptySidecar },
+    { label: "32-byte header-only WAL", suffix: "-wal", contents: walFixture.header },
+    { label: "zero-byte rollback journal", suffix: "-journal", contents: emptySidecar },
+  ] as const;
+
+  describe("non-blocking sidecars", () => {
+    for (const testCase of nonBlockingCases) {
+      for (const kind of databaseKinds) {
+        it(`creates a missing ${kind} database with a ${testCase.label}`, () => {
+          const { databasePath, env } = prepareCase(kind);
+          fs.writeFileSync(`${databasePath}${testCase.suffix}`, testCase.contents);
+
+          const database = openDatabase(kind, env);
+
+          expect(database.db.isOpen).toBe(true);
+          expect(fs.existsSync(databasePath)).toBe(true);
+          expect(database.db.prepare("PRAGMA integrity_check").get()).toEqual({
+            integrity_check: "ok",
+          });
+          expect(listQuarantinePaths(`${databasePath}${testCase.suffix}`)).toEqual([]);
+          expect(loggerMocks.warn).not.toHaveBeenCalled();
+        });
+      }
+    }
+  });
+
+  it("uses a unique suffix instead of overwriting an existing quarantine", () => {
+    const { databasePath, env } = prepareCase("state");
+    const sourcePath = `${databasePath}-wal`;
+    const epochMs = 1_786_738_000_000;
+    const existingQuarantinePath = `${sourcePath}.orphaned-${epochMs}`;
+    const newQuarantinePath = `${existingQuarantinePath}-1`;
+    const existingContents = Buffer.from("previously quarantined WAL");
+    fs.writeFileSync(sourcePath, walFixture.withFrames);
+    fs.writeFileSync(existingQuarantinePath, existingContents);
+    vi.spyOn(Date, "now").mockReturnValue(epochMs);
+
+    const database = openDatabase("state", env);
+
+    expect(database.db.isOpen).toBe(true);
+    expect(fs.readFileSync(existingQuarantinePath)).toEqual(existingContents);
+    expect(fs.readFileSync(newQuarantinePath)).toEqual(walFixture.withFrames);
+    expect(loggerMocks.warn).toHaveBeenCalledWith(
+      expect.stringContaining(newQuarantinePath),
+      expect.any(Object),
+    );
+  });
+
+  it("leaves the live sidecar untouched and reuses an identical preserved copy", () => {
+    const { databasePath } = prepareCase("state");
+    const sourcePath = `${databasePath}-wal`;
+    fs.writeFileSync(sourcePath, walFixture.withFrames);
+
+    quarantineOrphanedSqliteSidecars(databasePath);
+    const quarantinePaths = listQuarantinePaths(sourcePath);
+
+    expect(quarantinePaths).toHaveLength(1);
+    expect(fs.readFileSync(sourcePath)).toEqual(walFixture.withFrames);
+    expect(fs.readFileSync(quarantinePaths[0] ?? "")).toEqual(walFixture.withFrames);
+    loggerMocks.warn.mockClear();
+
+    quarantineOrphanedSqliteSidecars(databasePath);
+
+    expect(listQuarantinePaths(sourcePath)).toEqual(quarantinePaths);
+    expect(fs.readFileSync(sourcePath)).toEqual(walFixture.withFrames);
+    expect(loggerMocks.warn).not.toHaveBeenCalled();
+  });
+
+  it("fails closed with the typed error when quarantine copy fails", () => {
+    const { databasePath, env } = prepareCase("state");
+    const sourcePath = `${databasePath}-wal`;
+    fs.writeFileSync(sourcePath, walFixture.withFrames);
+    const copyError = Object.assign(new Error("read-only volume"), { code: "EROFS" });
+    vi.spyOn(fs, "copyFileSync").mockImplementationOnce(() => {
+      throw copyError;
+    });
+
+    let thrown: unknown;
+    try {
+      openDatabase("state", env);
+    } catch (error) {
+      thrown = error;
+    }
+
+    expect(thrown).toBeInstanceOf(Error);
+    expect((thrown as Error).name).toBe("SqliteOrphanedSidecarsError");
+    expect((thrown as Error).message).toContain(databasePath);
+    expect((thrown as Error).message).toContain(sourcePath);
+    expect((thrown as Error).message).toContain("restore the main database");
+    expect(fs.existsSync(databasePath)).toBe(false);
+    expect(fs.readFileSync(sourcePath)).toEqual(walFixture.withFrames);
+    expect(listQuarantinePaths(sourcePath)).toEqual([]);
+    expect(loggerMocks.warn).not.toHaveBeenCalled();
+  });
+});

src/state/openclaw-state-ownership.ts

@@ -16,6 +16,7 @@ import {
   runWithSqliteCoordinator,
   SqliteCoordinatorError,
 } from "../infra/sqlite-coordinator.js";
+import { quarantineOrphanedSqliteSidecars } from "../infra/sqlite-files.js";
 import {
   prepareSqliteReadOnlyLocation,
   prepareSqliteReadOnlyLocationSync,
@@ -244,6 +245,7 @@ function acquireOpenClawStateWriteAccess(options: {
   const resolvedPath = path.resolve(options.databasePath);
   const access = acquireOpenClawStateOwnershipCoordinator(resolvedPath);
   try {
+    quarantineOrphanedSqliteSidecars(resolvedPath);
     assertOwnershipAllowsWrite(
       inspectOpenClawStateOwnershipAtPathWhileCoordinatorHeld(resolvedPath),
       resolvedPath,
@@ -288,6 +290,7 @@ export async function assertOpenClawStateWriteAllowedAtPath(options: {
   env?: NodeJS.ProcessEnv;
 }): Promise<void> {
   const databasePath = path.resolve(options.databasePath);
+  quarantineOrphanedSqliteSidecars(databasePath);
   if (!existsSync(databasePath)) {
     return;
   }