diff --git a/src/snapshot/local-repository.test.ts b/src/snapshot/local-repository.test.ts deleted file mode 100644 index 4fbf0b948aa0..000000000000 --- a/src/snapshot/local-repository.test.ts +++ /dev/null @@ -1,958 +0,0 @@ -import fs from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; -import { afterEach, describe, expect, it, vi } from "vitest"; -import { requireNodeSqlite } from "../infra/node-sqlite.js"; -import { OPENCLAW_AGENT_SCHEMA_VERSION } from "../state/openclaw-agent-db.js"; -import { OPENCLAW_STATE_SCHEMA_VERSION } from "../state/openclaw-state-db.js"; -import { createLocalSqliteSnapshotProvider } from "./local-repository.js"; -import { hashSnapshotArtifact, parseSnapshotManifest, readSnapshotManifest } from "./manifest.js"; -import { - SNAPSHOT_MANIFEST_FILENAME, - SNAPSHOT_SQLITE_FILENAME, - type SnapshotManifest, - type SnapshotResult, -} from "./snapshot-provider.js"; - -const tempDirs: string[] = []; - -async function createTempDir(): Promise { - const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-snapshot-repository-")); - tempDirs.push(tempDir); - return tempDir; -} - -afterEach(async () => { - await Promise.all(tempDirs.splice(0).map((tempDir) => fs.rm(tempDir, { recursive: true }))); -}); - -function createGenericDatabase( - databasePath: string, - options: { userVersion?: number; values?: string[]; wal?: boolean } = {}, -): void { - const sqlite = requireNodeSqlite(); - const database = new sqlite.DatabaseSync(databasePath); - try { - database.exec(` - ${options.wal ? "PRAGMA journal_mode = WAL; PRAGMA wal_autocheckpoint = 0;" : ""} - PRAGMA user_version = ${options.userVersion ?? 7}; - CREATE TABLE entries ( - id INTEGER PRIMARY KEY, - value TEXT NOT NULL - ); - `); - const insert = database.prepare("INSERT INTO entries (value) VALUES (?)"); - for (const value of options.values ?? ["one"]) { - insert.run(value); - } - } finally { - database.close(); - } -} - -function createGlobalDatabase(databasePath: string): void { - const sqlite = requireNodeSqlite(); - const database = new sqlite.DatabaseSync(databasePath); - try { - database.exec(` - PRAGMA user_version = ${OPENCLAW_STATE_SCHEMA_VERSION}; - CREATE TABLE schema_meta ( - meta_key TEXT PRIMARY KEY, - role TEXT NOT NULL, - schema_version INTEGER NOT NULL - ); - INSERT INTO schema_meta VALUES ('primary', 'global', ${OPENCLAW_STATE_SCHEMA_VERSION}); - CREATE TABLE delivery_queue_entries ( - id TEXT PRIMARY KEY, - payload TEXT NOT NULL - ); - INSERT INTO delivery_queue_entries VALUES ('queued', 'do-not-restore'); - `); - } finally { - database.close(); - } -} - -function createAgentDatabase(databasePath: string, agentId: string): void { - const sqlite = requireNodeSqlite(); - const database = new sqlite.DatabaseSync(databasePath); - try { - database.exec(` - PRAGMA user_version = ${OPENCLAW_AGENT_SCHEMA_VERSION}; - CREATE TABLE schema_meta ( - meta_key TEXT PRIMARY KEY, - role TEXT NOT NULL, - schema_version INTEGER NOT NULL, - agent_id TEXT - ); - `); - database - .prepare("INSERT INTO schema_meta VALUES ('primary', 'agent', ?, ?)") - .run(OPENCLAW_AGENT_SCHEMA_VERSION, agentId); - } finally { - database.close(); - } -} - -function createUnsafeIndexDrift(databasePath: string): void { - const sqlite = requireNodeSqlite(); - const database = new sqlite.DatabaseSync(databasePath); - try { - database.exec(` - CREATE TABLE records ( - id INTEGER PRIMARY KEY, - indexed_value TEXT NOT NULL, - alternate_value TEXT NOT NULL - ); - CREATE INDEX records_value ON records(indexed_value); - INSERT INTO records (indexed_value, alternate_value) - VALUES ('alpha', 'zeta'), ('beta', 'eta'), ('gamma', 'theta'); - PRAGMA writable_schema = ON; - `); - database - .prepare( - "UPDATE sqlite_schema SET sql = 'CREATE INDEX records_value ON records(alternate_value)' WHERE name = 'records_value'", - ) - .run(); - const schemaVersion = Number( - Object.values(database.prepare("PRAGMA schema_version").get() as Record)[0], - ); - database.exec(`PRAGMA writable_schema = OFF; PRAGMA schema_version = ${schemaVersion + 1};`); - } finally { - database.close(); - } -} - -async function rewriteManifest( - result: SnapshotResult, - mutate: (manifest: SnapshotManifest) => SnapshotManifest, -): Promise { - const manifestPath = path.join(result.ref.path, SNAPSHOT_MANIFEST_FILENAME); - const manifest = await readSnapshotManifest(result.ref.path); - await fs.writeFile(manifestPath, `${JSON.stringify(mutate(manifest), null, 2)}\n`); -} - -async function refreshArtifactManifest(result: SnapshotResult): Promise { - const digest = await hashSnapshotArtifact(result.ref.path); - await rewriteManifest(result, (manifest) => ({ - ...manifest, - artifact: { - ...manifest.artifact, - sha256: digest.sha256, - sizeBytes: digest.sizeBytes, - }, - })); -} - -describe("local SQLite snapshot repository", () => { - it("creates, lists, verifies, and fresh-restores committed WAL state", async () => { - const tempDir = await createTempDir(); - const sourcePath = path.join(tempDir, "source.sqlite"); - const repositoryPath = path.join(tempDir, "snapshots"); - const restorePath = path.join(tempDir, "restore", "source.sqlite"); - const sqlite = requireNodeSqlite(); - const source = new sqlite.DatabaseSync(sourcePath); - try { - source.exec(` - PRAGMA journal_mode = WAL; - PRAGMA wal_autocheckpoint = 0; - PRAGMA user_version = 42; - CREATE TABLE entries (id INTEGER PRIMARY KEY, value TEXT NOT NULL); - INSERT INTO entries (value) VALUES ('checkpointed'); - PRAGMA wal_checkpoint(TRUNCATE); - INSERT INTO entries (value) VALUES ('committed-in-wal'); - `); - const provider = createLocalSqliteSnapshotProvider({ - repositoryPath, - now: () => new Date("2026-07-12T14:00:00.000Z"), - }); - const snapshot = await provider.create({ - path: sourcePath, - identity: { role: "generic", id: "test-database" }, - }); - - expect(snapshot.manifest).toMatchObject({ - schemaVersion: 1, - createdAt: "2026-07-12T14:00:00.000Z", - database: { - role: "generic", - id: "test-database", - basename: "source.sqlite", - userVersion: 42, - }, - artifact: { - path: SNAPSHOT_SQLITE_FILENAME, - }, - }); - expect(snapshot.manifest.artifact.sha256).toMatch(/^[a-f0-9]{64}$/u); - await expect(provider.verify(snapshot.ref)).resolves.toEqual({ - ok: true, - manifest: snapshot.manifest, - }); - await expect(provider.list()).resolves.toEqual([snapshot]); - await expect(provider.restoreFresh(snapshot.ref, restorePath)).resolves.toEqual({ - ok: true, - manifest: snapshot.manifest, - }); - await expect(fs.readFile(restorePath)).resolves.toEqual( - await fs.readFile(path.join(snapshot.ref.path, SNAPSHOT_SQLITE_FILENAME)), - ); - expect((await fs.readdir(repositoryPath)).every((name) => !name.startsWith(".tmp-"))).toBe( - true, - ); - await expect(fs.readdir(path.dirname(restorePath))).resolves.toEqual(["source.sqlite"]); - } finally { - source.close(); - } - - const restored = new sqlite.DatabaseSync(restorePath, { readOnly: true }); - try { - expect(restored.prepare("SELECT value FROM entries ORDER BY id").all()).toEqual([ - { value: "checkpointed" }, - { value: "committed-in-wal" }, - ]); - expect(restored.prepare("PRAGMA user_version").get()).toEqual({ user_version: 42 }); - } finally { - restored.close(); - } - if (process.platform !== "win32") { - expect((await fs.stat(repositoryPath)).mode & 0o777).toBe(0o700); - expect((await fs.stat(restorePath)).mode & 0o777).toBe(0o600); - } - }); - - it("sorts snapshots newest first and ignores incomplete staging directories", async () => { - const tempDir = await createTempDir(); - const sourcePath = path.join(tempDir, "source.sqlite"); - const repositoryPath = path.join(tempDir, "snapshots"); - createGenericDatabase(sourcePath); - const dates = [new Date("2026-07-12T14:00:00.000Z"), new Date("2026-07-12T14:01:00.000Z")]; - const provider = createLocalSqliteSnapshotProvider({ - repositoryPath, - now: () => dates.shift() ?? new Date("invalid"), - }); - const first = await provider.create({ - path: sourcePath, - identity: { role: "generic", id: "test-database" }, - }); - const second = await provider.create({ - path: sourcePath, - identity: { role: "generic", id: "test-database" }, - }); - await fs.mkdir(path.join(repositoryPath, ".tmp-interrupted")); - await fs.mkdir(path.join(repositoryPath, "interrupted-final")); - await fs.writeFile(path.join(repositoryPath, "interrupted-final", ".pending"), ""); - await fs.mkdir(path.join(repositoryPath, "empty-final")); - - await expect(provider.list()).resolves.toEqual([second, first]); - }); - - it("never replaces a snapshot directory raced into place", async () => { - const tempDir = await createTempDir(); - const sourcePath = path.join(tempDir, "source.sqlite"); - const repositoryPath = path.join(tempDir, "snapshots"); - createGenericDatabase(sourcePath); - const provider = createLocalSqliteSnapshotProvider({ - repositoryPath, - now: () => new Date("2026-07-12T14:00:00.000Z"), - }); - const originalMkdir = fs.mkdir.bind(fs); - let racedPath: string | undefined; - const mkdirSpy = vi.spyOn(fs, "mkdir").mockImplementation(async (directoryPath, options) => { - const resolvedPath = path.resolve(String(directoryPath)); - if ( - path.dirname(resolvedPath) === repositoryPath && - !path.basename(resolvedPath).startsWith(".tmp-") - ) { - racedPath = resolvedPath; - await originalMkdir(resolvedPath, options); - await fs.writeFile(path.join(resolvedPath, "keep"), "racer"); - } - return await originalMkdir(directoryPath, options); - }); - - try { - await expect( - provider.create({ - path: sourcePath, - identity: { role: "generic", id: "directory-race" }, - }), - ).rejects.toThrow(/directory already exists/u); - } finally { - mkdirSpy.mockRestore(); - } - expect(racedPath).toBeDefined(); - await expect(fs.readFile(path.join(racedPath!, "keep"), "utf8")).resolves.toBe("racer"); - }); - - it("rejects an artifact changed after entering the final directory", async () => { - const tempDir = await createTempDir(); - const sourcePath = path.join(tempDir, "source.sqlite"); - const repositoryPath = path.join(tempDir, "snapshots"); - createGenericDatabase(sourcePath); - const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); - const originalLink = fs.link.bind(fs); - const linkSpy = vi.spyOn(fs, "link").mockImplementation(async (source, target) => { - await originalLink(source, target); - if ( - path.basename(String(target)) === SNAPSHOT_MANIFEST_FILENAME && - !path.basename(path.dirname(String(target))).startsWith(".tmp-") - ) { - await fs.appendFile( - path.join(path.dirname(String(target)), SNAPSHOT_SQLITE_FILENAME), - "changed-after-final-move", - ); - } - }); - - try { - await expect( - provider.create({ - path: sourcePath, - identity: { role: "generic", id: "final-directory-race" }, - }), - ).rejects.toThrow(/size mismatch/u); - await expect(provider.list()).resolves.toEqual([]); - } finally { - linkSpy.mockRestore(); - } - }); - - it("cleans a linked entry when post-link inspection fails", async () => { - const tempDir = await createTempDir(); - const sourcePath = path.join(tempDir, "source.sqlite"); - const repositoryPath = path.join(tempDir, "snapshots"); - createGenericDatabase(sourcePath); - const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); - const originalLink = fs.link.bind(fs); - const originalLstat = fs.lstat.bind(fs); - let linkedArtifactPath: string | undefined; - let failedInspection = false; - const linkSpy = vi.spyOn(fs, "link").mockImplementation(async (source, target) => { - await originalLink(source, target); - if ( - path.basename(String(target)) === SNAPSHOT_SQLITE_FILENAME && - !path.basename(path.dirname(String(target))).startsWith(".tmp-") - ) { - linkedArtifactPath = path.resolve(String(target)); - } - }); - const lstatSpy = vi.spyOn(fs, "lstat").mockImplementation(async (filePath) => { - if ( - linkedArtifactPath && - !failedInspection && - path.resolve(String(filePath)) === linkedArtifactPath - ) { - failedInspection = true; - throw Object.assign(new Error("post-link inspection failed"), { code: "EIO" }); - } - return await originalLstat(filePath); - }); - - try { - await expect( - provider.create({ - path: sourcePath, - identity: { role: "generic", id: "post-link-inspection" }, - }), - ).rejects.toThrow(/post-link inspection failed/u); - await expect(provider.list()).resolves.toEqual([]); - await expect(fs.readdir(repositoryPath)).resolves.toEqual([]); - } finally { - lstatSpy.mockRestore(); - linkSpy.mockRestore(); - } - }); - - it("never overwrites a file raced into the final snapshot directory", async () => { - const tempDir = await createTempDir(); - const sourcePath = path.join(tempDir, "source.sqlite"); - const repositoryPath = path.join(tempDir, "snapshots"); - createGenericDatabase(sourcePath); - const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); - const originalLink = fs.link.bind(fs); - let racedPath: string | undefined; - const linkSpy = vi.spyOn(fs, "link").mockImplementation(async (source, target) => { - const targetPath = path.resolve(String(target)); - if ( - path.basename(targetPath) === SNAPSHOT_SQLITE_FILENAME && - path.dirname(targetPath) !== repositoryPath && - !path.basename(path.dirname(targetPath)).startsWith(".tmp-") - ) { - racedPath = targetPath; - await fs.writeFile(targetPath, "racer", { flag: "wx" }); - } - await originalLink(source, target); - }); - - try { - await expect( - provider.create({ - path: sourcePath, - identity: { role: "generic", id: "entry-race" }, - }), - ).rejects.toThrow(/EEXIST/u); - } finally { - linkSpy.mockRestore(); - } - expect(racedPath).toBeDefined(); - await expect(fs.readFile(racedPath!, "utf8")).resolves.toBe("racer"); - }); - - it("sanitizes transient global delivery rows and enforces the global owner", async () => { - const tempDir = await createTempDir(); - const sourcePath = path.join(tempDir, "openclaw.sqlite"); - const repositoryPath = path.join(tempDir, "snapshots"); - createGlobalDatabase(sourcePath); - const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); - const snapshot = await provider.create({ - path: sourcePath, - identity: { role: "global" }, - }); - const artifactPath = path.join(snapshot.ref.path, SNAPSHOT_SQLITE_FILENAME); - expect((await fs.readFile(artifactPath)).includes("do-not-restore")).toBe(false); - const sqlite = requireNodeSqlite(); - const artifact = new sqlite.DatabaseSync(artifactPath, { readOnly: true }); - try { - expect( - artifact.prepare("SELECT COUNT(*) AS count FROM delivery_queue_entries").get(), - ).toEqual({ count: 0 }); - } finally { - artifact.close(); - } - - const wrongRolePath = path.join(tempDir, "wrong-role.sqlite"); - createAgentDatabase(wrongRolePath, "main"); - const wrongRole = new sqlite.DatabaseSync(wrongRolePath); - wrongRole.exec(`PRAGMA user_version = ${OPENCLAW_STATE_SCHEMA_VERSION};`); - wrongRole.close(); - await expect( - provider.create({ path: wrongRolePath, identity: { role: "global" } }), - ).rejects.toThrow(/expected global/u); - }); - - it("enforces the exact agent owner and canonical agent id", async () => { - const tempDir = await createTempDir(); - const sourcePath = path.join(tempDir, "openclaw-agent.sqlite"); - const repositoryPath = path.join(tempDir, "snapshots"); - createAgentDatabase(sourcePath, "worker-1"); - const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); - - await expect( - provider.create({ - path: sourcePath, - identity: { role: "agent", agentId: "worker-2" }, - }), - ).rejects.toThrow(/belongs to agent worker-1/u); - await expect( - provider.create({ - path: sourcePath, - identity: { role: "agent", agentId: "Worker-1" }, - }), - ).rejects.toThrow(/must be canonical/u); - await expect( - provider.create({ - path: sourcePath, - identity: { role: "agent", agentId: "worker-1" }, - }), - ).resolves.toMatchObject({ - manifest: { - database: { - role: "agent", - agentId: "worker-1", - userVersion: OPENCLAW_AGENT_SCHEMA_VERSION, - }, - }, - }); - }); - - it("rejects foreign-key violations and unsafe index definitions at creation", async () => { - const tempDir = await createTempDir(); - const repositoryPath = path.join(tempDir, "snapshots"); - const foreignKeyPath = path.join(tempDir, "foreign-key.sqlite"); - const sqlite = requireNodeSqlite(); - const foreignKeyDatabase = new sqlite.DatabaseSync(foreignKeyPath); - try { - foreignKeyDatabase.exec(` - PRAGMA foreign_keys = OFF; - CREATE TABLE parents (id INTEGER PRIMARY KEY); - CREATE TABLE children ( - id INTEGER PRIMARY KEY, - parent_id INTEGER REFERENCES parents(id) - ); - INSERT INTO children VALUES (1, 99); - `); - } finally { - foreignKeyDatabase.close(); - } - const unsafeIndexPath = path.join(tempDir, "unsafe-index.sqlite"); - createUnsafeIndexDrift(unsafeIndexPath); - const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); - - await expect( - provider.create({ - path: foreignKeyPath, - identity: { role: "generic", id: "foreign-key" }, - }), - ).rejects.toThrow(/foreign_key_check failed/u); - await expect( - provider.create({ - path: unsafeIndexPath, - identity: { role: "generic", id: "unsafe-index" }, - }), - ).rejects.toThrow(/integrity_check failed|malformed database schema/iu); - await expect(provider.list()).resolves.toEqual([]); - }); - - it("detects artifact hash, user_version, and unsafe-index drift after creation", async () => { - const tempDir = await createTempDir(); - const sourcePath = path.join(tempDir, "source.sqlite"); - const repositoryPath = path.join(tempDir, "snapshots"); - createGenericDatabase(sourcePath); - const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); - - const hashSnapshot = await provider.create({ - path: sourcePath, - identity: { role: "generic", id: "hash" }, - }); - await fs.appendFile(path.join(hashSnapshot.ref.path, SNAPSHOT_SQLITE_FILENAME), "tamper"); - await expect(provider.verify(hashSnapshot.ref)).rejects.toThrow(/size mismatch/u); - - const versionSnapshot = await provider.create({ - path: sourcePath, - identity: { role: "generic", id: "version" }, - }); - const sqlite = requireNodeSqlite(); - const versionDatabase = new sqlite.DatabaseSync( - path.join(versionSnapshot.ref.path, SNAPSHOT_SQLITE_FILENAME), - ); - versionDatabase.exec("PRAGMA user_version = 99;"); - versionDatabase.close(); - await refreshArtifactManifest(versionSnapshot); - await expect(provider.verify(versionSnapshot.ref)).rejects.toThrow(/user_version mismatch/u); - - const unsafeSnapshot = await provider.create({ - path: sourcePath, - identity: { role: "generic", id: "unsafe" }, - }); - const unsafePath = path.join(unsafeSnapshot.ref.path, SNAPSHOT_SQLITE_FILENAME); - const unsafeDatabase = new sqlite.DatabaseSync(unsafePath); - try { - unsafeDatabase.exec(` - CREATE TABLE indexed_records ( - id INTEGER PRIMARY KEY, - indexed_value TEXT NOT NULL, - alternate_value TEXT NOT NULL - ); - CREATE INDEX indexed_records_value ON indexed_records(indexed_value); - INSERT INTO indexed_records (indexed_value, alternate_value) - VALUES ('alpha', 'zeta'), ('beta', 'eta'); - PRAGMA writable_schema = ON; - `); - unsafeDatabase - .prepare( - "UPDATE sqlite_schema SET sql = 'CREATE INDEX indexed_records_value ON indexed_records(alternate_value)' WHERE name = 'indexed_records_value'", - ) - .run(); - const schemaVersion = Number( - Object.values( - unsafeDatabase.prepare("PRAGMA schema_version").get() as Record, - )[0], - ); - unsafeDatabase.exec( - `PRAGMA writable_schema = OFF; PRAGMA schema_version = ${schemaVersion + 1};`, - ); - } finally { - unsafeDatabase.close(); - } - await refreshArtifactManifest(unsafeSnapshot); - await expect(provider.verify(unsafeSnapshot.ref)).rejects.toThrow( - /integrity_check failed|malformed database schema/iu, - ); - }); - - it("never overwrites an existing target or orphan SQLite sidecar", async () => { - const tempDir = await createTempDir(); - const sourcePath = path.join(tempDir, "source.sqlite"); - const repositoryPath = path.join(tempDir, "snapshots"); - const restorePath = path.join(tempDir, "restore", "source.sqlite"); - createGenericDatabase(sourcePath); - const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); - const snapshot = await provider.create({ - path: sourcePath, - identity: { role: "generic", id: "restore" }, - }); - await fs.mkdir(path.dirname(restorePath), { recursive: true }); - await fs.writeFile(restorePath, "keep"); - - await expect(provider.restoreFresh(snapshot.ref, restorePath)).rejects.toThrow( - /restore path already exists/u, - ); - await expect(fs.readFile(restorePath, "utf8")).resolves.toBe("keep"); - - await fs.unlink(restorePath); - await fs.writeFile(`${restorePath}-wal`, "keep-wal"); - await expect(provider.restoreFresh(snapshot.ref, restorePath)).rejects.toThrow( - /restore path already exists/u, - ); - await expect(fs.readFile(`${restorePath}-wal`, "utf8")).resolves.toBe("keep-wal"); - await expect(fs.access(restorePath)).rejects.toMatchObject({ code: "ENOENT" }); - }); - - it("fails closed when fresh restore cannot publish atomically", async () => { - const tempDir = await createTempDir(); - const sourcePath = path.join(tempDir, "source.sqlite"); - const repositoryPath = path.join(tempDir, "snapshots"); - const restorePath = path.join(tempDir, "restore", "source.sqlite"); - createGenericDatabase(sourcePath); - const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); - const snapshot = await provider.create({ - path: sourcePath, - identity: { role: "generic", id: "atomic-restore" }, - }); - const linkSpy = vi - .spyOn(fs, "link") - .mockRejectedValue(Object.assign(new Error("hard links unsupported"), { code: "ENOTSUP" })); - - try { - await expect(provider.restoreFresh(snapshot.ref, restorePath)).rejects.toThrow( - /requires hard-link support/u, - ); - await expect(fs.access(restorePath)).rejects.toMatchObject({ code: "ENOENT" }); - } finally { - linkSpy.mockRestore(); - } - }); - - it("rejects restore targets inside the snapshot repository", async () => { - const tempDir = await createTempDir(); - const sourcePath = path.join(tempDir, "source.sqlite"); - const repositoryPath = path.join(tempDir, "snapshots"); - createGenericDatabase(sourcePath); - const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); - const snapshot = await provider.create({ - path: sourcePath, - identity: { role: "generic", id: "repository-boundary" }, - }); - - await expect( - provider.restoreFresh(snapshot.ref, path.join(repositoryPath, "restored.sqlite")), - ).rejects.toThrow(/outside snapshot repository/u); - await expect( - provider.restoreFresh(snapshot.ref, path.join(snapshot.ref.path, "restored.sqlite")), - ).rejects.toThrow(/outside snapshot repository/u); - await expect(provider.verify(snapshot.ref)).resolves.toMatchObject({ ok: true }); - }); - - it.runIf(process.platform !== "win32")( - "binds restore to the exact artifact bytes recorded by the manifest", - async () => { - const tempDir = await createTempDir(); - const sourcePath = path.join(tempDir, "source.sqlite"); - const repositoryPath = path.join(tempDir, "snapshots"); - const restorePath = path.join(tempDir, "restore", "source.sqlite"); - createGenericDatabase(sourcePath); - const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); - const snapshot = await provider.create({ - path: sourcePath, - identity: { role: "generic", id: "verified-bytes" }, - }); - const artifactPath = path.join(snapshot.ref.path, SNAPSHOT_SQLITE_FILENAME); - const originalOpen = fs.open.bind(fs); - const openSpy = vi.spyOn(fs, "open").mockImplementation(async (filePath, flags, mode) => { - const handle = await originalOpen(filePath, flags, mode); - if ( - flags === "wx+" && - path.basename(String(filePath)) === SNAPSHOT_SQLITE_FILENAME && - path.basename(path.dirname(String(filePath))).startsWith(".tmp-restore-") - ) { - const sqlite = requireNodeSqlite(); - const database = new sqlite.DatabaseSync(artifactPath); - database.prepare("INSERT INTO entries (value) VALUES (?)").run("raced"); - database.close(); - } - return handle; - }); - - try { - await expect(provider.restoreFresh(snapshot.ref, restorePath)).rejects.toThrow( - /hash mismatch|size mismatch/u, - ); - await expect(fs.access(restorePath)).rejects.toMatchObject({ code: "ENOENT" }); - } finally { - openSpy.mockRestore(); - } - }, - ); - - it.runIf(process.platform !== "win32")( - "never publishes replacement bytes when the pinned staging pathname changes", - async () => { - const tempDir = await createTempDir(); - const sourcePath = path.join(tempDir, "source.sqlite"); - const repositoryPath = path.join(tempDir, "snapshots"); - const restorePath = path.join(tempDir, "restore", "source.sqlite"); - createGenericDatabase(sourcePath, { values: ["original"] }); - const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); - const snapshot = await provider.create({ - path: sourcePath, - identity: { role: "generic", id: "pinned-staging" }, - }); - const originalOpen = fs.open.bind(fs); - const openSpy = vi.spyOn(fs, "open").mockImplementation(async (filePath, flags, mode) => { - if ( - flags === "wx+" && - path.basename(path.dirname(String(filePath))).startsWith(".sqlite-publish-") - ) { - const stagingEntry = (await fs.readdir(repositoryPath)).find((entry) => - entry.startsWith(".tmp-restore-"), - ); - if (!stagingEntry) { - throw new Error("restore staging directory was not created"); - } - const stagedPath = path.join(repositoryPath, stagingEntry, SNAPSHOT_SQLITE_FILENAME); - await fs.unlink(stagedPath); - createGenericDatabase(stagedPath, { values: ["replacement"] }); - } - return await originalOpen(filePath, flags, mode); - }); - - let restored = false; - try { - await provider.restoreFresh(snapshot.ref, restorePath); - restored = true; - } catch (error) { - expect(String(error)).toMatch(/file changed while reading/u); - } finally { - openSpy.mockRestore(); - } - if (!restored) { - await expect(fs.access(restorePath)).rejects.toMatchObject({ code: "ENOENT" }); - return; - } - const sqlite = requireNodeSqlite(); - const database = new sqlite.DatabaseSync(restorePath, { readOnly: true }); - try { - expect(database.prepare("SELECT value FROM entries").all()).toEqual([ - { value: "original" }, - ]); - } finally { - database.close(); - } - }, - ); - - it.runIf(process.platform !== "win32")( - "removes only its restored target when a sidecar races publication", - async () => { - const tempDir = await createTempDir(); - const sourcePath = path.join(tempDir, "source.sqlite"); - const repositoryPath = path.join(tempDir, "snapshots"); - const restorePath = path.join(tempDir, "restore", "source.sqlite"); - createGenericDatabase(sourcePath); - const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); - const snapshot = await provider.create({ - path: sourcePath, - identity: { role: "generic", id: "restore-race" }, - }); - const originalLink = fs.link.bind(fs); - const linkSpy = vi.spyOn(fs, "link").mockImplementation(async (source, target) => { - await originalLink(source, target); - if (path.resolve(String(target)) === restorePath) { - await fs.writeFile(`${restorePath}-wal`, "racer"); - } - }); - - try { - await expect(provider.restoreFresh(snapshot.ref, restorePath)).rejects.toThrow( - /unexpected sidecar/u, - ); - await expect(fs.access(restorePath)).rejects.toMatchObject({ code: "ENOENT" }); - await expect(fs.readFile(`${restorePath}-wal`, "utf8")).resolves.toBe("racer"); - } finally { - linkSpy.mockRestore(); - } - }, - ); - - it("rejects snapshots outside the configured repository and unexpected contents", async () => { - const tempDir = await createTempDir(); - const sourcePath = path.join(tempDir, "source.sqlite"); - const repositoryPath = path.join(tempDir, "snapshots"); - createGenericDatabase(sourcePath); - const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); - const snapshot = await provider.create({ - path: sourcePath, - identity: { role: "generic", id: "boundary" }, - }); - - await expect(provider.verify({ path: tempDir })).rejects.toThrow(/immediate child/u); - await fs.writeFile(path.join(snapshot.ref.path, `${SNAPSHOT_SQLITE_FILENAME}-wal`), "orphan"); - await expect(provider.verify(snapshot.ref)).rejects.toThrow(/unexpected entry/u); - await expect(provider.list()).rejects.toThrow(/unexpected entry/u); - }); - - it("bounds manifest reads before parsing untrusted snapshot metadata", async () => { - const tempDir = await createTempDir(); - const sourcePath = path.join(tempDir, "source.sqlite"); - const repositoryPath = path.join(tempDir, "snapshots"); - createGenericDatabase(sourcePath); - const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); - const snapshot = await provider.create({ - path: sourcePath, - identity: { role: "generic", id: "bounded-manifest" }, - }); - await fs.writeFile( - path.join(snapshot.ref.path, SNAPSHOT_MANIFEST_FILENAME), - Buffer.alloc(1024 * 1024 + 1, 0x20), - ); - - await expect(provider.verify(snapshot.ref)).rejects.toThrow(/1048576 bytes/u); - }); - - it.runIf(process.platform !== "win32")( - "rejects symlinked repositories, snapshot files, restore parents, and hardlinked artifacts", - async () => { - const tempDir = await createTempDir(); - const sourcePath = path.join(tempDir, "source.sqlite"); - const realRepositoryPath = path.join(tempDir, "real-snapshots"); - const repositoryLink = path.join(tempDir, "snapshot-link"); - createGenericDatabase(sourcePath); - await fs.mkdir(realRepositoryPath); - await fs.symlink(realRepositoryPath, repositoryLink); - const linkedProvider = createLocalSqliteSnapshotProvider({ - repositoryPath: repositoryLink, - }); - await expect( - linkedProvider.create({ - path: sourcePath, - identity: { role: "generic", id: "symlink-repository" }, - }), - ).rejects.toThrow(/symlink|Invalid path/iu); - - const provider = createLocalSqliteSnapshotProvider({ - repositoryPath: realRepositoryPath, - }); - const snapshot = await provider.create({ - path: sourcePath, - identity: { role: "generic", id: "links" }, - }); - const artifactPath = path.join(snapshot.ref.path, SNAPSHOT_SQLITE_FILENAME); - const externalArtifact = path.join(tempDir, "external.sqlite"); - await fs.link(artifactPath, externalArtifact); - await expect(provider.verify(snapshot.ref)).rejects.toThrow(/hardlink/iu); - await fs.unlink(externalArtifact); - - const manifestPath = path.join(snapshot.ref.path, SNAPSHOT_MANIFEST_FILENAME); - const realManifest = path.join(tempDir, "manifest.json"); - await fs.rename(manifestPath, realManifest); - await fs.symlink(realManifest, manifestPath); - await expect(provider.verify(snapshot.ref)).rejects.toThrow(/regular file|symlink/iu); - - await fs.unlink(manifestPath); - await fs.rename(realManifest, manifestPath); - const realRestoreParent = path.join(tempDir, "real-restore"); - const restoreParentLink = path.join(tempDir, "restore-link"); - await fs.mkdir(realRestoreParent); - await fs.symlink(realRestoreParent, restoreParentLink); - await expect( - provider.restoreFresh(snapshot.ref, path.join(restoreParentLink, "restored.sqlite")), - ).rejects.toThrow(/symlink|Invalid path/iu); - }, - ); - - it.runIf(process.platform !== "win32")( - "rejects repository restore targets reached through another filesystem spelling", - async () => { - const tempDir = await createTempDir(); - const sourcePath = path.join(tempDir, "source.sqlite"); - const repositoryPath = path.join(tempDir, "snapshots"); - createGenericDatabase(sourcePath); - const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); - const snapshot = await provider.create({ - path: sourcePath, - identity: { role: "generic", id: "canonical-boundary" }, - }); - const aliasRoot = path.join(tempDir, "alias"); - await fs.symlink(tempDir, aliasRoot); - const aliasRepositoryPath = path.join(aliasRoot, "snapshots"); - const aliasProvider = createLocalSqliteSnapshotProvider({ - repositoryPath: aliasRepositoryPath, - }); - const aliasSnapshot = { - path: path.join(aliasRepositoryPath, path.basename(snapshot.ref.path)), - }; - - await expect( - aliasProvider.restoreFresh(aliasSnapshot, path.join(repositoryPath, "restored.sqlite")), - ).rejects.toThrow(/outside snapshot repository/u); - await expect(provider.verify(snapshot.ref)).resolves.toMatchObject({ ok: true }); - }, - ); -}); - -describe("snapshot manifest parser", () => { - const manifestPath = "/snapshots/snapshot/manifest.json"; - const snapshotId = "snapshot"; - const validManifest: SnapshotManifest = { - schemaVersion: 1, - snapshotId, - createdAt: "2026-07-12T14:00:00.000Z", - database: { - role: "agent", - agentId: "worker-1", - basename: "openclaw-agent.sqlite", - userVersion: OPENCLAW_AGENT_SCHEMA_VERSION, - }, - artifact: { - path: SNAPSHOT_SQLITE_FILENAME, - sha256: "a".repeat(64), - sizeBytes: 4096, - }, - }; - - it.each([ - ["unknown top-level field", { ...validManifest, extra: true }, /fields must be exactly/u], - ["wrong directory id", validManifest, /does not match directory/u, "other"], - [ - "noncanonical timestamp", - { ...validManifest, createdAt: "2026-07-12T14:00:00Z" }, - /not canonical/u, - ], - [ - "artifact path traversal", - { ...validManifest, artifact: { ...validManifest.artifact, path: "../database.sqlite" } }, - /artifact\.path must be database\.sqlite/u, - ], - [ - "prefixed digest", - { - ...validManifest, - artifact: { ...validManifest.artifact, sha256: `sha256:${"a".repeat(64)}` }, - }, - /sha256 is invalid/u, - ], - [ - "noncanonical agent id", - { ...validManifest, database: { ...validManifest.database, agentId: "Worker-1" } }, - /agentId is invalid/u, - ], - [ - "unsafe basename", - { ...validManifest, database: { ...validManifest.database, basename: "../db.sqlite" } }, - /basename is invalid/u, - ], - [ - "out-of-range user version", - { ...validManifest, database: { ...validManifest.database, userVersion: 2 ** 31 } }, - /userVersion is invalid/u, - ], - [ - "zero-byte artifact", - { ...validManifest, artifact: { ...validManifest.artifact, sizeBytes: 0 } }, - /sizeBytes is invalid/u, - ], - ])("rejects %s", (_name, value, error, expectedId = snapshotId) => { - expect(() => parseSnapshotManifest(value, manifestPath, expectedId)).toThrow(error); - }); -}); diff --git a/src/snapshot/local-repository.ts b/src/snapshot/local-repository.ts deleted file mode 100644 index bb3c07dee6a4..000000000000 --- a/src/snapshot/local-repository.ts +++ /dev/null @@ -1,863 +0,0 @@ -import { randomUUID } from "node:crypto"; -import fsSync, { type Stats } from "node:fs"; -import fs from "node:fs/promises"; -import type { FileHandle } from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; -import type { DatabaseSync } from "node:sqlite"; -import { isDeepStrictEqual } from "node:util"; -import { loadSqliteVecExtension } from "../../packages/memory-host-sdk/src/engine-storage.js"; -import { sameFileIdentity } from "../infra/fs-safe-advanced.js"; -import { - canonicalPathFromExistingAncestor, - ensureAbsoluteDirectory, - isPathInside, -} from "../infra/fs-safe.js"; -import { requireNodeSqlite } from "../infra/node-sqlite.js"; -import { applyPrivateModeSync } from "../infra/private-mode.js"; -import { assertSqliteIntegrity } from "../infra/sqlite-integrity.js"; -import { - createVerifiedSqliteSnapshot, - publishVerifiedSqliteFile, - syncDirectoryBestEffort, - type SqliteSnapshotValidator, -} from "../infra/sqlite-snapshot.js"; -import { readSqliteUserVersion } from "../infra/sqlite-user-version.js"; -import { isValidAgentId, normalizeAgentId } from "../routing/session-key.js"; -import { assertOpenClawAgentDatabaseForMaintenance } from "../state/openclaw-agent-db.js"; -import { assertOpenClawStateDatabaseForMaintenance } from "../state/openclaw-state-db.js"; -import { - containsAsciiControlCharacter, - copySnapshotArtifact, - hashSnapshotArtifact, - readSnapshotManifest, - type SnapshotArtifactDigest, - writeSnapshotManifest, -} from "./manifest.js"; -import { - SNAPSHOT_MANIFEST_FILENAME, - SNAPSHOT_SQLITE_FILENAME, - type SnapshotDatabaseIdentity, - type SnapshotDatabaseManifest, - type SnapshotDatabaseRef, - type SnapshotManifest, - type SnapshotRef, - type SnapshotResult, - type SnapshotSummary, - type SnapshotVerificationResult, - type SqliteSnapshotProvider, -} from "./snapshot-provider.js"; - -const SNAPSHOT_DIRECTORY_MODE = 0o700; -const SNAPSHOT_FILE_MODE = 0o600; -const SNAPSHOT_ID_BASENAME_MAX_LENGTH = 80; -const SNAPSHOT_PENDING_FILENAME = ".pending"; -const SQLITE_SIDECAR_SUFFIXES = ["-wal", "-shm", "-journal"] as const; -const SNAPSHOT_ARTIFACT_ENTRIES = new Set([ - SNAPSHOT_MANIFEST_FILENAME, - SNAPSHOT_PENDING_FILENAME, - SNAPSHOT_SQLITE_FILENAME, -]); -const RESTORE_STAGING_ENTRIES = new Set([SNAPSHOT_SQLITE_FILENAME]); - -export type LocalSqliteSnapshotProviderOptions = { - readonly repositoryPath: string; - readonly now?: () => Date; -}; - -export function createLocalSqliteSnapshotProvider( - options: LocalSqliteSnapshotProviderOptions, -): SqliteSnapshotProvider { - return new LocalSqliteSnapshotProvider(options); -} - -class LocalSqliteSnapshotProvider implements SqliteSnapshotProvider { - readonly #repositoryPath: string; - readonly #now: () => Date; - - constructor(options: LocalSqliteSnapshotProviderOptions) { - this.#repositoryPath = path.resolve(options.repositoryPath); - this.#now = options.now ?? (() => new Date()); - } - - async create(database: SnapshotDatabaseRef): Promise { - await ensurePrivateDirectory(this.#repositoryPath, "SQLite snapshot repository"); - const sourcePath = path.resolve(database.path); - const identity = normalizeSnapshotIdentity(database.identity); - const now = this.#now(); - if (!Number.isFinite(now.getTime())) { - throw new Error("SQLite snapshot timestamp is invalid."); - } - const snapshotId = buildSnapshotId(now, sourcePath); - const snapshotDir = path.join(this.#repositoryPath, snapshotId); - const stagingDir = path.join(this.#repositoryPath, `.tmp-${snapshotId}-${randomUUID()}`); - const artifactPath = path.join(stagingDir, SNAPSHOT_SQLITE_FILENAME); - await fs.mkdir(stagingDir, { mode: SNAPSHOT_DIRECTORY_MODE }); - - let stagingIdentity: Stats | undefined; - let publishedDirectory: FileHandle | undefined; - let publishedIdentity: Stats | undefined; - const publishedEntries = new Map(); - let snapshotDirectoryCreated = false; - try { - stagingIdentity = await fs.lstat(stagingDir); - applyPrivateModeSync(stagingDir, SNAPSHOT_DIRECTORY_MODE); - const result = await createVerifiedSqliteSnapshot({ - sourcePath, - targetPath: artifactPath, - transform: identity.role === "global" ? sanitizeGlobalStateSnapshot : undefined, - validate: buildDatabaseValidator(identity), - }); - applyPrivateModeSync(artifactPath, SNAPSHOT_FILE_MODE); - const artifact = await hashSnapshotArtifact(stagingDir); - const manifest: SnapshotManifest = { - schemaVersion: 1, - snapshotId, - createdAt: now.toISOString(), - database: buildDatabaseManifest(identity, sourcePath, result.userVersion), - artifact: { - path: SNAPSHOT_SQLITE_FILENAME, - sha256: artifact.sha256, - sizeBytes: artifact.sizeBytes, - }, - }; - await writeSnapshotManifest(stagingDir, manifest); - applyPrivateModeSync(path.join(stagingDir, SNAPSHOT_MANIFEST_FILENAME), SNAPSHOT_FILE_MODE); - await readSnapshotManifest(stagingDir, snapshotId); - await syncDirectoryBestEffort(stagingDir); - - try { - await fs.mkdir(snapshotDir, { mode: SNAPSHOT_DIRECTORY_MODE }); - snapshotDirectoryCreated = true; - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "EEXIST") { - throw new Error(`SQLite snapshot directory already exists: ${snapshotDir}`, { - cause: error, - }); - } - throw error; - } - publishedIdentity = await fs.lstat(snapshotDir); - applyPrivateModeSync(snapshotDir, SNAPSHOT_DIRECTORY_MODE); - publishedDirectory = await fs.open(snapshotDir, "r"); - await assertOpenDirectoryIdentity(publishedDirectory, snapshotDir, publishedIdentity); - const pendingPath = path.join(snapshotDir, SNAPSHOT_PENDING_FILENAME); - await fs.writeFile(pendingPath, "", { - flag: "wx", - mode: SNAPSHOT_FILE_MODE, - }); - publishedEntries.set(SNAPSHOT_PENDING_FILENAME, await fs.lstat(pendingPath)); - await assertOpenDirectoryIdentity(publishedDirectory, snapshotDir, publishedIdentity); - await publishSnapshotEntryNoOverwrite( - path.join(stagingDir, SNAPSHOT_SQLITE_FILENAME), - path.join(snapshotDir, SNAPSHOT_SQLITE_FILENAME), - SNAPSHOT_SQLITE_FILENAME, - publishedEntries, - ); - await assertOpenDirectoryIdentity(publishedDirectory, snapshotDir, publishedIdentity); - await publishSnapshotEntryNoOverwrite( - path.join(stagingDir, SNAPSHOT_MANIFEST_FILENAME), - path.join(snapshotDir, SNAPSHOT_MANIFEST_FILENAME), - SNAPSHOT_MANIFEST_FILENAME, - publishedEntries, - ); - await assertOpenDirectoryIdentity(publishedDirectory, snapshotDir, publishedIdentity); - await syncDirectoryBestEffort(snapshotDir); - await assertPendingSnapshotContents(snapshotDir); - const publishedManifest = await readSnapshotManifest(snapshotDir, snapshotId); - if (!isDeepStrictEqual(publishedManifest, manifest)) { - throw new Error(`SQLite snapshot manifest changed during publication: ${snapshotDir}`); - } - const publishedArtifact = await hashSnapshotArtifact(snapshotDir); - const publishedArtifactPath = path.join(snapshotDir, SNAPSHOT_SQLITE_FILENAME); - assertArtifactMatchesManifest(publishedArtifactPath, publishedArtifact, publishedManifest); - await verifySnapshotDatabaseFile( - publishedArtifactPath, - publishedArtifact.stat, - publishedManifest, - ); - const expectedPendingIdentity = publishedEntries.get(SNAPSHOT_PENDING_FILENAME); - const currentPendingIdentity = fsSync.lstatSync(pendingPath); - if ( - !expectedPendingIdentity || - !sameFileIdentity(expectedPendingIdentity, currentPendingIdentity) - ) { - throw new Error(`SQLite snapshot pending marker changed: ${pendingPath}`); - } - fsSync.unlinkSync(pendingPath); - publishedEntries.delete(SNAPSHOT_PENDING_FILENAME); - await syncDirectoryBestEffort(snapshotDir); - await publishedDirectory.close(); - publishedDirectory = undefined; - const committedManifest = await readSnapshotManifest(snapshotDir, snapshotId); - if (!isDeepStrictEqual(committedManifest, manifest)) { - throw new Error(`SQLite snapshot manifest changed after commit: ${snapshotDir}`); - } - const committedArtifact = await hashSnapshotArtifact(snapshotDir); - assertArtifactMatchesManifest( - path.join(snapshotDir, SNAPSHOT_SQLITE_FILENAME), - committedArtifact, - committedManifest, - ); - const currentIdentity = await fs.lstat(snapshotDir); - if (!sameFileIdentity(publishedIdentity, currentIdentity)) { - throw new Error(`SQLite snapshot directory changed during publication: ${snapshotDir}`); - } - await assertExactSnapshotContents(snapshotDir); - await syncDirectoryBestEffort(this.#repositoryPath); - return { ref: { path: snapshotDir }, manifest }; - } catch (error) { - await publishedDirectory?.close().catch(() => undefined); - publishedDirectory = undefined; - if (snapshotDirectoryCreated) { - publishedIdentity ??= await fs.lstat(snapshotDir).catch(() => undefined); - } - if (publishedIdentity) { - const removed = await removePublishedSnapshotDirectoryIfOwned( - snapshotDir, - publishedIdentity, - publishedEntries, - ); - if (removed) { - await syncDirectoryBestEffort(this.#repositoryPath); - } - } - throw error; - } finally { - const removed = stagingIdentity - ? await removePrivateDirectoryIfOwned( - stagingDir, - stagingIdentity, - SNAPSHOT_ARTIFACT_ENTRIES, - ).catch(() => false) - : await fs - .rmdir(stagingDir) - .then(() => true) - .catch(() => false); - if (removed) { - await syncDirectoryBestEffort(this.#repositoryPath).catch(() => undefined); - } - } - } - - async verify(snapshot: SnapshotRef): Promise { - const snapshotDir = await this.#resolveSnapshotDirectory(snapshot); - const manifest = await readVerifiedSnapshotManifest(snapshotDir); - const artifact = await hashSnapshotArtifact(snapshotDir); - const artifactPath = path.join(snapshotDir, SNAPSHOT_SQLITE_FILENAME); - assertArtifactMatchesManifest(artifactPath, artifact, manifest); - await verifySnapshotDatabaseFile(artifactPath, artifact.stat, manifest); - await assertExactSnapshotContents(snapshotDir); - return { ok: true, manifest }; - } - - async restoreFresh( - snapshot: SnapshotRef, - targetPath: string, - ): Promise { - const snapshotDir = await this.#resolveSnapshotDirectory(snapshot); - const manifest = await readVerifiedSnapshotManifest(snapshotDir); - const resolvedTargetPath = path.resolve(targetPath); - const canonicalRepositoryPath = await fs.realpath(this.#repositoryPath); - const canonicalTargetPath = await canonicalPathFromExistingAncestor(resolvedTargetPath); - if (isPathInside(canonicalRepositoryPath, canonicalTargetPath)) { - throw new Error( - `SQLite restore target must be outside snapshot repository ${this.#repositoryPath}: ${resolvedTargetPath}`, - ); - } - const restoreParentPath = path.dirname(resolvedTargetPath); - await ensureRestoreParentDirectory(restoreParentPath); - const restoreParentIdentity = await fs.lstat(restoreParentPath); - applyPrivateModeSync(this.#repositoryPath, SNAPSHOT_DIRECTORY_MODE); - // Existing databases need a crash-recoverable main/WAL/SHM swap protocol. - // This path is deliberately fresh-only and refuses every preexisting sidecar. - await assertFreshRestorePathsAbsent(resolvedTargetPath); - - const stagingDir = await fs.mkdtemp(path.join(this.#repositoryPath, ".tmp-restore-")); - const stagedSourcePath = path.join(stagingDir, SNAPSHOT_SQLITE_FILENAME); - let stagingIdentity: Stats | undefined; - try { - stagingIdentity = await fs.lstat(stagingDir); - applyPrivateModeSync(stagingDir, SNAPSHOT_DIRECTORY_MODE); - const stagedArtifact = await copySnapshotArtifact(snapshotDir, stagedSourcePath); - await assertDirectoryIdentity(stagingDir, stagingIdentity); - assertArtifactMatchesManifest(stagedSourcePath, stagedArtifact, manifest); - await assertExactSnapshotContents(snapshotDir); - await verifySnapshotDatabaseFile(stagedSourcePath, stagedArtifact.stat, manifest); - await publishVerifiedSqliteFile({ - sourceIdentity: stagedArtifact.stat, - sourcePath: stagedSourcePath, - targetPath: resolvedTargetPath, - expectedContent: manifest.artifact, - requireAtomicPublication: true, - beforePublish: async () => { - await assertDirectoryIdentity(restoreParentPath, restoreParentIdentity); - await assertFreshRestorePathsAbsent(resolvedTargetPath); - }, - afterPublish: (guard) => { - guard.assertTargetMatchesExpectedContent(() => { - assertDirectoryIdentitySync(restoreParentPath, restoreParentIdentity); - assertNoSqliteSidecarsSync(resolvedTargetPath); - }); - }, - }); - return { ok: true, manifest }; - } finally { - const removed = stagingIdentity - ? await removePrivateDirectoryIfOwned( - stagingDir, - stagingIdentity, - RESTORE_STAGING_ENTRIES, - ).catch(() => false) - : await fs - .rmdir(stagingDir) - .then(() => true) - .catch(() => false); - if (removed) { - await syncDirectoryBestEffort(this.#repositoryPath).catch(() => undefined); - } - } - } - - async list(): Promise { - const repositoryStat = await lstatIfExists(this.#repositoryPath); - if (!repositoryStat) { - return []; - } - assertDirectory(repositoryStat, this.#repositoryPath, "SQLite snapshot repository"); - applyPrivateModeSync(this.#repositoryPath, SNAPSHOT_DIRECTORY_MODE); - - const entries = await fs.readdir(this.#repositoryPath, { withFileTypes: true }); - const snapshots: SnapshotSummary[] = []; - for (const entry of entries) { - if (entry.name.startsWith(".tmp-")) { - if (entry.isSymbolicLink() || !entry.isDirectory()) { - throw new Error( - `SQLite snapshot repository contains unsafe staging entry: ${path.join(this.#repositoryPath, entry.name)}`, - ); - } - continue; - } - if (entry.isSymbolicLink() || !entry.isDirectory()) { - throw new Error( - `SQLite snapshot repository contains unexpected entry: ${path.join(this.#repositoryPath, entry.name)}`, - ); - } - const snapshotPath = path.join(this.#repositoryPath, entry.name); - if (await isIncompleteSnapshotDirectory(snapshotPath)) { - continue; - } - await assertExactSnapshotContents(snapshotPath); - snapshots.push({ - ref: { path: snapshotPath }, - manifest: await readSnapshotManifest(snapshotPath), - }); - } - return snapshots.toSorted( - (left, right) => - right.manifest.createdAt.localeCompare(left.manifest.createdAt) || - right.manifest.snapshotId.localeCompare(left.manifest.snapshotId), - ); - } - - async #resolveSnapshotDirectory(snapshot: SnapshotRef): Promise { - const snapshotDir = path.resolve(snapshot.path); - if (path.dirname(snapshotDir) !== this.#repositoryPath) { - throw new Error( - `SQLite snapshot must be an immediate child of repository ${this.#repositoryPath}: ${snapshotDir}`, - ); - } - const repositoryStat = await fs.lstat(this.#repositoryPath); - assertDirectory(repositoryStat, this.#repositoryPath, "SQLite snapshot repository"); - const snapshotStat = await fs.lstat(snapshotDir); - assertDirectory(snapshotStat, snapshotDir, "SQLite snapshot"); - return snapshotDir; - } -} - -async function readVerifiedSnapshotManifest(snapshotDir: string): Promise { - await assertExactSnapshotContents(snapshotDir); - return await readSnapshotManifest(snapshotDir); -} - -function assertArtifactMatchesManifest( - artifactPath: string, - artifact: SnapshotArtifactDigest, - manifest: SnapshotManifest, -): void { - if (artifact.sizeBytes !== manifest.artifact.sizeBytes) { - throw new Error( - `Snapshot artifact size mismatch for ${artifactPath}: expected ${manifest.artifact.sizeBytes}, got ${artifact.sizeBytes}`, - ); - } - if (artifact.sha256 !== manifest.artifact.sha256) { - throw new Error( - `Snapshot artifact hash mismatch for ${artifactPath}: expected ${manifest.artifact.sha256}, got ${artifact.sha256}`, - ); - } -} - -async function verifySnapshotDatabaseFile( - artifactPath: string, - expectedIdentity: Stats, - manifest: SnapshotManifest, -): Promise { - const beforeOpen = await fs.lstat(artifactPath); - if ( - beforeOpen.isSymbolicLink() || - !beforeOpen.isFile() || - beforeOpen.nlink > 1 || - !sameFileIdentity(expectedIdentity, beforeOpen) - ) { - throw new Error(`Snapshot artifact changed before SQLite verification: ${artifactPath}`); - } - - const validationDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-snapshot-verify-")); - const validationPath = path.join(validationDir, SNAPSHOT_SQLITE_FILENAME); - try { - await fs.chmod(validationDir, SNAPSHOT_DIRECTORY_MODE); - const validationArtifact = await copySnapshotArtifact( - path.dirname(artifactPath), - validationPath, - ); - assertArtifactMatchesManifest(validationPath, validationArtifact, manifest); - const sqlite = requireNodeSqlite(); - const database = new sqlite.DatabaseSync(validationPath, { - allowExtension: true, - readOnly: true, - }); - try { - database.exec("PRAGMA busy_timeout = 30000; PRAGMA trusted_schema = OFF;"); - await loadSqliteVecExtension({ db: database }); - assertSqliteIntegrity(database, artifactPath); - buildManifestDatabaseValidator(manifest.database)(database, artifactPath); - } finally { - database.close(); - } - const validatedArtifact = await hashSnapshotArtifact(validationDir); - if (!sameFileIdentity(validationArtifact.stat, validatedArtifact.stat)) { - throw new Error(`Snapshot validation copy changed: ${validationPath}`); - } - assertArtifactMatchesManifest(validationPath, validatedArtifact, manifest); - } finally { - await fs.unlink(validationPath).catch(() => undefined); - await fs.rmdir(validationDir).catch(() => undefined); - } - const afterOpen = await fs.lstat(artifactPath); - if ( - afterOpen.isSymbolicLink() || - !afterOpen.isFile() || - afterOpen.nlink > 1 || - !sameFileIdentity(expectedIdentity, afterOpen) - ) { - throw new Error(`Snapshot artifact changed during SQLite verification: ${artifactPath}`); - } - const verifiedArtifact = await hashSnapshotArtifact(path.dirname(artifactPath)); - if (!sameFileIdentity(expectedIdentity, verifiedArtifact.stat)) { - throw new Error(`Snapshot artifact changed after SQLite verification: ${artifactPath}`); - } - assertArtifactMatchesManifest(artifactPath, verifiedArtifact, manifest); -} - -function normalizeSnapshotIdentity(identity: SnapshotDatabaseIdentity): SnapshotDatabaseIdentity { - if (identity.role === "global") { - return identity; - } - if (identity.role === "agent") { - const agentId = normalizeAgentId(identity.agentId); - if (!isValidAgentId(identity.agentId) || agentId !== identity.agentId) { - throw new Error(`SQLite snapshot agent id must be canonical: ${identity.agentId}`); - } - return { role: "agent", agentId }; - } - const id = identity.id.trim(); - if (!id || id !== identity.id || id.length > 256 || containsAsciiControlCharacter(id)) { - throw new Error("SQLite snapshot generic database id is invalid."); - } - return { role: "generic", id }; -} - -function buildDatabaseManifest( - identity: SnapshotDatabaseIdentity, - sourcePath: string, - userVersion: number, -): SnapshotDatabaseManifest { - const basename = path.basename(sourcePath); - if (identity.role === "global") { - return { role: "global", basename, userVersion }; - } - if (identity.role === "agent") { - return { role: "agent", agentId: identity.agentId, basename, userVersion }; - } - return { role: "generic", id: identity.id, basename, userVersion }; -} - -function buildDatabaseValidator( - identity: SnapshotDatabaseIdentity | SnapshotDatabaseManifest, -): SqliteSnapshotValidator { - if (identity.role === "global") { - return (database, pathname) => - assertOpenClawStateDatabaseForMaintenance(database, { pathname }); - } - if (identity.role === "agent") { - return (database, pathname) => - assertOpenClawAgentDatabaseForMaintenance(database, { - agentId: identity.agentId, - pathname, - }); - } - return () => undefined; -} - -function buildManifestDatabaseValidator( - manifest: SnapshotDatabaseManifest, -): SqliteSnapshotValidator { - const validateOwner = buildDatabaseValidator(manifest); - return (database, pathname) => { - validateOwner(database, pathname); - const userVersion = readSqliteUserVersion(database); - if (userVersion !== manifest.userVersion) { - throw new Error( - `Snapshot database user_version mismatch for ${pathname}: expected ${manifest.userVersion}, got ${userVersion}`, - ); - } - }; -} - -function sanitizeGlobalStateSnapshot(database: DatabaseSync): void { - database.prepare("DELETE FROM delivery_queue_entries").run(); -} - -function buildSnapshotId(now: Date, sourcePath: string): string { - const timestamp = now.toISOString().replaceAll(/[:.]/g, "-"); - const basename = - path - .basename(sourcePath) - .replaceAll(/[^a-zA-Z0-9._-]/g, "-") - .slice(0, SNAPSHOT_ID_BASENAME_MAX_LENGTH) || "database.sqlite"; - return `${timestamp}-${basename}-${randomUUID()}`; -} - -async function ensurePrivateDirectory(directoryPath: string, scopeLabel: string): Promise { - const result = await ensureAbsoluteDirectory(directoryPath, { - mode: SNAPSHOT_DIRECTORY_MODE, - scopeLabel, - }); - if (!result.ok) { - throw result.error; - } - applyPrivateModeSync(result.path, SNAPSHOT_DIRECTORY_MODE); -} - -async function ensureRestoreParentDirectory(directoryPath: string): Promise { - const result = await ensureAbsoluteDirectory(directoryPath, { - mode: SNAPSHOT_DIRECTORY_MODE, - scopeLabel: "SQLite restore target", - }); - if (!result.ok) { - throw result.error; - } -} - -function assertDirectory(stat: Stats, pathname: string, label: string): void { - if (stat.isSymbolicLink() || !stat.isDirectory()) { - throw new Error(`${label} must be a real directory: ${pathname}`); - } -} - -async function assertDirectoryIdentity( - directoryPath: string, - expectedIdentity: Stats, -): Promise { - const currentIdentity = await fs.lstat(directoryPath); - assertDirectory(currentIdentity, directoryPath, "SQLite restore target directory"); - if (!sameFileIdentity(currentIdentity, expectedIdentity)) { - throw new Error(`SQLite restore target directory changed during restore: ${directoryPath}`); - } -} - -async function assertOpenDirectoryIdentity( - handle: FileHandle, - directoryPath: string, - expectedIdentity: Stats, -): Promise { - const openedIdentity = await handle.stat(); - const currentIdentity = await fs.lstat(directoryPath); - assertDirectory(openedIdentity, directoryPath, "SQLite snapshot directory"); - assertDirectory(currentIdentity, directoryPath, "SQLite snapshot directory"); - if ( - !sameFileIdentity(openedIdentity, expectedIdentity) || - !sameFileIdentity(currentIdentity, expectedIdentity) - ) { - throw new Error(`SQLite snapshot directory changed during publication: ${directoryPath}`); - } -} - -function assertDirectoryIdentitySync(directoryPath: string, expectedIdentity: Stats): void { - const currentIdentity = fsSync.lstatSync(directoryPath); - assertDirectory(currentIdentity, directoryPath, "SQLite restore target directory"); - if (!sameFileIdentity(currentIdentity, expectedIdentity)) { - throw new Error(`SQLite restore target directory changed during restore: ${directoryPath}`); - } -} - -function isSnapshotEntryLinkFallbackError(error: unknown): boolean { - const code = (error as NodeJS.ErrnoException).code; - return ( - code === "EPERM" || - code === "EXDEV" || - code === "ENOTSUP" || - code === "EOPNOTSUPP" || - code === "ENOSYS" - ); -} - -async function publishSnapshotEntryNoOverwrite( - sourcePath: string, - targetPath: string, - entryName: string, - publishedEntries: Map, -): Promise { - let linked = false; - let linkedSourceIdentity: Stats | undefined; - try { - linkedSourceIdentity = await fs.lstat(sourcePath); - await fs.link(sourcePath, targetPath); - publishedEntries.set(entryName, linkedSourceIdentity); - linked = true; - } catch (error) { - if (!isSnapshotEntryLinkFallbackError(error)) { - throw error; - } - const copiedIdentity = await copySnapshotEntryExclusive(sourcePath, targetPath); - publishedEntries.set(entryName, copiedIdentity); - } - const expectedTargetIdentity = publishedEntries.get(entryName); - const initialTargetIdentity = await fs.lstat(targetPath); - if (!expectedTargetIdentity || !sameFileIdentity(expectedTargetIdentity, initialTargetIdentity)) { - throw new Error(`SQLite snapshot entry changed during publication: ${targetPath}`); - } - if (linked) { - if (!linkedSourceIdentity || !sameFileIdentity(linkedSourceIdentity, initialTargetIdentity)) { - throw new Error(`SQLite snapshot entry changed during publication: ${targetPath}`); - } - const sourceIdentity = await fs.lstat(sourcePath); - if (!sameFileIdentity(sourceIdentity, initialTargetIdentity)) { - throw new Error(`SQLite snapshot entry changed during publication: ${targetPath}`); - } - } - await fs.unlink(sourcePath); - const finalTargetIdentity = await fs.lstat(targetPath); - if (!sameFileIdentity(initialTargetIdentity, finalTargetIdentity)) { - throw new Error(`SQLite snapshot entry changed after publication: ${targetPath}`); - } - publishedEntries.set(entryName, finalTargetIdentity); -} - -async function copySnapshotEntryExclusive(sourcePath: string, targetPath: string): Promise { - const source = await fs.open(sourcePath, "r"); - let target: FileHandle | undefined; - let targetIdentity: Stats | undefined; - try { - target = await fs.open(targetPath, "wx+", SNAPSHOT_FILE_MODE); - targetIdentity = await target.stat(); - const buffer = Buffer.allocUnsafe(1024 * 1024); - let offset = 0; - while (true) { - const { bytesRead } = await source.read(buffer, 0, buffer.length, offset); - if (bytesRead === 0) { - break; - } - let bytesWritten = 0; - while (bytesWritten < bytesRead) { - const result = await target.write( - buffer, - bytesWritten, - bytesRead - bytesWritten, - offset + bytesWritten, - ); - if (result.bytesWritten === 0) { - throw new Error(`SQLite snapshot entry copy made no progress: ${targetPath}`); - } - bytesWritten += result.bytesWritten; - } - offset += bytesRead; - } - await target.sync(); - const finalIdentity = await target.stat(); - const currentIdentity = await fs.lstat(targetPath); - if ( - !sameFileIdentity(targetIdentity, finalIdentity) || - !sameFileIdentity(targetIdentity, currentIdentity) - ) { - throw new Error(`SQLite snapshot entry changed during copy: ${targetPath}`); - } - return finalIdentity; - } catch (error) { - if (targetIdentity) { - const currentIdentity = await fs.lstat(targetPath).catch(() => undefined); - if (currentIdentity && sameFileIdentity(currentIdentity, targetIdentity)) { - await fs.unlink(targetPath).catch(() => undefined); - } - } - throw error; - } finally { - await target?.close().catch(() => undefined); - await source.close().catch(() => undefined); - } -} - -async function assertExactSnapshotContents(snapshotDir: string): Promise { - await assertSnapshotContents( - snapshotDir, - new Set([SNAPSHOT_MANIFEST_FILENAME, SNAPSHOT_SQLITE_FILENAME]), - ); -} - -async function assertPendingSnapshotContents(snapshotDir: string): Promise { - await assertSnapshotContents( - snapshotDir, - new Set([SNAPSHOT_MANIFEST_FILENAME, SNAPSHOT_PENDING_FILENAME, SNAPSHOT_SQLITE_FILENAME]), - ); -} - -async function assertSnapshotContents(snapshotDir: string, expected: Set): Promise { - const entries = await fs.readdir(snapshotDir, { withFileTypes: true }); - for (const entry of entries) { - if (!expected.delete(entry.name)) { - throw new Error( - `SQLite snapshot contains unexpected entry: ${path.join(snapshotDir, entry.name)}`, - ); - } - if (entry.isSymbolicLink() || !entry.isFile()) { - throw new Error( - `SQLite snapshot entry must be a regular file: ${path.join(snapshotDir, entry.name)}`, - ); - } - const stat = await fs.lstat(path.join(snapshotDir, entry.name)); - if (stat.nlink > 1) { - throw new Error( - `SQLite snapshot entry must not be hardlinked: ${path.join(snapshotDir, entry.name)}`, - ); - } - } - if (expected.size > 0) { - throw new Error(`SQLite snapshot is missing ${[...expected].join(", ")}: ${snapshotDir}`); - } -} - -async function isIncompleteSnapshotDirectory(snapshotDir: string): Promise { - const entries = await fs.readdir(snapshotDir, { withFileTypes: true }); - const names = new Set(entries.map((entry) => entry.name)); - if (names.has(SNAPSHOT_PENDING_FILENAME)) { - return true; - } - if (names.has(SNAPSHOT_MANIFEST_FILENAME)) { - return false; - } - return entries.length === 0; -} - -async function assertFreshRestorePathsAbsent(databasePath: string): Promise { - for (const candidate of [ - databasePath, - ...SQLITE_SIDECAR_SUFFIXES.map((suffix) => `${databasePath}${suffix}`), - ]) { - if (await lstatIfExists(candidate)) { - throw new Error(`Fresh SQLite restore path already exists: ${candidate}`); - } - } -} - -function assertNoSqliteSidecarsSync(databasePath: string): void { - for (const suffix of SQLITE_SIDECAR_SUFFIXES) { - const sidecarPath = `${databasePath}${suffix}`; - try { - fsSync.lstatSync(sidecarPath); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") { - continue; - } - throw error; - } - throw new Error(`Restored SQLite database has unexpected sidecar: ${sidecarPath}`); - } -} - -async function lstatIfExists(pathname: string): Promise { - try { - return await fs.lstat(pathname); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") { - return undefined; - } - throw error; - } -} - -async function removePrivateDirectoryIfOwned( - directoryPath: string, - expectedIdentity: Stats, - allowedEntries: ReadonlySet, -): Promise { - const currentIdentity = await lstatIfExists(directoryPath); - if (!currentIdentity) { - return false; - } - if ( - currentIdentity.isSymbolicLink() || - !currentIdentity.isDirectory() || - !sameFileIdentity(currentIdentity, expectedIdentity) - ) { - throw new Error(`Private SQLite staging directory changed before cleanup: ${directoryPath}`); - } - const entries = await fs.readdir(directoryPath, { withFileTypes: true }); - const verifiedPaths: string[] = []; - for (const entry of entries) { - const entryPath = path.join(directoryPath, entry.name); - if (!allowedEntries.has(entry.name) || entry.isSymbolicLink() || !entry.isFile()) { - throw new Error(`Private SQLite staging directory has unexpected entry: ${entryPath}`); - } - const stat = await fs.lstat(entryPath); - if (stat.nlink > 1) { - throw new Error(`Private SQLite staging file must not be hardlinked: ${entryPath}`); - } - verifiedPaths.push(entryPath); - } - await Promise.all(verifiedPaths.map(async (entryPath) => await fs.unlink(entryPath))); - await fs.rmdir(directoryPath); - return true; -} - -async function removePublishedSnapshotDirectoryIfOwned( - directoryPath: string, - expectedIdentity: Stats, - publishedEntries: ReadonlyMap, -): Promise { - const currentIdentity = await lstatIfExists(directoryPath); - if ( - !currentIdentity || - currentIdentity.isSymbolicLink() || - !currentIdentity.isDirectory() || - !sameFileIdentity(currentIdentity, expectedIdentity) - ) { - return false; - } - const entries = await fs.readdir(directoryPath, { withFileTypes: true }); - for (const entry of entries) { - const expectedEntryIdentity = publishedEntries.get(entry.name); - if (!expectedEntryIdentity || entry.isSymbolicLink() || !entry.isFile()) { - continue; - } - const entryPath = path.join(directoryPath, entry.name); - const currentEntryIdentity = await fs.lstat(entryPath); - if (sameFileIdentity(currentEntryIdentity, expectedEntryIdentity)) { - await fs.unlink(entryPath); - } - } - if ((await fs.readdir(directoryPath)).length > 0) { - return false; - } - await fs.rmdir(directoryPath); - return true; -} diff --git a/src/snapshot/manifest.ts b/src/snapshot/manifest.ts deleted file mode 100644 index 25d485fff89d..000000000000 --- a/src/snapshot/manifest.ts +++ /dev/null @@ -1,316 +0,0 @@ -import { createHash } from "node:crypto"; -import type { BigIntStats, Stats } from "node:fs"; -import fs from "node:fs/promises"; -import path from "node:path"; -import { sameFileIdentity } from "../infra/fs-safe-advanced.js"; -import { root } from "../infra/fs-safe.js"; -import { isValidAgentId, normalizeAgentId } from "../routing/session-key.js"; -import { - SNAPSHOT_MANIFEST_FILENAME, - SNAPSHOT_SQLITE_FILENAME, - type SnapshotDatabaseManifest, - type SnapshotManifest, -} from "./snapshot-provider.js"; - -const MAX_MANIFEST_BYTES = 1024 * 1024; -const MAX_SQLITE_USER_VERSION = 2_147_483_647; -const MIN_SQLITE_USER_VERSION = -2_147_483_648; -const SNAPSHOT_ID_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]{0,254}$/; -const SHA256_PATTERN = /^[a-f0-9]{64}$/; - -export type SnapshotArtifactDigest = { - sha256: string; - sizeBytes: number; - stat: Stats; -}; - -type OpenFileHandle = Awaited>; - -export function containsAsciiControlCharacter(value: string): boolean { - for (let index = 0; index < value.length; index += 1) { - const code = value.charCodeAt(index); - if (code <= 0x1f || code === 0x7f) { - return true; - } - } - return false; -} - -export async function hashSnapshotArtifact(snapshotDir: string): Promise { - const snapshotRoot = await root(snapshotDir); - const opened = await snapshotRoot.open(SNAPSHOT_SQLITE_FILENAME, { - hardlinks: "reject", - symlinks: "reject", - }); - try { - return { ...(await hashFileHandle(opened.handle)), stat: opened.stat }; - } finally { - await opened.handle.close(); - } -} - -export async function copySnapshotArtifact( - snapshotDir: string, - targetPath: string, -): Promise { - const snapshotRoot = await root(snapshotDir); - const source = await snapshotRoot.open(SNAPSHOT_SQLITE_FILENAME, { - hardlinks: "reject", - symlinks: "reject", - }); - let target: OpenFileHandle | undefined; - let targetIdentity: Stats | undefined; - try { - target = await fs.open(targetPath, "wx+", 0o600); - targetIdentity = await target.stat(); - const digest = await hashFileHandle(source.handle, target); - await target.sync(); - const finalIdentity = await target.stat(); - const currentIdentity = await fs.lstat(targetPath); - if ( - !sameFileIdentity(targetIdentity, finalIdentity) || - !sameFileIdentity(targetIdentity, currentIdentity) - ) { - throw new Error(`Snapshot restore staging file changed during copy: ${targetPath}`); - } - return { ...digest, stat: finalIdentity }; - } catch (error) { - await target?.close().catch(() => undefined); - target = undefined; - if (targetIdentity) { - const currentIdentity = await fs.lstat(targetPath).catch(() => undefined); - if (currentIdentity && sameFileIdentity(targetIdentity, currentIdentity)) { - await fs.unlink(targetPath).catch(() => undefined); - } - } - throw error; - } finally { - await target?.close().catch(() => undefined); - await source.handle.close().catch(() => undefined); - } -} - -async function hashFileHandle( - source: OpenFileHandle, - target?: OpenFileHandle, -): Promise> { - const initialStat = await source.stat({ bigint: true }); - const hash = createHash("sha256"); - const buffer = Buffer.allocUnsafe(1024 * 1024); - let sizeBytes = 0; - while (true) { - const { bytesRead } = await source.read(buffer, 0, buffer.length, sizeBytes); - if (bytesRead === 0) { - break; - } - hash.update(buffer.subarray(0, bytesRead)); - let bytesWritten = 0; - if (target) { - while (bytesWritten < bytesRead) { - const result = await target.write( - buffer, - bytesWritten, - bytesRead - bytesWritten, - sizeBytes + bytesWritten, - ); - if (result.bytesWritten === 0) { - throw new Error("Snapshot restore staging copy made no progress."); - } - bytesWritten += result.bytesWritten; - } - } - sizeBytes += bytesRead; - } - const finalStat = await source.stat({ bigint: true }); - if (!sameMutationFingerprint(initialStat, finalStat)) { - throw new Error("Snapshot artifact changed while being read."); - } - return { sha256: hash.digest("hex"), sizeBytes }; -} - -function sameMutationFingerprint(left: BigIntStats, right: BigIntStats): boolean { - return ( - left.birthtimeNs === right.birthtimeNs && - left.ctimeNs === right.ctimeNs && - left.dev === right.dev && - left.ino === right.ino && - left.mtimeNs === right.mtimeNs && - left.size === right.size - ); -} - -export async function writeSnapshotManifest( - snapshotDir: string, - manifest: SnapshotManifest, -): Promise { - const manifestPath = path.join(snapshotDir, SNAPSHOT_MANIFEST_FILENAME); - const handle = await fs.open(manifestPath, "wx+", 0o600); - try { - await handle.writeFile(`${JSON.stringify(manifest, null, 2)}\n`, "utf8"); - await handle.sync(); - } finally { - await handle.close(); - } -} - -export async function readSnapshotManifest( - snapshotDir: string, - expectedSnapshotId = path.basename(snapshotDir), -): Promise { - const snapshotRoot = await root(snapshotDir); - const manifestPath = path.join(snapshotDir, SNAPSHOT_MANIFEST_FILENAME); - const result = await snapshotRoot.read(SNAPSHOT_MANIFEST_FILENAME, { - hardlinks: "reject", - maxBytes: MAX_MANIFEST_BYTES, - symlinks: "reject", - }); - let parsed: unknown; - try { - parsed = JSON.parse(result.buffer.toString("utf8")) as unknown; - } catch (error) { - throw new Error(`Snapshot manifest is not valid JSON: ${manifestPath}`, { cause: error }); - } - return parseSnapshotManifest(parsed, manifestPath, expectedSnapshotId); -} - -export function parseSnapshotManifest( - value: unknown, - manifestPath: string, - expectedSnapshotId: string, -): SnapshotManifest { - const record = requireRecord(value, "manifest", manifestPath); - requireExactKeys(record, ["schemaVersion", "snapshotId", "createdAt", "database", "artifact"]); - if (record.schemaVersion !== 1) { - throw new Error( - `Unsupported snapshot manifest schemaVersion ${String(record.schemaVersion)}: ${manifestPath}`, - ); - } - const snapshotId = requireSnapshotId(record.snapshotId, manifestPath); - if (snapshotId !== expectedSnapshotId) { - throw new Error( - `Snapshot manifest id ${snapshotId} does not match directory ${expectedSnapshotId}: ${manifestPath}`, - ); - } - const createdAt = requireCanonicalTimestamp(record.createdAt, manifestPath); - const database = parseSnapshotDatabase(record.database, manifestPath); - const artifactRecord = requireRecord(record.artifact, "artifact", manifestPath); - requireExactKeys(artifactRecord, ["path", "sha256", "sizeBytes"]); - if (artifactRecord.path !== SNAPSHOT_SQLITE_FILENAME) { - throw new Error( - `Snapshot manifest artifact.path must be ${SNAPSHOT_SQLITE_FILENAME}: ${manifestPath}`, - ); - } - if (typeof artifactRecord.sha256 !== "string" || !SHA256_PATTERN.test(artifactRecord.sha256)) { - throw new Error(`Snapshot manifest artifact.sha256 is invalid: ${manifestPath}`); - } - if (!Number.isSafeInteger(artifactRecord.sizeBytes) || Number(artifactRecord.sizeBytes) <= 0) { - throw new Error(`Snapshot manifest artifact.sizeBytes is invalid: ${manifestPath}`); - } - return { - schemaVersion: 1, - snapshotId, - createdAt, - database, - artifact: { - path: SNAPSHOT_SQLITE_FILENAME, - sha256: artifactRecord.sha256, - sizeBytes: Number(artifactRecord.sizeBytes), - }, - }; -} - -function parseSnapshotDatabase(value: unknown, manifestPath: string): SnapshotDatabaseManifest { - const database = requireRecord(value, "database", manifestPath); - const role = database.role; - const basename = requireSafeText(database.basename, "database.basename", manifestPath, 255); - if (path.basename(basename) !== basename || basename === "." || basename === "..") { - throw new Error(`Snapshot manifest database.basename is invalid: ${manifestPath}`); - } - const userVersion = requireSqliteUserVersion(database.userVersion, manifestPath); - if (role === "global") { - requireExactKeys(database, ["role", "basename", "userVersion"]); - return { role, basename, userVersion }; - } - if (role === "agent") { - requireExactKeys(database, ["role", "agentId", "basename", "userVersion"]); - const agentId = requireSafeText(database.agentId, "database.agentId", manifestPath, 64); - if (!isValidAgentId(agentId) || normalizeAgentId(agentId) !== agentId) { - throw new Error(`Snapshot manifest database.agentId is invalid: ${manifestPath}`); - } - return { role, agentId, basename, userVersion }; - } - if (role === "generic") { - requireExactKeys(database, ["role", "id", "basename", "userVersion"]); - const id = requireSafeText(database.id, "database.id", manifestPath, 256); - return { role, id, basename, userVersion }; - } - throw new Error(`Snapshot manifest database.role is invalid: ${manifestPath}`); -} - -function requireRecord( - value: unknown, - field: string, - manifestPath: string, -): Record { - if (!value || typeof value !== "object" || Array.isArray(value)) { - throw new Error(`Snapshot manifest ${field} must be an object: ${manifestPath}`); - } - return value as Record; -} - -function requireExactKeys(record: Record, expectedKeys: readonly string[]): void { - const actual = Object.keys(record).toSorted(); - const expected = [...expectedKeys].toSorted(); - if (actual.length !== expected.length || actual.some((key, index) => key !== expected[index])) { - throw new Error( - `Snapshot manifest fields must be exactly ${expectedKeys.join(", ")}; got ${actual.join(", ")}`, - ); - } -} - -function requireSnapshotId(value: unknown, manifestPath: string): string { - if (typeof value !== "string" || !SNAPSHOT_ID_PATTERN.test(value)) { - throw new Error(`Snapshot manifest snapshotId is invalid: ${manifestPath}`); - } - return value; -} - -function requireCanonicalTimestamp(value: unknown, manifestPath: string): string { - if (typeof value !== "string") { - throw new Error(`Snapshot manifest createdAt is invalid: ${manifestPath}`); - } - const parsed = new Date(value); - if (!Number.isFinite(parsed.getTime()) || parsed.toISOString() !== value) { - throw new Error(`Snapshot manifest createdAt is not canonical ISO 8601: ${manifestPath}`); - } - return value; -} - -function requireSafeText( - value: unknown, - field: string, - manifestPath: string, - maxLength: number, -): string { - if ( - typeof value !== "string" || - value.length === 0 || - value.length > maxLength || - value.trim() !== value || - containsAsciiControlCharacter(value) - ) { - throw new Error(`Snapshot manifest ${field} is invalid: ${manifestPath}`); - } - return value; -} - -function requireSqliteUserVersion(value: unknown, manifestPath: string): number { - if ( - !Number.isSafeInteger(value) || - Number(value) < MIN_SQLITE_USER_VERSION || - Number(value) > MAX_SQLITE_USER_VERSION - ) { - throw new Error(`Snapshot manifest database.userVersion is invalid: ${manifestPath}`); - } - return Number(value); -} diff --git a/src/snapshot/snapshot-provider.ts b/src/snapshot/snapshot-provider.ts deleted file mode 100644 index 79fffaf4b2da..000000000000 --- a/src/snapshot/snapshot-provider.ts +++ /dev/null @@ -1,66 +0,0 @@ -export const SNAPSHOT_MANIFEST_FILENAME = "manifest.json"; -export const SNAPSHOT_SQLITE_FILENAME = "database.sqlite"; - -export type SnapshotDatabaseIdentity = - | { readonly role: "global" } - | { readonly role: "agent"; readonly agentId: string } - | { readonly role: "generic"; readonly id: string }; - -export type SnapshotDatabaseRef = { - readonly path: string; - readonly identity: SnapshotDatabaseIdentity; -}; - -export type SnapshotDatabaseManifest = - | { - readonly role: "global"; - readonly basename: string; - readonly userVersion: number; - } - | { - readonly role: "agent"; - readonly agentId: string; - readonly basename: string; - readonly userVersion: number; - } - | { - readonly role: "generic"; - readonly id: string; - readonly basename: string; - readonly userVersion: number; - }; - -export type SnapshotManifest = { - readonly schemaVersion: 1; - readonly snapshotId: string; - readonly createdAt: string; - readonly database: SnapshotDatabaseManifest; - readonly artifact: { - readonly path: typeof SNAPSHOT_SQLITE_FILENAME; - readonly sha256: string; - readonly sizeBytes: number; - }; -}; - -export type SnapshotRef = { - readonly path: string; -}; - -export type SnapshotResult = { - readonly ref: SnapshotRef; - readonly manifest: SnapshotManifest; -}; - -export type SnapshotVerificationResult = { - readonly ok: true; - readonly manifest: SnapshotManifest; -}; - -export type SnapshotSummary = SnapshotResult; - -export type SqliteSnapshotProvider = { - create(database: SnapshotDatabaseRef): Promise; - list(): Promise; - restoreFresh(snapshot: SnapshotRef, targetPath: string): Promise; - verify(snapshot: SnapshotRef): Promise; -};