From 63cadcf11bbe2a8a76be46ef64052dc8e349099f Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Thu, 6 Aug 2026 11:26:47 +0800 Subject: [PATCH] fix(backup): skip transient gateway lock databases (#119782) Reported by @AgentSolomon in #119757. Fixes #119757 Punchcard-Session: amber-meadow-timber-8r --- src/commands/backup-verify.test.ts | 5 + src/commands/backup-verify.ts | 8 +- src/infra/backup-create.test.ts | 142 ++++++++++++++++++++++- src/infra/backup-create.ts | 53 +++++---- src/infra/backup-volatile-filter.test.ts | 37 +++++- src/infra/backup-volatile-filter.ts | 19 +++ 6 files changed, 228 insertions(+), 36 deletions(-) diff --git a/src/commands/backup-verify.test.ts b/src/commands/backup-verify.test.ts index 48dacb5c8a2e..ff5f03b2c73d 100644 --- a/src/commands/backup-verify.test.ts +++ b/src/commands/backup-verify.test.ts @@ -836,6 +836,11 @@ describe("backupVerifyCommand", () => { contents: invalidSqlite, archivePath: `${stateAssetArchivePath}/memory/main.sqlite.reindex-lock.sqlite`, }, + { + fileName: "reindex-lock.sqlite-wal", + contents: invalidSqlite, + archivePath: `${stateAssetArchivePath}/memory/main.sqlite.reindex-lock.sqlite-wal`, + }, { fileName: "reindex-tmp", contents: invalidSqlite, diff --git a/src/commands/backup-verify.ts b/src/commands/backup-verify.ts index 1a2ac2ee6b07..12b7958f6e57 100644 --- a/src/commands/backup-verify.ts +++ b/src/commands/backup-verify.ts @@ -6,6 +6,7 @@ import type { DatabaseSync } from "node:sqlite"; import { readStringValue } from "@openclaw/normalization-core/string-coerce"; import * as tar from "tar"; import { loadSqliteVecExtension } from "../../packages/memory-host-sdk/src/engine-storage.js"; +import { isTransientSqliteBackupPath } from "../infra/backup-volatile-filter.js"; import { formatDiskSpaceBytes, tryReadDiskSpace } from "../infra/disk-space.js"; import { openNodeSqliteDatabase } from "../infra/node-sqlite.js"; import { assertSqliteIntegrity } from "../infra/sqlite-integrity.js"; @@ -18,9 +19,6 @@ const MAX_MANIFEST_BYTES = 1024 * 1024; const MAX_SQLITE_SNAPSHOT_EXTRACT_BYTES = 64 * 1024 * 1024 * 1024; const SQLITE_SNAPSHOT_FREE_SPACE_RESERVE_BYTES = 256 * 1024 * 1024; const SQLITE_SNAPSHOT_SIDECAR_SUFFIXES = ["-wal", "-shm", "-journal"] as const; -const SQLITE_BACKUP_EXCLUDED_SUFFIXES = [".reindex-lock.sqlite"] as const; -const SQLITE_BACKUP_REINDEX_TRANSIENT_PATTERN = - /\.sqlite\.(?:backup|memory-reindex|tmp)-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/iu; type BackupManifestAsset = { kind: string; @@ -400,9 +398,7 @@ function isSqliteSnapshotRelativePath(relativePath: string): boolean { return true; } return ( - !portablePath.split("/").includes("node_modules") && - !SQLITE_BACKUP_REINDEX_TRANSIENT_PATTERN.test(relativePath) && - !SQLITE_BACKUP_EXCLUDED_SUFFIXES.some((suffix) => portablePath.endsWith(suffix)) + !portablePath.split("/").includes("node_modules") && !isTransientSqliteBackupPath(portablePath) ); } diff --git a/src/infra/backup-create.test.ts b/src/infra/backup-create.test.ts index c10f410781ee..0e6466e3f478 100644 --- a/src/infra/backup-create.test.ts +++ b/src/infra/backup-create.test.ts @@ -8,7 +8,9 @@ import * as tar from "tar"; import { describe, expect, it, vi } from "vitest"; import { saveAuthProfileStore } from "../agents/auth-profiles/store.js"; import { backupVerifyCommand } from "../commands/backup-verify.js"; +import { isPathWithin } from "../commands/cleanup-utils.js"; import { CONFIG_AUDIT_MAX_ENTRIES, CONFIG_AUDIT_SCOPE } from "../config/io.audit.js"; +import { resolveGatewayLockDir } from "../config/paths.js"; import type { RuntimeEnv } from "../runtime.js"; import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; import { @@ -33,6 +35,7 @@ import { import { writeTarArchiveWithRetry } from "./backup-tar-retry.js"; import { isVolatileBackupPath } from "./backup-volatile-filter.js"; import { createBackupVolatileStatCache } from "./backup-volatile-stat-cache.js"; +import { acquireGatewayLock } from "./gateway-lock.js"; import { requireNodeSqlite } from "./node-sqlite.js"; import { createSqliteAuditRecordStore } from "./sqlite-audit-record-store.js"; import { detectLegacyAuditLogs, migrateLegacyAuditLogs } from "./state-migrations.audit-logs.js"; @@ -1659,7 +1662,7 @@ describe("createBackupArchive", () => { ); }); - it("snapshots nested live SQLite databases with transaction continuity", async () => { + it("snapshots lock-named plugin SQLite databases with transaction continuity", async () => { await withOpenClawTestState( { layout: "state-only", @@ -1669,7 +1672,7 @@ describe("createBackupArchive", () => { async (state) => { const outputDir = state.path("backups"); const extractDir = state.path("extract"); - const dbPath = state.statePath("plugins", "dedicated", "live.sqlite"); + const dbPath = state.statePath("plugins", "dedicated", "cache.lock.sqlite"); await fs.mkdir(path.dirname(dbPath), { recursive: true }); await fs.mkdir(outputDir, { recursive: true }); await fs.mkdir(extractDir, { recursive: true }); @@ -1714,13 +1717,13 @@ describe("createBackupArchive", () => { }); const entries = await listArchiveEntries(result.archivePath); const archivedDbEntries = entries.filter((entry) => - entry.endsWith("/state/plugins/dedicated/live.sqlite"), + entry.endsWith("/state/plugins/dedicated/cache.lock.sqlite"), ); expect(archivedDbEntries).toHaveLength(1); for (const suffix of ["-wal", "-shm", "-journal"]) { expect( entries.some((entry) => - entry.endsWith(`/state/plugins/dedicated/live.sqlite${suffix}`), + entry.endsWith(`/state/plugins/dedicated/cache.lock.sqlite${suffix}`), ), suffix, ).toBe(false); @@ -2025,6 +2028,137 @@ describe("createBackupArchive", () => { ); }); + it("backs up durable SQLite while live gateway coordinators remain held under state", async () => { + await withOpenClawTestState( + { + layout: "state-only", + prefix: "openclaw-backup-gateway-lock-sqlite-", + scenario: "minimal", + }, + async (state) => { + const outputDir = state.path("backups"); + const extractDir = state.path("extract"); + const lockDir = resolveGatewayLockDir(() => state.statePath("tmp")); + const pluginDbPath = state.statePath("plugins", "dedicated", "durable.sqlite"); + const producerShapedDbPath = state.statePath( + "plugins", + "dedicated", + "gateway.12345678.lock.sqlite", + ); + const colocatedDbPath = path.join(lockDir, "retained.sqlite"); + await fs.mkdir(outputDir, { recursive: true }); + await fs.mkdir(extractDir, { recursive: true }); + await fs.mkdir(path.dirname(pluginDbPath), { recursive: true }); + await fs.mkdir(lockDir, { recursive: true }); + + const sqlite = requireNodeSqlite(); + for (const [databasePath, value] of [ + [pluginDbPath, "plugin-state"], + [producerShapedDbPath, "producer-shaped-state"], + [colocatedDbPath, "colocated-state"], + ] as const) { + const database = new sqlite.DatabaseSync(databasePath); + try { + database.exec("CREATE TABLE durable_state (value TEXT NOT NULL)"); + database.prepare("INSERT INTO durable_state (value) VALUES (?)").run(value); + } finally { + database.close(); + } + } + + const gatewayLock = await acquireGatewayLock({ + allowInTests: true, + env: state.env, + lockDir, + timeoutMs: 100, + }); + if (!gatewayLock) { + throw new Error("expected test gateway lock"); + } + expect(isPathWithin(resolveGatewayLockDir(), state.stateDir)).toBe(false); + const gatewayCoordinatorPaths = [ + `${gatewayLock.lockPath}.sqlite`, + `${gatewayLock.stateLockPath}.sqlite`, + ]; + const extraTransientPaths = [ + path.join(lockDir, "device-identity.12345678.lock.sqlite"), + state.statePath("memory", "main.sqlite.reindex-lock.sqlite"), + ]; + + try { + for (const transientPath of [...gatewayCoordinatorPaths, ...extraTransientPaths]) { + await fs.mkdir(path.dirname(transientPath), { recursive: true }); + if (!gatewayCoordinatorPaths.includes(transientPath)) { + await fs.writeFile(transientPath, "transient coordinator database"); + } + for (const suffix of ["-wal", "-shm", "-journal"]) { + await fs.writeFile(`${transientPath}${suffix}`, "transient coordinator sidecar"); + } + } + + const result = await createBackupArchive({ + output: outputDir, + includeWorkspace: false, + nowMs: Date.UTC(2026, 7, 5, 12, 0, 0), + }); + const entries = await listArchiveEntries(result.archivePath); + for (const transientPath of [...gatewayCoordinatorPaths, ...extraTransientPaths]) { + const relativeTransientPath = path + .relative(state.stateDir, transientPath) + .split(path.sep) + .join("/"); + for (const suffix of ["", "-wal", "-shm", "-journal"]) { + expect( + entries.some((entry) => entry.endsWith(`/state/${relativeTransientPath}${suffix}`)), + `${relativeTransientPath}${suffix}`, + ).toBe(false); + } + } + expect( + entries.some((entry) => entry.endsWith("/state/plugins/dedicated/durable.sqlite")), + ).toBe(true); + expect( + entries.some((entry) => + entry.endsWith("/state/plugins/dedicated/gateway.12345678.lock.sqlite"), + ), + ).toBe(true); + expect( + entries.some((entry) => entry.endsWith(`/${path.basename(lockDir)}/retained.sqlite`)), + ).toBe(true); + + const runtime: RuntimeEnv = { log: vi.fn(), error: vi.fn(), exit: vi.fn() }; + await expect( + backupVerifyCommand(runtime, { archive: result.archivePath }), + ).resolves.toMatchObject({ ok: true }); + + await tar.x({ file: result.archivePath, gzip: true, cwd: extractDir }); + for (const [entrySuffix, value] of [ + ["/state/plugins/dedicated/durable.sqlite", "plugin-state"], + ["/state/plugins/dedicated/gateway.12345678.lock.sqlite", "producer-shaped-state"], + [`/${path.basename(lockDir)}/retained.sqlite`, "colocated-state"], + ] as const) { + const archivedEntry = expectDefined( + entries.find((entry) => entry.endsWith(entrySuffix)), + `archive entry ending with ${entrySuffix}`, + ); + const archivedDb = new sqlite.DatabaseSync(path.join(extractDir, archivedEntry), { + readOnly: true, + }); + try { + expect(archivedDb.prepare("SELECT value FROM durable_state").get()).toEqual({ + value, + }); + } finally { + archivedDb.close(); + } + } + } finally { + await gatewayLock.release(); + } + }, + ); + }); + it("preserves noncanonical symlinked SQLite paths without dereferencing them", async () => { if (process.platform === "win32") { return; diff --git a/src/infra/backup-create.ts b/src/infra/backup-create.ts index 79d7dffd494f..edbf89a8219f 100644 --- a/src/infra/backup-create.ts +++ b/src/infra/backup-create.ts @@ -12,6 +12,7 @@ import { resolveBackupPlanFromDisk, } from "../commands/backup-shared.js"; import { isPathWithin } from "../commands/cleanup-utils.js"; +import { resolveGatewayLockDir } from "../config/paths.js"; import { normalizeAgentId } from "../routing/session-key.js"; import { createLazyRuntimeModule } from "../shared/lazy-runtime.js"; import { assertOpenClawAgentDatabaseOwner } from "../state/openclaw-agent-db-maintenance.js"; @@ -31,7 +32,7 @@ import { } from "./backup-archive-publication.js"; import { removePreparedBackupArchive, writeArchiveStreamToFile } from "./backup-create-stream.js"; import { writeTarArchiveWithRetry } from "./backup-tar-retry.js"; -import { isVolatileBackupPath } from "./backup-volatile-filter.js"; +import { isTransientSqliteBackupPath, isVolatileBackupPath } from "./backup-volatile-filter.js"; import { createBackupLinkCache, createBackupVolatileStatCache, @@ -376,9 +377,6 @@ type StateSqliteBackupPlan = { }; const SQLITE_BACKUP_SOURCE_SUFFIXES = ["", "-wal", "-shm", "-journal"] as const; -const SQLITE_BACKUP_EXCLUDED_SUFFIXES = [".reindex-lock.sqlite"] as const; -const SQLITE_BACKUP_REINDEX_TRANSIENT_PATTERN = - /\.sqlite\.(?:backup|memory-reindex|tmp)-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/iu; function isCanonicalAgentSqlitePathOrAncestor(sourcePath: string, stateDir: string): boolean { const relativePath = path.relative(path.resolve(stateDir), path.resolve(sourcePath)); @@ -445,18 +443,10 @@ function resolveSqliteBackupDatabasePath(sourcePath: string): string | undefined return sourcePath.endsWith(".sqlite") ? sourcePath : undefined; } -function resolveSqliteBackupBasePath(sourcePath: string): string { - for (const suffix of SQLITE_BACKUP_SOURCE_SUFFIXES.slice(1)) { - if (sourcePath.endsWith(suffix)) { - return sourcePath.slice(0, -suffix.length); - } - } - return sourcePath; -} - function classifyStateSqliteBackupSourcePath( sourcePath: string, stateDir: string, + gatewayLockDirs: readonly string[], ): "excluded" | "sqlite" | undefined { const resolvedSourcePath = path.resolve(sourcePath); if (!isPathWithin(resolvedSourcePath, stateDir)) { @@ -465,18 +455,14 @@ function classifyStateSqliteBackupSourcePath( if (isStatePackageContentPath(resolvedSourcePath, stateDir)) { return undefined; } - if ( - SQLITE_BACKUP_REINDEX_TRANSIENT_PATTERN.test(resolveSqliteBackupBasePath(resolvedSourcePath)) - ) { + if (isTransientSqliteBackupPath(resolvedSourcePath, gatewayLockDirs)) { return "excluded"; } const databasePath = resolveSqliteBackupDatabasePath(resolvedSourcePath); if (!databasePath) { return undefined; } - return SQLITE_BACKUP_EXCLUDED_SUFFIXES.some((suffix) => databasePath.endsWith(suffix)) - ? "excluded" - : "sqlite"; + return "sqlite"; } function isBackupTarFilterFile(entry: import("node:fs").Stats | import("tar").ReadEntry): boolean { @@ -486,6 +472,7 @@ function isBackupTarFilterFile(entry: import("node:fs").Stats | import("tar").Re async function listStateSqlitePaths(params: { stateDir: string; globalStateSqlitePath: string; + gatewayLockDirs: readonly string[]; preservedStatePaths?: readonly string[]; }): Promise<{ snapshotPaths: string[]; discoveredSourcePaths: Set }> { const snapshotPaths = new Set(); @@ -534,13 +521,15 @@ async function listStateSqlitePaths(params: { !isStatePackageContentPath(entryPath, params.stateDir) ) { const resolvedEntryPath = path.resolve(entryPath); - if (resolveSqliteBackupDatabasePath(resolvedEntryPath)) { + const sqliteSourceKind = classifyStateSqliteBackupSourcePath( + resolvedEntryPath, + params.stateDir, + params.gatewayLockDirs, + ); + if (sqliteSourceKind === "sqlite") { discoveredSourcePaths.add(resolvedEntryPath); } - if ( - entry.name.endsWith(".sqlite") && - !SQLITE_BACKUP_EXCLUDED_SUFFIXES.some((suffix) => entry.name.endsWith(suffix)) - ) { + if (entry.name.endsWith(".sqlite") && sqliteSourceKind === "sqlite") { snapshotPaths.add(resolvedEntryPath); } } @@ -607,6 +596,12 @@ async function createStateSqliteBackupPlan(params: { const discovery = await listStateSqlitePaths({ stateDir: params.stateDir, globalStateSqlitePath, + // CLI and managed services use different temp roots for the same + // disposable gateway/device coordination databases. + gatewayLockDirs: [ + resolveGatewayLockDir(), + resolveGatewayLockDir(() => path.join(params.stateDir, "tmp")), + ], preservedStatePaths: params.preservedStatePaths, }); const globalStateIdentity = await fs.stat(globalStateSqlitePath).catch((error: unknown) => { @@ -871,6 +866,10 @@ export async function createBackupArchive( const stateFilter = stateAsset ? buildStateBackupFilter(stateAsset.sourcePath, preservedStatePaths) : undefined; + const gatewayLockDirs = [ + resolveGatewayLockDir(), + resolveGatewayLockDir(() => path.join(plan.stateDir, "tmp")), + ]; const volatilePlan = { stateDirs: [stateAsset?.sourcePath ?? plan.stateDir] }; let skippedVolatileCount = 0; // node-tar invokes filters from async stat callbacks, so throwing inside @@ -896,7 +895,11 @@ export async function createBackupArchive( return false; } const sqliteSourceKind = stateAsset - ? classifyStateSqliteBackupSourcePath(resolvedEntryPath, stateAsset.sourcePath) + ? classifyStateSqliteBackupSourcePath( + resolvedEntryPath, + stateAsset.sourcePath, + gatewayLockDirs, + ) : undefined; if (sqliteSourceKind === "excluded") { return false; diff --git a/src/infra/backup-volatile-filter.test.ts b/src/infra/backup-volatile-filter.test.ts index 25abfd7fa78a..65321d4dae81 100644 --- a/src/infra/backup-volatile-filter.test.ts +++ b/src/infra/backup-volatile-filter.test.ts @@ -1,6 +1,6 @@ // Tests volatile path filtering for backup operations. import { describe, expect, it } from "vitest"; -import { isVolatileBackupPath } from "./backup-volatile-filter.js"; +import { isTransientSqliteBackupPath, isVolatileBackupPath } from "./backup-volatile-filter.js"; const stateDir = "/opt/openclaw/state"; const plan = { stateDirs: [stateDir] }; @@ -126,3 +126,38 @@ describe("isVolatileBackupPath", () => { ).toBe(true); }); }); + +describe("isTransientSqliteBackupPath", () => { + it.each([ + "tmp/openclaw-502/gateway.12345678.lock.sqlite", + "tmp/openclaw-502/gateway.12345678.lock.sqlite-wal", + "tmp/openclaw-502/device-identity.12345678.lock.sqlite-journal", + ])("classifies transient coordinator state: %s", (filePath) => { + expect(isTransientSqliteBackupPath(filePath, ["tmp/openclaw-502"])).toBe(true); + }); + + it.each([ + "memory/main.sqlite.reindex-lock.sqlite", + "memory/main.sqlite.reindex-lock.sqlite-shm", + "memory/main.sqlite.tmp-11111111-2222-3333-4444-555555555555", + ])("classifies transient reindex state: %s", (filePath) => { + expect(isTransientSqliteBackupPath(filePath)).toBe(true); + }); + + it.each([ + "tmp/openclaw-502/retained.sqlite", + "plugins/dedicated/durable.sqlite", + "plugins/dedicated/cache.lock.sqlite", + "plugins/dedicated/durable.locked.sqlite", + "plugins/dedicated/lock.sqlite", + ])("preserves durable SQLite state: %s", (filePath) => { + expect(isTransientSqliteBackupPath(filePath, ["tmp/openclaw-502"])).toBe(false); + }); + + it.each([ + "plugins/dedicated/gateway.12345678.lock.sqlite", + "plugins/dedicated/device-identity.12345678.lock.sqlite", + ])("preserves coordinator-shaped databases outside the lock directory: %s", (filePath) => { + expect(isTransientSqliteBackupPath(filePath, ["tmp/openclaw-502"])).toBe(false); + }); +}); diff --git a/src/infra/backup-volatile-filter.ts b/src/infra/backup-volatile-filter.ts index 5d6bbcc10aa6..4b65e9ee278e 100644 --- a/src/infra/backup-volatile-filter.ts +++ b/src/infra/backup-volatile-filter.ts @@ -1,5 +1,6 @@ // Filters volatile files from backup manifests. import path from "node:path"; +import { isPathInside } from "./path-guards.js"; /** * Paths that are known to change during a live backup and commonly trigger @@ -13,6 +14,10 @@ import path from "node:path"; */ const STATE_TRANSIENT_EXTENSIONS = new Set([".sock", ".pid", ".tmp"]); +const SQLITE_COORDINATOR_BASENAME_PATTERN = + /^(?:gateway(?:\.state)?|device-identity)\.[0-9a-f]{8}\.lock\.sqlite(?:-wal|-shm|-journal)?$/iu; +const SQLITE_REINDEX_TRANSIENT_PATH_PATTERN = + /(?:^|\/)(?:[^/]+\.sqlite\.reindex-lock\.sqlite|[^/]+\.sqlite\.(?:backup|memory-reindex|tmp)-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:-wal|-shm|-journal)?$/iu; function normalizePosix(input: string): string { if (!input) { @@ -40,6 +45,20 @@ function hasExtensionInSet(filePosix: string, extensions: ReadonlySet): return extensions.has(path.posix.extname(filePosix).toLowerCase()); } +export function isTransientSqliteBackupPath( + filePath: string, + coordinatorDirs: readonly string[] = [], +): boolean { + const normalizedPath = normalizePosix(filePath); + if (SQLITE_REINDEX_TRANSIENT_PATH_PATTERN.test(normalizedPath)) { + return true; + } + if (!SQLITE_COORDINATOR_BASENAME_PATTERN.test(path.posix.basename(normalizedPath))) { + return false; + } + return coordinatorDirs.some((coordinatorDir) => isPathInside(coordinatorDir, filePath)); +} + function isAgentSessionTranscriptPath(filePosix: string, stateDirPosix: string): boolean { const agentsRoot = path.posix.join(stateDirPosix, "agents"); if (!isUnder(filePosix, agentsRoot)) {