fix(backup): skip transient gateway lock databases (#119782)

Reported by @AgentSolomon in #119757.

Fixes #119757

Punchcard-Session: amber-meadow-timber-8r

src/commands/backup-verify.test.ts

@@ -836,6 +836,11 @@ describe("backupVerifyCommand", () => {
             contents: invalidSqlite,
             archivePath: `${stateAssetArchivePath}/memory/main.sqlite.reindex-lock.sqlite`,
           },
+          {
+            fileName: "reindex-lock.sqlite-wal",
+            contents: invalidSqlite,
+            archivePath: `${stateAssetArchivePath}/memory/main.sqlite.reindex-lock.sqlite-wal`,
+          },
           {
             fileName: "reindex-tmp",
             contents: invalidSqlite,

src/commands/backup-verify.ts

@@ -6,6 +6,7 @@ import type { DatabaseSync } from "node:sqlite";
 import { readStringValue } from "@openclaw/normalization-core/string-coerce";
 import * as tar from "tar";
 import { loadSqliteVecExtension } from "../../packages/memory-host-sdk/src/engine-storage.js";
+import { isTransientSqliteBackupPath } from "../infra/backup-volatile-filter.js";
 import { formatDiskSpaceBytes, tryReadDiskSpace } from "../infra/disk-space.js";
 import { openNodeSqliteDatabase } from "../infra/node-sqlite.js";
 import { assertSqliteIntegrity } from "../infra/sqlite-integrity.js";
@@ -18,9 +19,6 @@ const MAX_MANIFEST_BYTES = 1024 * 1024;
 const MAX_SQLITE_SNAPSHOT_EXTRACT_BYTES = 64 * 1024 * 1024 * 1024;
 const SQLITE_SNAPSHOT_FREE_SPACE_RESERVE_BYTES = 256 * 1024 * 1024;
 const SQLITE_SNAPSHOT_SIDECAR_SUFFIXES = ["-wal", "-shm", "-journal"] as const;
-const SQLITE_BACKUP_EXCLUDED_SUFFIXES = [".reindex-lock.sqlite"] as const;
-const SQLITE_BACKUP_REINDEX_TRANSIENT_PATTERN =
-  /\.sqlite\.(?:backup|memory-reindex|tmp)-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/iu;
 
 type BackupManifestAsset = {
   kind: string;
@@ -400,9 +398,7 @@ function isSqliteSnapshotRelativePath(relativePath: string): boolean {
     return true;
   }
   return (
-    !portablePath.split("/").includes("node_modules") &&
-    !SQLITE_BACKUP_REINDEX_TRANSIENT_PATTERN.test(relativePath) &&
-    !SQLITE_BACKUP_EXCLUDED_SUFFIXES.some((suffix) => portablePath.endsWith(suffix))
+    !portablePath.split("/").includes("node_modules") && !isTransientSqliteBackupPath(portablePath)
   );
 }
 

src/infra/backup-create.test.ts

@@ -8,7 +8,9 @@ import * as tar from "tar";
 import { describe, expect, it, vi } from "vitest";
 import { saveAuthProfileStore } from "../agents/auth-profiles/store.js";
 import { backupVerifyCommand } from "../commands/backup-verify.js";
+import { isPathWithin } from "../commands/cleanup-utils.js";
 import { CONFIG_AUDIT_MAX_ENTRIES, CONFIG_AUDIT_SCOPE } from "../config/io.audit.js";
+import { resolveGatewayLockDir } from "../config/paths.js";
 import type { RuntimeEnv } from "../runtime.js";
 import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js";
 import {
@@ -33,6 +35,7 @@ import {
 import { writeTarArchiveWithRetry } from "./backup-tar-retry.js";
 import { isVolatileBackupPath } from "./backup-volatile-filter.js";
 import { createBackupVolatileStatCache } from "./backup-volatile-stat-cache.js";
+import { acquireGatewayLock } from "./gateway-lock.js";
 import { requireNodeSqlite } from "./node-sqlite.js";
 import { createSqliteAuditRecordStore } from "./sqlite-audit-record-store.js";
 import { detectLegacyAuditLogs, migrateLegacyAuditLogs } from "./state-migrations.audit-logs.js";
@@ -1659,7 +1662,7 @@ describe("createBackupArchive", () => {
     );
   });
 
-  it("snapshots nested live SQLite databases with transaction continuity", async () => {
+  it("snapshots lock-named plugin SQLite databases with transaction continuity", async () => {
     await withOpenClawTestState(
       {
         layout: "state-only",
@@ -1669,7 +1672,7 @@ describe("createBackupArchive", () => {
       async (state) => {
         const outputDir = state.path("backups");
         const extractDir = state.path("extract");
-        const dbPath = state.statePath("plugins", "dedicated", "live.sqlite");
+        const dbPath = state.statePath("plugins", "dedicated", "cache.lock.sqlite");
         await fs.mkdir(path.dirname(dbPath), { recursive: true });
         await fs.mkdir(outputDir, { recursive: true });
         await fs.mkdir(extractDir, { recursive: true });
@@ -1714,13 +1717,13 @@ describe("createBackupArchive", () => {
           });
           const entries = await listArchiveEntries(result.archivePath);
           const archivedDbEntries = entries.filter((entry) =>
-            entry.endsWith("/state/plugins/dedicated/live.sqlite"),
+            entry.endsWith("/state/plugins/dedicated/cache.lock.sqlite"),
           );
           expect(archivedDbEntries).toHaveLength(1);
           for (const suffix of ["-wal", "-shm", "-journal"]) {
             expect(
               entries.some((entry) =>
-                entry.endsWith(`/state/plugins/dedicated/live.sqlite${suffix}`),
+                entry.endsWith(`/state/plugins/dedicated/cache.lock.sqlite${suffix}`),
               ),
               suffix,
             ).toBe(false);
@@ -2025,6 +2028,137 @@ describe("createBackupArchive", () => {
     );
   });
 
+  it("backs up durable SQLite while live gateway coordinators remain held under state", async () => {
+    await withOpenClawTestState(
+      {
+        layout: "state-only",
+        prefix: "openclaw-backup-gateway-lock-sqlite-",
+        scenario: "minimal",
+      },
+      async (state) => {
+        const outputDir = state.path("backups");
+        const extractDir = state.path("extract");
+        const lockDir = resolveGatewayLockDir(() => state.statePath("tmp"));
+        const pluginDbPath = state.statePath("plugins", "dedicated", "durable.sqlite");
+        const producerShapedDbPath = state.statePath(
+          "plugins",
+          "dedicated",
+          "gateway.12345678.lock.sqlite",
+        );
+        const colocatedDbPath = path.join(lockDir, "retained.sqlite");
+        await fs.mkdir(outputDir, { recursive: true });
+        await fs.mkdir(extractDir, { recursive: true });
+        await fs.mkdir(path.dirname(pluginDbPath), { recursive: true });
+        await fs.mkdir(lockDir, { recursive: true });
+
+        const sqlite = requireNodeSqlite();
+        for (const [databasePath, value] of [
+          [pluginDbPath, "plugin-state"],
+          [producerShapedDbPath, "producer-shaped-state"],
+          [colocatedDbPath, "colocated-state"],
+        ] as const) {
+          const database = new sqlite.DatabaseSync(databasePath);
+          try {
+            database.exec("CREATE TABLE durable_state (value TEXT NOT NULL)");
+            database.prepare("INSERT INTO durable_state (value) VALUES (?)").run(value);
+          } finally {
+            database.close();
+          }
+        }
+
+        const gatewayLock = await acquireGatewayLock({
+          allowInTests: true,
+          env: state.env,
+          lockDir,
+          timeoutMs: 100,
+        });
+        if (!gatewayLock) {
+          throw new Error("expected test gateway lock");
+        }
+        expect(isPathWithin(resolveGatewayLockDir(), state.stateDir)).toBe(false);
+        const gatewayCoordinatorPaths = [
+          `${gatewayLock.lockPath}.sqlite`,
+          `${gatewayLock.stateLockPath}.sqlite`,
+        ];
+        const extraTransientPaths = [
+          path.join(lockDir, "device-identity.12345678.lock.sqlite"),
+          state.statePath("memory", "main.sqlite.reindex-lock.sqlite"),
+        ];
+
+        try {
+          for (const transientPath of [...gatewayCoordinatorPaths, ...extraTransientPaths]) {
+            await fs.mkdir(path.dirname(transientPath), { recursive: true });
+            if (!gatewayCoordinatorPaths.includes(transientPath)) {
+              await fs.writeFile(transientPath, "transient coordinator database");
+            }
+            for (const suffix of ["-wal", "-shm", "-journal"]) {
+              await fs.writeFile(`${transientPath}${suffix}`, "transient coordinator sidecar");
+            }
+          }
+
+          const result = await createBackupArchive({
+            output: outputDir,
+            includeWorkspace: false,
+            nowMs: Date.UTC(2026, 7, 5, 12, 0, 0),
+          });
+          const entries = await listArchiveEntries(result.archivePath);
+          for (const transientPath of [...gatewayCoordinatorPaths, ...extraTransientPaths]) {
+            const relativeTransientPath = path
+              .relative(state.stateDir, transientPath)
+              .split(path.sep)
+              .join("/");
+            for (const suffix of ["", "-wal", "-shm", "-journal"]) {
+              expect(
+                entries.some((entry) => entry.endsWith(`/state/${relativeTransientPath}${suffix}`)),
+                `${relativeTransientPath}${suffix}`,
+              ).toBe(false);
+            }
+          }
+          expect(
+            entries.some((entry) => entry.endsWith("/state/plugins/dedicated/durable.sqlite")),
+          ).toBe(true);
+          expect(
+            entries.some((entry) =>
+              entry.endsWith("/state/plugins/dedicated/gateway.12345678.lock.sqlite"),
+            ),
+          ).toBe(true);
+          expect(
+            entries.some((entry) => entry.endsWith(`/${path.basename(lockDir)}/retained.sqlite`)),
+          ).toBe(true);
+
+          const runtime: RuntimeEnv = { log: vi.fn(), error: vi.fn(), exit: vi.fn() };
+          await expect(
+            backupVerifyCommand(runtime, { archive: result.archivePath }),
+          ).resolves.toMatchObject({ ok: true });
+
+          await tar.x({ file: result.archivePath, gzip: true, cwd: extractDir });
+          for (const [entrySuffix, value] of [
+            ["/state/plugins/dedicated/durable.sqlite", "plugin-state"],
+            ["/state/plugins/dedicated/gateway.12345678.lock.sqlite", "producer-shaped-state"],
+            [`/${path.basename(lockDir)}/retained.sqlite`, "colocated-state"],
+          ] as const) {
+            const archivedEntry = expectDefined(
+              entries.find((entry) => entry.endsWith(entrySuffix)),
+              `archive entry ending with ${entrySuffix}`,
+            );
+            const archivedDb = new sqlite.DatabaseSync(path.join(extractDir, archivedEntry), {
+              readOnly: true,
+            });
+            try {
+              expect(archivedDb.prepare("SELECT value FROM durable_state").get()).toEqual({
+                value,
+              });
+            } finally {
+              archivedDb.close();
+            }
+          }
+        } finally {
+          await gatewayLock.release();
+        }
+      },
+    );
+  });
+
   it("preserves noncanonical symlinked SQLite paths without dereferencing them", async () => {
     if (process.platform === "win32") {
       return;

src/infra/backup-create.ts

@@ -12,6 +12,7 @@ import {
   resolveBackupPlanFromDisk,
 } from "../commands/backup-shared.js";
 import { isPathWithin } from "../commands/cleanup-utils.js";
+import { resolveGatewayLockDir } from "../config/paths.js";
 import { normalizeAgentId } from "../routing/session-key.js";
 import { createLazyRuntimeModule } from "../shared/lazy-runtime.js";
 import { assertOpenClawAgentDatabaseOwner } from "../state/openclaw-agent-db-maintenance.js";
@@ -31,7 +32,7 @@ import {
 } from "./backup-archive-publication.js";
 import { removePreparedBackupArchive, writeArchiveStreamToFile } from "./backup-create-stream.js";
 import { writeTarArchiveWithRetry } from "./backup-tar-retry.js";
-import { isVolatileBackupPath } from "./backup-volatile-filter.js";
+import { isTransientSqliteBackupPath, isVolatileBackupPath } from "./backup-volatile-filter.js";
 import {
   createBackupLinkCache,
   createBackupVolatileStatCache,
@@ -376,9 +377,6 @@ type StateSqliteBackupPlan = {
 };
 
 const SQLITE_BACKUP_SOURCE_SUFFIXES = ["", "-wal", "-shm", "-journal"] as const;
-const SQLITE_BACKUP_EXCLUDED_SUFFIXES = [".reindex-lock.sqlite"] as const;
-const SQLITE_BACKUP_REINDEX_TRANSIENT_PATTERN =
-  /\.sqlite\.(?:backup|memory-reindex|tmp)-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/iu;
 
 function isCanonicalAgentSqlitePathOrAncestor(sourcePath: string, stateDir: string): boolean {
   const relativePath = path.relative(path.resolve(stateDir), path.resolve(sourcePath));
@@ -445,18 +443,10 @@ function resolveSqliteBackupDatabasePath(sourcePath: string): string | undefined
   return sourcePath.endsWith(".sqlite") ? sourcePath : undefined;
 }
 
-function resolveSqliteBackupBasePath(sourcePath: string): string {
-  for (const suffix of SQLITE_BACKUP_SOURCE_SUFFIXES.slice(1)) {
-    if (sourcePath.endsWith(suffix)) {
-      return sourcePath.slice(0, -suffix.length);
-    }
-  }
-  return sourcePath;
-}
-
 function classifyStateSqliteBackupSourcePath(
   sourcePath: string,
   stateDir: string,
+  gatewayLockDirs: readonly string[],
 ): "excluded" | "sqlite" | undefined {
   const resolvedSourcePath = path.resolve(sourcePath);
   if (!isPathWithin(resolvedSourcePath, stateDir)) {
@@ -465,18 +455,14 @@ function classifyStateSqliteBackupSourcePath(
   if (isStatePackageContentPath(resolvedSourcePath, stateDir)) {
     return undefined;
   }
-  if (
-    SQLITE_BACKUP_REINDEX_TRANSIENT_PATTERN.test(resolveSqliteBackupBasePath(resolvedSourcePath))
-  ) {
+  if (isTransientSqliteBackupPath(resolvedSourcePath, gatewayLockDirs)) {
     return "excluded";
   }
   const databasePath = resolveSqliteBackupDatabasePath(resolvedSourcePath);
   if (!databasePath) {
     return undefined;
   }
-  return SQLITE_BACKUP_EXCLUDED_SUFFIXES.some((suffix) => databasePath.endsWith(suffix))
-    ? "excluded"
-    : "sqlite";
+  return "sqlite";
 }
 
 function isBackupTarFilterFile(entry: import("node:fs").Stats | import("tar").ReadEntry): boolean {
@@ -486,6 +472,7 @@ function isBackupTarFilterFile(entry: import("node:fs").Stats | import("tar").Re
 async function listStateSqlitePaths(params: {
   stateDir: string;
   globalStateSqlitePath: string;
+  gatewayLockDirs: readonly string[];
   preservedStatePaths?: readonly string[];
 }): Promise<{ snapshotPaths: string[]; discoveredSourcePaths: Set<string> }> {
   const snapshotPaths = new Set<string>();
@@ -534,13 +521,15 @@ async function listStateSqlitePaths(params: {
         !isStatePackageContentPath(entryPath, params.stateDir)
       ) {
         const resolvedEntryPath = path.resolve(entryPath);
-        if (resolveSqliteBackupDatabasePath(resolvedEntryPath)) {
+        const sqliteSourceKind = classifyStateSqliteBackupSourcePath(
+          resolvedEntryPath,
+          params.stateDir,
+          params.gatewayLockDirs,
+        );
+        if (sqliteSourceKind === "sqlite") {
           discoveredSourcePaths.add(resolvedEntryPath);
         }
-        if (
-          entry.name.endsWith(".sqlite") &&
-          !SQLITE_BACKUP_EXCLUDED_SUFFIXES.some((suffix) => entry.name.endsWith(suffix))
-        ) {
+        if (entry.name.endsWith(".sqlite") && sqliteSourceKind === "sqlite") {
           snapshotPaths.add(resolvedEntryPath);
         }
       }
@@ -607,6 +596,12 @@ async function createStateSqliteBackupPlan(params: {
   const discovery = await listStateSqlitePaths({
     stateDir: params.stateDir,
     globalStateSqlitePath,
+    // CLI and managed services use different temp roots for the same
+    // disposable gateway/device coordination databases.
+    gatewayLockDirs: [
+      resolveGatewayLockDir(),
+      resolveGatewayLockDir(() => path.join(params.stateDir, "tmp")),
+    ],
     preservedStatePaths: params.preservedStatePaths,
   });
   const globalStateIdentity = await fs.stat(globalStateSqlitePath).catch((error: unknown) => {
@@ -871,6 +866,10 @@ export async function createBackupArchive(
     const stateFilter = stateAsset
       ? buildStateBackupFilter(stateAsset.sourcePath, preservedStatePaths)
       : undefined;
+    const gatewayLockDirs = [
+      resolveGatewayLockDir(),
+      resolveGatewayLockDir(() => path.join(plan.stateDir, "tmp")),
+    ];
     const volatilePlan = { stateDirs: [stateAsset?.sourcePath ?? plan.stateDir] };
     let skippedVolatileCount = 0;
     // node-tar invokes filters from async stat callbacks, so throwing inside
@@ -896,7 +895,11 @@ export async function createBackupArchive(
         return false;
       }
       const sqliteSourceKind = stateAsset
-        ? classifyStateSqliteBackupSourcePath(resolvedEntryPath, stateAsset.sourcePath)
+        ? classifyStateSqliteBackupSourcePath(
+            resolvedEntryPath,
+            stateAsset.sourcePath,
+            gatewayLockDirs,
+          )
         : undefined;
       if (sqliteSourceKind === "excluded") {
         return false;

src/infra/backup-volatile-filter.test.ts

@@ -1,6 +1,6 @@
 // Tests volatile path filtering for backup operations.
 import { describe, expect, it } from "vitest";
-import { isVolatileBackupPath } from "./backup-volatile-filter.js";
+import { isTransientSqliteBackupPath, isVolatileBackupPath } from "./backup-volatile-filter.js";
 
 const stateDir = "/opt/openclaw/state";
 const plan = { stateDirs: [stateDir] };
@@ -126,3 +126,38 @@ describe("isVolatileBackupPath", () => {
     ).toBe(true);
   });
 });
+
+describe("isTransientSqliteBackupPath", () => {
+  it.each([
+    "tmp/openclaw-502/gateway.12345678.lock.sqlite",
+    "tmp/openclaw-502/gateway.12345678.lock.sqlite-wal",
+    "tmp/openclaw-502/device-identity.12345678.lock.sqlite-journal",
+  ])("classifies transient coordinator state: %s", (filePath) => {
+    expect(isTransientSqliteBackupPath(filePath, ["tmp/openclaw-502"])).toBe(true);
+  });
+
+  it.each([
+    "memory/main.sqlite.reindex-lock.sqlite",
+    "memory/main.sqlite.reindex-lock.sqlite-shm",
+    "memory/main.sqlite.tmp-11111111-2222-3333-4444-555555555555",
+  ])("classifies transient reindex state: %s", (filePath) => {
+    expect(isTransientSqliteBackupPath(filePath)).toBe(true);
+  });
+
+  it.each([
+    "tmp/openclaw-502/retained.sqlite",
+    "plugins/dedicated/durable.sqlite",
+    "plugins/dedicated/cache.lock.sqlite",
+    "plugins/dedicated/durable.locked.sqlite",
+    "plugins/dedicated/lock.sqlite",
+  ])("preserves durable SQLite state: %s", (filePath) => {
+    expect(isTransientSqliteBackupPath(filePath, ["tmp/openclaw-502"])).toBe(false);
+  });
+
+  it.each([
+    "plugins/dedicated/gateway.12345678.lock.sqlite",
+    "plugins/dedicated/device-identity.12345678.lock.sqlite",
+  ])("preserves coordinator-shaped databases outside the lock directory: %s", (filePath) => {
+    expect(isTransientSqliteBackupPath(filePath, ["tmp/openclaw-502"])).toBe(false);
+  });
+});

src/infra/backup-volatile-filter.ts

@@ -1,5 +1,6 @@
 // Filters volatile files from backup manifests.
 import path from "node:path";
+import { isPathInside } from "./path-guards.js";
 
 /**
  * Paths that are known to change during a live backup and commonly trigger
@@ -13,6 +14,10 @@ import path from "node:path";
  */
 
 const STATE_TRANSIENT_EXTENSIONS = new Set([".sock", ".pid", ".tmp"]);
+const SQLITE_COORDINATOR_BASENAME_PATTERN =
+  /^(?:gateway(?:\.state)?|device-identity)\.[0-9a-f]{8}\.lock\.sqlite(?:-wal|-shm|-journal)?$/iu;
+const SQLITE_REINDEX_TRANSIENT_PATH_PATTERN =
+  /(?:^|\/)(?:[^/]+\.sqlite\.reindex-lock\.sqlite|[^/]+\.sqlite\.(?:backup|memory-reindex|tmp)-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})(?:-wal|-shm|-journal)?$/iu;
 
 function normalizePosix(input: string): string {
   if (!input) {
@@ -40,6 +45,20 @@ function hasExtensionInSet(filePosix: string, extensions: ReadonlySet<string>):
   return extensions.has(path.posix.extname(filePosix).toLowerCase());
 }
 
+export function isTransientSqliteBackupPath(
+  filePath: string,
+  coordinatorDirs: readonly string[] = [],
+): boolean {
+  const normalizedPath = normalizePosix(filePath);
+  if (SQLITE_REINDEX_TRANSIENT_PATH_PATTERN.test(normalizedPath)) {
+    return true;
+  }
+  if (!SQLITE_COORDINATOR_BASENAME_PATTERN.test(path.posix.basename(normalizedPath))) {
+    return false;
+  }
+  return coordinatorDirs.some((coordinatorDir) => isPathInside(coordinatorDir, filePath));
+}
+
 function isAgentSessionTranscriptPath(filePosix: string, stateDirPosix: string): boolean {
   const agentsRoot = path.posix.join(stateDirPosix, "agents");
   if (!isUnder(filePosix, agentsRoot)) {