From 5a4004dfb76f2f1a7447862ff4e0d3a3f11ca262 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 16 Jul 2026 08:56:31 -0700 Subject: [PATCH] fix: state backups hang on dev-channel installs (#109090) * fix(backup): exclude managed runtime trees * chore: leave release notes to release automation * fix(backup): preserve managed-root workspaces * fix(backup): preserve configured state paths * test(gateway): isolate runtime service env --- docs/cli/backup.md | 4 +- src/gateway/server-runtime-services.test.ts | 5 ++ src/infra/backup-create-stream.ts | 52 +++++++++-- src/infra/backup-create.test.ts | 99 ++++++++++++++++++++- src/infra/backup-create.ts | 49 +++++++--- src/infra/backup-create.windows.test.ts | 52 ++++++++++- 6 files changed, 242 insertions(+), 19 deletions(-) diff --git a/docs/cli/backup.md b/docs/cli/backup.md index d2b55a06f77a..7725e584ee7a 100644 --- a/docs/cli/backup.md +++ b/docs/cli/backup.md @@ -107,6 +107,8 @@ SQLite databases under the state directory are compacted with `VACUUM INTO` so d Installed plugin source and manifest files under the state directory's `extensions/` tree are included, but their nested `node_modules/` dependency trees are skipped as rebuildable install artifacts. After restoring an archive, use `openclaw plugins update ` or reinstall with `openclaw plugins install --force` if a restored plugin reports missing dependencies. +Installer-managed and rebuildable runtime roots under the state directory are also skipped: `dev/`, `git/`, `npm/`, legacy `npm-runtime/`, and `tools/`. These contain managed checkouts, package trees, and downloaded runtimes rather than authoritative user state; reinstall or update the corresponding runtime or plugin after restore. An explicitly configured config file, credentials directory, or workspace inside one of these roots remains included. + ## Invalid config behavior `openclaw backup` bypasses the normal config preflight so it can still help during recovery. Workspace discovery depends on a valid config, so `openclaw backup create` fails fast when the config file exists but is invalid and workspace backup is still enabled. @@ -117,7 +119,7 @@ For a partial backup in that situation, rerun with `--no-include-workspace`: it ## Size and performance -OpenClaw does not enforce a built-in maximum backup size or per-file size limit. Practical limits come from: +OpenClaw does not enforce a built-in maximum backup size or per-file size limit. An archive write that produces no data for five minutes fails and removes its partial temporary file instead of hanging indefinitely. Practical limits otherwise come from: - Available space for the temporary archive write plus the final archive - Time to walk large workspace trees and compress them into a `.tar.gz` diff --git a/src/gateway/server-runtime-services.test.ts b/src/gateway/server-runtime-services.test.ts index 4869f1fab8bb..62d620498a10 100644 --- a/src/gateway/server-runtime-services.test.ts +++ b/src/gateway/server-runtime-services.test.ts @@ -108,6 +108,10 @@ const { describe("server-runtime-services", () => { beforeEach(() => { vi.useRealTimers(); + // Gateway test helpers set these at module load. Stub them off so a shared + // worker's import order cannot silently disable this suite's health monitor. + vi.stubEnv("OPENCLAW_SKIP_CHANNELS", ""); + vi.stubEnv("OPENCLAW_SKIP_PROVIDERS", ""); resetGatewayWorkAdmission(); hoisted.heartbeatRunner.stop.mockClear(); hoisted.heartbeatRunner.updateConfig.mockClear(); @@ -131,6 +135,7 @@ describe("server-runtime-services", () => { afterEach(() => { vi.useRealTimers(); + vi.unstubAllEnvs(); resetGatewayWorkAdmission(); }); diff --git a/src/infra/backup-create-stream.ts b/src/infra/backup-create-stream.ts index 5f28cfeba7a9..0f652ab4af21 100644 --- a/src/infra/backup-create-stream.ts +++ b/src/infra/backup-create-stream.ts @@ -1,15 +1,57 @@ import { createWriteStream } from "node:fs"; +import { Transform } from "node:stream"; import { pipeline } from "node:stream/promises"; +const BACKUP_ARCHIVE_IDLE_TIMEOUT_MS = 5 * 60_000; + +type DestroyableArchiveStream = (NodeJS.ReadableStream | AsyncIterable) & { + destroy(error?: Error): unknown; +}; + export async function writeArchiveStreamToFile(params: { archivePath: string; - archiveStream: AsyncIterable | NodeJS.ReadableStream; + archiveStream: DestroyableArchiveStream; + idleTimeoutMs?: number; }): Promise { // Own both stream lifecycles so a tar read error closes the output handle // before retry cleanup touches the partial archive. Exclusive creation also // refuses a pre-existing path instead of following a symlink. - await pipeline( - params.archiveStream, - createWriteStream(params.archivePath, { flags: "wx", mode: 0o600 }), - ); + const idleTimeoutMs = params.idleTimeoutMs ?? BACKUP_ARCHIVE_IDLE_TIMEOUT_MS; + const controller = new AbortController(); + let idleTimer: ReturnType | undefined; + let idleTimeoutError: Error | undefined; + const armIdleTimer = () => { + if (idleTimer) { + clearTimeout(idleTimer); + } + idleTimer = setTimeout(() => { + idleTimeoutError = new Error( + `Backup archive write stalled: no data produced for ${idleTimeoutMs}ms`, + ); + params.archiveStream.destroy(idleTimeoutError); + controller.abort(idleTimeoutError); + }, idleTimeoutMs); + }; + const progress = new Transform({ + transform(chunk, _encoding, callback) { + armIdleTimer(); + callback(null, chunk); + }, + }); + + armIdleTimer(); + try { + await pipeline( + params.archiveStream, + progress, + createWriteStream(params.archivePath, { flags: "wx", mode: 0o600 }), + { signal: controller.signal }, + ); + } catch (err) { + throw idleTimeoutError ?? err; + } finally { + if (idleTimer) { + clearTimeout(idleTimer); + } + } } diff --git a/src/infra/backup-create.test.ts b/src/infra/backup-create.test.ts index 35384e063892..8347a9eb630a 100644 --- a/src/infra/backup-create.test.ts +++ b/src/infra/backup-create.test.ts @@ -1369,7 +1369,7 @@ describe("createBackupArchive", () => { ); }); - it("omits installed plugin node_modules from the real archive while keeping plugin files", async () => { + it("omits reinstallable runtime trees and plugin dependencies while keeping plugin files", async () => { await withOpenClawTestState( { layout: "state-only", @@ -1387,6 +1387,14 @@ describe("createBackupArchive", () => { await fs.mkdir(path.join(stateDir, "npm", "projects", "demo", "node_modules", "dep"), { recursive: true, }); + for (const managedRoot of ["dev", "git", "npm-runtime", "tools"]) { + await fs.mkdir(path.join(stateDir, managedRoot, "runtime"), { recursive: true }); + await fs.writeFile( + path.join(stateDir, managedRoot, "runtime", "fixture.sqlite"), + "reinstallable runtime content\n", + "utf8", + ); + } await fs.writeFile( path.join(stateDir, "extensions", "demo", "openclaw.plugin.json"), '{"id":"demo"}\n', @@ -1436,7 +1444,15 @@ describe("createBackupArchive", () => { expect(entrySuffixes).toContain("/state/extensions/demo/src/index.js"); expect(entrySuffixes).toContain("/state/node_modules/root-dep/index.js"); expect(entrySuffixes).toContain("/state/node_modules/root-dep/fixture.sqlite"); - expect(entrySuffixes).toContain("/state/npm/projects/demo/node_modules/dep/fixture.sqlite"); + for (const managedRoot of ["dev", "git", "npm", "npm-runtime", "tools"]) { + expect( + entrySuffixes.some( + (entry) => + entry === `/state/${managedRoot}` || entry.startsWith(`/state/${managedRoot}/`), + ), + managedRoot, + ).toBe(false); + } const pluginNodeModuleEntries = entries.filter((entry) => entry.includes("/state/extensions/demo/node_modules/"), ); @@ -1449,6 +1465,85 @@ describe("createBackupArchive", () => { ); }); + it("preserves configured state paths nested under managed runtime roots", async () => { + await withOpenClawTestState( + { + layout: "state-only", + prefix: "openclaw-backup-managed-root-workspace-", + scenario: "minimal", + env: { OPENCLAW_OAUTH_DIR: undefined }, + }, + async (state) => { + const stateDir = state.stateDir; + const workspaceDir = path.join(stateDir, "dev", "workspace"); + const runtimeDir = path.join(stateDir, "dev", "openclaw"); + const configPath = path.join(stateDir, "git", "config", "openclaw.json"); + const oauthDir = path.join(stateDir, "tools", "oauth"); + const toolRuntimeDir = path.join(stateDir, "tools", "runtime"); + const workspaceDbPath = path.join(workspaceDir, "workspace.sqlite"); + const outputDir = state.path("backups"); + state.envVars.OPENCLAW_CONFIG_PATH = configPath; + state.envVars.OPENCLAW_OAUTH_DIR = oauthDir; + state.applyEnv(); + await fs.mkdir(workspaceDir, { recursive: true }); + await fs.mkdir(runtimeDir, { recursive: true }); + await fs.mkdir(path.dirname(configPath), { recursive: true }); + await fs.mkdir(oauthDir, { recursive: true }); + await fs.mkdir(toolRuntimeDir, { recursive: true }); + await fs.writeFile( + configPath, + `${JSON.stringify({ + agents: { + list: [{ id: "main", default: true, workspace: workspaceDir }], + }, + })}\n`, + "utf8", + ); + await fs.writeFile(path.join(oauthDir, "credentials.json"), "{}\n", "utf8"); + await fs.writeFile(path.join(workspaceDir, "AGENTS.md"), "durable workspace\n", "utf8"); + await fs.writeFile(path.join(runtimeDir, "package.json"), "{}\n", "utf8"); + await fs.writeFile(path.join(toolRuntimeDir, "tool.bin"), "runtime\n", "utf8"); + const sqlite = requireNodeSqlite(); + const workspaceDb = new sqlite.DatabaseSync(workspaceDbPath); + try { + workspaceDb.exec( + "CREATE TABLE durable_state (value TEXT NOT NULL); INSERT INTO durable_state VALUES ('keep');", + ); + } finally { + workspaceDb.close(); + } + await fs.mkdir(outputDir, { recursive: true }); + + const result = await createBackupArchive({ + output: outputDir, + includeWorkspace: true, + nowMs: Date.UTC(2026, 3, 28, 12, 30, 0), + }); + const entries = await listArchiveEntries(result.archivePath); + + expect(entries.some((entry) => entry.endsWith("/state/dev/workspace/AGENTS.md"))).toBe( + true, + ); + expect( + entries.some((entry) => entry.endsWith("/state/dev/workspace/workspace.sqlite")), + ).toBe(true); + expect(entries.some((entry) => entry.endsWith("/state/git/config/openclaw.json"))).toBe( + true, + ); + expect(entries.some((entry) => entry.endsWith("/state/tools/oauth/credentials.json"))).toBe( + true, + ); + expect(entries.some((entry) => entry.includes("/state/dev/openclaw/"))).toBe(false); + expect(entries.some((entry) => entry.includes("/state/tools/runtime/"))).toBe(false); + + const runtime: RuntimeEnv = { log: vi.fn(), error: vi.fn(), exit: vi.fn() }; + await expect( + backupVerifyCommand(runtime, { archive: result.archivePath }), + ).resolves.toMatchObject({ ok: true }); + }, + ); + }); + it("dereferences hardlinks instead of emitting restore-hostile Link entries", async () => { await withOpenClawTestState( { diff --git a/src/infra/backup-create.ts b/src/infra/backup-create.ts index f10e3d793b68..05e9ecb28c4a 100644 --- a/src/infra/backup-create.ts +++ b/src/infra/backup-create.ts @@ -363,17 +363,35 @@ function normalizeBackupFilterPath(value: string): string { return value.replaceAll("\\", "/").replace(/\/+$/u, ""); } -function buildExtensionsNodeModulesFilter(stateDir: string): (filePath: string) => boolean { +const REINSTALLABLE_STATE_ROOTS = new Set(["dev", "git", "npm", "npm-runtime", "tools"]); + +function buildStateBackupFilter( + stateDir: string, + preservedStatePaths: readonly string[] = [], +): (filePath: string) => boolean { const normalizedStateDir = normalizeBackupFilterPath(stateDir); - const extensionsPrefix = `${normalizedStateDir}/extensions/`; + const statePrefix = `${normalizedStateDir}/`; + const resolvedPreservedPaths = preservedStatePaths.map((entry) => path.resolve(entry)); return (filePath: string): boolean => { const normalizedFilePath = normalizeBackupFilterPath(filePath); - if (!normalizedFilePath.startsWith(extensionsPrefix)) { + if (!normalizedFilePath.startsWith(statePrefix)) { return true; } - return !normalizedFilePath.slice(extensionsPrefix.length).split("/").includes("node_modules"); + const segments = normalizedFilePath.slice(statePrefix.length).split("/"); + if (REINSTALLABLE_STATE_ROOTS.has(segments[0] ?? "")) { + const resolvedFilePath = path.resolve(filePath); + // Configured workspaces nested under a managed root remain authoritative + // user state. Keep their ancestors traversable without admitting siblings. + return resolvedPreservedPaths.some( + (preservedPath) => + isPathWithin(resolvedFilePath, preservedPath) || + isPathWithin(preservedPath, resolvedFilePath), + ); + } + + return segments[0] !== "extensions" || !segments.includes("node_modules"); }; } @@ -490,10 +508,11 @@ function sanitizeGlobalStateSqliteSnapshot(db: DatabaseSync): void { async function listStateSqlitePaths(params: { stateDir: string; globalStateSqlitePath: string; + preservedStatePaths?: readonly string[]; }): Promise<{ snapshotPaths: string[]; discoveredSourcePaths: Set }> { const snapshotPaths = new Set(); const discoveredSourcePaths = new Set(); - const extensionsFilter = buildExtensionsNodeModulesFilter(params.stateDir); + const stateFilter = buildStateBackupFilter(params.stateDir, params.preservedStatePaths); async function visit(dir: string): Promise { let entries: import("node:fs").Dirent[]; try { @@ -509,12 +528,12 @@ async function listStateSqlitePaths(params: { continue; } if (entry.isDirectory()) { - if (extensionsFilter(entryPath) && !isStatePackageContentPath(entryPath, params.stateDir)) { + if (stateFilter(entryPath) && !isStatePackageContentPath(entryPath, params.stateDir)) { await visit(entryPath); } } else if ( entry.isFile() && - extensionsFilter(entryPath) && + stateFilter(entryPath) && !isStatePackageContentPath(entryPath, params.stateDir) ) { const resolvedEntryPath = path.resolve(entryPath); @@ -576,6 +595,7 @@ async function listStateSqlitePaths(params: { async function createStateSqliteBackupPlan(params: { stateDir: string; tempDir: string; + preservedStatePaths?: readonly string[]; }): Promise { // Complete discovery before writing snapshots. chooseBackupTempRoot keeps // tempDir outside stateDir, and this ordering prevents future overlap from @@ -589,6 +609,7 @@ async function createStateSqliteBackupPlan(params: { const discovery = await listStateSqlitePaths({ stateDir: params.stateDir, globalStateSqlitePath, + preservedStatePaths: params.preservedStatePaths, }); const snapshots: SqliteBackupAsset[] = []; for (const archiveSourcePath of discovery.snapshotPaths) { @@ -692,11 +713,19 @@ export async function createBackupArchive( const tempArchivePath = buildTempArchivePath(outputPath); const tempArchiveCleanupPaths = resolveBackupTarAttemptTempPaths(tempArchivePath); const stateAsset = result.assets.find((asset) => asset.kind === "state"); + const preservedStatePaths = [ + plan.configPath, + plan.oauthDir, + ...plan.skipped + .filter((asset) => asset.kind === "workspace" && asset.reason === "covered") + .map((asset) => asset.sourcePath), + ].filter((entry) => stateAsset && isPathWithin(entry, stateAsset.sourcePath)); try { const stateSqliteBackup = stateAsset ? await createStateSqliteBackupPlan({ stateDir: stateAsset.sourcePath, tempDir, + preservedStatePaths, }) : { snapshots: [], discoveredSourcePaths: new Set() }; const sourcePathRemaps = new Map(); @@ -722,8 +751,8 @@ export async function createBackupArchive( await writeJson(manifestPath, manifest, { trailingNewline: true }); const tar = await loadTarRuntime(); - const extensionsFilter = stateAsset - ? buildExtensionsNodeModulesFilter(stateAsset.sourcePath) + const stateFilter = stateAsset + ? buildStateBackupFilter(stateAsset.sourcePath, preservedStatePaths) : undefined; const volatilePlan = { stateDirs: [stateAsset?.sourcePath ?? plan.stateDir] }; let skippedVolatileCount = 0; @@ -740,7 +769,7 @@ export async function createBackupArchive( if (resolvedEntryPath === manifestPath) { return true; } - if (extensionsFilter && !extensionsFilter(entryPath)) { + if (stateFilter && !stateFilter(entryPath)) { return false; } const sqliteSourceKind = stateAsset diff --git a/src/infra/backup-create.windows.test.ts b/src/infra/backup-create.windows.test.ts index 3d3085baf05c..4d1cac35bd45 100644 --- a/src/infra/backup-create.windows.test.ts +++ b/src/infra/backup-create.windows.test.ts @@ -1,7 +1,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { PassThrough } from "node:stream"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; import { writeArchiveStreamToFile } from "./backup-create-stream.js"; @@ -22,4 +22,54 @@ describe("writeArchiveStreamToFile", () => { await expect(writePromise).rejects.toThrow("injected tar read failure"); await expect(fs.rm(archivePath)).resolves.toBeUndefined(); }); + + it("aborts and closes a partial archive when the source stops producing data", async () => { + vi.useFakeTimers(); + try { + const tempDir = tempDirs.make("openclaw-backup-stream-timeout-"); + const archivePath = path.join(tempDir, "partial.tar.gz"); + const archiveStream = new PassThrough(); + const writePromise = writeArchiveStreamToFile({ + archivePath, + archiveStream, + idleTimeoutMs: 50, + }); + archiveStream.write("partial archive"); + + const rejection = expect(writePromise).rejects.toThrow( + "Backup archive write stalled: no data produced for 50ms", + ); + await vi.advanceTimersByTimeAsync(60); + await rejection; + expect(archiveStream.destroyed).toBe(true); + await expect(fs.rm(archivePath)).resolves.toBeUndefined(); + } finally { + vi.useRealTimers(); + } + }); + + it("resets the idle timeout when archive data keeps arriving", async () => { + vi.useFakeTimers(); + try { + const tempDir = tempDirs.make("openclaw-backup-stream-progress-"); + const archivePath = path.join(tempDir, "complete.tar.gz"); + const archiveStream = new PassThrough(); + const writePromise = writeArchiveStreamToFile({ + archivePath, + archiveStream, + idleTimeoutMs: 50, + }); + + archiveStream.write("first"); + await vi.advanceTimersByTimeAsync(40); + archiveStream.write("second"); + await vi.advanceTimersByTimeAsync(40); + archiveStream.end("third"); + + await expect(writePromise).resolves.toBeUndefined(); + await expect(fs.readFile(archivePath, "utf8")).resolves.toBe("firstsecondthird"); + } finally { + vi.useRealTimers(); + } + }); });