diff --git a/docs/cli/backup.md b/docs/cli/backup.md index f6c58331b21f..89df597d29a0 100644 --- a/docs/cli/backup.md +++ b/docs/cli/backup.md @@ -112,6 +112,8 @@ Installed plugin source and manifest files under the state directory's `extensio Installer-managed and rebuildable runtime roots under the state directory are also skipped: `dev/`, `git/`, `npm/`, legacy `npm-runtime/`, and `tools/`. These contain managed checkouts, package trees, and downloaded runtimes rather than authoritative user state; reinstall or update the corresponding runtime or plugin after restore. An explicitly configured config file, credentials directory, or workspace inside one of these roots remains included. +Local edits inside a managed `dev/` checkout are developer source, not OpenClaw product state, and are not included. Commit and push those edits or copy the checkout separately before relying on a state backup. + ## Invalid config behavior `openclaw backup` bypasses the normal config preflight so it can still help during recovery. Workspace discovery depends on a valid config, so `openclaw backup create` fails fast when the config file exists but is invalid and workspace backup is still enabled. diff --git a/src/infra/backup-create-stream.ts b/src/infra/backup-create-stream.ts index 8620d3cc14a2..b0402e71e6c0 100644 --- a/src/infra/backup-create-stream.ts +++ b/src/infra/backup-create-stream.ts @@ -10,6 +10,18 @@ type DestroyableArchiveStream = (NodeJS.ReadableStream | AsyncIterable void): unknown; + pause(): unknown; +}; + +type BackupArchiveProgress = { + bytes?: number; + entryPath?: string; + phase: "entry" | "output" | "raw" | "traversal"; +}; + export type BackupArchiveCleanupReceipt = { archivePath: string; identity?: Stats; @@ -19,6 +31,21 @@ export type PreparedBackupArchive = BackupArchiveCleanupReceipt & { identity: Stats; }; +export function observeBackupTarEntryProgress( + entry: BackupTarEntryProgressStream, + reportProgress: (bytes: number) => void, +): void { + const wasFlowing = entry.flowing; + entry.on("data", (chunk) => { + reportProgress(typeof chunk === "string" ? Buffer.byteLength(chunk) : chunk.length); + }); + if (!wasFlowing) { + // node-tar calls onWriteEntry before emitting the header. Adding a Minipass + // data listener starts flow, so pause until Pack attaches its own consumer. + entry.pause(); + } +} + // OpenClaw's one-user trust model treats hostile same-UID pathname rewrites as // trusted host mutation. Keep the check and unlink synchronous so cooperative // processes cannot interleave through an in-process await boundary. @@ -42,7 +69,9 @@ export function removePreparedBackupArchive(prepared: PreparedBackupArchive): bo export async function writeArchiveStreamToFile(params: { archivePath: string; - createArchiveStream: (reportProgress: () => void) => DestroyableArchiveStream; + createArchiveStream: ( + reportProgress: (progress?: BackupArchiveProgress) => void, + ) => DestroyableArchiveStream; idleTimeoutMs?: number; onPartialArchive?: (receipt: BackupArchiveCleanupReceipt) => void; }): Promise { @@ -55,19 +84,39 @@ export async function writeArchiveStreamToFile(params: { let openedIdentity: Stats | undefined; let idleTimer: ReturnType | undefined; let idleTimeoutError: Error | undefined; + let lastEntryPath: string | undefined; + let lastProgress: BackupArchiveProgress | undefined; + let outputBytes = 0; + let producerBytes = 0; let settled = false; - const armIdleTimer = () => { + const reportProgress = (progress?: BackupArchiveProgress) => { // A destroyed producer may finish an in-flight filesystem callback later; - // never let that callback retain the process after cleanup has completed. + // do not let that callback rearm the watchdog after cleanup has completed. if (settled) { return; } + if (progress) { + lastProgress = progress; + if (progress.entryPath) { + lastEntryPath = progress.entryPath; + } + if (progress.bytes) { + if (progress.phase === "output") { + outputBytes += progress.bytes; + } else if (progress.phase === "raw") { + producerBytes += progress.bytes; + } + } + } if (idleTimer) { clearTimeout(idleTimer); } idleTimer = setTimeout(() => { + const entrySuffix = lastEntryPath + ? `, entry=${JSON.stringify(lastEntryPath.slice(-512))}` + : ""; idleTimeoutError = new Error( - `Backup archive write stalled: no progress observed for ${idleTimeoutMs}ms`, + `Backup archive write stalled: no progress observed for ${idleTimeoutMs}ms (phase=${lastProgress?.phase ?? "starting"}${entrySuffix}, rawBytes=${producerBytes}, outputBytes=${outputBytes})`, ); archiveStream?.destroy(idleTimeoutError); controller.abort(idleTimeoutError); @@ -75,7 +124,7 @@ export async function writeArchiveStreamToFile(params: { }; const progress = new Transform({ transform(chunk, _encoding, callback) { - armIdleTimer(); + reportProgress({ phase: "output", bytes: chunk.length }); callback(null, chunk); }, }); @@ -93,11 +142,11 @@ export async function writeArchiveStreamToFile(params: { } }); try { - archiveStream = params.createArchiveStream(armIdleTimer); + archiveStream = params.createArchiveStream(reportProgress); const pipelinePromise = pipeline(archiveStream, progress, archiveWriteStream, { signal: controller.signal, }); - armIdleTimer(); + reportProgress(); await pipelinePromise; const currentIdentity = await fs.lstat(params.archivePath); if ( diff --git a/src/infra/backup-create.test.ts b/src/infra/backup-create.test.ts index 41c16eadc9ec..6f4a4a344462 100644 --- a/src/infra/backup-create.test.ts +++ b/src/infra/backup-create.test.ts @@ -2515,6 +2515,15 @@ describe("createBackupArchive", () => { await fs.mkdir(path.join(stateDir, "npm", "projects", "demo", "node_modules", "dep"), { recursive: true, }); + await fs.mkdir(path.join(stateDir, "dev", "openclaw", ".git", "objects", "pack"), { + recursive: true, + }); + await fs.mkdir(path.join(stateDir, "dev", "openclaw", "node_modules", "dep"), { + recursive: true, + }); + await fs.mkdir(path.join(stateDir, "dev", "openclaw", "dist"), { recursive: true }); + await fs.mkdir(path.join(stateDir, "developer"), { recursive: true }); + await fs.mkdir(path.join(stateDir, "dev-backup"), { recursive: true }); for (const managedRoot of ["dev", "git", "npm-runtime", "tools"]) { await fs.mkdir(path.join(stateDir, managedRoot, "runtime"), { recursive: true }); await fs.writeFile( @@ -2558,6 +2567,28 @@ describe("createBackupArchive", () => { "managed-package sqlite-named asset\n", "utf8", ); + await fs.writeFile( + path.join(stateDir, "dev", "openclaw", ".git", "objects", "pack", "pack-fixture.pack"), + "reinstallable git pack\n", + "utf8", + ); + await fs.writeFile( + path.join(stateDir, "dev", "openclaw", "node_modules", "dep", "index.js"), + "module.exports = {}\n", + "utf8", + ); + await fs.writeFile( + path.join(stateDir, "dev", "openclaw", "dist", "entry.js"), + "export {};\n", + "utf8", + ); + await fs.writeFile( + path.join(stateDir, "dev", "openclaw", "invalid.sqlite"), + "reinstallable sqlite-named artifact\n", + "utf8", + ); + await fs.writeFile(path.join(stateDir, "developer", "keep.txt"), "keep\n", "utf8"); + await fs.writeFile(path.join(stateDir, "dev-backup", "keep.txt"), "keep\n", "utf8"); await fs.mkdir(outputDir, { recursive: true }); const result = await createBackupArchive({ @@ -2581,6 +2612,8 @@ describe("createBackupArchive", () => { managedRoot, ).toBe(false); } + expect(entrySuffixes).toContain("/state/developer/keep.txt"); + expect(entrySuffixes).toContain("/state/dev-backup/keep.txt"); const pluginNodeModuleEntries = entries.filter((entry) => entry.includes("/state/extensions/demo/node_modules/"), ); diff --git a/src/infra/backup-create.ts b/src/infra/backup-create.ts index b9d3f88fb0e6..98c6e9bb9a9c 100644 --- a/src/infra/backup-create.ts +++ b/src/infra/backup-create.ts @@ -30,7 +30,11 @@ import { publishPreparedBackupArchive, type BackupArchivePublication, } from "./backup-archive-publication.js"; -import { removePreparedBackupArchive, writeArchiveStreamToFile } from "./backup-create-stream.js"; +import { + observeBackupTarEntryProgress, + removePreparedBackupArchive, + writeArchiveStreamToFile, +} from "./backup-create-stream.js"; import { writeTarArchiveWithRetry } from "./backup-tar-retry.js"; import { isTransientSqliteBackupPath, isVolatileBackupPath } from "./backup-volatile-filter.js"; import { @@ -936,11 +940,17 @@ export async function createBackupArchive( linkCache: createBackupLinkCache(), statCache: createBackupVolatileStatCache(volatilePlan), filter: (entryPath, entryStat) => { - reportProgress(); + reportProgress({ phase: "traversal", entryPath }); return tarFilter(entryPath, entryStat); }, onWriteEntry: (entry) => { - reportProgress(); + const sourceEntryPath = entry.path; + reportProgress({ phase: "entry", entryPath: sourceEntryPath }); + if (entry.type === "File" && (entry.stat?.size ?? 0) > 0) { + observeBackupTarEntryProgress(entry, (bytes) => { + reportProgress({ phase: "raw", entryPath: sourceEntryPath, bytes }); + }); + } entry.path = remapArchiveEntryPath({ entryPath: entry.path, manifestPath, diff --git a/src/infra/backup-create.windows.test.ts b/src/infra/backup-create.windows.test.ts index 0ecd69893564..77565c23d56a 100644 --- a/src/infra/backup-create.windows.test.ts +++ b/src/infra/backup-create.windows.test.ts @@ -2,11 +2,15 @@ import fsSync from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; import { PassThrough } from "node:stream"; +import { Minipass } from "minipass"; import { afterEach, describe, expect, it, vi } from "vitest"; import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; -import { writeArchiveStreamToFile } from "./backup-create-stream.js"; +import { observeBackupTarEntryProgress, writeArchiveStreamToFile } from "./backup-create-stream.js"; const tempDirs = useAutoCleanupTempDirTracker(afterEach); +type ReportBackupProgress = Parameters< + Parameters[0]["createArchiveStream"] +>[0]; describe("writeArchiveStreamToFile", () => { it("removes the exclusive partial archive when its initial descriptor stat fails", async () => { @@ -101,7 +105,7 @@ describe("writeArchiveStreamToFile", () => { const tempDir = tempDirs.make("openclaw-backup-stream-traversal-progress-"); const archivePath = path.join(tempDir, "complete.tar.gz"); const archiveStream = new PassThrough(); - let reportProgress: (() => void) | undefined; + let reportProgress: ReportBackupProgress | undefined; const writePromise = writeArchiveStreamToFile({ archivePath, createArchiveStream: (progress) => { @@ -122,4 +126,91 @@ describe("writeArchiveStreamToFile", () => { vi.useRealTimers(); } }); + + it("keeps the archive alive through more than five minutes of one entry's raw bytes", async () => { + vi.useFakeTimers(); + try { + const tempDir = tempDirs.make("openclaw-backup-stream-entry-progress-"); + const archivePath = path.join(tempDir, "complete.tar.gz"); + const archiveStream = new PassThrough(); + const entry = new Minipass(); + let reportProgress: ReportBackupProgress | undefined; + const writePromise = writeArchiveStreamToFile({ + archivePath, + createArchiveStream: (progress) => { + reportProgress = progress; + return archiveStream; + }, + }); + observeBackupTarEntryProgress(entry, (bytes) => { + reportProgress?.({ phase: "raw", entryPath: "/source/large.pack", bytes }); + }); + entry.on("data", () => {}); + + for (let elapsed = 0; elapsed < 360_000; elapsed += 60_000) { + await vi.advanceTimersByTimeAsync(60_000); + entry.write(Buffer.alloc(16)); + } + entry.end(); + archiveStream.end("archive after one large entry"); + + await expect(writePromise).resolves.toMatchObject({ archivePath }); + await expect(fs.readFile(archivePath, "utf8")).resolves.toBe("archive after one large entry"); + } finally { + vi.useRealTimers(); + } + }); + + it("keeps non-current tar entries paused until the archive consumer attaches", async () => { + const firstEntry = new Minipass(); + const secondEntry = new Minipass(); + const reportProgress = vi.fn(); + observeBackupTarEntryProgress(firstEntry, reportProgress); + observeBackupTarEntryProgress(secondEntry, reportProgress); + + firstEntry.end("first entry"); + secondEntry.end("second entry"); + const firstChunks: Buffer[] = []; + const secondChunks: Buffer[] = []; + firstEntry.on("data", (chunk) => firstChunks.push(chunk)); + secondEntry.on("data", (chunk) => secondChunks.push(chunk)); + + await Promise.all([firstEntry.promise(), secondEntry.promise()]); + expect(Buffer.concat(firstChunks).toString()).toBe("first entry"); + expect(Buffer.concat(secondChunks).toString()).toBe("second entry"); + expect(reportProgress).toHaveBeenCalledTimes(2); + }); + + it("cleans a partial archive when one entry stops producing raw bytes", async () => { + vi.useFakeTimers(); + try { + const tempDir = tempDirs.make("openclaw-backup-stream-entry-timeout-"); + const archivePath = path.join(tempDir, "partial.tar.gz"); + const archiveStream = new PassThrough(); + const entry = new Minipass(); + let reportProgress: ReportBackupProgress | undefined; + const writePromise = writeArchiveStreamToFile({ + archivePath, + createArchiveStream: (progress) => { + reportProgress = progress; + return archiveStream; + }, + }); + observeBackupTarEntryProgress(entry, (bytes) => { + reportProgress?.({ phase: "raw", entryPath: "/source/stalled.pack", bytes }); + }); + entry.on("data", () => {}); + entry.write(Buffer.alloc(16)); + archiveStream.write("partial archive"); + + const rejection = expect(writePromise).rejects.toThrow( + 'Backup archive write stalled: no progress observed for 300000ms (phase=output, entry="/source/stalled.pack", rawBytes=16, outputBytes=15)', + ); + await vi.advanceTimersByTimeAsync(300_001); + await rejection; + await expect(fs.lstat(archivePath)).rejects.toMatchObject({ code: "ENOENT" }); + } finally { + vi.useRealTimers(); + } + }); });