fix(backup): publish archives durably (#113302)

This commit is contained in:
Vincent Koc
2026-07-24 19:12:11 +08:00
committed by GitHub
parent 218c2fe3a5
commit 3bfc23c676
9 changed files with 982 additions and 177 deletions
+3 -1
View File
@@ -124,7 +124,9 @@ OpenClaw does not enforce a built-in maximum backup size or per-file size limit.
- Available space for the temporary archive write plus the final archive
- Time to walk large workspace trees and compress them into a `.tar.gz`
- Time to rescan the archive with `--verify` or `openclaw backup verify`
- Destination filesystem behavior: OpenClaw prefers a no-overwrite hard-link publish step and falls back to exclusive copy when hard links are unsupported
- Destination filesystem behavior: OpenClaw requires no-overwrite hard-link publication so a final archive path never exposes an in-progress copy; unsupported filesystems fail with an actionable error
If final-directory durability confirmation fails after publication, the command reports failure but preserves the complete final entry rather than risk deleting a concurrent replacement.
Large workspaces are usually the main driver of archive size. Use `--no-include-workspace` for a smaller/faster backup, or `--only-config` for the smallest archive.
+27 -35
View File
@@ -1,8 +1,8 @@
// Backup atomicity tests cover temp-file writes, rollback behavior, and backup archive consistency.
import fsSync from "node:fs";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { expectDefined } from "@openclaw/normalization-core";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { createTempHomeEnv, type TempHomeEnv } from "../test-utils/temp-home.js";
import {
@@ -95,28 +95,26 @@ describe("backupCreateCommand atomic archive write", () => {
}
});
it("cleans intermediate retry temp archives after cleanup races", async () => {
it("cleans intermediate retry archives after a later attempt succeeds", async () => {
const { archiveDir, outputPath, runtime } = await prepareAtomicBackupScenario({
archivePrefix: "openclaw-backup-retry-cleanup-",
});
const realRm = fs.rm.bind(fs);
const rmAttempts = new Map<string, number>();
const attemptFiles: string[] = [];
const rmSpy = vi.spyOn(fs, "rm").mockImplementation((async (
targetPath: Parameters<typeof fs.rm>[0],
options?: Parameters<typeof fs.rm>[1],
) => {
const key = String(targetPath);
const attempt = (rmAttempts.get(key) ?? 0) + 1;
rmAttempts.set(key, attempt);
if (key.startsWith(`${outputPath}.`) && !attemptFiles.includes(key)) {
attemptFiles.push(key);
const originalUnlinkSync = fsSync.unlinkSync.bind(fsSync);
let blockedPartialPath: string | undefined;
let blockedPartialCleanupAttempts = 0;
const unlinkSpy = vi.spyOn(fsSync, "unlinkSync").mockImplementation((target) => {
const targetPath = path.resolve(String(target));
if (!blockedPartialPath && targetPath.endsWith("archive.tar.gz.tmp")) {
blockedPartialPath = targetPath;
}
if (attemptFiles.length <= 2 && key === attemptFiles.at(-1) && attempt === 1) {
throw Object.assign(new Error("resource busy"), { code: "EBUSY" });
if (targetPath === blockedPartialPath) {
blockedPartialCleanupAttempts += 1;
if (blockedPartialCleanupAttempts === 1) {
throw Object.assign(new Error("busy"), { code: "EBUSY" });
}
}
await realRm(targetPath, options);
}) as typeof fs.rm);
return originalUnlinkSync(target);
});
try {
let tarAttempt = 0;
tarCreateMock.mockImplementation(() => {
@@ -139,17 +137,10 @@ describe("backupCreateCommand atomic archive write", () => {
expect(result.archivePath).toBe(outputPath);
expect(sleepMock.mock.calls).toStrictEqual([[10_000], [20_000]]);
expect(attemptFiles).toStrictEqual([
attemptFiles[0],
`${attemptFiles[0]}.retry-2`,
`${attemptFiles[0]}.retry-3`,
]);
expect(
rmAttempts.get(expectDefined(attemptFiles[1], "attemptFiles[1] test invariant")),
).toBeGreaterThanOrEqual(2);
expect(blockedPartialCleanupAttempts).toBeGreaterThanOrEqual(2);
expect((await fs.readdir(archiveDir)).toSorted()).toStrictEqual([path.basename(outputPath)]);
} finally {
rmSpy.mockRestore();
unlinkSpy.mockRestore();
await fs.rm(archiveDir, { recursive: true, force: true });
}
});
@@ -180,9 +171,9 @@ describe("backupCreateCommand atomic archive write", () => {
}
});
it("falls back to exclusive copy when hard-link publication is unsupported", async () => {
it("fails closed when hard-link publication is unsupported", async () => {
const { archiveDir, outputPath, runtime } = await prepareAtomicBackupScenario({
archivePrefix: "openclaw-backup-copy-fallback-",
archivePrefix: "openclaw-backup-no-hardlink-",
});
const linkSpy = vi.spyOn(fs, "link");
try {
@@ -191,12 +182,13 @@ describe("backupCreateCommand atomic archive write", () => {
Object.assign(new Error("hard links not supported"), { code: "EOPNOTSUPP" }),
);
const result = await backupCreateCommand(runtime, {
output: outputPath,
});
expect(result.archivePath).toBe(outputPath);
expect(await fs.readFile(outputPath, "utf8")).toBe("archive-bytes");
await expect(
backupCreateCommand(runtime, {
output: outputPath,
}),
).rejects.toThrow(/requires hard-link support/iu);
await expectPathMissing(outputPath);
await expect(fs.readdir(archiveDir)).resolves.toEqual([]);
} finally {
linkSpy.mockRestore();
await fs.rm(archiveDir, { recursive: true, force: true });
@@ -0,0 +1,320 @@
import fsSync from "node:fs";
import fs from "node:fs/promises";
import type { FileHandle } from "node:fs/promises";
import path from "node:path";
import { PassThrough } from "node:stream";
import { afterEach, describe, expect, it, vi } from "vitest";
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
import {
cleanupBackupArchivePublication,
createBackupArchivePublication,
publishPreparedBackupArchive,
type BackupArchivePublication,
} from "./backup-archive-publication.js";
import { writeArchiveStreamToFile, type PreparedBackupArchive } from "./backup-create-stream.js";
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
async function createPublication(
prefix: string,
): Promise<{ outputDir: string; outputPath: string; plan: BackupArchivePublication }> {
const root = tempDirs.make(prefix);
const outputDir = path.join(root, "backups");
const outputPath = path.join(outputDir, "backup.tar.gz");
await fs.mkdir(outputDir, { recursive: true });
const plan = await createBackupArchivePublication(outputPath);
return { outputDir, outputPath, plan };
}
async function prepareArchive(
plan: BackupArchivePublication,
content = "complete archive",
): Promise<PreparedBackupArchive> {
const archiveStream = new PassThrough();
const preparedPromise = writeArchiveStreamToFile({
archivePath: plan.tempArchivePath,
archiveStream,
});
archiveStream.end(content);
return await preparedPromise;
}
describe("backup archive publication", () => {
it("publishes a complete archive and removes its private staging directory", async () => {
const { outputPath, plan } = await createPublication("openclaw-backup-publish-");
const prepared = await prepareArchive(plan);
const originalOpen = fs.open.bind(fs);
const openedPaths: string[] = [];
const openSpy = vi.spyOn(fs, "open").mockImplementation(async (target, flags, mode) => {
openedPaths.push(path.resolve(String(target)));
return await originalOpen(target, flags, mode);
});
try {
await publishPreparedBackupArchive({ plan, prepared });
await expect(fs.readFile(outputPath, "utf8")).resolves.toBe("complete archive");
await expect(fs.lstat(prepared.archivePath)).rejects.toMatchObject({ code: "ENOENT" });
await expect(fs.lstat(plan.stagingDir)).rejects.toMatchObject({ code: "ENOENT" });
expect(openedPaths).not.toContain(path.resolve(outputPath));
} finally {
openSpy.mockRestore();
}
});
it("removes its staging directory when private setup fails", async () => {
const root = tempDirs.make("openclaw-backup-setup-failure-");
const outputDir = path.join(root, "backups");
await fs.mkdir(outputDir);
const chmodSpy = vi
.spyOn(fs, "chmod")
.mockRejectedValue(Object.assign(new Error("chmod failed"), { code: "EIO" }));
try {
await expect(
createBackupArchivePublication(path.join(outputDir, "backup.tar.gz")),
).rejects.toThrow(/chmod failed/iu);
await expect(fs.readdir(outputDir)).resolves.toEqual([]);
} finally {
chmodSpy.mockRestore();
}
});
it.each(["EPERM", "EXDEV", "ENOTSUP", "EOPNOTSUPP", "ENOSYS"])(
"fails closed when hard-link publication returns %s",
async (code) => {
const { outputPath, plan } = await createPublication("openclaw-backup-no-link-");
const prepared = await prepareArchive(plan);
const linkSpy = vi
.spyOn(fs, "link")
.mockRejectedValue(Object.assign(new Error("unsupported"), { code }));
try {
await expect(publishPreparedBackupArchive({ plan, prepared })).rejects.toThrow(
/requires hard-link support/iu,
);
await expect(fs.lstat(outputPath)).rejects.toMatchObject({ code: "ENOENT" });
await expect(fs.lstat(prepared.archivePath)).rejects.toMatchObject({ code: "ENOENT" });
} finally {
linkSpy.mockRestore();
}
},
);
it("preserves a destination raced in before publication", async () => {
const { outputPath, plan } = await createPublication("openclaw-backup-destination-race-");
const prepared = await prepareArchive(plan);
await fs.writeFile(outputPath, "racer", "utf8");
await expect(publishPreparedBackupArchive({ plan, prepared })).rejects.toThrow(
/Refusing to overwrite existing backup archive/iu,
);
await expect(fs.readFile(outputPath, "utf8")).resolves.toBe("racer");
});
it("rejects a replaced staging pathname without publishing replacement bytes", async () => {
const { outputPath, plan } = await createPublication("openclaw-backup-staging-race-");
const prepared = await prepareArchive(plan);
const originalPath = `${prepared.archivePath}.original`;
await fs.rename(prepared.archivePath, originalPath);
await fs.writeFile(prepared.archivePath, "replacement", "utf8");
await expect(publishPreparedBackupArchive({ plan, prepared })).rejects.toThrow(
/staging file changed/iu,
);
await expect(fs.lstat(outputPath)).rejects.toMatchObject({ code: "ENOENT" });
await expect(fs.readFile(prepared.archivePath, "utf8")).resolves.toBe("replacement");
});
it.runIf(process.platform !== "win32")(
"rejects a requested output-parent symlink retarget",
async () => {
const root = tempDirs.make("openclaw-backup-parent-retarget-");
const firstDir = path.join(root, "first");
const secondDir = path.join(root, "second");
const requestedDir = path.join(root, "current");
await fs.mkdir(firstDir);
await fs.mkdir(secondDir);
await fs.symlink(firstDir, requestedDir);
const outputPath = path.join(requestedDir, "backup.tar.gz");
const plan = await createBackupArchivePublication(outputPath);
const prepared = await prepareArchive(plan);
await fs.unlink(requestedDir);
await fs.symlink(secondDir, requestedDir);
await expect(publishPreparedBackupArchive({ plan, prepared })).rejects.toThrow(
/output directory changed/iu,
);
await expect(fs.lstat(path.join(firstDir, "backup.tar.gz"))).rejects.toMatchObject({
code: "ENOENT",
});
await expect(fs.lstat(path.join(secondDir, "backup.tar.gz"))).rejects.toMatchObject({
code: "ENOENT",
});
},
);
it.runIf(process.platform !== "win32")(
"rejects a canonical output-parent directory replacement",
async () => {
const { outputDir, outputPath, plan } = await createPublication(
"openclaw-backup-parent-replace-",
);
const prepared = await prepareArchive(plan);
const movedOutputDir = `${outputDir}.moved`;
await fs.rename(outputDir, movedOutputDir);
await fs.mkdir(outputDir);
await expect(
publishPreparedBackupArchive({
plan,
prepared,
}),
).rejects.toThrow(/output directory changed/iu);
await expect(fs.lstat(outputPath)).rejects.toMatchObject({ code: "ENOENT" });
},
);
it.runIf(process.platform !== "win32").each(["EIO", "EINVAL"])(
"preserves the complete final archive when commit directory sync fails with %s",
async (code) => {
const { outputPath, plan } = await createPublication("openclaw-backup-sync-failure-");
const prepared = await prepareArchive(plan);
const log = vi.fn();
const originalOpen = fs.open.bind(fs);
const openSpy = vi.spyOn(fs, "open").mockImplementation(async (target, flags, mode) => {
if (path.resolve(String(target)) === path.resolve(plan.canonicalParentPath)) {
return {
close: vi.fn().mockResolvedValue(undefined),
sync: vi.fn().mockRejectedValue(Object.assign(new Error("sync failed"), { code })),
} as unknown as FileHandle;
}
return await originalOpen(target, flags, mode);
});
try {
await expect(publishPreparedBackupArchive({ plan, prepared, log })).rejects.toThrow(
/sync failed/iu,
);
await expect(fs.readFile(outputPath, "utf8")).resolves.toBe("complete archive");
expect(log).toHaveBeenCalledWith(expect.stringContaining("concurrent replacement"));
} finally {
openSpy.mockRestore();
}
},
);
it("preserves a destination that replaces the linked archive before validation", async () => {
const { outputPath, plan } = await createPublication("openclaw-backup-linked-race-");
const prepared = await prepareArchive(plan);
const displacedPath = `${outputPath}.displaced`;
const originalLstat = fs.lstat.bind(fs);
let targetLstatCount = 0;
const lstatSpy = vi.spyOn(fs, "lstat").mockImplementation(async (target, options) => {
if (path.resolve(String(target)) === path.resolve(plan.canonicalOutputPath)) {
targetLstatCount += 1;
}
if (targetLstatCount === 2) {
targetLstatCount += 1;
await fs.rename(plan.canonicalOutputPath, displacedPath);
await fs.writeFile(plan.canonicalOutputPath, "racer", "utf8");
}
return await originalLstat(target, options);
});
try {
await expect(publishPreparedBackupArchive({ plan, prepared })).rejects.toThrow(
/Backup archive changed during publication/iu,
);
await expect(fs.readFile(outputPath, "utf8")).resolves.toBe("racer");
await expect(fs.readFile(displacedPath, "utf8")).resolves.toBe("complete archive");
} finally {
lstatSpy.mockRestore();
}
});
it("keeps the committed final archive when staging cleanup fails", async () => {
const { outputPath, plan } = await createPublication("openclaw-backup-cleanup-failure-");
const prepared = await prepareArchive(plan);
const log = vi.fn();
const originalUnlinkSync = fsSync.unlinkSync.bind(fsSync);
const unlinkSpy = vi.spyOn(fsSync, "unlinkSync").mockImplementation((target) => {
if (path.resolve(String(target)) === path.resolve(prepared.archivePath)) {
throw Object.assign(new Error("busy"), { code: "EBUSY" });
}
return originalUnlinkSync(target);
});
try {
await expect(publishPreparedBackupArchive({ plan, prepared, log })).resolves.toBeUndefined();
await expect(fs.readFile(outputPath, "utf8")).resolves.toBe("complete archive");
expect(log).toHaveBeenCalledWith(
`Backup archiver preserved changed staging file ${prepared.archivePath}.`,
);
} finally {
unlinkSpy.mockRestore();
await cleanupBackupArchivePublication(plan);
await expect(fs.lstat(prepared.archivePath)).rejects.toMatchObject({ code: "ENOENT" });
await expect(fs.lstat(plan.stagingDir)).rejects.toMatchObject({ code: "ENOENT" });
}
});
it("retries cleanup when descriptor and pathname identity reads initially fail", async () => {
const { plan } = await createPublication("openclaw-backup-unidentified-partial-");
const archiveStream = new PassThrough();
const originalLstatSync = fsSync.lstatSync.bind(fsSync);
const fstatSpy = vi.spyOn(fsSync, "fstatSync").mockImplementationOnce(() => {
throw Object.assign(new Error("fstat failed"), { code: "EIO" });
});
let stagedLstatAttempts = 0;
const lstatSpy = vi.spyOn(fsSync, "lstatSync").mockImplementation((target, options) => {
if (path.resolve(String(target)) === path.resolve(plan.tempArchivePath)) {
stagedLstatAttempts += 1;
if (stagedLstatAttempts === 1) {
throw Object.assign(new Error("lstat failed"), { code: "EIO" });
}
}
return originalLstatSync(target, options);
});
try {
const writePromise = writeArchiveStreamToFile({
archivePath: plan.tempArchivePath,
archiveStream,
onPartialArchive: (receipt) => {
plan.pendingCleanupArchives.push(receipt);
},
});
archiveStream.end("partial archive");
await expect(writePromise).rejects.toThrow("fstat failed");
expect(plan.pendingCleanupArchives).toEqual([{ archivePath: plan.tempArchivePath }]);
} finally {
fstatSpy.mockRestore();
lstatSpy.mockRestore();
}
await cleanupBackupArchivePublication(plan);
await expect(fs.lstat(plan.tempArchivePath)).rejects.toMatchObject({ code: "ENOENT" });
await expect(fs.lstat(plan.stagingDir)).rejects.toMatchObject({ code: "ENOENT" });
});
it("preserves a final-path replacement detected after the commit point", async () => {
const { outputPath, plan } = await createPublication("openclaw-backup-final-race-");
const prepared = await prepareArchive(plan);
const displacedPath = `${outputPath}.displaced`;
const originalUnlinkSync = fsSync.unlinkSync.bind(fsSync);
let replaced = false;
const unlinkSpy = vi.spyOn(fsSync, "unlinkSync").mockImplementation((target) => {
if (!replaced && path.resolve(String(target)) === path.resolve(prepared.archivePath)) {
replaced = true;
fsSync.renameSync(plan.canonicalOutputPath, displacedPath);
fsSync.writeFileSync(plan.canonicalOutputPath, "racer", "utf8");
}
return originalUnlinkSync(target);
});
try {
await expect(publishPreparedBackupArchive({ plan, prepared })).rejects.toThrow(
/Published backup archive changed/iu,
);
await expect(fs.readFile(outputPath, "utf8")).resolves.toBe("racer");
await expect(fs.readFile(displacedPath, "utf8")).resolves.toBe("complete archive");
} finally {
unlinkSpy.mockRestore();
}
});
});
+398
View File
@@ -0,0 +1,398 @@
import { randomUUID } from "node:crypto";
import { constants as fsConstants, type Stats } from "node:fs";
import fs from "node:fs/promises";
import type { FileHandle } from "node:fs/promises";
import path from "node:path";
import {
removePreparedBackupArchive,
type BackupArchiveCleanupReceipt,
type PreparedBackupArchive,
} from "./backup-create-stream.js";
import { sameFileIdentity } from "./fs-safe-advanced.js";
type BackupArchiveLogger = (message: string) => void;
export type BackupArchivePublication = {
canonicalOutputPath: string;
canonicalParentPath: string;
parentIdentity: Stats;
pendingCleanupArchives: BackupArchiveCleanupReceipt[];
requestedOutputPath: string;
requestedParentPath: string;
stagingDir: string;
stagingIdentity: Stats;
tempArchivePath: string;
};
function pathsEqual(left: string, right: string): boolean {
const resolvedLeft = path.resolve(left);
const resolvedRight = path.resolve(right);
return process.platform === "win32"
? resolvedLeft.toLowerCase() === resolvedRight.toLowerCase()
: resolvedLeft === resolvedRight;
}
async function assertTargetAbsent(targetPath: string): Promise<void> {
try {
await fs.lstat(targetPath);
} catch (error) {
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
return;
}
throw error;
}
throw new Error(`Refusing to overwrite existing backup archive: ${targetPath}`);
}
async function assertPublicationParentUnchanged(plan: BackupArchivePublication): Promise<void> {
const currentCanonicalParent = await fs.realpath(plan.requestedParentPath);
const currentParentIdentity = await fs.lstat(plan.canonicalParentPath);
if (
!pathsEqual(currentCanonicalParent, plan.canonicalParentPath) ||
!currentParentIdentity.isDirectory() ||
!sameFileIdentity(plan.parentIdentity, currentParentIdentity)
) {
throw new Error(
`Backup output directory changed during archive creation: ${plan.requestedParentPath}`,
);
}
}
async function removeDirectoryIfOwned(
directoryPath: string,
expectedIdentity: Stats,
): Promise<boolean> {
// This is a cooperative same-user fence, not hostile local-user isolation;
// SECURITY.md treats co-equal host mutation as inside the operator boundary.
const currentIdentity = await fs.lstat(directoryPath).catch(() => undefined);
if (
!currentIdentity ||
!currentIdentity.isDirectory() ||
!sameFileIdentity(expectedIdentity, currentIdentity)
) {
return false;
}
try {
await fs.rmdir(directoryPath);
return true;
} catch {
return false;
}
}
async function removeStagingDirectoryIfOwned(plan: BackupArchivePublication): Promise<boolean> {
return await removeDirectoryIfOwned(plan.stagingDir, plan.stagingIdentity);
}
function isUnsupportedDirectorySyncError(error: unknown): boolean {
const code = (error as NodeJS.ErrnoException).code;
return (
code === "EINVAL" ||
code === "ENOTSUP" ||
code === "ENOSYS" ||
(process.platform === "win32" && (code === "EISDIR" || code === "EPERM" || code === "EACCES"))
);
}
async function syncDirectoryBestEffort(directoryPath: string): Promise<void> {
const handle = await fs.open(directoryPath, "r").catch((error: unknown) => {
if (isUnsupportedDirectorySyncError(error)) {
return undefined;
}
throw error;
});
if (!handle) {
return;
}
try {
await handle.sync();
} catch (error) {
if (!isUnsupportedDirectorySyncError(error)) {
throw error;
}
} finally {
await handle.close();
}
}
async function syncPublishedArchiveCommit(
plan: BackupArchivePublication,
preparedHandle: FileHandle,
): Promise<void> {
if (process.platform === "win32") {
// Windows FlushFileBuffers requires a writable file handle and flushes
// buffered file metadata. The prepared handle pins the published inode.
await preparedHandle.sync();
return;
}
const directoryHandle = await fs.open(plan.canonicalParentPath, "r");
try {
// Publication success requires a real directory fsync. Unsupported
// filesystems fail closed instead of weakening crash durability.
await directoryHandle.sync();
} finally {
await directoryHandle.close();
}
}
function isUnsupportedHardLinkError(error: unknown): boolean {
const code = (error as NodeJS.ErrnoException).code;
return (
code === "EPERM" ||
code === "EXDEV" ||
code === "ENOTSUP" ||
code === "EOPNOTSUPP" ||
code === "ENOSYS"
);
}
async function openPreparedArchive(
plan: BackupArchivePublication,
prepared: PreparedBackupArchive,
): Promise<FileHandle> {
const accessMode = process.platform === "win32" ? fsConstants.O_RDWR : fsConstants.O_RDONLY;
const flags = accessMode | (fsConstants.O_NOFOLLOW ?? 0) | (fsConstants.O_NONBLOCK ?? 0);
const handle = await fs.open(prepared.archivePath, flags);
try {
const openedIdentity = await handle.stat();
const currentIdentity = await fs.lstat(prepared.archivePath);
if (
!pathsEqual(path.dirname(prepared.archivePath), plan.stagingDir) ||
!openedIdentity.isFile() ||
!currentIdentity.isFile() ||
!sameFileIdentity(prepared.identity, openedIdentity) ||
!sameFileIdentity(prepared.identity, currentIdentity)
) {
throw new Error(
`Backup archive staging file changed before publication: ${prepared.archivePath}`,
);
}
return handle;
} catch (error) {
await handle.close().catch(() => undefined);
throw error;
}
}
async function assertPublishedArchiveUnchanged(
plan: BackupArchivePublication,
handle: FileHandle,
expectedIdentity: Stats,
): Promise<void> {
const openedIdentity = await handle.stat();
const currentIdentity = await fs.lstat(plan.canonicalOutputPath);
if (
!openedIdentity.isFile() ||
!currentIdentity.isFile() ||
!sameFileIdentity(expectedIdentity, openedIdentity) ||
!sameFileIdentity(expectedIdentity, currentIdentity)
) {
throw new Error(`Published backup archive changed: ${plan.requestedOutputPath}`);
}
}
export async function createBackupArchivePublication(
outputPath: string,
): Promise<BackupArchivePublication> {
const requestedOutputPath = path.resolve(outputPath);
const requestedParentPath = path.dirname(requestedOutputPath);
const canonicalParentPath = await fs.realpath(requestedParentPath);
const parentIdentity = await fs.lstat(canonicalParentPath);
if (!parentIdentity.isDirectory()) {
throw new Error(`Backup output parent is not a directory: ${requestedParentPath}`);
}
const canonicalOutputPath = path.join(canonicalParentPath, path.basename(requestedOutputPath));
await assertTargetAbsent(canonicalOutputPath);
const stagingDir = await fs.mkdtemp(
path.join(canonicalParentPath, `.openclaw-backup-publish-${randomUUID()}-`),
);
let stagingIdentity: Stats | undefined;
try {
stagingIdentity = await fs.lstat(stagingDir);
await fs.chmod(stagingDir, 0o700);
return {
canonicalOutputPath,
canonicalParentPath,
parentIdentity,
pendingCleanupArchives: [],
requestedOutputPath,
requestedParentPath,
stagingDir,
stagingIdentity,
tempArchivePath: path.join(stagingDir, "archive.tar.gz.tmp"),
};
} catch (error) {
if (stagingIdentity) {
await removeDirectoryIfOwned(stagingDir, stagingIdentity);
}
throw error;
}
}
function retainArchiveForCleanup(
plan: BackupArchivePublication,
receipt: BackupArchiveCleanupReceipt,
): void {
for (const [index, candidate] of plan.pendingCleanupArchives.entries()) {
if (!pathsEqual(candidate.archivePath, receipt.archivePath)) {
continue;
}
if (!candidate.identity || !receipt.identity) {
if (!candidate.identity && receipt.identity) {
plan.pendingCleanupArchives[index] = receipt;
}
return;
}
if (sameFileIdentity(candidate.identity, receipt.identity)) {
return;
}
}
plan.pendingCleanupArchives.push(receipt);
}
async function removePendingBackupArchive(
plan: BackupArchivePublication,
receipt: BackupArchiveCleanupReceipt,
): Promise<boolean> {
if (!pathsEqual(path.dirname(receipt.archivePath), plan.stagingDir)) {
return false;
}
if (receipt.identity) {
return removePreparedBackupArchive(receipt as PreparedBackupArchive);
}
let currentIdentity: Stats;
try {
currentIdentity = await fs.lstat(receipt.archivePath);
} catch (error) {
return (error as NodeJS.ErrnoException).code === "ENOENT";
}
if (!currentIdentity.isFile()) {
return false;
}
return removePreparedBackupArchive({
archivePath: receipt.archivePath,
identity: currentIdentity,
});
}
export async function cleanupBackupArchivePublication(
plan: BackupArchivePublication,
log?: BackupArchiveLogger,
): Promise<void> {
const retainedArchives = plan.pendingCleanupArchives.splice(0);
for (const receipt of retainedArchives) {
if (!(await removePendingBackupArchive(plan, receipt))) {
retainArchiveForCleanup(plan, receipt);
}
}
if (await removeStagingDirectoryIfOwned(plan)) {
await syncDirectoryBestEffort(plan.canonicalParentPath).catch(() => undefined);
return;
}
const currentIdentity = await fs.lstat(plan.stagingDir).catch(() => undefined);
if (currentIdentity) {
log?.(`Backup archiver preserved changed or non-empty staging directory ${plan.stagingDir}.`);
}
}
export async function publishPreparedBackupArchive(params: {
plan: BackupArchivePublication;
prepared: PreparedBackupArchive;
log?: BackupArchiveLogger;
}): Promise<void> {
const { plan, prepared } = params;
let preparedHandle: FileHandle | undefined;
let publishedIdentity: Stats | undefined;
let hardLinkCreated = false;
let committed = false;
try {
await assertPublicationParentUnchanged(plan);
preparedHandle = await openPreparedArchive(plan, prepared);
await assertTargetAbsent(plan.canonicalOutputPath);
// Node has no portable link-by-handle primitive. Under OpenClaw's one-user
// host trust model, post-link identity checks fence cooperative replacement
// races and ensure a changed staging pathname can never produce success.
try {
await fs.link(prepared.archivePath, plan.canonicalOutputPath);
hardLinkCreated = true;
} catch (error) {
if ((error as NodeJS.ErrnoException).code === "EEXIST") {
throw new Error(
`Refusing to overwrite existing backup archive: ${plan.requestedOutputPath}`,
{ cause: error },
);
}
if (isUnsupportedHardLinkError(error)) {
throw new Error(
`Atomic backup publication requires hard-link support in ${plan.requestedParentPath}.`,
{ cause: error },
);
}
throw error;
}
await assertPublicationParentUnchanged(plan);
const currentTargetIdentity = await fs.lstat(plan.canonicalOutputPath);
const currentStagingIdentity = await fs.lstat(prepared.archivePath);
if (
!currentTargetIdentity.isFile() ||
!currentStagingIdentity.isFile() ||
!sameFileIdentity(prepared.identity, currentTargetIdentity) ||
!sameFileIdentity(prepared.identity, currentStagingIdentity)
) {
throw new Error(`Backup archive changed during publication: ${plan.requestedOutputPath}`);
}
publishedIdentity = currentTargetIdentity;
await assertPublishedArchiveUnchanged(plan, preparedHandle, publishedIdentity);
// The first parent sync commits the final pathname. After this point,
// cleanup failures must not remove or invalidate the durable archive.
await syncPublishedArchiveCommit(plan, preparedHandle);
committed = true;
if (!removePreparedBackupArchive(prepared)) {
retainArchiveForCleanup(plan, prepared);
params.log?.(`Backup archiver preserved changed staging file ${prepared.archivePath}.`);
}
if (!(await removeStagingDirectoryIfOwned(plan))) {
params.log?.(
`Backup archiver preserved changed or non-empty staging directory ${plan.stagingDir}.`,
);
}
await syncDirectoryBestEffort(plan.canonicalParentPath).catch((error: unknown) => {
params.log?.(
`Backup archiver could not sync cleanup in ${plan.canonicalParentPath}: ${
(error as NodeJS.ErrnoException).code ?? String(error)
}.`,
);
});
await assertPublicationParentUnchanged(plan);
await assertPublishedArchiveUnchanged(plan, preparedHandle, publishedIdentity);
} catch (error) {
if (!committed) {
if (!publishedIdentity && hardLinkCreated) {
const currentTargetIdentity = await fs
.lstat(plan.canonicalOutputPath)
.catch(() => undefined);
if (
currentTargetIdentity?.isFile() &&
sameFileIdentity(currentTargetIdentity, prepared.identity)
) {
publishedIdentity = currentTargetIdentity;
}
}
if (publishedIdentity) {
params.log?.(
`Backup archiver preserved the final archive after publication failed so a concurrent replacement could not be deleted: ${plan.requestedOutputPath}.`,
);
}
if (!removePreparedBackupArchive(prepared)) {
retainArchiveForCleanup(plan, prepared);
}
await removeStagingDirectoryIfOwned(plan);
}
throw error;
} finally {
await preparedHandle?.close().catch(() => undefined);
}
}
+107 -9
View File
@@ -1,6 +1,8 @@
import { createWriteStream } from "node:fs";
import fsSync, { createWriteStream, type Stats } from "node:fs";
import fs from "node:fs/promises";
import { Transform } from "node:stream";
import { pipeline } from "node:stream/promises";
import { sameFileIdentity } from "./fs-safe-advanced.js";
const BACKUP_ARCHIVE_IDLE_TIMEOUT_MS = 5 * 60_000;
@@ -8,16 +10,48 @@ type DestroyableArchiveStream = (NodeJS.ReadableStream | AsyncIterable<Uint8Arra
destroy(error?: Error): unknown;
};
export type BackupArchiveCleanupReceipt = {
archivePath: string;
identity?: Stats;
};
export type PreparedBackupArchive = BackupArchiveCleanupReceipt & {
identity: Stats;
};
// OpenClaw's one-user trust model treats hostile same-UID pathname rewrites as
// trusted host mutation. Keep the check and unlink synchronous so cooperative
// processes cannot interleave through an in-process await boundary.
export function removePreparedBackupArchive(prepared: PreparedBackupArchive): boolean {
let currentIdentity: Stats;
try {
currentIdentity = fsSync.lstatSync(prepared.archivePath);
} catch {
return false;
}
if (!currentIdentity.isFile() || !sameFileIdentity(prepared.identity, currentIdentity)) {
return false;
}
try {
fsSync.unlinkSync(prepared.archivePath);
return true;
} catch {
return false;
}
}
export async function writeArchiveStreamToFile(params: {
archivePath: string;
archiveStream: DestroyableArchiveStream;
idleTimeoutMs?: number;
}): Promise<void> {
onPartialArchive?: (receipt: BackupArchiveCleanupReceipt) => void;
}): Promise<PreparedBackupArchive> {
// Own both stream lifecycles so a tar read error closes the output handle
// before retry cleanup touches the partial archive. Exclusive creation also
// refuses a pre-existing path instead of following a symlink.
const idleTimeoutMs = params.idleTimeoutMs ?? BACKUP_ARCHIVE_IDLE_TIMEOUT_MS;
const controller = new AbortController();
let openedIdentity: Stats | undefined;
let idleTimer: ReturnType<typeof setTimeout> | undefined;
let idleTimeoutError: Error | undefined;
const armIdleTimer = () => {
@@ -39,15 +73,79 @@ export async function writeArchiveStreamToFile(params: {
},
});
armIdleTimer();
const archiveWriteStream = createWriteStream(params.archivePath, {
flags: "wx",
flush: true,
mode: 0o600,
});
archiveWriteStream.once("open", (fileDescriptor) => {
try {
openedIdentity = fsSync.fstatSync(fileDescriptor);
} catch (error) {
archiveWriteStream.destroy(error as Error);
}
});
try {
await pipeline(
params.archiveStream,
progress,
createWriteStream(params.archivePath, { flags: "wx", mode: 0o600 }),
{ signal: controller.signal },
);
const pipelinePromise = pipeline(params.archiveStream, progress, archiveWriteStream, {
signal: controller.signal,
});
armIdleTimer();
await pipelinePromise;
const currentIdentity = await fs.lstat(params.archivePath);
if (
!openedIdentity?.isFile() ||
!currentIdentity.isFile() ||
!sameFileIdentity(openedIdentity, currentIdentity)
) {
throw new Error(`Backup archive path changed while writing: ${params.archivePath}`);
}
return { archivePath: params.archivePath, identity: currentIdentity };
} catch (err) {
archiveWriteStream.destroy();
let cleanupReceipt: BackupArchiveCleanupReceipt | undefined = openedIdentity
? { archivePath: params.archivePath, identity: openedIdentity }
: undefined;
if (!cleanupReceipt) {
try {
const currentIdentity = fsSync.lstatSync(params.archivePath);
cleanupReceipt = currentIdentity.isFile()
? {
archivePath: params.archivePath,
identity: currentIdentity,
}
: { archivePath: params.archivePath };
} catch (cleanupError) {
if ((cleanupError as NodeJS.ErrnoException).code !== "ENOENT") {
// Preserve the cleanup obligation even when the filesystem cannot
// supply an identity until a later outer-cleanup attempt.
cleanupReceipt = { archivePath: params.archivePath };
}
}
}
if (
cleanupReceipt &&
(!cleanupReceipt.identity ||
!removePreparedBackupArchive(cleanupReceipt as PreparedBackupArchive))
) {
params.onPartialArchive?.(cleanupReceipt);
}
if (cleanupReceipt && !cleanupReceipt.identity) {
// The outer cleanup owns the retry because this scope cannot safely
// unlink a pathname whose identity is temporarily unavailable.
if (!params.onPartialArchive) {
try {
const currentIdentity = fsSync.lstatSync(cleanupReceipt.archivePath);
if (currentIdentity.isFile()) {
removePreparedBackupArchive({
archivePath: cleanupReceipt.archivePath,
identity: currentIdentity,
});
}
} catch {
// No outer owner was provided; preserve the original write error.
}
}
}
throw idleTimeoutError ?? err;
} finally {
if (idleTimer) {
+65 -36
View File
@@ -1,5 +1,5 @@
// Covers backup archive creation and verification filtering.
import { rmSync } from "node:fs";
import fsSync, { rmSync } from "node:fs";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
@@ -394,23 +394,21 @@ describe("writeTarArchiveWithRetry", () => {
expect(log).toHaveBeenCalledTimes(2);
});
it("uses a fresh temp archive path when cleanup cannot remove a failed attempt", async () => {
it("uses a fresh temp archive path without pathname-based cleanup", async () => {
const eofErr = Object.assign(new Error("did not encounter expected EOF"), {
path: "/state/sessions/s-abc/transcript.jsonl",
});
const tempArchivePath = "/tmp/backup.tar.gz.tmp";
const runTar = vi
.fn<(attemptTempArchivePath: string) => Promise<void>>()
.fn<(attemptTempArchivePath: string) => Promise<string>>()
.mockRejectedValueOnce(eofErr)
.mockResolvedValueOnce(undefined);
.mockResolvedValueOnce("complete");
const log = vi.fn();
const sleep = vi.fn<(ms: number) => Promise<void>>().mockResolvedValue(undefined);
const rmSpy = vi.spyOn(fs, "rm").mockImplementation(async () => {
throw Object.assign(new Error("resource busy"), { code: "EBUSY" });
});
const rmSpy = vi.spyOn(fs, "rm");
try {
const completedTempArchivePath = await writeTarArchiveWithRetry({
const result = await writeTarArchiveWithRetry({
tempArchivePath,
runTar,
log,
@@ -419,17 +417,15 @@ describe("writeTarArchiveWithRetry", () => {
expect(runTar).toHaveBeenNthCalledWith(1, tempArchivePath);
expect(runTar).toHaveBeenNthCalledWith(2, `${tempArchivePath}.retry-2`);
expect(completedTempArchivePath).toBe(`${tempArchivePath}.retry-2`);
expect(rmSpy).toHaveBeenCalledWith(tempArchivePath, { force: true });
expect(log).toHaveBeenCalledWith(
`Backup archiver could not remove temp archive ${tempArchivePath} between retries: EBUSY. Continuing.`,
);
expect(result).toBe("complete");
expect(rmSpy).not.toHaveBeenCalled();
expect(log).toHaveBeenCalledOnce();
} finally {
rmSpy.mockRestore();
}
});
it("cleans retry temp archive paths when a later attempt fails", async () => {
it("does not remove retry paths by pathname when a later attempt fails", async () => {
const eofErr = Object.assign(new Error("did not encounter expected EOF"), {
path: "/state/sessions/s-abc/transcript.jsonl",
});
@@ -452,7 +448,7 @@ describe("writeTarArchiveWithRetry", () => {
expect(runTar).toHaveBeenNthCalledWith(1, tempArchivePath);
expect(runTar).toHaveBeenNthCalledWith(2, `${tempArchivePath}.retry-2`);
expect(rmSpy).toHaveBeenCalledWith(`${tempArchivePath}.retry-2`, { force: true });
expect(rmSpy).not.toHaveBeenCalled();
} finally {
rmSpy.mockRestore();
}
@@ -1891,6 +1887,7 @@ describe("createBackupArchive", () => {
const originalReaddir = fs.readdir.bind(fs);
let createdLatePath = false;
let stagedArchiveCleanupAttempts = 0;
const readdirSpy = vi.spyOn(fs, "readdir").mockImplementation((async (
...args: unknown[]
) => {
@@ -1906,6 +1903,20 @@ describe("createBackupArchive", () => {
}
return entries;
}) as typeof fs.readdir);
const originalUnlinkSync = fsSync.unlinkSync.bind(fsSync);
const unlinkSpy = vi.spyOn(fsSync, "unlinkSync").mockImplementation((target) => {
const targetPath = path.resolve(String(target));
if (
targetPath.startsWith(path.resolve(outputDir)) &&
targetPath.includes(".openclaw-backup-publish-")
) {
stagedArchiveCleanupAttempts += 1;
if (stagedArchiveCleanupAttempts === 1) {
throw Object.assign(new Error("busy"), { code: "EBUSY" });
}
}
return originalUnlinkSync(target);
});
try {
await expect(
@@ -1916,8 +1927,10 @@ describe("createBackupArchive", () => {
}),
).rejects.toThrow(/SQLite state appeared after snapshot discovery/);
expect(createdLatePath).toBe(true);
expect(stagedArchiveCleanupAttempts).toBeGreaterThanOrEqual(2);
expect(await fs.readdir(outputDir)).toEqual([]);
} finally {
unlinkSpy.mockRestore();
readdirSpy.mockRestore();
}
},
@@ -2626,40 +2639,56 @@ describe("createBackupArchive", () => {
});
describe.runIf(process.platform !== "win32")("archive permissions", () => {
it.each([
["hard link", false],
["copy fallback", true],
] as const)("publishes via %s with owner-only 0o600 permissions", async (_name, forceCopy) => {
const linkSpy = forceCopy
? vi
.spyOn(fs, "link")
.mockRejectedValue(
Object.assign(new Error("hard links unsupported"), { code: "EPERM" }),
)
: undefined;
it("publishes via hard link with owner-only 0o600 permissions", async () => {
await withOpenClawTestState(
{
layout: "state-only",
prefix: "openclaw-backup-mode-",
scenario: "minimal",
},
async (state) => {
const outputDir = state.path("backups");
await fs.mkdir(outputDir, { recursive: true });
const result = await createBackupArchive({
output: outputDir,
includeWorkspace: false,
nowMs: Date.UTC(2026, 4, 9, 12, 0, 0),
});
const stat = await fs.stat(result.archivePath);
expect(stat.mode & 0o777).toBe(0o600);
},
);
});
it("fails closed when the destination does not support hard links", async () => {
const linkSpy = vi
.spyOn(fs, "link")
.mockRejectedValue(Object.assign(new Error("hard links unsupported"), { code: "EPERM" }));
try {
await withOpenClawTestState(
{
layout: "state-only",
prefix: "openclaw-backup-mode-",
prefix: "openclaw-backup-no-hardlinks-",
scenario: "minimal",
},
async (state) => {
const outputDir = state.path("backups");
await fs.mkdir(outputDir, { recursive: true });
const result = await createBackupArchive({
output: outputDir,
includeWorkspace: false,
nowMs: Date.UTC(2026, 4, 9, 12, 0, 0),
});
const stat = await fs.stat(result.archivePath);
expect(stat.mode & 0o777).toBe(0o600);
await expect(
createBackupArchive({
output: outputDir,
includeWorkspace: false,
nowMs: Date.UTC(2026, 4, 9, 12, 0, 0),
}),
).rejects.toThrow(/requires hard-link support/iu);
await expect(fs.readdir(outputDir)).resolves.toEqual([]);
},
);
} finally {
linkSpy?.mockRestore();
linkSpy.mockRestore();
}
});
});
+31 -59
View File
@@ -1,6 +1,5 @@
// Creates backup archives while filtering volatile runtime state.
import { randomUUID } from "node:crypto";
import { constants as fsConstants, type Stats } from "node:fs";
import type { Stats } from "node:fs";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
@@ -24,12 +23,14 @@ import {
} from "../state/openclaw-state-snapshot-sanitizer.js";
import { resolveHomeDir, resolveUserPath } from "../utils.js";
import { resolveRuntimeServiceVersion } from "../version.js";
import { writeArchiveStreamToFile } from "./backup-create-stream.js";
import {
removeBackupTempArchiveBestEffort,
resolveBackupTarAttemptTempPaths,
writeTarArchiveWithRetry,
} from "./backup-tar-retry.js";
cleanupBackupArchivePublication,
createBackupArchivePublication,
publishPreparedBackupArchive,
type BackupArchivePublication,
} from "./backup-archive-publication.js";
import { removePreparedBackupArchive, writeArchiveStreamToFile } from "./backup-create-stream.js";
import { writeTarArchiveWithRetry } from "./backup-tar-retry.js";
import { isVolatileBackupPath } from "./backup-volatile-filter.js";
import { createBackupVolatileStatCache } from "./backup-volatile-stat-cache.js";
import { formatErrorMessage } from "./errors.js";
@@ -179,10 +180,6 @@ async function assertOutputPathReady(outputPath: string): Promise<void> {
}
}
function buildTempArchivePath(outputPath: string): string {
return `${outputPath}.${randomUUID()}.tmp`;
}
// The temp manifest is passed to `tar.c` alongside the asset source paths. If
// the temp file lives inside any asset, recursive traversal pulls it in a
// second time and both copies remap to `<archiveRoot>/manifest.json`, which
@@ -220,46 +217,6 @@ async function chooseBackupTempRoot(params: {
return fallback;
}
function isLinkUnsupportedError(code: string | undefined): boolean {
return code === "ENOTSUP" || code === "EOPNOTSUPP" || code === "EPERM";
}
async function publishTempArchive(params: {
tempArchivePath: string;
outputPath: string;
}): Promise<void> {
try {
await fs.link(params.tempArchivePath, params.outputPath);
} catch (err) {
const code = (err as NodeJS.ErrnoException | undefined)?.code;
if (code === "EEXIST") {
throw new Error(`Refusing to overwrite existing backup archive: ${params.outputPath}`, {
cause: err,
});
}
if (!isLinkUnsupportedError(code)) {
throw err;
}
try {
// Some backup targets support ordinary files but not hard links.
await fs.copyFile(params.tempArchivePath, params.outputPath, fsConstants.COPYFILE_EXCL);
} catch (copyErr) {
const copyCode = (copyErr as NodeJS.ErrnoException | undefined)?.code;
if (copyCode !== "EEXIST") {
await fs.rm(params.outputPath, { force: true }).catch(() => undefined);
}
if (copyCode === "EEXIST") {
throw new Error(`Refusing to overwrite existing backup archive: ${params.outputPath}`, {
cause: copyErr,
});
}
throw copyErr;
}
}
await fs.rm(params.tempArchivePath, { force: true });
}
async function canonicalizePathForContainment(targetPath: string): Promise<string> {
const resolved = path.resolve(targetPath);
const suffix: string[] = [];
@@ -845,8 +802,14 @@ export async function createBackupArchive(
await fs.mkdir(tempRoot, { recursive: true });
const tempDir = await fs.mkdtemp(path.join(tempRoot, "openclaw-backup-"));
const manifestPath = path.join(tempDir, "manifest.json");
const tempArchivePath = buildTempArchivePath(outputPath);
const tempArchiveCleanupPaths = resolveBackupTarAttemptTempPaths(tempArchivePath);
let publication: BackupArchivePublication;
try {
publication = await createBackupArchivePublication(outputPath);
} catch (error) {
await fs.rm(tempDir, { recursive: true, force: true }).catch(() => undefined);
throw error;
}
const tempArchivePath = publication.tempArchivePath;
const stateAsset = result.assets.find((asset) => asset.kind === "state");
const preservedStatePaths = [
plan.configPath,
@@ -966,7 +929,7 @@ export async function createBackupArchive(
}
return true;
};
const completedTempArchivePath = await writeTarArchiveWithRetry({
const completedArchive = await writeTarArchiveWithRetry({
tempArchivePath,
log: opts.log,
runTar: async (attemptTempArchivePath) => {
@@ -975,7 +938,7 @@ export async function createBackupArchive(
// cumulative skip counts across attempts instead of the final one.
skippedVolatileCount = 0;
unexpectedSqliteSourcePaths.length = 0;
await writeArchiveStreamToFile({
const prepared = await writeArchiveStreamToFile({
archivePath: attemptTempArchivePath,
archiveStream: tar.c(
{
@@ -1001,13 +964,20 @@ export async function createBackupArchive(
...result.assets.map((asset) => asset.sourcePath),
],
),
onPartialArchive: (partialArchive) => {
publication.pendingCleanupArchives.push(partialArchive);
},
});
const unexpectedSqliteSourcePath = unexpectedSqliteSourcePaths[0];
if (unexpectedSqliteSourcePath) {
if (!removePreparedBackupArchive(prepared)) {
publication.pendingCleanupArchives.push(prepared);
}
throw new Error(
`SQLite state appeared after snapshot discovery: ${unexpectedSqliteSourcePath}. Retry backup so it can be snapshotted.`,
);
}
return prepared;
},
});
result.skippedVolatileCount = skippedVolatileCount;
@@ -1018,11 +988,13 @@ export async function createBackupArchive(
} (live sessions, cron logs, queues, sockets, pid/tmp).`,
);
}
await publishTempArchive({ tempArchivePath: completedTempArchivePath, outputPath });
await publishPreparedBackupArchive({
plan: publication,
prepared: completedArchive,
log: opts.log,
});
} finally {
for (const cleanupPath of tempArchiveCleanupPaths) {
await removeBackupTempArchiveBestEffort(cleanupPath);
}
await cleanupBackupArchivePublication(publication, opts.log);
await fs.rm(tempDir, { recursive: true, force: true }).catch(() => undefined);
}
+25 -3
View File
@@ -1,3 +1,4 @@
import fsSync from "node:fs";
import fs from "node:fs/promises";
import path from "node:path";
import { PassThrough } from "node:stream";
@@ -8,6 +9,27 @@ import { writeArchiveStreamToFile } from "./backup-create-stream.js";
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
describe("writeArchiveStreamToFile", () => {
it("removes the exclusive partial archive when its initial descriptor stat fails", async () => {
const tempDir = tempDirs.make("openclaw-backup-stream-fstat-");
const archivePath = path.join(tempDir, "partial.tar.gz");
const archiveStream = new PassThrough();
const fstatSpy = vi.spyOn(fsSync, "fstatSync").mockImplementationOnce(() => {
throw Object.assign(new Error("fstat failed"), { code: "EIO" });
});
try {
const writePromise = writeArchiveStreamToFile({
archivePath,
archiveStream,
});
archiveStream.end("partial archive");
await expect(writePromise).rejects.toThrow("fstat failed");
await expect(fs.lstat(archivePath)).rejects.toMatchObject({ code: "ENOENT" });
} finally {
fstatSpy.mockRestore();
}
});
it("closes a partial archive before propagating a stream error", async () => {
const tempDir = tempDirs.make("openclaw-backup-stream-");
const archivePath = path.join(tempDir, "partial.tar.gz");
@@ -20,7 +42,7 @@ describe("writeArchiveStreamToFile", () => {
archiveStream.destroy(new Error("injected tar read failure"));
await expect(writePromise).rejects.toThrow("injected tar read failure");
await expect(fs.rm(archivePath)).resolves.toBeUndefined();
await expect(fs.lstat(archivePath)).rejects.toMatchObject({ code: "ENOENT" });
});
it("aborts and closes a partial archive when the source stops producing data", async () => {
@@ -42,7 +64,7 @@ describe("writeArchiveStreamToFile", () => {
await vi.advanceTimersByTimeAsync(60);
await rejection;
expect(archiveStream.destroyed).toBe(true);
await expect(fs.rm(archivePath)).resolves.toBeUndefined();
await expect(fs.lstat(archivePath)).rejects.toMatchObject({ code: "ENOENT" });
} finally {
vi.useRealTimers();
}
@@ -66,7 +88,7 @@ describe("writeArchiveStreamToFile", () => {
await vi.advanceTimersByTimeAsync(40);
archiveStream.end("third");
await expect(writePromise).resolves.toBeUndefined();
await expect(writePromise).resolves.toMatchObject({ archivePath });
await expect(fs.readFile(archivePath, "utf8")).resolves.toBe("firstsecondthird");
} finally {
vi.useRealTimers();
+6 -34
View File
@@ -1,4 +1,3 @@
import fs from "node:fs/promises";
import { sleep } from "../utils/sleep.js";
const BACKUP_TAR_MAX_ATTEMPTS = 3;
@@ -24,52 +23,25 @@ function resolveBackupTarAttemptTempPath(tempArchivePath: string, attempt: numbe
return attempt === 1 ? tempArchivePath : `${tempArchivePath}.retry-${attempt}`;
}
export function resolveBackupTarAttemptTempPaths(tempArchivePath: string): string[] {
return Array.from({ length: BACKUP_TAR_MAX_ATTEMPTS }, (_value, index) =>
resolveBackupTarAttemptTempPath(tempArchivePath, index + 1),
);
}
export async function removeBackupTempArchiveBestEffort(tempArchivePath: string): Promise<void> {
await fs.rm(tempArchivePath, { force: true }).catch(() => undefined);
}
export async function writeTarArchiveWithRetry(params: {
export async function writeTarArchiveWithRetry<T>(params: {
tempArchivePath: string;
runTar: (tempArchivePath: string) => Promise<void>;
runTar: (tempArchivePath: string) => Promise<T>;
log?: BackupTarRetryLogger;
sleepMs?: (ms: number) => Promise<void>;
}): Promise<string> {
}): Promise<T> {
const sleepFn = params.sleepMs ?? sleep;
let lastErr: unknown;
const attemptTempArchivePaths: string[] = [];
for (let attempt = 1; attempt <= BACKUP_TAR_MAX_ATTEMPTS; attempt += 1) {
const attemptTempArchivePath = resolveBackupTarAttemptTempPath(params.tempArchivePath, attempt);
attemptTempArchivePaths.push(attemptTempArchivePath);
try {
await params.runTar(attemptTempArchivePath);
for (const staleTempArchivePath of attemptTempArchivePaths.slice(0, -1)) {
await removeBackupTempArchiveBestEffort(staleTempArchivePath);
}
return attemptTempArchivePath;
return await params.runTar(attemptTempArchivePath);
} catch (err) {
lastErr = err;
if (!isTarEofRaceError(err) || attempt === BACKUP_TAR_MAX_ATTEMPTS) {
for (const staleTempArchivePath of attemptTempArchivePaths) {
await removeBackupTempArchiveBestEffort(staleTempArchivePath);
}
break;
}
try {
await fs.rm(attemptTempArchivePath, { force: true });
} catch (cleanupErr) {
const code = (cleanupErr as NodeJS.ErrnoException).code;
if (code && code !== "ENOENT") {
params.log?.(
`Backup archiver could not remove temp archive ${attemptTempArchivePath} between retries: ${code}. Continuing.`,
);
}
}
// The writer owns checked cleanup inside the private staging directory.
// A fresh path keeps retries independent when a changed entry is preserved.
const backoff = BACKUP_TAR_BACKOFF_MS[attempt - 1] ?? 0;
const offendingPath = (err as NodeJS.ErrnoException).path;
params.log?.(