From 725f50b883a210c1233bf23c2c27549753a857dd Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 6 Aug 2026 15:57:36 -0700 Subject: [PATCH] fix(doctor): recover terminal NUL-only tails in archived session JSONL (#120041) * fix(doctor): recover terminal NUL-only tails in archived session JSONL * test(infra): split NUL-tail recovery tests to satisfy max-lines --- ...rations.media-persistence.nul-tail.test.ts | 120 ++++++++++++++++++ .../state-migrations.media-persistence.ts | 27 +++- 2 files changed, 140 insertions(+), 7 deletions(-) create mode 100644 src/infra/state-migrations.media-persistence.nul-tail.test.ts diff --git a/src/infra/state-migrations.media-persistence.nul-tail.test.ts b/src/infra/state-migrations.media-persistence.nul-tail.test.ts new file mode 100644 index 000000000000..688061be8e57 --- /dev/null +++ b/src/infra/state-migrations.media-persistence.nul-tail.test.ts @@ -0,0 +1,120 @@ +import fs from "node:fs"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { cleanupTempDirs, makeTempDir } from "../../test/helpers/temp-dir.js"; +import { + closeOpenClawAgentDatabasesForTest, + openOpenClawAgentDatabase, +} from "../state/openclaw-agent-db.js"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; +import { migrateLegacyMediaPersistence } from "./state-migrations.media-persistence.js"; + +const tempDirs: string[] = []; + +function createArchiveFixture(bytes: Uint8Array): { + archivePath: string; + env: NodeJS.ProcessEnv; +} { + const stateDir = fs.realpathSync(makeTempDir(tempDirs, "media-persistence-archive-")); + const env = { OPENCLAW_STATE_DIR: stateDir }; + openOpenClawAgentDatabase({ agentId: "main", env }); + closeOpenClawAgentDatabasesForTest(); + const archivePath = path.join( + stateDir, + "agents", + "main", + "sessions", + "fixture.jsonl.deleted.2026-07-24T01-02-03.000Z", + ); + fs.mkdirSync(path.dirname(archivePath), { recursive: true }); + fs.writeFileSync(archivePath, bytes); + return { archivePath, env }; +} + +afterEach(() => { + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + cleanupTempDirs(tempDirs); +}); + +describe("legacy media persistence NUL-tail recovery", () => { + it("atomically removes a terminal NUL suffix from an otherwise valid archive", () => { + const valid = Buffer.from(`${JSON.stringify({ type: "event", id: "event-1" })}\n`); + const { archivePath, env } = createArchiveFixture(Buffer.concat([valid, Buffer.alloc(284)])); + let replacements = 0; + + const result = migrateLegacyMediaPersistence({ + env, + hooks: { beforeArchiveReplace: () => (replacements += 1) }, + }); + + expect(result.warnings).toEqual([]); + expect(result.changes).toContain(`Migrated archived transcript media in ${archivePath}.`); + expect(replacements).toBe(1); + expect(fs.readFileSync(archivePath)).toEqual(valid); + }); + + it.each([ + { + name: "an interior NUL", + bytes: Buffer.concat([ + Buffer.from(JSON.stringify({ type: "event", id: "event-1" })), + Buffer.from([0]), + Buffer.from(`\n${JSON.stringify({ type: "event", id: "event-2" })}\n`), + ]), + }, + { name: "an all-NUL file", bytes: Buffer.alloc(284) }, + { + name: "a truncated JSON tail before terminal NULs", + bytes: Buffer.concat([ + Buffer.from(`${JSON.stringify({ type: "event", id: "event-1" })}\n{"type":`), + Buffer.alloc(32), + ]), + }, + { + name: "a blank record", + bytes: Buffer.from(`${JSON.stringify({ type: "event", id: "event-1" })}\n\n`), + }, + ])("rejects and preserves $name", ({ bytes }) => { + const { archivePath, env } = createArchiveFixture(bytes); + let replacements = 0; + + const result = migrateLegacyMediaPersistence({ + env, + hooks: { beforeArchiveReplace: () => (replacements += 1) }, + }); + + expect(result.warnings).toHaveLength(1); + expect(result.warnings[0]).toContain("Skipped archived transcript media migration"); + expect(replacements).toBe(0); + expect(fs.readFileSync(archivePath)).toEqual(bytes); + }); + + it.each([ + { name: "an empty file", bytes: Buffer.alloc(0) }, + { + name: "a valid archive without a NUL tail", + bytes: Buffer.from(`${JSON.stringify({ type: "event", id: "event-1" })}\n`), + }, + ])("does not rewrite $name", ({ bytes }) => { + const { archivePath, env } = createArchiveFixture(bytes); + const before = fs.lstatSync(archivePath); + let replacements = 0; + + const result = migrateLegacyMediaPersistence({ + env, + hooks: { beforeArchiveReplace: () => (replacements += 1) }, + }); + + const after = fs.lstatSync(archivePath); + expect(result).toEqual({ changes: [], warnings: [] }); + expect(replacements).toBe(0); + expect(fs.readFileSync(archivePath)).toEqual(bytes); + expect({ dev: after.dev, ino: after.ino, mtimeMs: after.mtimeMs, size: after.size }).toEqual({ + dev: before.dev, + ino: before.ino, + mtimeMs: before.mtimeMs, + size: before.size, + }); + }); +}); diff --git a/src/infra/state-migrations.media-persistence.ts b/src/infra/state-migrations.media-persistence.ts index f138275c84ad..a86eaa41ae5e 100644 --- a/src/infra/state-migrations.media-persistence.ts +++ b/src/infra/state-migrations.media-persistence.ts @@ -486,10 +486,10 @@ function archiveSourceMatches(filePath: string, expected: ArchiveSourceSnapshot) } function parseArchiveContent(content: string, filePath: string): TranscriptEvent[] { - const lines = content.endsWith("\n") ? content.slice(0, -1).split("\n") : content.split("\n"); - if (lines.length === 1 && lines[0] === "") { + if (content === "") { return []; } + const lines = content.endsWith("\n") ? content.slice(0, -1).split("\n") : content.split("\n"); return lines.map((line, index) => { if (!line) { throw new Error(`${filePath} contains a blank JSONL record at line ${index + 1}`); @@ -514,18 +514,31 @@ function migrateTranscriptArchive( ): boolean { const source = readArchiveSourceSnapshot(filePath); const content = readSessionArchiveContentSync(filePath); - const events = parseArchiveContent(content, filePath); - let changed = false; + let nulTailStart = content.length; + while (nulTailStart > 0 && content.charCodeAt(nulTailStart - 1) === 0) { + nulTailStart -= 1; + } + const hasTerminalNulSuffix = nulTailStart < content.length; + if (hasTerminalNulSuffix && nulTailStart === 0) { + throw new Error(`${filePath} contains no JSONL records before its terminal NUL suffix`); + } + // Torn writes may leave only preallocated NUL bytes after complete JSONL records. + // Recovery stays doctor-owned and reaches the same verified atomic replacement as media repair. + const recoveredContent = hasTerminalNulSuffix ? content.slice(0, nulTailStart) : content; + const events = parseArchiveContent(recoveredContent, filePath); + let mediaChanged = false; const transformed = events.map((event) => { const result = transformTranscriptEvent(event); - changed ||= result.changed; + mediaChanged ||= result.changed; return result.event; }); - if (!changed) { + if (!hasTerminalNulSuffix && !mediaChanged) { return false; } assertEventIdentitiesUnchanged(events, transformed, filePath); - const rewritten = serializeArchiveEvents(transformed, content.endsWith("\n")); + const rewritten = mediaChanged + ? serializeArchiveEvents(transformed, recoveredContent.endsWith("\n")) + : recoveredContent; const compressed = filePath.endsWith(SESSION_ARCHIVE_ZSTD_SUFFIX); const encoded = compressed ? encodeSessionArchiveContent(rewritten)