From e474d23e653c91fb3364976f6ecd8fb61b4cf0f5 Mon Sep 17 00:00:00 2001 From: ClawSweeper Date: Wed, 19 Aug 2026 10:29:54 -0700 Subject: [PATCH] fix(doctor): recover recreated legacy workspace state (#126059) * test(doctor): cover interrupted recreated workspace state * fix(doctor): recover recreated legacy workspace state --------- Co-authored-by: Josh Lehman <550978+jalehman@users.noreply.github.com> Co-authored-by: RoboClaw <309084314+roboclaw-bot@users.noreply.github.com> --- ...grations.workspace-setup-recreated.test.ts | 161 ++++++++++++++++++ .../state-migrations.workspace-setup-store.ts | 19 ++- ...migrations.workspace-setup.test-support.ts | 63 +++++++ .../state-migrations.workspace-setup.test.ts | 58 +------ src/infra/state-migrations.workspace-setup.ts | 26 +-- 5 files changed, 261 insertions(+), 66 deletions(-) create mode 100644 src/infra/state-migrations.workspace-setup-recreated.test.ts create mode 100644 src/infra/state-migrations.workspace-setup.test-support.ts diff --git a/src/infra/state-migrations.workspace-setup-recreated.test.ts b/src/infra/state-migrations.workspace-setup-recreated.test.ts new file mode 100644 index 000000000000..0d80c0518bfa --- /dev/null +++ b/src/infra/state-migrations.workspace-setup-recreated.test.ts @@ -0,0 +1,161 @@ +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import fsp from "node:fs/promises"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { resolveWorkspaceStateIdentity } from "../agents/workspace-state-store.js"; +import { openOpenClawStateDatabase } from "../state/openclaw-state-db.js"; +import { useWorkspaceMigrationTestFixture } from "./state-migrations.workspace-setup.test-support.js"; + +describe("recreated legacy workspace state migration", () => { + const { migrate, setup } = useWorkspaceMigrationTestFixture(); + + it("cleans a covered setup marker recreated after completed migration", async () => { + const context = setup(); + const identity = resolveWorkspaceStateIdentity(context.workspaceDir); + const setupPath = path.join(context.workspaceDir, "openclaw-workspace-state.json"); + const seededAt = "2026-07-15T10:00:00.000Z"; + const completedAt = "2026-07-15T10:01:00.000Z"; + await fsp.writeFile( + setupPath, + JSON.stringify({ version: 1, bootstrapSeededAt: seededAt, setupCompletedAt: completedAt }), + "utf8", + ); + expect((await migrate(context)).warnings).toEqual([]); + + const recreated = JSON.stringify({ version: 1, setupCompletedAt: completedAt }); + await fsp.writeFile(setupPath, recreated, "utf8"); + + const result = await migrate(context); + + expect(result.warnings).toEqual([]); + expect(fs.existsSync(setupPath)).toBe(false); + const db = openOpenClawStateDatabase({ env: context.env }).db; + expect( + db + .prepare( + "SELECT bootstrap_seeded_at, setup_completed_at FROM workspace_setup_state WHERE workspace_key = ?", + ) + .get(identity.workspaceKey), + ).toEqual({ bootstrap_seeded_at: seededAt, setup_completed_at: completedAt }); + const receipt = db + .prepare( + "SELECT source_sha256, removed_source, report_json FROM migration_sources WHERE source_path = ?", + ) + .get(setupPath) as { + source_sha256: string; + removed_source: number; + report_json: string; + }; + expect(receipt).toMatchObject({ + source_sha256: createHash("sha256").update(recreated).digest("hex"), + removed_source: 1, + }); + expect(JSON.parse(receipt.report_json)).toMatchObject({ + authoritative: true, + resolution: "verified", + }); + }); + + it.each([ + { claimed: false, state: "source" }, + { claimed: true, state: "interrupted claim" }, + ])("imports an attestation recreated as a sole $state", async ({ claimed }) => { + const context = setup(); + const identity = resolveWorkspaceStateIdentity(context.workspaceDir); + const attestationPath = path.join( + context.stateDir, + "workspace-attestations", + `${identity.workspaceKey}.attested`, + ); + await fsp.mkdir(path.dirname(attestationPath), { recursive: true }); + await fsp.writeFile( + attestationPath, + "openclaw-workspace-attestation:v1\n2026-07-15T11:00:00.000Z\n", + "utf8", + ); + const originalMtime = new Date("2026-07-15T11:01:00.000Z"); + await fsp.utimes(attestationPath, originalMtime, originalMtime); + expect((await migrate(context)).warnings).toEqual([]); + + const recreated = "openclaw-workspace-attestation:v1\n2026-07-16T11:00:00.000Z\n"; + await fsp.mkdir(path.dirname(attestationPath), { recursive: true }); + await fsp.writeFile(attestationPath, recreated, "utf8"); + const recreatedMtime = new Date("2026-07-16T11:01:00.000Z"); + await fsp.utimes(attestationPath, recreatedMtime, recreatedMtime); + const claimPath = `${attestationPath}.doctor-importing`; + if (claimed) { + await fsp.rename(attestationPath, claimPath); + } + + const result = await migrate(context); + + expect(result.warnings).toEqual([]); + expect(fs.existsSync(attestationPath)).toBe(false); + expect(fs.existsSync(claimPath)).toBe(false); + const db = openOpenClawStateDatabase({ env: context.env }).db; + expect( + db + .prepare("SELECT attested_at_ms FROM workspace_attestations WHERE workspace_key = ?") + .get(identity.workspaceKey), + ).toEqual({ attested_at_ms: recreatedMtime.getTime() }); + expect( + db + .prepare( + "SELECT source_sha256, removed_source FROM migration_sources WHERE source_path = ?", + ) + .get(attestationPath), + ).toEqual({ + source_sha256: createHash("sha256").update(recreated).digest("hex"), + removed_source: 1, + }); + }); + + it("retains colliding source and claim from a recreated generation", async () => { + const context = setup(); + const identity = resolveWorkspaceStateIdentity(context.workspaceDir); + const attestationPath = path.join( + context.stateDir, + "workspace-attestations", + `${identity.workspaceKey}.attested`, + ); + const original = "openclaw-workspace-attestation:v1\n2026-07-15T11:00:00.000Z\n"; + await fsp.mkdir(path.dirname(attestationPath), { recursive: true }); + await fsp.writeFile(attestationPath, original, "utf8"); + const originalMtime = new Date("2026-07-15T11:01:00.000Z"); + await fsp.utimes(attestationPath, originalMtime, originalMtime); + expect((await migrate(context)).warnings).toEqual([]); + + const recreated = "openclaw-workspace-attestation:v1\n2026-07-16T11:00:00.000Z\n"; + const claimPath = `${attestationPath}.doctor-importing`; + await fsp.mkdir(path.dirname(attestationPath), { recursive: true }); + await Promise.all([ + fsp.writeFile(attestationPath, recreated, "utf8"), + fsp.writeFile(claimPath, recreated, "utf8"), + ]); + + const result = await migrate(context); + + expect(result.warnings).toEqual([ + "Workspace state is in SQLite, but source and interrupted claim both exist.", + ]); + expect(fs.existsSync(attestationPath)).toBe(true); + expect(fs.existsSync(claimPath)).toBe(true); + const db = openOpenClawStateDatabase({ env: context.env }).db; + expect( + db + .prepare("SELECT attested_at_ms FROM workspace_attestations WHERE workspace_key = ?") + .get(identity.workspaceKey), + ).toEqual({ attested_at_ms: originalMtime.getTime() }); + expect( + db + .prepare( + "SELECT source_sha256, removed_source FROM migration_sources WHERE source_path = ?", + ) + .get(attestationPath), + ).toEqual({ + source_sha256: createHash("sha256").update(original).digest("hex"), + removed_source: 1, + }); + }); +}); diff --git a/src/infra/state-migrations.workspace-setup-store.ts b/src/infra/state-migrations.workspace-setup-store.ts index 32cae3dc97ad..039f2f152ded 100644 --- a/src/infra/state-migrations.workspace-setup-store.ts +++ b/src/infra/state-migrations.workspace-setup-store.ts @@ -1,5 +1,6 @@ // SQLite import and receipt semantics for retired workspace state. import { createHash } from "node:crypto"; +import { safeParseJsonRecord } from "@openclaw/normalization-core/json-coercion"; import { LEGACY_WORKSPACE_ATTESTATION_HEADER } from "../agents/workspace-legacy-state.js"; import { WORKSPACE_LEGACY_STATE_MIGRATION_KIND, @@ -201,6 +202,14 @@ function attestationFingerprint(params: { }); } +function receiptPreservesAuthority( + receipt: { reportJson: string } | null, + expectedFingerprint: string, +): boolean { + const report = receipt ? safeParseJsonRecord(receipt.reportJson) : undefined; + return report?.authoritative === true && report.canonicalFingerprint === expectedFingerprint; +} + function findMigrationAuthority(params: { db: ReturnType["db"]; kysely: ReturnType>; @@ -332,6 +341,7 @@ export function importAndRecordReceipt(params: { snapshot: SourceSnapshot; parsed: ParsedSource; env: NodeJS.ProcessEnv; + replaceRemovedReceipt?: boolean; }): { sourceKey: string; imported: boolean } { const key = resolveWorkspaceMigrationSourceKey(params.source); const runId = `${key}:${params.snapshot.sha256.slice(0, 16)}`; @@ -341,7 +351,8 @@ export function importAndRecordReceipt(params: { const { db } = database; const kysely = getNodeSqliteKysely(db); const existingReceipt = readLegacyMigrationReceiptFromDatabase(db, key); - if (existingReceipt) { + // Only a receipt whose source was fully removed can be replaced by a later generation. + if (existingReceipt && (!params.replaceRemovedReceipt || !existingReceipt.removedSource)) { throw new Error("workspace migration receipt appeared concurrently; retry Doctor"); } @@ -654,7 +665,10 @@ export function importAndRecordReceipt(params: { // Only a whole-source insert or precedence replacement can establish // authority. Verification and complementary merges may cover cleanup, // but must not let a legacy source overwrite unrelated canonical data. - authoritative: resolution === "inserted" || resolution === "replaced", + authoritative: + resolution === "inserted" || + resolution === "replaced" || + receiptPreservesAuthority(existingReceipt, verifiedFingerprint), resolution, imported, }); @@ -669,6 +683,7 @@ export function importAndRecordReceipt(params: { runId, now, reportJson, + upsert: existingReceipt !== null, }); return { sourceKey: key, imported }; }, diff --git a/src/infra/state-migrations.workspace-setup.test-support.ts b/src/infra/state-migrations.workspace-setup.test-support.ts new file mode 100644 index 000000000000..bfae2711aa64 --- /dev/null +++ b/src/infra/state-migrations.workspace-setup.test-support.ts @@ -0,0 +1,63 @@ +import fs from "node:fs"; +import path from "node:path"; +import { afterEach } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; +import { captureEnv, setTestEnvValue } from "../test-utils/env.js"; +import { + detectLegacyWorkspaceState, + migrateLegacyWorkspaceState, +} from "./state-migrations.workspace-setup.js"; + +export function useWorkspaceMigrationTestFixture() { + let envSnapshot: ReturnType | undefined; + const tempDirs = useAutoCleanupTempDirTracker((cleanup) => { + afterEach(() => { + closeOpenClawStateDatabaseForTest(); + envSnapshot?.restore(); + envSnapshot = undefined; + cleanup(); + }); + }); + + function setup() { + const homeDir = tempDirs.make("openclaw-workspace-migration-home-"); + const stateDir = path.join(homeDir, ".openclaw"); + const workspaceDir = path.join(homeDir, "workspace"); + fs.mkdirSync(workspaceDir, { recursive: true }); + envSnapshot ??= captureEnv(["HOME", "OPENCLAW_HOME", "OPENCLAW_STATE_DIR"]); + setTestEnvValue("HOME", homeDir); + setTestEnvValue("OPENCLAW_STATE_DIR", stateDir); + const cfg = { + agents: { defaults: { workspace: workspaceDir } }, + } satisfies OpenClawConfig; + return { + cfg, + env: { ...process.env, HOME: homeDir, OPENCLAW_STATE_DIR: stateDir }, + homeDir, + stateDir, + workspaceDir, + }; + } + + function detect(context: ReturnType) { + return detectLegacyWorkspaceState({ + cfg: context.cfg, + stateDir: context.stateDir, + env: context.env, + homedir: () => context.homeDir, + doctorOnlyStateMigrations: true, + }); + } + + async function migrate(context: ReturnType) { + return await migrateLegacyWorkspaceState({ + detected: detect(context), + env: context.env, + stateDir: context.stateDir, + }); + } + + return { detect, migrate, setup }; +} diff --git a/src/infra/state-migrations.workspace-setup.test.ts b/src/infra/state-migrations.workspace-setup.test.ts index cb22263b3286..88615a9a6d9b 100644 --- a/src/infra/state-migrations.workspace-setup.test.ts +++ b/src/infra/state-migrations.workspace-setup.test.ts @@ -2,8 +2,7 @@ import { createHash } from "node:crypto"; import fs from "node:fs"; import fsp from "node:fs/promises"; import path from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; -import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import { describe, expect, it } from "vitest"; import { deleteWorkspaceState, prepareWorkspaceStateDeletion, @@ -11,66 +10,17 @@ import { resolveWorkspaceStateIdentity, } from "../agents/workspace-state-store.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; -import { - closeOpenClawStateDatabaseForTest, - openOpenClawStateDatabase, -} from "../state/openclaw-state-db.js"; -import { captureEnv, setTestEnvValue } from "../test-utils/env.js"; +import { openOpenClawStateDatabase } from "../state/openclaw-state-db.js"; import { detectLegacyWorkspaceState, migrateLegacyWorkspaceState, } from "./state-migrations.workspace-setup.js"; +import { useWorkspaceMigrationTestFixture } from "./state-migrations.workspace-setup.test-support.js"; const HASH = "a".repeat(64); describe("legacy workspace Doctor migration", () => { - let envSnapshot: ReturnType | undefined; - const tempDirs = useAutoCleanupTempDirTracker((cleanup) => { - afterEach(() => { - closeOpenClawStateDatabaseForTest(); - envSnapshot?.restore(); - envSnapshot = undefined; - cleanup(); - }); - }); - - function setup() { - const homeDir = tempDirs.make("openclaw-workspace-migration-home-"); - const stateDir = path.join(homeDir, ".openclaw"); - const workspaceDir = path.join(homeDir, "workspace"); - fs.mkdirSync(workspaceDir, { recursive: true }); - envSnapshot ??= captureEnv(["HOME", "OPENCLAW_HOME", "OPENCLAW_STATE_DIR"]); - setTestEnvValue("HOME", homeDir); - setTestEnvValue("OPENCLAW_STATE_DIR", stateDir); - const cfg = { - agents: { defaults: { workspace: workspaceDir } }, - } satisfies OpenClawConfig; - return { - cfg, - env: { ...process.env, HOME: homeDir, OPENCLAW_STATE_DIR: stateDir }, - homeDir, - stateDir, - workspaceDir, - }; - } - - function detect(context: ReturnType) { - return detectLegacyWorkspaceState({ - cfg: context.cfg, - stateDir: context.stateDir, - env: context.env, - homedir: () => context.homeDir, - doctorOnlyStateMigrations: true, - }); - } - - async function migrate(context: ReturnType) { - return await migrateLegacyWorkspaceState({ - detected: detect(context), - env: context.env, - stateDir: context.stateDir, - }); - } + const { detect, migrate, setup } = useWorkspaceMigrationTestFixture(); it("detects configured and orphan sources only for explicit Doctor repair", async () => { const context = setup(); diff --git a/src/infra/state-migrations.workspace-setup.ts b/src/infra/state-migrations.workspace-setup.ts index 3bcf645776bf..c117bb6c1a94 100644 --- a/src/infra/state-migrations.workspace-setup.ts +++ b/src/infra/state-migrations.workspace-setup.ts @@ -378,12 +378,13 @@ async function cleanupReceiptSource(params: { source: LegacyWorkspaceStateSource; receipt: MigrationReceipt; env: NodeJS.ProcessEnv; + hasSource: boolean; + hasClaim: boolean; }): Promise { try { assertConfiguredWorkspaceIdentity(params.source); const sourceClaim = params.sourceClaim; - const hasSource = await sourceClaim.exists(); - const hasClaim = await sourceClaim.exists(true); + const { hasSource, hasClaim } = params; if (!hasSource && !hasClaim) { if (!params.receipt.removedSource) { markLegacyMigrationSourceRemoved(params.receipt.sourceKey, params.env); @@ -470,14 +471,6 @@ async function migrateOneSource(params: { }; } const receipt = readReceipt(params.source, params.env); - if (receipt) { - return cleanupReceiptSource({ - sourceClaim, - source: params.source, - receipt, - env: params.env, - }); - } let hasSource: boolean; let hasClaim: boolean; try { @@ -489,6 +482,18 @@ async function migrateOneSource(params: { warnings: [`Failed reading legacy workspace state: ${formatErrorMessage(error)}`], }; } + // One artifact after verified removal is a new generation, including a source + // already renamed before a crash. Collisions keep the stricter receipt check. + if (receipt && !(receipt.removedSource && hasSource !== hasClaim)) { + return cleanupReceiptSource({ + sourceClaim, + source: params.source, + receipt, + env: params.env, + hasSource, + hasClaim, + }); + } if (hasSource && hasClaim) { return { changes: [], @@ -544,6 +549,7 @@ async function migrateOneSource(params: { snapshot, parsed, env: params.env, + replaceRemovedReceipt: receipt?.removedSource === true, }); } catch (error) { const restoreError = claimedByThisRun ? await sourceClaim.restore() : null;