diff --git a/extensions/codex/doctor-contract-api.test.ts b/extensions/codex/doctor-contract-api.test.ts index eb79ede1794e..e815c36ebedb 100644 --- a/extensions/codex/doctor-contract-api.test.ts +++ b/extensions/codex/doctor-contract-api.test.ts @@ -703,7 +703,8 @@ describe("codex doctor contract", () => { const result = await fixture.migration.migrateLegacyState({ ...fixture.params, context }); - expect(result.warnings).toEqual([ + expect(result.warnings).toEqual([]); + expect(result.notices).toEqual([ expect.stringContaining("session owner changed before Codex ownership could be recorded"), ]); await expect(fs.access(fixture.sidecarPath)).resolves.toBeUndefined(); @@ -761,7 +762,8 @@ describe("codex doctor contract", () => { const result = await fixture.migration.migrateLegacyState({ ...fixture.params, context }); - expect(result.warnings).toEqual([ + expect(result.warnings).toEqual([]); + expect(result.notices).toEqual([ expect.stringContaining("session owner changed before Codex ownership could be recorded"), ]); await expect(fs.access(fixture.sidecarPath)).resolves.toBeUndefined(); @@ -979,7 +981,8 @@ describe("codex doctor contract", () => { const result = await fixture.migration.migrateLegacyState(fixture.params); expect(result.changes).toEqual([]); - expect(result.warnings).toEqual([expect.stringContaining("owned by agent harness pi")]); + expect(result.warnings).toEqual([]); + expect(result.notices).toEqual([expect.stringContaining("owned by agent harness pi")]); await expect(fs.access(fixture.sidecarPath)).resolves.toBeUndefined(); await expect(openBindingStore(fixture.env).entries()).resolves.toEqual([]); diff --git a/extensions/codex/src/migration/session-binding-sidecar-archive.ts b/extensions/codex/src/migration/session-binding-sidecar-archive.ts new file mode 100644 index 000000000000..27dd3a79ff48 --- /dev/null +++ b/extensions/codex/src/migration/session-binding-sidecar-archive.ts @@ -0,0 +1,36 @@ +import fs from "node:fs/promises"; + +async function pathExists(filePath: string): Promise { + try { + await fs.access(filePath); + return true; + } catch { + return false; + } +} + +async function firstFreeArchivePath(sourcePath: string): Promise { + for (let index = 2; ; index++) { + const candidate = `${sourcePath}.migrated.${index}`; + if (!(await pathExists(candidate))) { + return candidate; + } + } +} + +export async function archiveBindingSidecar(sourcePath: string): Promise { + const archivePath = `${sourcePath}.migrated`; + if (!(await pathExists(archivePath))) { + await fs.rename(sourcePath, archivePath); + return; + } + const [sourceBytes, archiveBytes] = await Promise.all([ + fs.readFile(sourcePath), + fs.readFile(archivePath), + ]); + if (sourceBytes.equals(archiveBytes)) { + await fs.rm(sourcePath, { force: true }); + return; + } + await fs.rename(sourcePath, await firstFreeArchivePath(sourcePath)); +} diff --git a/extensions/codex/src/migration/session-binding-sidecars.ts b/extensions/codex/src/migration/session-binding-sidecars.ts index f9ee76698ccd..79068e0af227 100644 --- a/extensions/codex/src/migration/session-binding-sidecars.ts +++ b/extensions/codex/src/migration/session-binding-sidecars.ts @@ -14,6 +14,7 @@ import { CODEX_APP_SERVER_BINDING_MAX_ENTRIES, CODEX_APP_SERVER_BINDING_NAMESPACE, } from "../app-server/session-binding-meta.js"; +import { archiveBindingSidecar } from "./session-binding-sidecar-archive.js"; const LEGACY_BINDING_SUFFIX = ".codex-app-server.json"; const CODEX_AGENT_HARNESS_ID = "codex"; @@ -66,6 +67,7 @@ type BindingOwnerCollection = { type SourceMigrationResult = { archived: boolean; importedKeys: number; + notice?: string; warning?: string; }; @@ -429,6 +431,11 @@ async function migrateSource( importedKeys, warning: `Left Codex binding sidecar in place because ${reason}: ${source.sidecarPath}`, }); + const retainNotice = (reason: string): SourceMigrationResult => ({ + archived: false, + importedKeys, + notice: `Left Codex binding sidecar in place because ${reason}: ${source.sidecarPath}`, + }); const owner = candidates.length === 1 ? candidates[0] : undefined; try { return await withFileLock(source.sidecarPath, LEGACY_BINDING_LOCK_OPTIONS, async () => { @@ -467,7 +474,10 @@ async function migrateSource( return retain(`${candidates.length} matching session owners make ownership ambiguous`); } if (owner?.agentHarnessId && owner.agentHarnessId !== CODEX_AGENT_HARNESS_ID) { - return retain(`its session is owned by agent harness ${owner.agentHarnessId}`); + // Explicit foreign ownership is a complete decision, not an unsafe + // migration conflict. Preserve the sidecar for that harness without + // blocking every later Gateway startup. + return retainNotice(`its session is owned by agent harness ${owner.agentHarnessId}`); } const sourceSessionFile = typeof raw.sessionFile === "string" && raw.sessionFile.trim() @@ -611,7 +621,10 @@ async function migrateSource( return retain(`${ownershipWarning}; its stale session binding could not be retired`); } } - return retain(ownershipWarning); + // Imported active session state is retired before reaching here. + // The remaining sidecar may belong to the new owner, so preserve it + // as a note; failed retirement and revalidation stay warnings above. + return retainNotice(ownershipWarning); } for (const entry of entries) { if (!hasExpected(await store.lookup(entry.key), entry.value)) { @@ -808,32 +821,6 @@ async function pathExists(filePath: string): Promise { } } -async function firstFreeArchivePath(sourcePath: string): Promise { - for (let index = 2; ; index++) { - const candidate = `${sourcePath}.migrated.${index}`; - if (!(await pathExists(candidate))) { - return candidate; - } - } -} - -async function archiveBindingSidecar(sourcePath: string): Promise { - const archivePath = `${sourcePath}.migrated`; - if (await pathExists(archivePath)) { - const [sourceBytes, archiveBytes] = await Promise.all([ - fs.readFile(sourcePath), - fs.readFile(archivePath), - ]); - if (sourceBytes.equals(archiveBytes)) { - await fs.rm(sourcePath, { force: true }); - return; - } - await fs.rename(sourcePath, await firstFreeArchivePath(sourcePath)); - return; - } - await fs.rename(sourcePath, archivePath); -} - export const stateMigrations: PluginDoctorStateMigration[] = [ { id: "codex-app-server-sidecars-to-plugin-state", @@ -851,6 +838,7 @@ export const stateMigrations: PluginDoctorStateMigration[] = [ async migrateLegacyState(params) { const changes: string[] = []; const warnings: string[] = []; + const notices: string[] = []; const { sources, surfaces } = await collectLegacyBindingSources(params); if (sources.length === 0) { return { changes, warnings }; @@ -876,6 +864,9 @@ export const stateMigrations: PluginDoctorStateMigration[] = [ if (result.warning) { warnings.push(result.warning); } + if (result.notice) { + notices.push(result.notice); + } if (result.archived) { migrated++; } else { @@ -892,7 +883,11 @@ export const stateMigrations: PluginDoctorStateMigration[] = [ `Migrated ${partialImports} safe Codex app-server binding row(s) to plugin state; retained legacy sidecars needing review`, ); } - return { changes, warnings }; + return { + changes, + warnings, + ...(notices.length > 0 ? { notices } : {}), + }; }, }, ]; diff --git a/extensions/matrix/doctor-contract-api.test.ts b/extensions/matrix/doctor-contract-api.test.ts index 779b7bc79476..4a18899e1c77 100644 --- a/extensions/matrix/doctor-contract-api.test.ts +++ b/extensions/matrix/doctor-contract-api.test.ts @@ -117,7 +117,51 @@ describe("matrix doctor contract state migrations", () => { expect(store.hasSavedSync()).toBe(true); expect(store.hasSavedSyncFromCleanShutdown()).toBe(true); await expect(store.getSavedSyncToken()).resolves.toBe("legacy-token"); - expect(fs.existsSync(path.join(storageRootDir, "bot-storage.json"))).toBe(false); + const sourcePath = path.join(storageRootDir, "bot-storage.json"); + const archivePath = `${sourcePath}.migrated`; + expect(fs.existsSync(sourcePath)).toBe(false); + + fs.copyFileSync(archivePath, sourcePath); + await expect(migration.migrateLegacyState(createMigrationParams(stateDir))).resolves.toEqual({ + changes: [`Removed already-archived Matrix sync cache legacy source ${sourcePath}`], + warnings: [], + notices: [ + `Kept existing Matrix sync cache in SQLite and archived the legacy source for ${storageRootDir}`, + ], + }); + + fs.writeFileSync( + sourcePath, + JSON.stringify({ + version: 1, + savedSync: { + nextBatch: "newer-legacy-token", + accountData: [], + roomsData: { join: {}, invite: {}, leave: {}, knock: {} }, + }, + cleanShutdown: true, + }), + ); + await expect(migration.migrateLegacyState(createMigrationParams(stateDir))).resolves.toEqual({ + changes: [`Archived Matrix sync cache legacy source -> ${sourcePath}.migrated.2`], + warnings: [], + notices: [ + `Kept existing Matrix sync cache in SQLite and archived the legacy source for ${storageRootDir}`, + ], + }); + await expect(migration.migrateLegacyState(createMigrationParams(stateDir))).resolves.toEqual({ + changes: [], + warnings: [], + }); + + fs.writeFileSync(sourcePath, `${fs.readFileSync(`${sourcePath}.migrated.2`, "utf8")} `, "utf8"); + fs.mkdirSync(`${sourcePath}.migrated.3`); + const failedArchive = await migration.migrateLegacyState(createMigrationParams(stateDir)); + expect(failedArchive.changes).toEqual([]); + expect(failedArchive.warnings).toEqual([ + expect.stringContaining("Failed archiving Matrix sync cache legacy source"), + ]); + expect(failedArchive.notices).toBeUndefined(); }); it("migrates Matrix storage metadata JSON to SQLite plugin state", async () => { diff --git a/extensions/matrix/doctor-contract-api.ts b/extensions/matrix/doctor-contract-api.ts index 155fee9e0c93..6ca36941d2a4 100644 --- a/extensions/matrix/doctor-contract-api.ts +++ b/extensions/matrix/doctor-contract-api.ts @@ -3,7 +3,7 @@ import type { Dirent } from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; import { - legacyStateFileExists, + archiveLegacyStateSource, type PluginDoctorStateMigration, } from "openclaw/plugin-sdk/runtime-doctor"; import { @@ -106,21 +106,14 @@ async function archiveLegacySyncCache(params: { storageRootDir: string; changes: string[]; warnings: string[]; + notices?: string[]; + notice?: string; }): Promise { - const sourcePath = path.join(params.storageRootDir, MATRIX_SYNC_CACHE_FILENAME); - const archivedPath = `${sourcePath}.migrated`; - if (await legacyStateFileExists(archivedPath)) { - params.warnings.push( - `Left migrated Matrix sync cache in place because ${archivedPath} already exists`, - ); - return; - } - try { - await fs.rename(sourcePath, archivedPath); - params.changes.push(`Archived Matrix sync cache legacy source -> ${archivedPath}`); - } catch (err) { - params.warnings.push(`Failed archiving Matrix sync cache legacy source: ${String(err)}`); - } + await archiveLegacyMatrixStateFile({ + ...params, + filename: MATRIX_SYNC_CACHE_FILENAME, + label: "Matrix sync cache", + }); } async function archiveLegacyMatrixStateFile(params: { @@ -129,20 +122,18 @@ async function archiveLegacyMatrixStateFile(params: { label: string; changes: string[]; warnings: string[]; + notices?: string[]; + notice?: string; }): Promise { - const sourcePath = path.join(params.storageRootDir, params.filename); - const archivedPath = `${sourcePath}.migrated`; - if (await legacyStateFileExists(archivedPath)) { - params.warnings.push( - `Left migrated ${params.label} in place because ${archivedPath} already exists`, - ); - return; - } - try { - await fs.rename(sourcePath, archivedPath); - params.changes.push(`Archived ${params.label} legacy source -> ${archivedPath}`); - } catch (err) { - params.warnings.push(`Failed archiving ${params.label} legacy source: ${String(err)}`); + const warningCount = params.warnings.length; + await archiveLegacyStateSource({ + filePath: path.join(params.storageRootDir, params.filename), + label: params.label, + changes: params.changes, + warnings: params.warnings, + }); + if (params.notice && params.warnings.length === warningCount) { + params.notices?.push(params.notice); } } @@ -278,6 +269,7 @@ export const stateMigrations: PluginDoctorStateMigration[] = [ async migrateLegacyState(params) { const changes: string[] = []; const warnings: string[] = []; + const notices: string[] = []; for (const storageRootDir of await collectLegacyMatrixStateRoots( params.stateDir, MATRIX_STORAGE_META_FILENAME, @@ -290,15 +282,14 @@ export const stateMigrations: PluginDoctorStateMigration[] = [ openMatrixStorageMetaStoreOptions(storageRootDir), ); if (await hasMatrixStorageMetaStateInStore({ store })) { - warnings.push( - `Skipped Matrix storage metadata import for ${storageRootDir} because SQLite already has metadata`, - ); await archiveLegacyMatrixStateFile({ storageRootDir, filename: MATRIX_STORAGE_META_FILENAME, label: "Matrix storage metadata", changes, warnings, + notices, + notice: `Kept existing Matrix storage metadata in SQLite and archived the legacy source for ${storageRootDir}`, }); continue; } @@ -312,7 +303,7 @@ export const stateMigrations: PluginDoctorStateMigration[] = [ warnings, }); } - return { changes, warnings }; + return { changes, warnings, ...(notices.length > 0 ? { notices } : {}) }; }, }, { @@ -332,6 +323,7 @@ export const stateMigrations: PluginDoctorStateMigration[] = [ async migrateLegacyState(params) { const changes: string[] = []; const warnings: string[] = []; + const notices: string[] = []; for (const storageRootDir of await collectLegacySyncCacheRoots(params.stateDir)) { const persisted = await readLegacyMatrixSyncCacheState(storageRootDir); if (!persisted) { @@ -341,10 +333,13 @@ export const stateMigrations: PluginDoctorStateMigration[] = [ openMatrixSyncCacheStoreOptions(storageRootDir), ); if (await hasMatrixSyncCacheStateInStore({ storageRootDir, store })) { - warnings.push( - `Skipped Matrix sync cache import for ${storageRootDir} because SQLite already has sync cache state`, - ); - await archiveLegacySyncCache({ storageRootDir, changes, warnings }); + await archiveLegacySyncCache({ + storageRootDir, + changes, + warnings, + notices, + notice: `Kept existing Matrix sync cache in SQLite and archived the legacy source for ${storageRootDir}`, + }); continue; } await writeMatrixSyncCacheStateToStore({ @@ -355,7 +350,7 @@ export const stateMigrations: PluginDoctorStateMigration[] = [ changes.push(`Migrated Matrix sync cache JSON to SQLite for ${storageRootDir}`); await archiveLegacySyncCache({ storageRootDir, changes, warnings }); } - return { changes, warnings }; + return { changes, warnings, ...(notices.length > 0 ? { notices } : {}) }; }, }, { @@ -377,6 +372,7 @@ export const stateMigrations: PluginDoctorStateMigration[] = [ async migrateLegacyState(params) { const changes: string[] = []; const warnings: string[] = []; + const notices: string[] = []; for (const storageRootDir of await collectLegacyMatrixStateRoots( params.stateDir, MATRIX_RECOVERY_KEY_FILENAME, @@ -389,15 +385,14 @@ export const stateMigrations: PluginDoctorStateMigration[] = [ openMatrixRecoveryKeyStoreOptions(storageRootDir), ); if (await hasMatrixRecoveryKeyStateInStore({ store })) { - warnings.push( - `Skipped Matrix recovery-key import for ${storageRootDir} because SQLite already has recovery-key state`, - ); await archiveLegacyMatrixStateFile({ storageRootDir, filename: MATRIX_RECOVERY_KEY_FILENAME, label: "Matrix recovery key", changes, warnings, + notices, + notice: `Kept existing Matrix recovery key in SQLite and archived the legacy source for ${storageRootDir}`, }); continue; } @@ -411,7 +406,7 @@ export const stateMigrations: PluginDoctorStateMigration[] = [ warnings, }); } - return { changes, warnings }; + return { changes, warnings, ...(notices.length > 0 ? { notices } : {}) }; }, }, { @@ -434,6 +429,7 @@ export const stateMigrations: PluginDoctorStateMigration[] = [ async migrateLegacyState(params) { const changes: string[] = []; const warnings: string[] = []; + const notices: string[] = []; for (const storageRootDir of await collectLegacyMatrixStateRoots( params.stateDir, MATRIX_IDB_SNAPSHOT_FILENAME, @@ -446,15 +442,14 @@ export const stateMigrations: PluginDoctorStateMigration[] = [ openMatrixIdbSnapshotStoreOptions(storageRootDir), ); if (await hasMatrixIdbSnapshotStateInStore({ store })) { - warnings.push( - `Skipped Matrix IndexedDB snapshot import for ${storageRootDir} because SQLite already has snapshot state`, - ); await archiveLegacyMatrixStateFile({ storageRootDir, filename: MATRIX_IDB_SNAPSHOT_FILENAME, label: "Matrix IndexedDB snapshot", changes, warnings, + notices, + notice: `Kept existing Matrix IndexedDB snapshot in SQLite and archived the legacy source for ${storageRootDir}`, }); continue; } @@ -472,7 +467,7 @@ export const stateMigrations: PluginDoctorStateMigration[] = [ warnings, }); } - return { changes, warnings }; + return { changes, warnings, ...(notices.length > 0 ? { notices } : {}) }; }, }, { @@ -496,6 +491,7 @@ export const stateMigrations: PluginDoctorStateMigration[] = [ async migrateLegacyState(params) { const changes: string[] = []; const warnings: string[] = []; + const notices: string[] = []; for (const storageRootDir of await collectLegacyMatrixStateRoots( params.stateDir, MATRIX_LEGACY_CRYPTO_MIGRATION_FILENAME, @@ -508,15 +504,14 @@ export const stateMigrations: PluginDoctorStateMigration[] = [ openMatrixLegacyCryptoMigrationStoreOptions(storageRootDir), ); if (await hasMatrixLegacyCryptoMigrationStateInStore({ store })) { - warnings.push( - `Skipped Matrix legacy crypto migration import for ${storageRootDir} because SQLite already has migration state`, - ); await archiveLegacyMatrixStateFile({ storageRootDir, filename: MATRIX_LEGACY_CRYPTO_MIGRATION_FILENAME, label: "Matrix legacy crypto migration", changes, warnings, + notices, + notice: `Kept existing Matrix legacy crypto migration in SQLite and archived the legacy source for ${storageRootDir}`, }); continue; } @@ -532,7 +527,7 @@ export const stateMigrations: PluginDoctorStateMigration[] = [ warnings, }); } - return { changes, warnings }; + return { changes, warnings, ...(notices.length > 0 ? { notices } : {}) }; }, }, ]; diff --git a/extensions/memory-core/doctor-contract-api.test.ts b/extensions/memory-core/doctor-contract-api.test.ts index 49a925994a3f..458ddde7a527 100644 --- a/extensions/memory-core/doctor-contract-api.test.ts +++ b/extensions/memory-core/doctor-contract-api.test.ts @@ -19,7 +19,11 @@ import type { } from "openclaw/plugin-sdk/runtime-doctor"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { stateMigrations } from "./doctor-contract-api.js"; -import { configureMemoryCoreDreamingState } from "./src/dreaming-state.js"; +import { + DREAMING_DAILY_INGESTION_NAMESPACE, + configureMemoryCoreDreamingState, + writeMemoryCoreWorkspaceEntry, +} from "./src/dreaming-state.js"; import { bm25RankToScore, buildFtsQuery } from "./src/memory/hybrid.js"; import { searchKeyword, searchVector } from "./src/memory/manager-search.js"; import { @@ -484,8 +488,6 @@ describe("memory-core doctor dreaming migration", () => { recallCount: 1, totalScore: 0.9, maxScore: 0.9, - firstRecalledAt: "2026-04-05T12:00:00.000Z", - lastRecalledAt: "2026-04-05T12:00:00.000Z", queryHashes: ["hash-a"], }, }, @@ -553,6 +555,40 @@ describe("memory-core doctor dreaming migration", () => { "2026-04-05T13:00:00.000Z", ); expect(phase.entries["memory:memory/2026-04-05.md:1:1"]?.remHits).toBe(2); + + for (const sourcePath of [dailyPath, sessionPath, recallPath, phasePath]) { + await fs.copyFile(`${sourcePath}.migrated`, sourcePath); + } + await fs.copyFile(dailyPath, `${dailyPath}.migrated.2`); + await fs.writeFile(`${dailyPath}.migrated`, "older archive", "utf8"); + await writeMemoryCoreWorkspaceEntry({ + namespace: DREAMING_DAILY_INGESTION_NAMESPACE, + workspaceDir, + key: "memory/2026-04-05.md", + value: { ...daily.files["memory/2026-04-05.md"], mtimeMs: 2 }, + }); + const matchingResult = await migration.migrateLegacyState(migrationParams()); + expect(matchingResult.changes).toEqual([]); + expect(matchingResult.warnings).toEqual([]); + expect(matchingResult.notices).toEqual([ + expect.stringContaining("Retained acknowledged Memory Core daily ingestion"), + expect.stringContaining("Retained acknowledged Memory Core session ingestion"), + expect.stringContaining("Retained acknowledged Memory Core short-term recall"), + expect.stringContaining("Retained acknowledged Memory Core phase signals"), + ]); + + const changedDaily = JSON.parse(await fs.readFile(dailyPath, "utf8")) as { + files: Record; + }; + changedDaily.files["memory/2026-04-05.md"]!.mtimeMs = 999; + await fs.writeFile(dailyPath, JSON.stringify(changedDaily), "utf8"); + const conflictResult = await migration.migrateLegacyState(migrationParams()); + expect(conflictResult.changes).toEqual([]); + expect(conflictResult.warnings).toEqual([ + expect.stringContaining("SQLite rows conflict with the legacy source"), + ]); + expect(conflictResult.notices).toHaveLength(3); + await expect(fs.access(dailyPath)).resolves.toBeUndefined(); }); it("leaves invalid legacy JSON in place", async () => { diff --git a/extensions/memory-core/doctor-contract-api.ts b/extensions/memory-core/doctor-contract-api.ts index 73261f0ed8d2..d0ce302edab9 100644 --- a/extensions/memory-core/doctor-contract-api.ts +++ b/extensions/memory-core/doctor-contract-api.ts @@ -42,10 +42,10 @@ import { SHORT_TERM_PHASE_SIGNAL_NAMESPACE, SHORT_TERM_RECALL_NAMESPACE, configureMemoryCoreDreamingState, - readMemoryCoreWorkspaceEntries, writeMemoryCoreWorkspaceEntries, writeMemoryCoreWorkspaceEntry, } from "./src/dreaming-state.js"; +import { dreamingStateComparison } from "./src/migration/dreaming-state-comparison.js"; import { SHORT_TERM_PHASE_SIGNAL_RELATIVE_PATH, SHORT_TERM_STORE_RELATIVE_PATH, @@ -1032,10 +1032,6 @@ async function collectLegacySources( return sources; } -async function workspaceHasRows(namespace: string, workspaceDir: string): Promise { - return (await readMemoryCoreWorkspaceEntries({ namespace, workspaceDir })).length > 0; -} - async function migrateDailyIngestion(source: LegacySource): Promise { const state = normalizeDailyIngestionState(await readJsonFile(source.filePath)); await writeMemoryCoreWorkspaceEntries({ @@ -1117,19 +1113,6 @@ async function migratePhaseSignals(source: LegacySource): Promise { return Object.keys(state.entries).length; } -function targetNamespacesForSource(label: string): string[] { - if (label === "daily ingestion") { - return [DREAMING_DAILY_INGESTION_NAMESPACE]; - } - if (label === "session ingestion") { - return [DREAMING_SESSION_INGESTION_FILES_NAMESPACE, DREAMING_SESSION_INGESTION_SEEN_NAMESPACE]; - } - if (label === "short-term recall") { - return [SHORT_TERM_RECALL_NAMESPACE]; - } - return [SHORT_TERM_PHASE_SIGNAL_NAMESPACE]; -} - async function migrateSource(source: LegacySource): Promise { if (source.label === "daily ingestion") { return await migrateDailyIngestion(source); @@ -1163,17 +1146,29 @@ export const stateMigrations: PluginDoctorStateMigration[] = [ configureMemoryCoreDreamingState(params.context.openPluginStateKeyedStore); const changes: string[] = []; const warnings: string[] = []; + const notices: string[] = []; for (const source of await collectLegacySources(params.config, params.env)) { - const targetHasRows = ( - await Promise.all( - targetNamespacesForSource(source.label).map((namespace) => - workspaceHasRows(namespace, source.workspaceDir), - ), - ) - ).some(Boolean); + const targetHasRows = await dreamingStateComparison.targetHasRows(source); if (targetHasRows) { + let sourceAcknowledged: boolean; + try { + sourceAcknowledged = await dreamingStateComparison.sourceIsAcknowledged(source); + } catch (err) { + warnings.push( + `Skipped Memory Core ${source.label} import for ${source.workspaceDir} because the legacy source could not be compared: ${String(err)}`, + ); + continue; + } + if (sourceAcknowledged) { + // Older releases may rewrite these rollback sources. The stored hash + // keeps unchanged sources informational; rewritten sources fail closed. + notices.push( + `Retained acknowledged Memory Core ${source.label} legacy source for rollback: ${source.filePath}`, + ); + continue; + } warnings.push( - `Skipped Memory Core ${source.label} import for ${source.workspaceDir} because SQLite rows already exist; left legacy source in place`, + `Skipped Memory Core ${source.label} import for ${source.workspaceDir} because SQLite rows conflict with the legacy source; left legacy source in place`, ); continue; } @@ -1196,7 +1191,11 @@ export const stateMigrations: PluginDoctorStateMigration[] = [ warnings, }); } - return { changes, warnings }; + return { + changes, + warnings, + ...(notices.length > 0 ? { notices } : {}), + }; }, }, { diff --git a/extensions/memory-core/src/migration/dreaming-state-comparison.ts b/extensions/memory-core/src/migration/dreaming-state-comparison.ts new file mode 100644 index 000000000000..a9d7dea070ae --- /dev/null +++ b/extensions/memory-core/src/migration/dreaming-state-comparison.ts @@ -0,0 +1,223 @@ +import { createHash } from "node:crypto"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { isDeepStrictEqual } from "node:util"; +import { + normalizeDailyIngestionState, + normalizeSessionIngestionState, +} from "../dreaming-phases.js"; +import { + DREAMING_DAILY_INGESTION_NAMESPACE, + DREAMING_SESSION_INGESTION_FILES_NAMESPACE, + DREAMING_SESSION_INGESTION_SEEN_NAMESPACE, + SHORT_TERM_META_NAMESPACE, + SHORT_TERM_PHASE_SIGNAL_NAMESPACE, + SHORT_TERM_RECALL_NAMESPACE, + readMemoryCoreWorkspaceEntries, + writeMemoryCoreWorkspaceEntry, +} from "../dreaming-state.js"; +import { + normalizeShortTermPhaseSignalStore, + normalizeShortTermRecallStore, +} from "../short-term-promotion.js"; + +type LegacyDreamingSource = { + workspaceDir: string; + label: string; + filePath: string; +}; + +const LEGACY_SOURCE_ACKNOWLEDGEMENT_NAMESPACE = "legacy-dreaming-source-acknowledgements"; + +function targetNamespacesForSource(label: string): string[] { + if (label === "daily ingestion") { + return [DREAMING_DAILY_INGESTION_NAMESPACE]; + } + if (label === "session ingestion") { + return [DREAMING_SESSION_INGESTION_FILES_NAMESPACE, DREAMING_SESSION_INGESTION_SEEN_NAMESPACE]; + } + return [ + label === "short-term recall" ? SHORT_TERM_RECALL_NAMESPACE : SHORT_TERM_PHASE_SIGNAL_NAMESPACE, + ]; +} + +function asRecord(value: unknown): Record | undefined { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : undefined; +} + +async function memoryCoreLegacyTargetHasRows(source: LegacyDreamingSource): Promise { + const counts = await Promise.all( + targetNamespacesForSource(source.label).map( + async (namespace) => + ( + await readMemoryCoreWorkspaceEntries({ + namespace, + workspaceDir: source.workspaceDir, + }) + ).length, + ), + ); + return counts.some((count) => count > 0); +} + +async function memoryCoreLegacySourceMatchesCanonical( + source: LegacyDreamingSource, + raw: unknown, +): Promise { + if (source.label === "daily ingestion") { + const rows = await readMemoryCoreWorkspaceEntries({ + namespace: DREAMING_DAILY_INGESTION_NAMESPACE, + workspaceDir: source.workspaceDir, + }); + return isDeepStrictEqual( + normalizeDailyIngestionState(raw), + normalizeDailyIngestionState({ + version: 1, + files: Object.fromEntries(rows.map((row) => [row.key, row.value])), + }), + ); + } + if (source.label === "session ingestion") { + const [fileRows, seenRows] = await Promise.all([ + readMemoryCoreWorkspaceEntries({ + namespace: DREAMING_SESSION_INGESTION_FILES_NAMESPACE, + workspaceDir: source.workspaceDir, + }), + readMemoryCoreWorkspaceEntries<{ scope: string; index: number; hashes: string[] }>({ + namespace: DREAMING_SESSION_INGESTION_SEEN_NAMESPACE, + workspaceDir: source.workspaceDir, + }), + ]); + const chunksByScope = new Map>(); + for (const row of seenRows) { + const chunks = chunksByScope.get(row.value.scope) ?? []; + chunks.push({ index: row.value.index, hashes: row.value.hashes }); + chunksByScope.set(row.value.scope, chunks); + } + return isDeepStrictEqual( + normalizeSessionIngestionState(raw), + normalizeSessionIngestionState({ + version: 3, + files: Object.fromEntries(fileRows.map((row) => [row.key, row.value])), + seenMessages: Object.fromEntries( + [...chunksByScope].map(([scope, chunks]) => [ + scope, + chunks.toSorted((left, right) => left.index - right.index).flatMap((row) => row.hashes), + ]), + ), + }), + ); + } + const [entryRows, metaRows] = await Promise.all([ + readMemoryCoreWorkspaceEntries({ + namespace: + source.label === "short-term recall" + ? SHORT_TERM_RECALL_NAMESPACE + : SHORT_TERM_PHASE_SIGNAL_NAMESPACE, + workspaceDir: source.workspaceDir, + }), + readMemoryCoreWorkspaceEntries<{ updatedAt?: unknown }>({ + namespace: SHORT_TERM_META_NAMESPACE, + workspaceDir: source.workspaceDir, + }), + ]); + const metaKey = source.label === "short-term recall" ? "recall" : "phase"; + const updatedAt = metaRows.find((row) => row.key === metaKey)?.value.updatedAt; + if (typeof updatedAt !== "string" || updatedAt.length === 0) { + return false; + } + const canonicalRaw = { + version: 1, + updatedAt, + entries: Object.fromEntries(entryRows.map((row) => [row.key, row.value])), + }; + if (source.label === "short-term recall") { + const canonical = normalizeShortTermRecallStore(canonicalRaw, updatedAt); + const fallbackCandidates = new Set([updatedAt]); + for (const row of entryRows) { + const value = asRecord(row.value); + for (const key of ["firstRecalledAt", "lastRecalledAt"] as const) { + if (typeof value?.[key] === "string") { + fallbackCandidates.add(value[key]); + } + } + } + // Missing legacy timestamps received one migration-time value, preserved only in rows. + return [...fallbackCandidates].some((fallback) => + isDeepStrictEqual(normalizeShortTermRecallStore(raw, fallback), canonical), + ); + } + return isDeepStrictEqual( + normalizeShortTermPhaseSignalStore(raw, updatedAt), + normalizeShortTermPhaseSignalStore(canonicalRaw, updatedAt), + ); +} + +async function memoryCoreLegacySourceIsAcknowledged( + source: LegacyDreamingSource, +): Promise { + const contents = await fs.readFile(source.filePath); + const sha256 = createHash("sha256").update(contents).digest("hex"); + const markerKey = `legacy-source:${source.label}`; + const markers = await readMemoryCoreWorkspaceEntries<{ sha256?: unknown }>({ + namespace: LEGACY_SOURCE_ACKNOWLEDGEMENT_NAMESPACE, + workspaceDir: source.workspaceDir, + }); + if (markers.find((row) => row.key === markerKey)?.value.sha256 === sha256) { + return true; + } + let archiveNames: string[] = []; + try { + const archivePrefix = `${path.basename(source.filePath)}.migrated`; + archiveNames = (await fs.readdir(path.dirname(source.filePath))).filter((name) => { + if (name === archivePrefix) { + return true; + } + const suffix = name.slice(archivePrefix.length + 1); + const archiveIndex = Number(suffix); + return ( + name.startsWith(`${archivePrefix}.`) && + Number.isSafeInteger(archiveIndex) && + archiveIndex >= 2 && + String(archiveIndex) === suffix + ); + }); + } catch { + // The archive is optional provenance; canonical comparison remains authoritative. + } + let matchesArchive = false; + for (const archiveName of archiveNames) { + try { + if ( + contents.equals(await fs.readFile(path.join(path.dirname(source.filePath), archiveName))) + ) { + matchesArchive = true; + break; + } + } catch { + // One unreadable archive must not hide another valid provenance snapshot. + } + } + if ( + !matchesArchive && + !(await memoryCoreLegacySourceMatchesCanonical(source, JSON.parse(contents.toString("utf8")))) + ) { + return false; + } + // The migration archive bootstraps existing installs after SQLite has drifted. + // The stored hash then detects an older process rewriting the rollback source. + await writeMemoryCoreWorkspaceEntry({ + namespace: LEGACY_SOURCE_ACKNOWLEDGEMENT_NAMESPACE, + workspaceDir: source.workspaceDir, + key: markerKey, + value: { sha256 }, + }); + return true; +} + +export const dreamingStateComparison = { + targetHasRows: memoryCoreLegacyTargetHasRows, + sourceIsAcknowledged: memoryCoreLegacySourceIsAcknowledged, +}; diff --git a/src/commands/doctor-state-migrations.test.ts b/src/commands/doctor-state-migrations.test.ts index d73ddbbb89b0..a2a659762008 100644 --- a/src/commands/doctor-state-migrations.test.ts +++ b/src/commands/doctor-state-migrations.test.ts @@ -16,11 +16,15 @@ import { } from "../plugin-state/plugin-state-store.js"; import { setMaxPluginStateEntriesPerPluginForTests } from "../plugin-state/plugin-state-store.sqlite.js"; import { seedPluginStateEntriesForTests } from "../plugin-state/plugin-state-store.test-helpers.js"; +import { hashJson } from "../plugins/installed-plugin-index-hash.js"; import { readPersistedInstalledPluginIndex, writePersistedInstalledPluginIndex, } from "../plugins/installed-plugin-index-store.js"; -import type { InstalledPluginInstallRecordInfo } from "../plugins/installed-plugin-index.js"; +import type { + InstalledPluginIndexRecord, + InstalledPluginInstallRecordInfo, +} from "../plugins/installed-plugin-index.js"; import { closeOpenClawStateDatabaseForTest, openOpenClawStateDatabase, @@ -393,7 +397,32 @@ function writeLegacyDebugProxyCaptureSidecar( async function writeExistingPluginInstallIndex( root: string, installRecords: Record, + options: { canonicalPluginIds?: readonly string[] } = {}, ): Promise { + const canonicalPluginIds = new Set(options.canonicalPluginIds ?? []); + const plugins: InstalledPluginIndexRecord[] = Object.entries(installRecords).flatMap( + ([pluginId, installRecord]) => + canonicalPluginIds.has(pluginId) + ? [ + { + pluginId, + installRecordHash: hashJson(installRecord), + manifestPath: `/plugins/${pluginId}/openclaw.plugin.json`, + manifestHash: "test", + rootDir: `/plugins/${pluginId}`, + origin: "global", + enabled: false, + startup: { + sidecar: false, + memory: false, + deferConfiguredChannelFullLoadUntilAfterListen: false, + agentHarnesses: [], + }, + compat: [], + }, + ] + : [], + ); await writePersistedInstalledPluginIndex( { version: 1, @@ -403,7 +432,7 @@ async function writeExistingPluginInstallIndex( policyHash: "test", generatedAtMs: 1, installRecords, - plugins: [], + plugins, diagnostics: [], }, { stateDir: root }, @@ -2366,10 +2395,51 @@ describe("doctor legacy state migrations", () => { expect(result.warnings).toStrictEqual([ "Left plugin install index in place because shared SQLite state has conflicting plugin install metadata for: demo", ]); + expect(result.notices).toBeUndefined(); expect(fs.existsSync(sourcePath)).toBe(true); expect(fs.existsSync(`${sourcePath}.migrated`)).toBe(false); }); + it("archives differing legacy metadata for a disabled canonical plugin", async () => { + const root = await makeTempRoot(); + await writeExistingPluginInstallIndex( + root, + { + demo: { + source: "npm", + spec: "demo@latest", + version: "1.0.0", + }, + }, + { canonicalPluginIds: ["demo"] }, + ); + const sourcePath = writeLegacyPluginInstallIndex(root, { + demo: { + source: "npm", + spec: "demo@1.0.0", + version: "1.0.0", + }, + }); + + const result = await runLegacyStateMigrationsForRoot(root); + + expect(result.warnings).toStrictEqual([]); + expect(result.notices).toStrictEqual([ + "Kept canonical shared SQLite plugin install metadata despite differing legacy records for: demo", + ]); + expect(fs.existsSync(sourcePath)).toBe(false); + expect(fs.existsSync(`${sourcePath}.migrated`)).toBe(true); + + const retry = await runLegacyStateMigrationsForRoot(root); + expect(retry.warnings).toStrictEqual([]); + expect(retry.notices).toBeUndefined(); + await expect(readPersistedInstalledPluginIndex({ stateDir: root })).resolves.toMatchObject({ + installRecords: { + demo: { source: "npm", spec: "demo@latest", version: "1.0.0" }, + }, + }); + }); + for (const fixture of [ { label: "name different packages", @@ -2483,6 +2553,7 @@ describe("doctor legacy state migrations", () => { expect(result.warnings).toStrictEqual([ "Left plugin install index in place because shared SQLite state has conflicting plugin install metadata for: demo", ]); + expect(result.notices).toBeUndefined(); expect(fs.existsSync(sourcePath)).toBe(true); expect(fs.existsSync(`${sourcePath}.migrated`)).toBe(false); }); diff --git a/src/infra/state-migrations.doctor.ts b/src/infra/state-migrations.doctor.ts index c77a38eda57e..7f6f433ac753 100644 --- a/src/infra/state-migrations.doctor.ts +++ b/src/infra/state-migrations.doctor.ts @@ -48,6 +48,7 @@ import { migrateLegacyAgentDir, migrateLegacySessions, } from "./state-migrations.legacy-sessions.js"; +import { mergeNotices } from "./state-migrations.messages.js"; import { migrateLegacyInstalledPluginIndex, migrateLegacyPluginStateSidecar, @@ -57,12 +58,10 @@ import { migrateLegacyConfigHealth, migrateLegacyCurrentConversationBindings, migrateLegacyPluginBindingApprovals, - migrateLegacyUpdateCheckState, migrateLegacyVoiceWakeSettings, resolveLegacyConfigHealthPath, resolveLegacyCurrentConversationBindingsPath, resolveLegacyPluginBindingApprovalsPath, - resolveLegacyUpdateCheckPath, resolveLegacyVoiceWakeRoutingPath, resolveLegacyVoiceWakeTriggersPath, } from "./state-migrations.runtime-state.js"; @@ -100,6 +99,10 @@ import type { MigrationLogger, MigrationMessages, } from "./state-migrations.types.js"; +import { + migrateLegacyUpdateCheckState, + resolveLegacyUpdateCheckPath, +} from "./state-migrations.update-check.js"; let autoMigrateChecked = false; @@ -800,6 +803,7 @@ export async function runLegacyStateMigrations(params: { const channelPlans = await runLegacyMigrationPlans( detected.channelPlans.plans.filter((plan) => plan.kind !== "plugin-state-import"), ); + const notices = mergeNotices([pluginInstallIndex, updateCheck, pluginPlans]); return { changes: [ ...stateSchema.changes, @@ -844,9 +848,7 @@ export async function runLegacyStateMigrations(params: { ...agentDir.warnings, ...channelPlans.warnings, ], - ...(pluginPlans.notices && pluginPlans.notices.length > 0 - ? { notices: [...pluginPlans.notices] } - : {}), + ...(notices.length > 0 ? { notices } : {}), }; } @@ -1051,11 +1053,8 @@ export async function autoMigrateLegacyState(params: { ...preSessionChannelPlans.warnings, ...pluginPlans.warnings, ]; - const notices = [ - ...(stateDirResult.notices ?? []), - ...detected.notices, - ...(pluginPlans.notices ?? []), - ]; + const noticeSources = [stateDirResult, detected, pluginInstallIndex, updateCheck, pluginPlans]; + const notices = mergeNotices(noticeSources); logMigrationResults(changes, warnings, notices); return { migrated: @@ -1242,11 +1241,8 @@ export async function autoMigrateLegacyState(params: { ...agentDir.warnings, ...channelPlans.warnings, ]; - const notices = [ - ...(stateDirResult.notices ?? []), - ...detected.notices, - ...(pluginPlans.notices ?? []), - ]; + const noticeSources = [stateDirResult, detected, pluginInstallIndex, updateCheck, pluginPlans]; + const notices = mergeNotices(noticeSources); logMigrationResults(changes, warnings, notices); diff --git a/src/infra/state-migrations.messages.ts b/src/infra/state-migrations.messages.ts new file mode 100644 index 000000000000..281447343801 --- /dev/null +++ b/src/infra/state-migrations.messages.ts @@ -0,0 +1,5 @@ +type NoticeSource = { notices?: readonly string[] } | undefined; + +export function mergeNotices(sources: NoticeSource[]): string[] { + return [...new Set(sources.flatMap((source) => (source?.notices ? [...source.notices] : [])))]; +} diff --git a/src/infra/state-migrations.plugin-state.ts b/src/infra/state-migrations.plugin-state.ts index d7173b028ca6..a06937a8216f 100644 --- a/src/infra/state-migrations.plugin-state.ts +++ b/src/infra/state-migrations.plugin-state.ts @@ -6,11 +6,13 @@ import { createPluginStateKeyedStore, MAX_PLUGIN_STATE_ENTRIES_PER_PLUGIN, } from "../plugin-state/plugin-state-store.js"; +import { hashJson } from "../plugins/installed-plugin-index-hash.js"; import { readPersistedInstalledPluginIndexSync, resolveLegacyInstalledPluginIndexStorePath, writePersistedInstalledPluginIndexSync, } from "../plugins/installed-plugin-index-store.js"; +import type { InstalledPluginIndex } from "../plugins/installed-plugin-index.js"; import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; import { runOpenClawStateWriteTransaction } from "../state/openclaw-state-db.js"; import { @@ -35,9 +37,23 @@ import { resolveLegacyPluginStateSidecarPath, type LegacyPluginStateSidecarRow, } from "./state-migrations.storage.js"; +import type { MigrationMessages } from "./state-migrations.types.js"; type LegacyPluginStateImportDatabase = Pick; +function hasCanonicalInstallRecord(current: InstalledPluginIndex, pluginId: string): boolean { + const installRecord = current.installRecords[pluginId]; + if (!installRecord) { + return false; + } + // A manifest record bound to this exact install record proves SQLite owns the plugin, + // even when disabled. Keep all other conflicts blocking so stale metadata is not accepted. + return current.plugins.some( + (plugin) => + plugin.pluginId === pluginId && plugin.installRecordHash === hashJson(installRecord), + ); +} + export async function migrateLegacyPluginStateSidecar(params: { stateDir: string; }): Promise<{ changes: string[]; warnings: string[] }> { @@ -162,7 +178,7 @@ export async function migrateLegacyPluginStateSidecar(params: { export async function migrateLegacyInstalledPluginIndex(params: { stateDir: string; -}): Promise<{ changes: string[]; warnings: string[] }> { +}): Promise { const sourcePath = resolveLegacyInstalledPluginIndexStorePath({ stateDir: params.stateDir }); if (!fileExists(sourcePath)) { return { changes: [], warnings: [] }; @@ -196,11 +212,28 @@ export async function migrateLegacyInstalledPluginIndex(params: { } } if (merged.conflicts.length > 0) { + const acknowledged = merged.conflicts.filter((pluginId) => + hasCanonicalInstallRecord(current, pluginId), + ); + const unresolved = merged.conflicts.filter((pluginId) => !acknowledged.includes(pluginId)); + if (unresolved.length === 0) { + archiveLegacyInstalledPluginIndex({ sourcePath, changes, warnings }); + } return { changes, - warnings: [ - `Left plugin install index in place because shared SQLite state has conflicting plugin install metadata for: ${merged.conflicts.join(", ")}`, - ], + warnings: + unresolved.length > 0 + ? [ + `Left plugin install index in place because shared SQLite state has conflicting plugin install metadata for: ${unresolved.join(", ")}`, + ] + : [], + ...(acknowledged.length > 0 + ? { + notices: [ + `Kept canonical shared SQLite plugin install metadata despite differing legacy records for: ${acknowledged.join(", ")}`, + ], + } + : {}), }; } } diff --git a/src/infra/state-migrations.runtime-state.ts b/src/infra/state-migrations.runtime-state.ts index e1b80af749fc..e289bfcdbe01 100644 --- a/src/infra/state-migrations.runtime-state.ts +++ b/src/infra/state-migrations.runtime-state.ts @@ -21,7 +21,6 @@ type LegacyVoiceWakeImportDatabase = Pick< OpenClawStateKyselyDatabase, "voicewake_routing_config" | "voicewake_routing_routes" | "voicewake_triggers" >; -type LegacyUpdateCheckImportDatabase = Pick; type LegacyConfigHealthImportDatabase = Pick; type LegacyPluginBindingApprovalsImportDatabase = Pick< OpenClawStateKyselyDatabase, @@ -321,172 +320,6 @@ export function migrateLegacyVoiceWakeSettings(params: { return { changes, warnings }; } -const UPDATE_CHECK_STATE_KEY = "default"; - -type LegacyUpdateCheckState = { - lastCheckedAt?: string; - lastNotifiedVersion?: string; - lastNotifiedTag?: string; - lastAvailableVersion?: string; - lastAvailableTag?: string; - autoInstallId?: string; - autoFirstSeenVersion?: string; - autoFirstSeenTag?: string; - autoFirstSeenAt?: string; - autoLastAttemptVersion?: string; - autoLastAttemptAt?: string; - autoLastSuccessVersion?: string; - autoLastSuccessAt?: string; -}; - -export function resolveLegacyUpdateCheckPath(stateDir: string): string { - return path.join(stateDir, "update-check.json"); -} - -function optionalLegacyString(record: Record, key: string): string | undefined { - const value = record[key]; - return typeof value === "string" && value.trim().length > 0 ? value : undefined; -} - -function normalizeLegacyUpdateCheckState(input: unknown): LegacyUpdateCheckState { - const record = input && typeof input === "object" ? (input as Record) : {}; - return { - lastCheckedAt: optionalLegacyString(record, "lastCheckedAt"), - lastNotifiedVersion: optionalLegacyString(record, "lastNotifiedVersion"), - lastNotifiedTag: optionalLegacyString(record, "lastNotifiedTag"), - lastAvailableVersion: optionalLegacyString(record, "lastAvailableVersion"), - lastAvailableTag: optionalLegacyString(record, "lastAvailableTag"), - autoInstallId: optionalLegacyString(record, "autoInstallId"), - autoFirstSeenVersion: optionalLegacyString(record, "autoFirstSeenVersion"), - autoFirstSeenTag: optionalLegacyString(record, "autoFirstSeenTag"), - autoFirstSeenAt: optionalLegacyString(record, "autoFirstSeenAt"), - autoLastAttemptVersion: optionalLegacyString(record, "autoLastAttemptVersion"), - autoLastAttemptAt: optionalLegacyString(record, "autoLastAttemptAt"), - autoLastSuccessVersion: optionalLegacyString(record, "autoLastSuccessVersion"), - autoLastSuccessAt: optionalLegacyString(record, "autoLastSuccessAt"), - }; -} - -function legacyUpdateCheckStateMatches( - row: { - last_checked_at: string | null; - last_notified_version: string | null; - last_notified_tag: string | null; - last_available_version: string | null; - last_available_tag: string | null; - auto_install_id: string | null; - auto_first_seen_version: string | null; - auto_first_seen_tag: string | null; - auto_first_seen_at: string | null; - auto_last_attempt_version: string | null; - auto_last_attempt_at: string | null; - auto_last_success_version: string | null; - auto_last_success_at: string | null; - }, - state: LegacyUpdateCheckState, -): boolean { - return ( - (state.lastCheckedAt ?? null) === row.last_checked_at && - (state.lastNotifiedVersion ?? null) === row.last_notified_version && - (state.lastNotifiedTag ?? null) === row.last_notified_tag && - (state.lastAvailableVersion ?? null) === row.last_available_version && - (state.lastAvailableTag ?? null) === row.last_available_tag && - (state.autoInstallId ?? null) === row.auto_install_id && - (state.autoFirstSeenVersion ?? null) === row.auto_first_seen_version && - (state.autoFirstSeenTag ?? null) === row.auto_first_seen_tag && - (state.autoFirstSeenAt ?? null) === row.auto_first_seen_at && - (state.autoLastAttemptVersion ?? null) === row.auto_last_attempt_version && - (state.autoLastAttemptAt ?? null) === row.auto_last_attempt_at && - (state.autoLastSuccessVersion ?? null) === row.auto_last_success_version && - (state.autoLastSuccessAt ?? null) === row.auto_last_success_at - ); -} - -export function migrateLegacyUpdateCheckState(params: { - detected: LegacyStateDetection["updateCheck"]; - stateDir: string; -}): { changes: string[]; warnings: string[] } { - const changes: string[] = []; - const warnings: string[] = []; - if (!fileExists(params.detected.sourcePath)) { - return { changes, warnings }; - } - - let state: LegacyUpdateCheckState; - try { - state = normalizeLegacyUpdateCheckState(readLegacyJsonObject(params.detected.sourcePath)); - } catch (err) { - warnings.push( - `Failed reading legacy update-check state ${params.detected.sourcePath}: ${String(err)}`, - ); - return { changes, warnings }; - } - - let imported = false; - let shouldArchive = false; - try { - runOpenClawStateWriteTransaction( - ({ db }) => { - const stateDb = getNodeSqliteKysely(db); - const existing = executeSqliteQueryTakeFirstSync( - db, - stateDb - .selectFrom("update_check_state") - .selectAll() - .where("state_key", "=", UPDATE_CHECK_STATE_KEY), - ); - if (existing) { - if (legacyUpdateCheckStateMatches(existing, state)) { - shouldArchive = true; - } else { - warnings.push( - `Left legacy update-check state in place because shared SQLite state already differs: ${params.detected.sourcePath}`, - ); - } - return; - } - executeSqliteQuerySync( - db, - stateDb.insertInto("update_check_state").values({ - state_key: UPDATE_CHECK_STATE_KEY, - last_checked_at: state.lastCheckedAt ?? null, - last_notified_version: state.lastNotifiedVersion ?? null, - last_notified_tag: state.lastNotifiedTag ?? null, - last_available_version: state.lastAvailableVersion ?? null, - last_available_tag: state.lastAvailableTag ?? null, - auto_install_id: state.autoInstallId ?? null, - auto_first_seen_version: state.autoFirstSeenVersion ?? null, - auto_first_seen_tag: state.autoFirstSeenTag ?? null, - auto_first_seen_at: state.autoFirstSeenAt ?? null, - auto_last_attempt_version: state.autoLastAttemptVersion ?? null, - auto_last_attempt_at: state.autoLastAttemptAt ?? null, - auto_last_success_version: state.autoLastSuccessVersion ?? null, - auto_last_success_at: state.autoLastSuccessAt ?? null, - updated_at_ms: Date.now(), - }), - ); - imported = true; - shouldArchive = true; - }, - { env: { ...process.env, OPENCLAW_STATE_DIR: params.stateDir } }, - ); - } catch (err) { - warnings.push(`Failed migrating legacy update-check state: ${String(err)}`); - } - if (imported) { - changes.push("Migrated update-check state → shared SQLite state"); - } - if (shouldArchive) { - archiveLegacyImportSource({ - sourcePath: params.detected.sourcePath, - label: "update-check state", - changes, - warnings, - }); - } - return { changes, warnings }; -} - type LegacyConfigHealthFile = { entries?: unknown; }; diff --git a/src/infra/state-migrations.state-dir.test.ts b/src/infra/state-migrations.state-dir.test.ts index 71638c0b47b7..27efb1a657bf 100644 --- a/src/infra/state-migrations.state-dir.test.ts +++ b/src/infra/state-migrations.state-dir.test.ts @@ -2,7 +2,11 @@ import fs from "node:fs"; import path from "node:path"; import { describe, expect, it } from "vitest"; -import { readPersistedInstalledPluginIndex } from "../plugins/installed-plugin-index-store.js"; +import { hashJson } from "../plugins/installed-plugin-index-hash.js"; +import { + readPersistedInstalledPluginIndex, + writePersistedInstalledPluginIndex, +} from "../plugins/installed-plugin-index-store.js"; import { withTempDir } from "../test-helpers/temp-dir.js"; import { autoMigrateLegacyStateDir, @@ -105,6 +109,73 @@ describe("legacy state dir auto-migration", () => { }); }); + it("reports conflicting plugin install metadata as a notice from the early state-dir pass", async () => { + await withStateDirFixture(async (root) => { + const stateDir = path.join(root, "custom-state"); + const sourcePath = path.join(stateDir, "plugins", "installs.json"); + await writePersistedInstalledPluginIndex( + { + version: 1, + hostContractVersion: "test", + compatRegistryVersion: "test", + migrationVersion: 1, + policyHash: "test", + generatedAtMs: 1, + installRecords: { + demo: { source: "npm", spec: "demo@latest", version: "1.0.0" }, + }, + plugins: [ + { + pluginId: "demo", + installRecordHash: hashJson({ + source: "npm", + spec: "demo@latest", + version: "1.0.0", + }), + manifestPath: "/plugins/demo/openclaw.plugin.json", + manifestHash: "test", + rootDir: "/plugins/demo", + origin: "global", + enabled: false, + startup: { + sidecar: false, + memory: false, + deferConfiguredChannelFullLoadUntilAfterListen: false, + agentHarnesses: [], + }, + compat: [], + }, + ], + diagnostics: [], + }, + { stateDir }, + ); + fs.mkdirSync(path.dirname(sourcePath), { recursive: true }); + fs.writeFileSync( + sourcePath, + JSON.stringify({ + records: { + demo: { source: "npm", spec: "demo@1.0.0", version: "1.0.0" }, + }, + }), + "utf8", + ); + + const result = await autoMigrateLegacyStateDir({ + env: { OPENCLAW_STATE_DIR: stateDir } as NodeJS.ProcessEnv, + homedir: () => root, + }); + + expect(result.warnings).toStrictEqual([]); + expect(result.notices).toStrictEqual([ + "Kept canonical shared SQLite plugin install metadata despite differing legacy records for: demo", + ]); + expect(result.skipped).toBe(false); + expect(fs.existsSync(sourcePath)).toBe(false); + expect(fs.existsSync(`${sourcePath}.migrated`)).toBe(true); + }); + }); + it("removes legacy plugin install index source when the existing archive has identical bytes", async () => { await withStateDirFixture(async (root) => { const stateDir = path.join(root, "custom-state"); diff --git a/src/infra/state-migrations.state-dir.ts b/src/infra/state-migrations.state-dir.ts index b78992f7f44d..da13f5087f01 100644 --- a/src/infra/state-migrations.state-dir.ts +++ b/src/infra/state-migrations.state-dir.ts @@ -123,20 +123,23 @@ export async function autoMigrateLegacyStateDir(params: { const env = params.env ?? process.env; const warnings: string[] = []; const changes: string[] = []; + const notices: string[] = []; const hasCustomStateDir = Boolean(env.OPENCLAW_STATE_DIR?.trim()); const targetDir = hasCustomStateDir ? resolveStateDir(env, homedir) : resolveNewStateDir(homedir); const migratePluginInstallIndex = async () => { const result = await migrateLegacyInstalledPluginIndex({ stateDir: targetDir }); changes.push(...result.changes); warnings.push(...result.warnings); + notices.push(...(result.notices ?? [])); }; if (hasCustomStateDir) { await migratePluginInstallIndex(); return { migrated: changes.length > 0, - skipped: changes.length === 0 && warnings.length === 0, + skipped: changes.length === 0 && warnings.length === 0 && notices.length === 0, changes, warnings, + ...(notices.length > 0 ? { notices } : {}), }; } @@ -157,7 +160,13 @@ export async function autoMigrateLegacyStateDir(params: { } if (!legacyStat) { await migratePluginInstallIndex(); - return { migrated: changes.length > 0, skipped: false, changes, warnings }; + return { + migrated: changes.length > 0, + skipped: false, + changes, + warnings, + ...(notices.length > 0 ? { notices } : {}), + }; } if (!legacyStat.isDirectory() && !legacyStat.isSymbolicLink()) { warnings.push(`Legacy state path is not a directory: ${legacyDir}`); @@ -175,7 +184,13 @@ export async function autoMigrateLegacyStateDir(params: { } if (path.resolve(legacyTarget) === path.resolve(targetDir)) { await migratePluginInstallIndex(); - return { migrated: changes.length > 0, skipped: false, changes, warnings }; + return { + migrated: changes.length > 0, + skipped: false, + changes, + warnings, + ...(notices.length > 0 ? { notices } : {}), + }; } if (legacyDirs.some((dir) => path.resolve(dir) === path.resolve(legacyTarget))) { legacyDir = legacyTarget; @@ -208,13 +223,25 @@ export async function autoMigrateLegacyStateDir(params: { if (isDirPath(targetDir)) { if (legacyDir && isLegacyDirSymlinkMirror(legacyDir, targetDir)) { await migratePluginInstallIndex(); - return { migrated: changes.length > 0, skipped: false, changes, warnings }; + return { + migrated: changes.length > 0, + skipped: false, + changes, + warnings, + ...(notices.length > 0 ? { notices } : {}), + }; } await migratePluginInstallIndex(); warnings.push( `State dir migration skipped: target already exists (${targetDir}). Remove or merge manually.`, ); - return { migrated: changes.length > 0, skipped: false, changes, warnings }; + return { + migrated: changes.length > 0, + skipped: false, + changes, + warnings, + ...(notices.length > 0 ? { notices } : {}), + }; } try { @@ -269,7 +296,13 @@ export async function autoMigrateLegacyStateDir(params: { } await migratePluginInstallIndex(); - return { migrated: changes.length > 0, skipped: false, changes, warnings }; + return { + migrated: changes.length > 0, + skipped: false, + changes, + warnings, + ...(notices.length > 0 ? { notices } : {}), + }; } export async function autoMigrateLegacyTaskStateSidecars(params: { diff --git a/src/infra/state-migrations.test.ts b/src/infra/state-migrations.test.ts index b49823db43dc..e2ee600a0f7a 100644 --- a/src/infra/state-migrations.test.ts +++ b/src/infra/state-migrations.test.ts @@ -2104,6 +2104,28 @@ describe("state migrations", () => { }); await expectMissingPath(sourcePath); await expect(fs.readFile(`${sourcePath}.migrated`, "utf8")).resolves.toContain("2.0.0"); + + await fs.writeFile( + sourcePath, + JSON.stringify({ + lastCheckedAt: "2026-01-18T09:30:00.000Z", + lastAvailableVersion: "3.0.0", + lastAvailableTag: "latest", + }), + "utf8", + ); + const conflictResult = await runLegacyStateMigrations({ detected, config: cfg }); + expect(conflictResult.warnings).toStrictEqual([]); + expect(conflictResult.notices).toEqual([ + expect.stringContaining("Kept shared SQLite update-check state because legacy cache differs"), + ]); + expect(readUpdateCheckState(env)?.last_available_version).toBe("2.0.0"); + await expectMissingPath(sourcePath); + await expect(fs.readFile(`${sourcePath}.migrated.2`, "utf8")).resolves.toContain("3.0.0"); + + const convergedResult = await runLegacyStateMigrations({ detected, config: cfg }); + expect(convergedResult.warnings).toStrictEqual([]); + expect(convergedResult.notices).toBeUndefined(); }); it("migrates legacy config health JSON into shared SQLite state", async () => { diff --git a/src/infra/state-migrations.update-check.ts b/src/infra/state-migrations.update-check.ts new file mode 100644 index 000000000000..87de610ac651 --- /dev/null +++ b/src/infra/state-migrations.update-check.ts @@ -0,0 +1,180 @@ +import fs from "node:fs"; +import path from "node:path"; +import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; +import { runOpenClawStateWriteTransaction } from "../state/openclaw-state-db.js"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "./kysely-sync.js"; +import { fileExists } from "./state-migrations.fs.js"; +import { archiveLegacyImportSource } from "./state-migrations.storage.js"; +import type { LegacyStateDetection, MigrationMessages } from "./state-migrations.types.js"; + +type LegacyUpdateCheckImportDatabase = Pick; + +type LegacyUpdateCheckState = { + lastCheckedAt?: string; + lastNotifiedVersion?: string; + lastNotifiedTag?: string; + lastAvailableVersion?: string; + lastAvailableTag?: string; + autoInstallId?: string; + autoFirstSeenVersion?: string; + autoFirstSeenTag?: string; + autoFirstSeenAt?: string; + autoLastAttemptVersion?: string; + autoLastAttemptAt?: string; + autoLastSuccessVersion?: string; + autoLastSuccessAt?: string; +}; + +const UPDATE_CHECK_STATE_KEY = "default"; + +export function resolveLegacyUpdateCheckPath(stateDir: string): string { + return path.join(stateDir, "update-check.json"); +} + +function normalizeLegacyUpdateCheckState(input: unknown): LegacyUpdateCheckState { + const record = input && typeof input === "object" ? (input as Record) : {}; + const readString = (key: string): string | undefined => { + const value = record[key]; + return typeof value === "string" && value.trim().length > 0 ? value : undefined; + }; + return { + lastCheckedAt: readString("lastCheckedAt"), + lastNotifiedVersion: readString("lastNotifiedVersion"), + lastNotifiedTag: readString("lastNotifiedTag"), + lastAvailableVersion: readString("lastAvailableVersion"), + lastAvailableTag: readString("lastAvailableTag"), + autoInstallId: readString("autoInstallId"), + autoFirstSeenVersion: readString("autoFirstSeenVersion"), + autoFirstSeenTag: readString("autoFirstSeenTag"), + autoFirstSeenAt: readString("autoFirstSeenAt"), + autoLastAttemptVersion: readString("autoLastAttemptVersion"), + autoLastAttemptAt: readString("autoLastAttemptAt"), + autoLastSuccessVersion: readString("autoLastSuccessVersion"), + autoLastSuccessAt: readString("autoLastSuccessAt"), + }; +} + +function legacyUpdateCheckStateMatches( + row: { + last_checked_at: string | null; + last_notified_version: string | null; + last_notified_tag: string | null; + last_available_version: string | null; + last_available_tag: string | null; + auto_install_id: string | null; + auto_first_seen_version: string | null; + auto_first_seen_tag: string | null; + auto_first_seen_at: string | null; + auto_last_attempt_version: string | null; + auto_last_attempt_at: string | null; + auto_last_success_version: string | null; + auto_last_success_at: string | null; + }, + state: LegacyUpdateCheckState, +): boolean { + return ( + (state.lastCheckedAt ?? null) === row.last_checked_at && + (state.lastNotifiedVersion ?? null) === row.last_notified_version && + (state.lastNotifiedTag ?? null) === row.last_notified_tag && + (state.lastAvailableVersion ?? null) === row.last_available_version && + (state.lastAvailableTag ?? null) === row.last_available_tag && + (state.autoInstallId ?? null) === row.auto_install_id && + (state.autoFirstSeenVersion ?? null) === row.auto_first_seen_version && + (state.autoFirstSeenTag ?? null) === row.auto_first_seen_tag && + (state.autoFirstSeenAt ?? null) === row.auto_first_seen_at && + (state.autoLastAttemptVersion ?? null) === row.auto_last_attempt_version && + (state.autoLastAttemptAt ?? null) === row.auto_last_attempt_at && + (state.autoLastSuccessVersion ?? null) === row.auto_last_success_version && + (state.autoLastSuccessAt ?? null) === row.auto_last_success_at + ); +} + +export function migrateLegacyUpdateCheckState(params: { + detected: LegacyStateDetection["updateCheck"]; + stateDir: string; +}): MigrationMessages { + const changes: string[] = []; + const warnings: string[] = []; + let notice: string | undefined; + if (!fileExists(params.detected.sourcePath)) { + return { changes, warnings }; + } + + let state: LegacyUpdateCheckState; + try { + state = normalizeLegacyUpdateCheckState( + JSON.parse(fs.readFileSync(params.detected.sourcePath, "utf8")) as unknown, + ); + } catch (err) { + warnings.push( + `Failed reading legacy update-check state ${params.detected.sourcePath}: ${String(err)}`, + ); + return { changes, warnings }; + } + + let imported = false; + let shouldArchive = false; + try { + runOpenClawStateWriteTransaction( + ({ db }) => { + const stateDb = getNodeSqliteKysely(db); + const existing = executeSqliteQueryTakeFirstSync( + db, + stateDb + .selectFrom("update_check_state") + .selectAll() + .where("state_key", "=", UPDATE_CHECK_STATE_KEY), + ); + if (existing) { + if (!legacyUpdateCheckStateMatches(existing, state)) { + // SQLite is the canonical cache; retaining divergent JSON would block every startup. + notice = `Kept shared SQLite update-check state because legacy cache differs: ${params.detected.sourcePath}`; + } + shouldArchive = true; + return; + } + executeSqliteQuerySync( + db, + stateDb.insertInto("update_check_state").values({ + state_key: UPDATE_CHECK_STATE_KEY, + last_checked_at: state.lastCheckedAt ?? null, + last_notified_version: state.lastNotifiedVersion ?? null, + last_notified_tag: state.lastNotifiedTag ?? null, + last_available_version: state.lastAvailableVersion ?? null, + last_available_tag: state.lastAvailableTag ?? null, + auto_install_id: state.autoInstallId ?? null, + auto_first_seen_version: state.autoFirstSeenVersion ?? null, + auto_first_seen_tag: state.autoFirstSeenTag ?? null, + auto_first_seen_at: state.autoFirstSeenAt ?? null, + auto_last_attempt_version: state.autoLastAttemptVersion ?? null, + auto_last_attempt_at: state.autoLastAttemptAt ?? null, + auto_last_success_version: state.autoLastSuccessVersion ?? null, + auto_last_success_at: state.autoLastSuccessAt ?? null, + updated_at_ms: Date.now(), + }), + ); + imported = true; + shouldArchive = true; + }, + { env: { ...process.env, OPENCLAW_STATE_DIR: params.stateDir } }, + ); + } catch (err) { + warnings.push(`Failed migrating legacy update-check state: ${String(err)}`); + } + if (imported) { + changes.push("Migrated update-check state → shared SQLite state"); + } + if (shouldArchive) { + archiveLegacyImportSource({ + sourcePath: params.detected.sourcePath, + label: "update-check state", + changes, + warnings, + }); + } + return { changes, warnings, ...(notice ? { notices: [notice] } : {}) }; +}