diff --git a/extensions/memory-core/doctor-contract-api.test.ts b/extensions/memory-core/doctor-contract-api.test.ts index 077d093218cf..f53283a7dc25 100644 --- a/extensions/memory-core/doctor-contract-api.test.ts +++ b/extensions/memory-core/doctor-contract-api.test.ts @@ -61,6 +61,8 @@ async function writeLegacyMemorySidecar( fileHash?: string; filePath?: string; text?: string; + cacheEmbedding?: string; + cacheDims?: number | null; } = {}, ): Promise { await fs.mkdir(path.dirname(legacyPath), { recursive: true }); @@ -71,6 +73,8 @@ async function writeLegacyMemorySidecar( const chunkId = params.chunkId ?? "chunk-1"; const chunkHash = params.chunkHash ?? "chunk-hash"; const text = params.text ?? "remember this"; + const cacheEmbedding = params.cacheEmbedding ?? "[1,0,0]"; + const cacheDims = params.cacheDims === undefined ? 3 : params.cacheDims; db.exec(` CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL); CREATE TABLE files ( @@ -109,8 +113,8 @@ async function writeLegacyMemorySidecar( "INSERT INTO chunks VALUES (?, ?, 'memory', 1, 2, ?, 'embed-model', ?, '[1,0,0]', 30)", ).run(chunkId, filePath, chunkHash, text); db.prepare( - "INSERT INTO embedding_cache VALUES ('openai', 'embed-model', 'key', ?, '[1,0,0]', 3, 40)", - ).run(chunkHash); + "INSERT INTO embedding_cache VALUES ('openai', 'embed-model', 'key', ?, ?, ?, 40)", + ).run(chunkHash, cacheEmbedding, cacheDims); if (params.vector === "vec0") { const loaded = await loadSqliteVecExtension({ db }); expect(loaded.ok, loaded.error).toBe(true); @@ -330,6 +334,19 @@ function readMemoryRows(agentPath: string) { } } +function readMemoryCacheRows(agentPath: string) { + const db = new DatabaseSync(agentPath); + try { + return db + .prepare( + "SELECT provider, model, provider_key, hash, embedding, dims, updated_at FROM memory_embedding_cache ORDER BY provider, hash", + ) + .all(); + } finally { + db.close(); + } +} + function readMemoryFtsSql(agentPath: string): string | undefined { const db = new DatabaseSync(agentPath); try { @@ -1309,7 +1326,7 @@ describe("memory-core doctor dreaming migration", () => { await expect(fs.access(retryPath)).resolves.toBeUndefined(); }); - it("leaves the legacy memory sidecar in place when metadata conflicts", async () => { + it("keeps canonical metadata and archives a conflicting derived legacy index", async () => { const stateDir = path.join(rootDir, "state"); const legacyPath = path.join(stateDir, "memory", "main.sqlite"); const agentPath = path.join(stateDir, "agents", "main", "agent", "openclaw-agent.sqlite"); @@ -1318,17 +1335,50 @@ describe("memory-core doctor dreaming migration", () => { const result = await legacyMemoryIndexMigration().migrateLegacyState(migrationParams()); - expect(result.warnings).toEqual([ - expect.stringContaining( - "Skipped Memory Core legacy memory index import for agent main because legacy rows could not be imported: Error: legacy memory meta rows conflict with canonical memory index rows", - ), + expect(result.warnings).toEqual([]); + expect(result.changes).toEqual([ + "Resolved Memory Core legacy memory index conflict for agent main by keeping canonical per-agent SQLite rows", + expect.stringContaining("Archived Memory Core legacy memory index sidecar"), ]); - expect(result.changes).toEqual([]); expect(readMemoryRows(agentPath).chunks).toEqual([ { id: "canonical-other-chunk", text: "canonical unrelated memory" }, ]); - await expect(fs.access(legacyPath)).resolves.toBeUndefined(); - await expect(fs.access(`${legacyPath}.migrated`)).rejects.toThrow(); + await expect(fs.access(legacyPath)).rejects.toThrow(); + await expect(fs.access(`${legacyPath}.migrated`)).resolves.toBeUndefined(); + + const secondRun = await legacyMemoryIndexMigration().migrateLegacyState(migrationParams()); + expect(secondRun).toEqual({ changes: [], warnings: [] }); + }); + + it("keeps canonical chunks and archives a conflicting derived legacy index", async () => { + const stateDir = path.join(rootDir, "state"); + const legacyPath = path.join(stateDir, "memory", "main.sqlite"); + const agentPath = path.join(stateDir, "agents", "main", "agent", "openclaw-agent.sqlite"); + await writeLegacyMemorySidecar(legacyPath); + await createCanonicalLegacyMemoryRowsWithFts(agentPath, "remember this"); + const canonicalDb = new DatabaseSync(agentPath); + try { + canonicalDb + .prepare("UPDATE memory_index_chunks SET text = ? WHERE id = ?") + .run("canonical memory remains authoritative", "chunk-1"); + } finally { + canonicalDb.close(); + } + + const result = await legacyMemoryIndexMigration().migrateLegacyState(migrationParams()); + + expect(result.warnings).toEqual([]); + expect(result.changes).toEqual([ + "Resolved Memory Core legacy memory index conflict for agent main by keeping canonical per-agent SQLite rows", + expect.stringContaining("Archived Memory Core legacy memory index sidecar"), + ]); + expect(readMemoryRows(agentPath)).toEqual({ + sources: [{ path: "MEMORY.md", source: "memory", hash: "file-hash" }], + chunks: [{ id: "chunk-1", text: "canonical memory remains authoritative" }], + cache: [], + }); + await expect(fs.access(legacyPath)).rejects.toThrow(); + await expect(fs.access(`${legacyPath}.migrated`)).resolves.toBeUndefined(); }); it("merges legacy sidecar rows into a non-empty canonical index when rows do not conflict", async () => { @@ -1361,6 +1411,78 @@ describe("memory-core doctor dreaming migration", () => { await expect(fs.access(`${legacyPath}.migrated`)).resolves.toBeUndefined(); }); + it("keeps a valid canonical cache collision while importing remaining legacy rows", async () => { + const stateDir = path.join(rootDir, "state"); + const legacyPath = path.join(stateDir, "memory", "main.sqlite"); + const agentPath = path.join(stateDir, "agents", "main", "agent", "openclaw-agent.sqlite"); + await writeLegacyMemorySidecar(legacyPath); + await createUnrelatedCanonicalMemoryIndex(agentPath); + const canonicalDb = new DatabaseSync(agentPath); + try { + canonicalDb + .prepare( + "INSERT INTO memory_embedding_cache (provider, model, provider_key, hash, embedding, dims, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)", + ) + .run("openai", "embed-model", "key", "chunk-hash", "[0,1,0]", 3, 99); + } finally { + canonicalDb.close(); + } + + const result = await legacyMemoryIndexMigration().migrateLegacyState(migrationParams()); + + expect(result.warnings).toEqual([]); + expect(result.changes).toEqual([ + "Migrated Memory Core legacy memory index for agent main -> per-agent SQLite (1 source(s), 1 chunk(s), 1 cache row(s))", + expect.stringContaining("Archived Memory Core legacy memory index sidecar"), + ]); + expect(readMemoryCacheRows(agentPath)).toEqual([ + { + provider: "openai", + model: "embed-model", + provider_key: "key", + hash: "chunk-hash", + embedding: "[0,1,0]", + dims: 3, + updated_at: 99, + }, + ]); + await expect(fs.access(legacyPath)).rejects.toThrow(); + await expect(fs.access(`${legacyPath}.migrated`)).resolves.toBeUndefined(); + }); + + it("archives a malformed canonical cache collision as a derived conflict", async () => { + const stateDir = path.join(rootDir, "state"); + const legacyPath = path.join(stateDir, "memory", "main.sqlite"); + const agentPath = path.join(stateDir, "agents", "main", "agent", "openclaw-agent.sqlite"); + await writeLegacyMemorySidecar(legacyPath); + await createUnrelatedCanonicalMemoryIndex(agentPath); + const canonicalDb = new DatabaseSync(agentPath); + try { + canonicalDb + .prepare( + "INSERT INTO memory_embedding_cache (provider, model, provider_key, hash, embedding, dims, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)", + ) + .run("openai", "embed-model", "key", "chunk-hash", "not-json", 3, 99); + } finally { + canonicalDb.close(); + } + + const result = await legacyMemoryIndexMigration().migrateLegacyState(migrationParams()); + + expect(result.warnings).toEqual([]); + expect(result.changes).toEqual([ + "Resolved Memory Core legacy memory index conflict for agent main by keeping canonical per-agent SQLite rows", + expect.stringContaining("Archived Memory Core legacy memory index sidecar"), + ]); + expect(readMemoryCacheRows(agentPath)[0]).toMatchObject({ + embedding: "not-json", + dims: 3, + updated_at: 99, + }); + await expect(fs.access(legacyPath)).rejects.toThrow(); + await expect(fs.access(`${legacyPath}.migrated`)).resolves.toBeUndefined(); + }); + it("leaves legacy vector sidecars in place when vector dimensions conflict", async () => { const stateDir = path.join(rootDir, "state"); const legacyPath = path.join(stateDir, "memory", "main.sqlite"); @@ -1380,7 +1502,7 @@ describe("memory-core doctor dreaming migration", () => { await expect(fs.access(`${legacyPath}.migrated`)).rejects.toThrow(); }); - it("leaves legacy vector sidecars in place when canonical vector rows conflict", async () => { + it("keeps canonical vector rows and archives a conflicting derived legacy index", async () => { const stateDir = path.join(rootDir, "state"); const legacyPath = path.join(stateDir, "memory", "main.sqlite"); const agentPath = path.join(stateDir, "agents", "main", "agent", "openclaw-agent.sqlite"); @@ -1389,14 +1511,13 @@ describe("memory-core doctor dreaming migration", () => { const result = await legacyMemoryIndexMigration().migrateLegacyState(migrationParams()); - expect(result.warnings).toEqual([ - expect.stringContaining( - "Skipped Memory Core legacy memory index import for agent main because legacy rows could not be imported: Error: legacy memory chunks_vec rows conflict with canonical memory index rows", - ), + expect(result.warnings).toEqual([]); + expect(result.changes).toEqual([ + "Resolved Memory Core legacy memory index conflict for agent main by keeping canonical per-agent SQLite rows", + expect.stringContaining("Archived Memory Core legacy memory index sidecar"), ]); - expect(result.changes).toEqual([]); - await expect(fs.access(legacyPath)).resolves.toBeUndefined(); - await expect(fs.access(`${legacyPath}.migrated`)).rejects.toThrow(); + await expect(fs.access(legacyPath)).rejects.toThrow(); + await expect(fs.access(`${legacyPath}.migrated`)).resolves.toBeUndefined(); }); it("leaves legacy vector sidecars in place when vector rows have no chunk", async () => { @@ -1414,7 +1535,7 @@ describe("memory-core doctor dreaming migration", () => { expect(result.warnings).toEqual([ expect.stringContaining( - "Skipped Memory Core legacy memory index import for agent main because legacy rows could not be imported: Error: legacy memory chunks_vec chunk references rows conflict", + "Skipped Memory Core legacy memory index import for agent main because legacy rows could not be imported: Error: legacy memory chunks_vec rows reference missing chunks", ), ]); expect(result.changes).toEqual([]); @@ -1422,7 +1543,7 @@ describe("memory-core doctor dreaming migration", () => { await expect(fs.access(`${legacyPath}.migrated`)).rejects.toThrow(); }); - it("leaves legacy sidecars in place when canonical FTS rows conflict", async () => { + it("keeps canonical FTS rows and archives a conflicting derived legacy index", async () => { const stateDir = path.join(rootDir, "state"); const legacyPath = path.join(stateDir, "memory", "main.sqlite"); const agentPath = path.join(stateDir, "agents", "main", "agent", "openclaw-agent.sqlite"); @@ -1431,17 +1552,18 @@ describe("memory-core doctor dreaming migration", () => { const result = await legacyMemoryIndexMigration().migrateLegacyState(migrationParams()); - expect(result.warnings).toEqual([ - expect.stringContaining( - "Skipped Memory Core legacy memory index import for agent main because legacy rows could not be imported: Error: legacy memory fts rows conflict", - ), + expect(result.warnings).toEqual([]); + expect(result.changes).toEqual([ + "Resolved Memory Core legacy memory index conflict for agent main by keeping canonical per-agent SQLite rows", + expect.stringContaining("Archived Memory Core legacy memory index sidecar"), ]); - expect(result.changes).toEqual([]); - await expect(fs.access(legacyPath)).resolves.toBeUndefined(); - await expect(fs.access(`${legacyPath}.migrated`)).rejects.toThrow(); + const keywordRows = await searchMigratedKeywordRows(agentPath, "stale"); + expect(keywordRows.map((row) => row.id)).toEqual(["chunk-1"]); + await expect(fs.access(legacyPath)).rejects.toThrow(); + await expect(fs.access(`${legacyPath}.migrated`)).resolves.toBeUndefined(); }); - it("leaves legacy vector sidecars in place when canonical metadata dimensions conflict", async () => { + it("keeps canonical vector metadata and archives a conflicting derived legacy index", async () => { const stateDir = path.join(rootDir, "state"); const legacyPath = path.join(stateDir, "memory", "main.sqlite"); const agentPath = path.join(stateDir, "agents", "main", "agent", "openclaw-agent.sqlite"); @@ -1450,16 +1572,15 @@ describe("memory-core doctor dreaming migration", () => { const result = await legacyMemoryIndexMigration().migrateLegacyState(migrationParams()); - expect(result.warnings).toEqual([ - expect.stringContaining( - "Skipped Memory Core legacy memory index import for agent main because legacy rows could not be imported: Error: legacy memory meta rows conflict with canonical memory index rows", - ), + expect(result.warnings).toEqual([]); + expect(result.changes).toEqual([ + "Resolved Memory Core legacy memory index conflict for agent main by keeping canonical per-agent SQLite rows", + expect.stringContaining("Archived Memory Core legacy memory index sidecar"), ]); - expect(result.changes).toEqual([]); expect(readMemoryRows(agentPath).chunks).toEqual([ { id: "canonical-other-chunk", text: "canonical unrelated memory" }, ]); - await expect(fs.access(legacyPath)).resolves.toBeUndefined(); - await expect(fs.access(`${legacyPath}.migrated`)).rejects.toThrow(); + await expect(fs.access(legacyPath)).rejects.toThrow(); + await expect(fs.access(`${legacyPath}.migrated`)).resolves.toBeUndefined(); }); }); diff --git a/extensions/memory-core/doctor-contract-api.ts b/extensions/memory-core/doctor-contract-api.ts index 73261f0ed8d2..5ab7152e17db 100644 --- a/extensions/memory-core/doctor-contract-api.ts +++ b/extensions/memory-core/doctor-contract-api.ts @@ -106,7 +106,7 @@ type LegacyMemorySidecarImportResult = { type MemoryFtsTokenizer = "unicode61" | "trigram"; -class LegacyMemoryRowsConflictError extends Error { +class LegacyMemoryDerivedRowsConflictError extends Error { constructor(readonly tableName: string) { super(`legacy memory ${tableName} rows conflict with canonical memory index rows`); } @@ -211,10 +211,26 @@ function formatLegacyVectorRows(count: number | undefined): string { return count === undefined ? "legacy vector rows" : `${count} vector row(s)`; } -function assertLegacyRowsCopied(db: DatabaseSync, query: string, tableName: string): void { +function assertLegacyDerivedRowsCopied(db: DatabaseSync, query: string, tableName: string): void { const row = db.prepare(query).get() as { missing?: unknown } | undefined; if (Number(row?.missing ?? 0) > 0) { - throw new LegacyMemoryRowsConflictError(tableName); + throw new LegacyMemoryDerivedRowsConflictError(tableName); + } +} + +function assertLegacyVectorRowsReferenceChunks(db: DatabaseSync, schema: string): void { + const row = db + .prepare( + `SELECT COUNT(*) AS missing + FROM ${schema}.${LEGACY_MEMORY_VECTOR_TABLE} AS legacy + WHERE NOT EXISTS ( + SELECT 1 FROM main.${MEMORY_INDEX_CHUNKS_TABLE} AS chunk + WHERE chunk.id = legacy.id + )`, + ) + .get() as { missing?: unknown } | undefined; + if (Number(row?.missing ?? 0) > 0) { + throw new Error(`legacy memory ${LEGACY_MEMORY_VECTOR_TABLE} rows reference missing chunks`); } } @@ -340,17 +356,8 @@ function copyLegacyMemoryVectorRows(db: DatabaseSync, schema: string): void { if (!tableExists(db, "main", MEMORY_INDEX_VECTOR_TABLE)) { return; } - assertLegacyRowsCopied( - db, - `SELECT COUNT(*) AS missing - FROM ${schema}.${LEGACY_MEMORY_VECTOR_TABLE} AS legacy - WHERE NOT EXISTS ( - SELECT 1 FROM main.${MEMORY_INDEX_CHUNKS_TABLE} AS chunk - WHERE chunk.id = legacy.id - )`, - `${LEGACY_MEMORY_VECTOR_TABLE} chunk references`, - ); - assertLegacyRowsCopied( + assertLegacyVectorRowsReferenceChunks(db, schema); + assertLegacyDerivedRowsCopied( db, `SELECT COUNT(*) AS missing FROM ${schema}.${LEGACY_MEMORY_VECTOR_TABLE} AS legacy @@ -368,7 +375,7 @@ function copyLegacyMemoryVectorRows(db: DatabaseSync, schema: string): void { WHERE canonical.id = legacy.id ); `); - assertLegacyRowsCopied( + assertLegacyDerivedRowsCopied( db, `SELECT COUNT(*) AS missing FROM ${schema}.${LEGACY_MEMORY_VECTOR_TABLE} AS legacy @@ -398,7 +405,7 @@ function copyLegacyMemoryFtsRows(db: DatabaseSync, schema: string): void { WHERE canonical.id = legacy.id ); `); - assertLegacyRowsCopied( + assertLegacyDerivedRowsCopied( db, `SELECT COUNT(*) AS missing FROM ${schema}.chunks AS legacy @@ -435,7 +442,7 @@ function copyLegacyMemoryIndexRows( SELECT id, path, source, start_line, end_line, hash, model, text, embedding, updated_at FROM ${schema}.chunks; `); - assertLegacyRowsCopied( + assertLegacyDerivedRowsCopied( db, `SELECT COUNT(*) AS missing FROM ${schema}.meta AS legacy @@ -445,7 +452,7 @@ function copyLegacyMemoryIndexRows( )`, "meta", ); - assertLegacyRowsCopied( + assertLegacyDerivedRowsCopied( db, `SELECT COUNT(*) AS missing FROM ${schema}.files AS legacy @@ -459,7 +466,7 @@ function copyLegacyMemoryIndexRows( )`, "files", ); - assertLegacyRowsCopied( + assertLegacyDerivedRowsCopied( db, `SELECT COUNT(*) AS missing FROM ${schema}.chunks AS legacy @@ -500,7 +507,9 @@ function copyLegacyMemoryIndexRows( SELECT provider, model, provider_key, hash, embedding, dims, updated_at FROM ${schema}.embedding_cache; `); - assertLegacyRowsCopied( + // Matching cache keys are derived rows. Validate shape before deciding whether the + // entire stale sidecar should yield to the canonical index. + assertLegacyDerivedRowsCopied( db, `SELECT COUNT(*) AS missing FROM ${schema}.embedding_cache AS legacy @@ -510,9 +519,8 @@ function copyLegacyMemoryIndexRows( AND canonical.model = legacy.model AND canonical.provider_key = legacy.provider_key AND canonical.hash = legacy.hash - AND canonical.embedding IS legacy.embedding AND canonical.dims IS legacy.dims - AND canonical.updated_at IS legacy.updated_at + AND CASE WHEN json_valid(canonical.embedding) AND json_valid(legacy.embedding) THEN json_type(canonical.embedding) = 'array' AND json_array_length(canonical.embedding) = canonical.dims AND json_type(legacy.embedding) = 'array' AND json_array_length(legacy.embedding) = legacy.dims ELSE 0 END )`, "embedding_cache", ); @@ -939,9 +947,9 @@ async function migrateLegacyMemorySidecarSource(params: { requireVectorRows: vectorEnabled, }); } catch (err) { - if (err instanceof LegacyMemoryRowsConflictError && err.tableName === "files") { - // Memory index rows are derived from canonical memory sources. Keep the - // current per-agent index and let normal sync rebuild any stale entries. + if (err instanceof LegacyMemoryDerivedRowsConflictError) { + // Every imported table is a derived search index. A same-identity mismatch means the + // current per-agent row wins; normal sync rebuilds any rows skipped with the sidecar. params.changes.push( `Resolved Memory Core legacy memory index conflict for agent ${params.source.agentId} by keeping canonical per-agent SQLite rows`, );