fix(memory): recover derived sidecar conflicts

Backport #108652 for the 2026.7.1 correction release.
2026-08-28 05:16:23 -06:00 · 2026-07-18 03:43:48 +01:00
parent ad807549c6
commit f47a2e001f
2 changed files with 189 additions and 60 deletions
@@ -61,6 +61,8 @@ async function writeLegacyMemorySidecar(
    fileHash?: string;
    filePath?: string;
    text?: string;
+    cacheEmbedding?: string;
+    cacheDims?: number | null;
  } = {},
 ): Promise<void> {
  await fs.mkdir(path.dirname(legacyPath), { recursive: true });
@@ -71,6 +73,8 @@ async function writeLegacyMemorySidecar(
    const chunkId = params.chunkId ?? "chunk-1";
    const chunkHash = params.chunkHash ?? "chunk-hash";
    const text = params.text ?? "remember this";
+    const cacheEmbedding = params.cacheEmbedding ?? "[1,0,0]";
+    const cacheDims = params.cacheDims === undefined ? 3 : params.cacheDims;
    db.exec(`
      CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
      CREATE TABLE files (
@@ -109,8 +113,8 @@ async function writeLegacyMemorySidecar(
      "INSERT INTO chunks VALUES (?, ?, 'memory', 1, 2, ?, 'embed-model', ?, '[1,0,0]', 30)",
    ).run(chunkId, filePath, chunkHash, text);
    db.prepare(
-      "INSERT INTO embedding_cache VALUES ('openai', 'embed-model', 'key', ?, '[1,0,0]', 3, 40)",
-    ).run(chunkHash);
+      "INSERT INTO embedding_cache VALUES ('openai', 'embed-model', 'key', ?, ?, ?, 40)",
+    ).run(chunkHash, cacheEmbedding, cacheDims);
    if (params.vector === "vec0") {
      const loaded = await loadSqliteVecExtension({ db });
      expect(loaded.ok, loaded.error).toBe(true);
@@ -330,6 +334,19 @@ function readMemoryRows(agentPath: string) {
  }
 }

+function readMemoryCacheRows(agentPath: string) {
+  const db = new DatabaseSync(agentPath);
+  try {
+    return db
+      .prepare(
+        "SELECT provider, model, provider_key, hash, embedding, dims, updated_at FROM memory_embedding_cache ORDER BY provider, hash",
+      )
+      .all();
+  } finally {
+    db.close();
+  }
+}
+
 function readMemoryFtsSql(agentPath: string): string | undefined {
  const db = new DatabaseSync(agentPath);
  try {
@@ -1309,7 +1326,7 @@ describe("memory-core doctor dreaming migration", () => {
    await expect(fs.access(retryPath)).resolves.toBeUndefined();
  });

-  it("leaves the legacy memory sidecar in place when metadata conflicts", async () => {
+  it("keeps canonical metadata and archives a conflicting derived legacy index", async () => {
    const stateDir = path.join(rootDir, "state");
    const legacyPath = path.join(stateDir, "memory", "main.sqlite");
    const agentPath = path.join(stateDir, "agents", "main", "agent", "openclaw-agent.sqlite");
@@ -1318,17 +1335,50 @@ describe("memory-core doctor dreaming migration", () => {

    const result = await legacyMemoryIndexMigration().migrateLegacyState(migrationParams());

-    expect(result.warnings).toEqual([
-      expect.stringContaining(
-        "Skipped Memory Core legacy memory index import for agent main because legacy rows could not be imported: Error: legacy memory meta rows conflict with canonical memory index rows",
-      ),
+    expect(result.warnings).toEqual([]);
+    expect(result.changes).toEqual([
+      "Resolved Memory Core legacy memory index conflict for agent main by keeping canonical per-agent SQLite rows",
+      expect.stringContaining("Archived Memory Core legacy memory index sidecar"),
    ]);
-    expect(result.changes).toEqual([]);
    expect(readMemoryRows(agentPath).chunks).toEqual([
      { id: "canonical-other-chunk", text: "canonical unrelated memory" },
    ]);
-    await expect(fs.access(legacyPath)).resolves.toBeUndefined();
-    await expect(fs.access(`${legacyPath}.migrated`)).rejects.toThrow();
+    await expect(fs.access(legacyPath)).rejects.toThrow();
+    await expect(fs.access(`${legacyPath}.migrated`)).resolves.toBeUndefined();
+
+    const secondRun = await legacyMemoryIndexMigration().migrateLegacyState(migrationParams());
+    expect(secondRun).toEqual({ changes: [], warnings: [] });
+  });
+
+  it("keeps canonical chunks and archives a conflicting derived legacy index", async () => {
+    const stateDir = path.join(rootDir, "state");
+    const legacyPath = path.join(stateDir, "memory", "main.sqlite");
+    const agentPath = path.join(stateDir, "agents", "main", "agent", "openclaw-agent.sqlite");
+    await writeLegacyMemorySidecar(legacyPath);
+    await createCanonicalLegacyMemoryRowsWithFts(agentPath, "remember this");
+    const canonicalDb = new DatabaseSync(agentPath);
+    try {
+      canonicalDb
+        .prepare("UPDATE memory_index_chunks SET text = ? WHERE id = ?")
+        .run("canonical memory remains authoritative", "chunk-1");
+    } finally {
+      canonicalDb.close();
+    }
+
+    const result = await legacyMemoryIndexMigration().migrateLegacyState(migrationParams());
+
+    expect(result.warnings).toEqual([]);
+    expect(result.changes).toEqual([
+      "Resolved Memory Core legacy memory index conflict for agent main by keeping canonical per-agent SQLite rows",
+      expect.stringContaining("Archived Memory Core legacy memory index sidecar"),
+    ]);
+    expect(readMemoryRows(agentPath)).toEqual({
+      sources: [{ path: "MEMORY.md", source: "memory", hash: "file-hash" }],
+      chunks: [{ id: "chunk-1", text: "canonical memory remains authoritative" }],
+      cache: [],
+    });
+    await expect(fs.access(legacyPath)).rejects.toThrow();
+    await expect(fs.access(`${legacyPath}.migrated`)).resolves.toBeUndefined();
  });

  it("merges legacy sidecar rows into a non-empty canonical index when rows do not conflict", async () => {
@@ -1361,6 +1411,78 @@ describe("memory-core doctor dreaming migration", () => {
    await expect(fs.access(`${legacyPath}.migrated`)).resolves.toBeUndefined();
  });

+  it("keeps a valid canonical cache collision while importing remaining legacy rows", async () => {
+    const stateDir = path.join(rootDir, "state");
+    const legacyPath = path.join(stateDir, "memory", "main.sqlite");
+    const agentPath = path.join(stateDir, "agents", "main", "agent", "openclaw-agent.sqlite");
+    await writeLegacyMemorySidecar(legacyPath);
+    await createUnrelatedCanonicalMemoryIndex(agentPath);
+    const canonicalDb = new DatabaseSync(agentPath);
+    try {
+      canonicalDb
+        .prepare(
+          "INSERT INTO memory_embedding_cache (provider, model, provider_key, hash, embedding, dims, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)",
+        )
+        .run("openai", "embed-model", "key", "chunk-hash", "[0,1,0]", 3, 99);
+    } finally {
+      canonicalDb.close();
+    }
+
+    const result = await legacyMemoryIndexMigration().migrateLegacyState(migrationParams());
+
+    expect(result.warnings).toEqual([]);
+    expect(result.changes).toEqual([
+      "Migrated Memory Core legacy memory index for agent main -> per-agent SQLite (1 source(s), 1 chunk(s), 1 cache row(s))",
+      expect.stringContaining("Archived Memory Core legacy memory index sidecar"),
+    ]);
+    expect(readMemoryCacheRows(agentPath)).toEqual([
+      {
+        provider: "openai",
+        model: "embed-model",
+        provider_key: "key",
+        hash: "chunk-hash",
+        embedding: "[0,1,0]",
+        dims: 3,
+        updated_at: 99,
+      },
+    ]);
+    await expect(fs.access(legacyPath)).rejects.toThrow();
+    await expect(fs.access(`${legacyPath}.migrated`)).resolves.toBeUndefined();
+  });
+
+  it("archives a malformed canonical cache collision as a derived conflict", async () => {
+    const stateDir = path.join(rootDir, "state");
+    const legacyPath = path.join(stateDir, "memory", "main.sqlite");
+    const agentPath = path.join(stateDir, "agents", "main", "agent", "openclaw-agent.sqlite");
+    await writeLegacyMemorySidecar(legacyPath);
+    await createUnrelatedCanonicalMemoryIndex(agentPath);
+    const canonicalDb = new DatabaseSync(agentPath);
+    try {
+      canonicalDb
+        .prepare(
+          "INSERT INTO memory_embedding_cache (provider, model, provider_key, hash, embedding, dims, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)",
+        )
+        .run("openai", "embed-model", "key", "chunk-hash", "not-json", 3, 99);
+    } finally {
+      canonicalDb.close();
+    }
+
+    const result = await legacyMemoryIndexMigration().migrateLegacyState(migrationParams());
+
+    expect(result.warnings).toEqual([]);
+    expect(result.changes).toEqual([
+      "Resolved Memory Core legacy memory index conflict for agent main by keeping canonical per-agent SQLite rows",
+      expect.stringContaining("Archived Memory Core legacy memory index sidecar"),
+    ]);
+    expect(readMemoryCacheRows(agentPath)[0]).toMatchObject({
+      embedding: "not-json",
+      dims: 3,
+      updated_at: 99,
+    });
+    await expect(fs.access(legacyPath)).rejects.toThrow();
+    await expect(fs.access(`${legacyPath}.migrated`)).resolves.toBeUndefined();
+  });
+
  it("leaves legacy vector sidecars in place when vector dimensions conflict", async () => {
    const stateDir = path.join(rootDir, "state");
    const legacyPath = path.join(stateDir, "memory", "main.sqlite");
@@ -1380,7 +1502,7 @@ describe("memory-core doctor dreaming migration", () => {
    await expect(fs.access(`${legacyPath}.migrated`)).rejects.toThrow();
  });

-  it("leaves legacy vector sidecars in place when canonical vector rows conflict", async () => {
+  it("keeps canonical vector rows and archives a conflicting derived legacy index", async () => {
    const stateDir = path.join(rootDir, "state");
    const legacyPath = path.join(stateDir, "memory", "main.sqlite");
    const agentPath = path.join(stateDir, "agents", "main", "agent", "openclaw-agent.sqlite");
@@ -1389,14 +1511,13 @@ describe("memory-core doctor dreaming migration", () => {

    const result = await legacyMemoryIndexMigration().migrateLegacyState(migrationParams());

-    expect(result.warnings).toEqual([
-      expect.stringContaining(
-        "Skipped Memory Core legacy memory index import for agent main because legacy rows could not be imported: Error: legacy memory chunks_vec rows conflict with canonical memory index rows",
-      ),
+    expect(result.warnings).toEqual([]);
+    expect(result.changes).toEqual([
+      "Resolved Memory Core legacy memory index conflict for agent main by keeping canonical per-agent SQLite rows",
+      expect.stringContaining("Archived Memory Core legacy memory index sidecar"),
    ]);
-    expect(result.changes).toEqual([]);
-    await expect(fs.access(legacyPath)).resolves.toBeUndefined();
-    await expect(fs.access(`${legacyPath}.migrated`)).rejects.toThrow();
+    await expect(fs.access(legacyPath)).rejects.toThrow();
+    await expect(fs.access(`${legacyPath}.migrated`)).resolves.toBeUndefined();
  });

  it("leaves legacy vector sidecars in place when vector rows have no chunk", async () => {
@@ -1414,7 +1535,7 @@ describe("memory-core doctor dreaming migration", () => {

    expect(result.warnings).toEqual([
      expect.stringContaining(
-        "Skipped Memory Core legacy memory index import for agent main because legacy rows could not be imported: Error: legacy memory chunks_vec chunk references rows conflict",
+        "Skipped Memory Core legacy memory index import for agent main because legacy rows could not be imported: Error: legacy memory chunks_vec rows reference missing chunks",
      ),
    ]);
    expect(result.changes).toEqual([]);
@@ -1422,7 +1543,7 @@ describe("memory-core doctor dreaming migration", () => {
    await expect(fs.access(`${legacyPath}.migrated`)).rejects.toThrow();
  });

-  it("leaves legacy sidecars in place when canonical FTS rows conflict", async () => {
+  it("keeps canonical FTS rows and archives a conflicting derived legacy index", async () => {
    const stateDir = path.join(rootDir, "state");
    const legacyPath = path.join(stateDir, "memory", "main.sqlite");
    const agentPath = path.join(stateDir, "agents", "main", "agent", "openclaw-agent.sqlite");
@@ -1431,17 +1552,18 @@ describe("memory-core doctor dreaming migration", () => {

    const result = await legacyMemoryIndexMigration().migrateLegacyState(migrationParams());

-    expect(result.warnings).toEqual([
-      expect.stringContaining(
-        "Skipped Memory Core legacy memory index import for agent main because legacy rows could not be imported: Error: legacy memory fts rows conflict",
-      ),
+    expect(result.warnings).toEqual([]);
+    expect(result.changes).toEqual([
+      "Resolved Memory Core legacy memory index conflict for agent main by keeping canonical per-agent SQLite rows",
+      expect.stringContaining("Archived Memory Core legacy memory index sidecar"),
    ]);
-    expect(result.changes).toEqual([]);
-    await expect(fs.access(legacyPath)).resolves.toBeUndefined();
-    await expect(fs.access(`${legacyPath}.migrated`)).rejects.toThrow();
+    const keywordRows = await searchMigratedKeywordRows(agentPath, "stale");
+    expect(keywordRows.map((row) => row.id)).toEqual(["chunk-1"]);
+    await expect(fs.access(legacyPath)).rejects.toThrow();
+    await expect(fs.access(`${legacyPath}.migrated`)).resolves.toBeUndefined();
  });

-  it("leaves legacy vector sidecars in place when canonical metadata dimensions conflict", async () => {
+  it("keeps canonical vector metadata and archives a conflicting derived legacy index", async () => {
    const stateDir = path.join(rootDir, "state");
    const legacyPath = path.join(stateDir, "memory", "main.sqlite");
    const agentPath = path.join(stateDir, "agents", "main", "agent", "openclaw-agent.sqlite");
@@ -1450,16 +1572,15 @@ describe("memory-core doctor dreaming migration", () => {

    const result = await legacyMemoryIndexMigration().migrateLegacyState(migrationParams());

-    expect(result.warnings).toEqual([
-      expect.stringContaining(
-        "Skipped Memory Core legacy memory index import for agent main because legacy rows could not be imported: Error: legacy memory meta rows conflict with canonical memory index rows",
-      ),
+    expect(result.warnings).toEqual([]);
+    expect(result.changes).toEqual([
+      "Resolved Memory Core legacy memory index conflict for agent main by keeping canonical per-agent SQLite rows",
+      expect.stringContaining("Archived Memory Core legacy memory index sidecar"),
    ]);
-    expect(result.changes).toEqual([]);
    expect(readMemoryRows(agentPath).chunks).toEqual([
      { id: "canonical-other-chunk", text: "canonical unrelated memory" },
    ]);
-    await expect(fs.access(legacyPath)).resolves.toBeUndefined();
-    await expect(fs.access(`${legacyPath}.migrated`)).rejects.toThrow();
+    await expect(fs.access(legacyPath)).rejects.toThrow();
+    await expect(fs.access(`${legacyPath}.migrated`)).resolves.toBeUndefined();
  });
 });
@@ -106,7 +106,7 @@ type LegacyMemorySidecarImportResult = {

 type MemoryFtsTokenizer = "unicode61" | "trigram";

-class LegacyMemoryRowsConflictError extends Error {
+class LegacyMemoryDerivedRowsConflictError extends Error {
  constructor(readonly tableName: string) {
    super(`legacy memory ${tableName} rows conflict with canonical memory index rows`);
  }
@@ -211,10 +211,26 @@ function formatLegacyVectorRows(count: number | undefined): string {
  return count === undefined ? "legacy vector rows" : `${count} vector row(s)`;
 }

-function assertLegacyRowsCopied(db: DatabaseSync, query: string, tableName: string): void {
+function assertLegacyDerivedRowsCopied(db: DatabaseSync, query: string, tableName: string): void {
  const row = db.prepare(query).get() as { missing?: unknown } | undefined;
  if (Number(row?.missing ?? 0) > 0) {
-    throw new LegacyMemoryRowsConflictError(tableName);
+    throw new LegacyMemoryDerivedRowsConflictError(tableName);
+  }
+}
+
+function assertLegacyVectorRowsReferenceChunks(db: DatabaseSync, schema: string): void {
+  const row = db
+    .prepare(
+      `SELECT COUNT(*) AS missing
+       FROM ${schema}.${LEGACY_MEMORY_VECTOR_TABLE} AS legacy
+       WHERE NOT EXISTS (
+         SELECT 1 FROM main.${MEMORY_INDEX_CHUNKS_TABLE} AS chunk
+         WHERE chunk.id = legacy.id
+       )`,
+    )
+    .get() as { missing?: unknown } | undefined;
+  if (Number(row?.missing ?? 0) > 0) {
+    throw new Error(`legacy memory ${LEGACY_MEMORY_VECTOR_TABLE} rows reference missing chunks`);
  }
 }

@@ -340,17 +356,8 @@ function copyLegacyMemoryVectorRows(db: DatabaseSync, schema: string): void {
  if (!tableExists(db, "main", MEMORY_INDEX_VECTOR_TABLE)) {
    return;
  }
-  assertLegacyRowsCopied(
-    db,
-    `SELECT COUNT(*) AS missing
-     FROM ${schema}.${LEGACY_MEMORY_VECTOR_TABLE} AS legacy
-     WHERE NOT EXISTS (
-       SELECT 1 FROM main.${MEMORY_INDEX_CHUNKS_TABLE} AS chunk
-       WHERE chunk.id = legacy.id
-     )`,
-    `${LEGACY_MEMORY_VECTOR_TABLE} chunk references`,
-  );
-  assertLegacyRowsCopied(
+  assertLegacyVectorRowsReferenceChunks(db, schema);
+  assertLegacyDerivedRowsCopied(
    db,
    `SELECT COUNT(*) AS missing
     FROM ${schema}.${LEGACY_MEMORY_VECTOR_TABLE} AS legacy
@@ -368,7 +375,7 @@ function copyLegacyMemoryVectorRows(db: DatabaseSync, schema: string): void {
      WHERE canonical.id = legacy.id
    );
  `);
-  assertLegacyRowsCopied(
+  assertLegacyDerivedRowsCopied(
    db,
    `SELECT COUNT(*) AS missing
     FROM ${schema}.${LEGACY_MEMORY_VECTOR_TABLE} AS legacy
@@ -398,7 +405,7 @@ function copyLegacyMemoryFtsRows(db: DatabaseSync, schema: string): void {
      WHERE canonical.id = legacy.id
    );
  `);
-  assertLegacyRowsCopied(
+  assertLegacyDerivedRowsCopied(
    db,
    `SELECT COUNT(*) AS missing
     FROM ${schema}.chunks AS legacy
@@ -435,7 +442,7 @@ function copyLegacyMemoryIndexRows(
    SELECT id, path, source, start_line, end_line, hash, model, text, embedding, updated_at
    FROM ${schema}.chunks;
  `);
-  assertLegacyRowsCopied(
+  assertLegacyDerivedRowsCopied(
    db,
    `SELECT COUNT(*) AS missing
     FROM ${schema}.meta AS legacy
@@ -445,7 +452,7 @@ function copyLegacyMemoryIndexRows(
     )`,
    "meta",
  );
-  assertLegacyRowsCopied(
+  assertLegacyDerivedRowsCopied(
    db,
    `SELECT COUNT(*) AS missing
     FROM ${schema}.files AS legacy
@@ -459,7 +466,7 @@ function copyLegacyMemoryIndexRows(
     )`,
    "files",
  );
-  assertLegacyRowsCopied(
+  assertLegacyDerivedRowsCopied(
    db,
    `SELECT COUNT(*) AS missing
     FROM ${schema}.chunks AS legacy
@@ -500,7 +507,9 @@ function copyLegacyMemoryIndexRows(
      SELECT provider, model, provider_key, hash, embedding, dims, updated_at
      FROM ${schema}.embedding_cache;
    `);
-    assertLegacyRowsCopied(
+    // Matching cache keys are derived rows. Validate shape before deciding whether the
+    // entire stale sidecar should yield to the canonical index.
+    assertLegacyDerivedRowsCopied(
      db,
      `SELECT COUNT(*) AS missing
       FROM ${schema}.embedding_cache AS legacy
@@ -510,9 +519,8 @@ function copyLegacyMemoryIndexRows(
           AND canonical.model = legacy.model
           AND canonical.provider_key = legacy.provider_key
           AND canonical.hash = legacy.hash
-           AND canonical.embedding IS legacy.embedding
           AND canonical.dims IS legacy.dims
-           AND canonical.updated_at IS legacy.updated_at
+           AND CASE WHEN json_valid(canonical.embedding) AND json_valid(legacy.embedding) THEN json_type(canonical.embedding) = 'array' AND json_array_length(canonical.embedding) = canonical.dims AND json_type(legacy.embedding) = 'array' AND json_array_length(legacy.embedding) = legacy.dims ELSE 0 END
       )`,
      "embedding_cache",
    );
@@ -939,9 +947,9 @@ async function migrateLegacyMemorySidecarSource(params: {
        requireVectorRows: vectorEnabled,
      });
    } catch (err) {
-      if (err instanceof LegacyMemoryRowsConflictError && err.tableName === "files") {
-        // Memory index rows are derived from canonical memory sources. Keep the
-        // current per-agent index and let normal sync rebuild any stale entries.
+      if (err instanceof LegacyMemoryDerivedRowsConflictError) {
+        // Every imported table is a derived search index. A same-identity mismatch means the
+        // current per-agent row wins; normal sync rebuilds any rows skipped with the sidecar.
        params.changes.push(
          `Resolved Memory Core legacy memory index conflict for agent ${params.source.agentId} by keeping canonical per-agent SQLite rows`,
        );