From 9cfd1cd2878d9b33d57dd63019d5fededce2538e Mon Sep 17 00:00:00 2001 From: Josh Lehman Date: Wed, 17 Jun 2026 06:55:34 -0700 Subject: [PATCH] fix(memory): key index sources by path and source --- .../memory-core/src/memory/manager-db.test.ts | 8 +++--- .../src/memory/manager-embedding-ops.ts | 3 +-- ...manager.self-heal-missing-identity.test.ts | 5 ++-- .../src/host/memory-schema.test.ts | 27 +++++++++++++++++++ .../memory-host-sdk/src/host/memory-schema.ts | 9 +++++-- src/state/openclaw-agent-db.generated.d.ts | 2 +- src/state/openclaw-agent-schema.generated.ts | 11 ++++++-- src/state/openclaw-agent-schema.sql | 11 ++++++-- 8 files changed, 62 insertions(+), 14 deletions(-) diff --git a/extensions/memory-core/src/memory/manager-db.test.ts b/extensions/memory-core/src/memory/manager-db.test.ts index b0535f891654..7dda247657de 100644 --- a/extensions/memory-core/src/memory/manager-db.test.ts +++ b/extensions/memory-core/src/memory/manager-db.test.ts @@ -140,8 +140,8 @@ describe("memory manager database publication", () => { concurrentDb = new DatabaseSync(targetPath); concurrentDb - .prepare("UPDATE memory_index_sources SET hash = ? WHERE path = ?") - .run("newer", "memory.md"); + .prepare("UPDATE memory_index_sources SET hash = ? WHERE path = ? AND source = ?") + .run("newer", "memory.md", "memory"); concurrentDb.close(); concurrentDb = undefined; @@ -154,7 +154,9 @@ describe("memory manager database publication", () => { }), ).rejects.toThrow(/changed while full reindex was building/); expect( - targetDb.prepare("SELECT hash FROM memory_index_sources WHERE path = ?").get("memory.md"), + targetDb + .prepare("SELECT hash FROM memory_index_sources WHERE path = ? AND source = ?") + .get("memory.md", "memory"), ).toEqual({ hash: "newer" }); } finally { try { diff --git a/extensions/memory-core/src/memory/manager-embedding-ops.ts b/extensions/memory-core/src/memory/manager-embedding-ops.ts index bd0cec9b29b1..0569ed8758d6 100644 --- a/extensions/memory-core/src/memory/manager-embedding-ops.ts +++ b/extensions/memory-core/src/memory/manager-embedding-ops.ts @@ -719,8 +719,7 @@ export abstract class MemoryManagerEmbeddingOps extends MemoryManagerSyncOps { this.db .prepare( `INSERT INTO memory_index_sources (path, source, hash, mtime, size) VALUES (?, ?, ?, ?, ?) - ON CONFLICT(path) DO UPDATE SET - source=excluded.source, + ON CONFLICT(path, source) DO UPDATE SET hash=excluded.hash, mtime=excluded.mtime, size=excluded.size`, diff --git a/extensions/memory-core/src/memory/manager.self-heal-missing-identity.test.ts b/extensions/memory-core/src/memory/manager.self-heal-missing-identity.test.ts index 9dfc4b8dd24a..6804ee09a96e 100644 --- a/extensions/memory-core/src/memory/manager.self-heal-missing-identity.test.ts +++ b/extensions/memory-core/src/memory/manager.self-heal-missing-identity.test.ts @@ -118,11 +118,12 @@ describe("memory manager self-heal missing identity with FTS-only chunks", () => updated_at INTEGER NOT NULL ); CREATE TABLE IF NOT EXISTS memory_index_sources ( - path TEXT PRIMARY KEY, + path TEXT NOT NULL, source TEXT NOT NULL DEFAULT 'memory', hash TEXT NOT NULL, mtime INTEGER NOT NULL, - size INTEGER NOT NULL + size INTEGER NOT NULL, + PRIMARY KEY (path, source) ); INSERT INTO memory_index_chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at) VALUES ('chunk-1', 'MEMORY.md', 'memory', 1, 3, 'hash-1', '${model}', 'Alpha topic keep note', '[]', ${Date.now()}); diff --git a/packages/memory-host-sdk/src/host/memory-schema.test.ts b/packages/memory-host-sdk/src/host/memory-schema.test.ts index 5a0b41af1677..d731797294c8 100644 --- a/packages/memory-host-sdk/src/host/memory-schema.test.ts +++ b/packages/memory-host-sdk/src/host/memory-schema.test.ts @@ -84,6 +84,33 @@ describe("memory index schema", () => { } }); + it("stores source records with the same path in separate sources", () => { + const db = new DatabaseSync(":memory:"); + try { + ensureMemoryIndexSchema({ + db, + cacheEnabled: false, + ftsEnabled: false, + }); + + db.prepare( + "INSERT INTO memory_index_sources (path, source, hash, mtime, size) VALUES (?, ?, ?, ?, ?)", + ).run("shared.md", "memory", "memory-hash", 10, 20); + db.prepare( + "INSERT INTO memory_index_sources (path, source, hash, mtime, size) VALUES (?, ?, ?, ?, ?)", + ).run("shared.md", "sessions", "session-hash", 30, 40); + + expect( + db.prepare("SELECT path, source, hash FROM memory_index_sources ORDER BY source").all(), + ).toEqual([ + { path: "shared.md", source: "memory", hash: "memory-hash" }, + { path: "shared.md", source: "sessions", hash: "session-hash" }, + ]); + } finally { + db.close(); + } + }); + it("leaves unrelated generic tables untouched", () => { const db = new DatabaseSync(":memory:"); try { diff --git a/packages/memory-host-sdk/src/host/memory-schema.ts b/packages/memory-host-sdk/src/host/memory-schema.ts index a930a6c5f80c..6b84d092d72f 100644 --- a/packages/memory-host-sdk/src/host/memory-schema.ts +++ b/packages/memory-host-sdk/src/host/memory-schema.ts @@ -194,11 +194,12 @@ export function ensureMemoryIndexSchema(params: { value TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS ${MEMORY_INDEX_SOURCES_TABLE} ( - path TEXT PRIMARY KEY, + path TEXT NOT NULL, source TEXT NOT NULL DEFAULT 'memory', hash TEXT NOT NULL, mtime INTEGER NOT NULL, - size INTEGER NOT NULL + size INTEGER NOT NULL, + PRIMARY KEY (path, source) ); CREATE TABLE IF NOT EXISTS ${MEMORY_INDEX_CHUNKS_TABLE} ( id TEXT PRIMARY KEY, @@ -250,6 +251,10 @@ export function ensureMemoryIndexSchema(params: { UPDATE ${MEMORY_INDEX_STATE_TABLE} SET revision = revision + 1 WHERE id = 1; END; + CREATE INDEX IF NOT EXISTS idx_memory_index_sources_source + ON ${MEMORY_INDEX_SOURCES_TABLE}(source); + CREATE INDEX IF NOT EXISTS idx_memory_index_chunks_path_source + ON ${MEMORY_INDEX_CHUNKS_TABLE}(path, source); CREATE INDEX IF NOT EXISTS idx_memory_index_chunks_path ON ${MEMORY_INDEX_CHUNKS_TABLE}(path); CREATE INDEX IF NOT EXISTS idx_memory_index_chunks_source diff --git a/src/state/openclaw-agent-db.generated.d.ts b/src/state/openclaw-agent-db.generated.d.ts index 45836a2dd0ff..1f13f4730fab 100644 --- a/src/state/openclaw-agent-db.generated.d.ts +++ b/src/state/openclaw-agent-db.generated.d.ts @@ -62,7 +62,7 @@ export interface MemoryIndexMeta { export interface MemoryIndexSources { hash: string; mtime: number; - path: string | null; + path: string; size: number; source: Generated; } diff --git a/src/state/openclaw-agent-schema.generated.ts b/src/state/openclaw-agent-schema.generated.ts index 214c602b03ee..34d6642ae8f8 100644 --- a/src/state/openclaw-agent-schema.generated.ts +++ b/src/state/openclaw-agent-schema.generated.ts @@ -48,11 +48,12 @@ CREATE TABLE IF NOT EXISTS memory_index_meta ( ); CREATE TABLE IF NOT EXISTS memory_index_sources ( - path TEXT PRIMARY KEY, + path TEXT NOT NULL, source TEXT NOT NULL DEFAULT 'memory', hash TEXT NOT NULL, mtime INTEGER NOT NULL, - size INTEGER NOT NULL + size INTEGER NOT NULL, + PRIMARY KEY (path, source) ); CREATE TABLE IF NOT EXISTS memory_index_chunks ( @@ -125,6 +126,12 @@ END; CREATE INDEX IF NOT EXISTS idx_memory_embedding_cache_updated_at ON memory_embedding_cache(updated_at); +CREATE INDEX IF NOT EXISTS idx_memory_index_sources_source + ON memory_index_sources(source); + +CREATE INDEX IF NOT EXISTS idx_memory_index_chunks_path_source + ON memory_index_chunks(path, source); + CREATE INDEX IF NOT EXISTS idx_memory_index_chunks_path ON memory_index_chunks(path); diff --git a/src/state/openclaw-agent-schema.sql b/src/state/openclaw-agent-schema.sql index 8c1bfa892e37..ae920526e47c 100644 --- a/src/state/openclaw-agent-schema.sql +++ b/src/state/openclaw-agent-schema.sql @@ -43,11 +43,12 @@ CREATE TABLE IF NOT EXISTS memory_index_meta ( ); CREATE TABLE IF NOT EXISTS memory_index_sources ( - path TEXT PRIMARY KEY, + path TEXT NOT NULL, source TEXT NOT NULL DEFAULT 'memory', hash TEXT NOT NULL, mtime INTEGER NOT NULL, - size INTEGER NOT NULL + size INTEGER NOT NULL, + PRIMARY KEY (path, source) ); CREATE TABLE IF NOT EXISTS memory_index_chunks ( @@ -120,6 +121,12 @@ END; CREATE INDEX IF NOT EXISTS idx_memory_embedding_cache_updated_at ON memory_embedding_cache(updated_at); +CREATE INDEX IF NOT EXISTS idx_memory_index_sources_source + ON memory_index_sources(source); + +CREATE INDEX IF NOT EXISTS idx_memory_index_chunks_path_source + ON memory_index_chunks(path, source); + CREATE INDEX IF NOT EXISTS idx_memory_index_chunks_path ON memory_index_chunks(path);