diff --git a/extensions/memory-core/src/memory/index.test.ts b/extensions/memory-core/src/memory/index.test.ts index cdac86da1e68..88b573912cd7 100644 --- a/extensions/memory-core/src/memory/index.test.ts +++ b/extensions/memory-core/src/memory/index.test.ts @@ -1828,6 +1828,33 @@ describe("memory index", () => { expect(status.vector?.available).toBe(available); }); + it("rebuilds vector tables created before completeness markers", async () => { + const cfg = createCfg({ provider: "gemini", vectorEnabled: true }); + const legacyManager = await getFreshManager(cfg); + const available = await legacyManager.probeVectorStoreAvailability?.(); + if (!available) { + await legacyManager.close?.(); + return; + } + const legacyDb = Reflect.get(legacyManager, "db") as DatabaseSync; + legacyDb.exec(` + CREATE VIRTUAL TABLE memory_index_chunks_vec USING vec0( + id TEXT PRIMARY KEY, + embedding FLOAT[3] + ); + INSERT INTO memory_index_chunks_vec VALUES ('orphan-before-marker', '[1,0,0]'); + `); + await legacyManager.close?.(); + + const manager = await getFreshManager(cfg); + try { + await expect(manager.probeVectorStoreAvailability?.()).resolves.toBe(false); + expect(Reflect.get(manager, "memoryFullRetryDirty")).toBe(true); + } finally { + await manager.close?.(); + } + }); + it("drops the shipped legacy vector table and schedules a full reindex", async () => { const cfg = createCfg({ vectorEnabled: true }); const manager = await getPersistentManager(cfg); @@ -1887,6 +1914,48 @@ describe("memory index", () => { } }); + it("forces a rebuild after incremental writes while vectors are disabled", async () => { + const enabledCfg = createCfg({ provider: "gemini", vectorEnabled: true }); + const initialManager = await getFreshManager(enabledCfg); + await initialManager.sync({ reason: "test", force: true }); + await initialManager.close?.(); + + await fs.writeFile( + path.join(memoryDir, "2026-01-12.md"), + "# Updated\n\nvector writes were disabled for this update\n", + ); + const disabledManager = await getFreshManager( + createCfg({ provider: "gemini", vectorEnabled: false }), + ); + Reflect.set(disabledManager, "dirty", true); + await disabledManager.sync({ reason: "test" }); + const disabledDb = Reflect.get(disabledManager, "db") as DatabaseSync; + expect( + disabledDb + .prepare("SELECT value FROM memory_index_meta WHERE key = 'memory_vector_rebuild_v1'") + .get(), + ).toEqual({ value: "1" }); + await disabledManager.close?.(); + + const reloadedManager = await getFreshManager(enabledCfg); + try { + await expect(reloadedManager.probeVectorStoreAvailability?.()).resolves.toBe(false); + expect(Reflect.get(reloadedManager, "memoryFullRetryDirty")).toBe(true); + expect(reloadedManager.status().dirty).toBe(true); + + await reloadedManager.sync({ reason: "test" }); + const rebuiltDb = Reflect.get(reloadedManager, "db") as DatabaseSync; + expect( + rebuiltDb + .prepare("SELECT value FROM memory_index_meta WHERE key = 'memory_vector_rebuild_v1'") + .get(), + ).toEqual({ value: "clean" }); + await expect(reloadedManager.probeVectorStoreAvailability?.()).resolves.toBe(true); + } finally { + await reloadedManager.close?.(); + } + }); + it("keeps empty vector indexes clean after vector store probing", async () => { await fs.rm(path.join(memoryDir, "2026-01-12.md")); const legacyCfg = createCfg({ diff --git a/extensions/memory-core/src/memory/manager-embedding-ops.ts b/extensions/memory-core/src/memory/manager-embedding-ops.ts index 44bb437f0c55..5d41a8ea8278 100644 --- a/extensions/memory-core/src/memory/manager-embedding-ops.ts +++ b/extensions/memory-core/src/memory/manager-embedding-ops.ts @@ -704,15 +704,7 @@ export abstract class MemoryManagerEmbeddingOps extends MemoryManagerSyncOps { } private clearIndexedFileData(pathname: string, source: MemorySource): void { - if (this.vector.enabled) { - try { - this.db - .prepare( - `DELETE FROM ${VECTOR_TABLE} WHERE id IN (SELECT id FROM memory_index_chunks WHERE path = ? AND source = ?)`, - ) - .run(pathname, source); - } catch {} - } + this.deleteVectorRowsForSource(pathname, source); if (this.fts.enabled && this.fts.available) { try { deleteMemoryFtsRows({ @@ -761,6 +753,7 @@ export abstract class MemoryManagerEmbeddingOps extends MemoryManagerSyncOps { vectorReady: boolean, ): void { const now = Date.now(); + const needsVectorRebuild = !vectorReady && embeddings.some((embedding) => embedding.length > 0); runSqliteImmediateTransactionSync(this.db, () => { this.clearIndexedFileData(entry.path, source); for (const [i, chunk] of chunks.entries()) { @@ -809,6 +802,9 @@ export abstract class MemoryManagerEmbeddingOps extends MemoryManagerSyncOps { } } this.upsertFileRecord(entry, source); + if (needsVectorRebuild) { + this.markVectorRebuildRequired(); + } }); this.vectorDegradedWriteWarningShown = logMemoryVectorDegradedWrite({ vectorEnabled: this.vector.enabled, diff --git a/extensions/memory-core/src/memory/manager-session-sync-ops.ts b/extensions/memory-core/src/memory/manager-session-sync-ops.ts index dbdbe0d5200b..da415d82cf9f 100644 --- a/extensions/memory-core/src/memory/manager-session-sync-ops.ts +++ b/extensions/memory-core/src/memory/manager-session-sync-ops.ts @@ -179,7 +179,7 @@ export abstract class MemoryManagerSessionSyncOps extends MemoryManagerWatchOps protected async runSessionStartupCatchup(): Promise { const dirtyFiles = await this.markSessionStartupCatchupDirtyFiles(); - if (dirtyFiles.length === 0 || this.closed) { + if ((dirtyFiles.length === 0 && !this.sessionsFullRetryDirty) || this.closed) { return dirtyFiles; } void this.sync({ reason: "session-startup-catchup" }).catch((err: unknown) => { diff --git a/extensions/memory-core/src/memory/manager-session-sync-state.test.ts b/extensions/memory-core/src/memory/manager-session-sync-state.test.ts index 19ec2a60dfd4..80d674014a42 100644 --- a/extensions/memory-core/src/memory/manager-session-sync-state.test.ts +++ b/extensions/memory-core/src/memory/manager-session-sync-state.test.ts @@ -94,6 +94,12 @@ describe("memory session sync state", () => { mtimeMs: 100.75, size: 10, }, + { + absPath: "/tmp/sessions/invalidated.jsonl", + path: "sessions/invalidated.jsonl", + mtimeMs: 200, + size: 20, + }, { absPath: "/tmp/sessions/newer.jsonl", path: "sessions/newer.jsonl", @@ -116,6 +122,7 @@ describe("memory session sync state", () => { existingRows: [ { path: "sessions/unchanged.jsonl", hash: "hash-unchanged", mtime: 100.75, size: 10 }, { path: "sessions/sub-ms-newer.jsonl", hash: "hash-sub-ms", mtime: 100.25, size: 10 }, + { path: "sessions/invalidated.jsonl", hash: "", mtime: 200, size: 20 }, { path: "sessions/newer.jsonl", hash: "hash-newer", mtime: 200, size: 20 }, { path: "sessions/resized.jsonl", hash: "hash-resized", mtime: 300, size: 30 }, ], @@ -123,6 +130,7 @@ describe("memory session sync state", () => { expect(dirtyFiles).toEqual([ "/tmp/sessions/sub-ms-newer.jsonl", + "/tmp/sessions/invalidated.jsonl", "/tmp/sessions/newer.jsonl", "/tmp/sessions/resized.jsonl", "/tmp/sessions/missing.jsonl", diff --git a/extensions/memory-core/src/memory/manager-session-sync-state.ts b/extensions/memory-core/src/memory/manager-session-sync-state.ts index 37a1ff064a87..7857762185c6 100644 --- a/extensions/memory-core/src/memory/manager-session-sync-state.ts +++ b/extensions/memory-core/src/memory/manager-session-sync-state.ts @@ -16,7 +16,7 @@ export function resolveMemorySessionStartupDirtyFiles(params: { const dirtyFiles: string[] = []; for (const file of params.files) { const existing = indexedRows.get(file.path); - if (!existing) { + if (!existing || existing.hash === "") { dirtyFiles.push(file.absPath); continue; } diff --git a/extensions/memory-core/src/memory/manager-source-sync-ops.ts b/extensions/memory-core/src/memory/manager-source-sync-ops.ts index f042f770a9d0..7f7ada062111 100644 --- a/extensions/memory-core/src/memory/manager-source-sync-ops.ts +++ b/extensions/memory-core/src/memory/manager-source-sync-ops.ts @@ -14,7 +14,6 @@ import { buildFileEntry, listMemoryFiles, MEMORY_INDEX_FTS_TABLE, - MEMORY_INDEX_VECTOR_TABLE, runWithConcurrency, } from "openclaw/plugin-sdk/memory-core-host-engine-storage"; import { MemoryManagerSessionSyncOps } from "./manager-session-sync-ops.js"; @@ -30,7 +29,6 @@ import type { MemorySyncProgressState, } from "./manager-sync-base.js"; -const VECTOR_TABLE = MEMORY_INDEX_VECTOR_TABLE; const FTS_TABLE = MEMORY_INDEX_FTS_TABLE; const SESSION_SYNC_YIELD_EVERY = 10; const SOURCE_WIDE_SESSION_INDEX_FLUSH_FILES = 128; @@ -60,12 +58,6 @@ export abstract class MemoryManagerSourceSyncOps extends MemoryManagerSessionSyn const deleteChunksByPathAndSource = this.db.prepare( `DELETE FROM memory_index_chunks WHERE path = ? AND source = ?`, ); - const deleteVectorRowsByPathAndSource = - this.vector.enabled && this.vector.available - ? this.db.prepare( - `DELETE FROM ${VECTOR_TABLE} WHERE id IN (SELECT id FROM memory_index_chunks WHERE path = ? AND source = ?)`, - ) - : null; const deleteFtsRowsByPathAndSource = this.fts.enabled && this.fts.available ? this.db.prepare(`DELETE FROM ${FTS_TABLE} WHERE path = ? AND source = ?`) @@ -113,11 +105,7 @@ export abstract class MemoryManagerSourceSyncOps extends MemoryManagerSessionSyn continue; } deleteFileByPathAndSource.run(stale.path, "memory"); - if (deleteVectorRowsByPathAndSource) { - try { - deleteVectorRowsByPathAndSource.run(stale.path, "memory"); - } catch {} - } + this.deleteVectorRowsForSource(stale.path, "memory"); deleteChunksByPathAndSource.run(stale.path, "memory"); if (deleteFtsRowsByPathAndSource) { try { @@ -190,12 +178,6 @@ export abstract class MemoryManagerSourceSyncOps extends MemoryManagerSessionSyn const deleteChunksByPathAndSource = this.db.prepare( `DELETE FROM memory_index_chunks WHERE path = ? AND source = ?`, ); - const deleteVectorRowsByPathAndSource = - this.vector.enabled && this.vector.available - ? this.db.prepare( - `DELETE FROM ${VECTOR_TABLE} WHERE id IN (SELECT id FROM memory_index_chunks WHERE path = ? AND source = ?)`, - ) - : null; const deleteFtsRowsByPathAndSource = this.fts.enabled && this.fts.available ? this.db.prepare(`DELETE FROM ${FTS_TABLE} WHERE path = ? AND source = ?`) @@ -254,11 +236,7 @@ export abstract class MemoryManagerSourceSyncOps extends MemoryManagerSessionSyn const yieldAfterSessionFile = createSessionSyncYield(files.length); const deleteIndexedSessionPath = (memoryPath: string) => { deleteFileByPathAndSource.run(memoryPath, "sessions"); - if (deleteVectorRowsByPathAndSource) { - try { - deleteVectorRowsByPathAndSource.run(memoryPath, "sessions"); - } catch {} - } + this.deleteVectorRowsForSource(memoryPath, "sessions"); deleteChunksByPathAndSource.run(memoryPath, "sessions"); if (deleteFtsRowsByPathAndSource) { try { diff --git a/extensions/memory-core/src/memory/manager-sync-base.ts b/extensions/memory-core/src/memory/manager-sync-base.ts index 3130bc9f0ea3..17520016513c 100644 --- a/extensions/memory-core/src/memory/manager-sync-base.ts +++ b/extensions/memory-core/src/memory/manager-sync-base.ts @@ -43,6 +43,10 @@ import { type MemoryIndexMeta, type MemoryIndexProviderIdentity, } from "./manager-reindex-state.js"; +import { + markMemoryVectorRebuildRequired, + requiresMemoryVectorRebuild, +} from "./manager-vector-rebuild-state.js"; import type { MemoryWatchSettleQueue } from "./watch-settle.js"; export type MemorySyncProgressState = { @@ -488,6 +492,10 @@ export abstract class MemoryManagerSyncBase { } private async loadVectorExtension(): Promise { + if (this.vector.available === true && this.hasVectorRebuildMarker()) { + this.markConfiguredSourcesForFullReindex(); + return false; + } if (this.vector.available !== null) { return this.vector.available; } @@ -505,6 +513,13 @@ export abstract class MemoryManagerSyncBase { } this.vector.extensionPath = loaded.extensionPath; this.vector.available = true; + if (this.hasVectorRebuildMarker()) { + // A skipped vector write/delete can leave both missing and extra rows. + // Refuse partial KNN results and let the normal shadow reindex rebuild all + // configured sources before this manager treats vectors as ready. + this.markConfiguredSourcesForFullReindex(); + return false; + } if (this.dropLegacyVectorTable()) { // A broad dirty sync can skip unchanged files whose source hashes were // migrated. Force the next sync to republish the derived vector rows. @@ -521,6 +536,53 @@ export abstract class MemoryManagerSyncBase { } } + protected deleteVectorRowsForSource(pathname: string, source: MemorySource): void { + if (!memoryTableExists(this.db, VECTOR_TABLE)) { + return; + } + if (!this.vector.enabled || this.vector.available !== true) { + this.markVectorRebuildRequired(); + return; + } + try { + this.db + .prepare( + `DELETE FROM ${VECTOR_TABLE} WHERE id IN ( + SELECT id FROM memory_index_chunks WHERE path = ? AND source = ? + )`, + ) + .run(pathname, source); + } catch { + this.markVectorRebuildRequired(); + } + } + + protected markVectorRebuildRequired(): void { + markMemoryVectorRebuildRequired(this.db); + } + + private hasVectorRebuildMarker(): boolean { + return requiresMemoryVectorRebuild({ + db: this.db, + vectorTable: VECTOR_TABLE, + metaVectorDims: this.readMeta()?.vectorDims, + hasSemanticChunks: this.hasSemanticChunks(), + }); + } + + private markConfiguredSourcesForFullReindex(): void { + // This flag selects the shadow-reindex path even for a sessions-only index; + // the rebuild itself still filters work through the configured sources. + this.memoryFullRetryDirty = true; + if (this.sources.has("memory")) { + this.dirty = true; + } + if (this.sources.has("sessions")) { + this.sessionsDirty = true; + this.sessionsFullRetryDirty = true; + } + } + private ensureVectorTable(dimensions: number): void { if (this.vector.dims === dimensions && memoryTableExists(this.db, VECTOR_TABLE)) { return; diff --git a/extensions/memory-core/src/memory/manager-sync-ops.startup-catchup.test.ts b/extensions/memory-core/src/memory/manager-sync-ops.startup-catchup.test.ts index 702fc7f0e3c2..673578c56153 100644 --- a/extensions/memory-core/src/memory/manager-sync-ops.startup-catchup.test.ts +++ b/extensions/memory-core/src/memory/manager-sync-ops.startup-catchup.test.ts @@ -212,6 +212,11 @@ class SessionStartupCatchupHarness extends MemoryManagerSyncOps { return this.sessionsDirty; } + markFullSessionRetry(): void { + this.sessionsDirty = true; + this.sessionsFullRetryDirty = true; + } + startTranscriptListener(): void { this.ensureSessionListener(); } @@ -389,6 +394,14 @@ describe("session startup catch-up", () => { expect(harness.syncCalls).toEqual([{ reason: "session-startup-catchup" }]); }); + it("schedules a full retry when invalidated sessions no longer exist", async () => { + const harness = new SessionStartupCatchupHarness([]); + harness.markFullSessionRetry(); + + await expect(harness.catchUp()).resolves.toEqual([]); + expect(harness.syncCalls).toEqual([{ reason: "session-startup-catchup" }]); + }); + it("retries transient session transcript reads during session indexing", async () => { const session = await writeSessionFile("thread.jsonl.deleted.2026-02-16T22-27-33.000Z"); const harness = new SessionStartupCatchupHarness([]); diff --git a/extensions/memory-core/src/memory/manager-sync-ops.ts b/extensions/memory-core/src/memory/manager-sync-ops.ts index 137f727d2b12..50236816819e 100644 --- a/extensions/memory-core/src/memory/manager-sync-ops.ts +++ b/extensions/memory-core/src/memory/manager-sync-ops.ts @@ -40,6 +40,7 @@ import { markMemoryTargetArchiveFilesDirty, runMemoryTargetedSessionSync, } from "./manager-targeted-sync.js"; +import { markMemoryVectorIndexClean } from "./manager-vector-rebuild-state.js"; export type { MemoryIndexWorkItem } from "./manager-sync-base.js"; @@ -444,9 +445,9 @@ export abstract class MemoryManagerSyncOps extends MemoryManagerSourceSyncOps { } } if (!shouldSyncMemory) { - this.dirty = false; + this.clearMemoryRetryState(); } - + const vectorIndexComplete = this.vector.available === true; const nextMeta: MemoryIndexMeta = { model: this.provider?.model ?? "fts-only", provider: this.provider?.id ?? "none", @@ -487,6 +488,11 @@ export abstract class MemoryManagerSyncOps extends MemoryManagerSourceSyncOps { }); this.db = originalDb; + if (vectorIndexComplete) { + // Publish completeness only after the shadow tables committed. A crash + // before this point leaves the rebuild marker conservative and retryable. + markMemoryVectorIndexClean(originalDb); + } this.resetVectorState(); this.fts.available = nextFtsState.available; this.fts.loadError = nextFtsState.loadError; diff --git a/extensions/memory-core/src/memory/manager-vector-rebuild-state.ts b/extensions/memory-core/src/memory/manager-vector-rebuild-state.ts new file mode 100644 index 000000000000..abea46596f14 --- /dev/null +++ b/extensions/memory-core/src/memory/manager-vector-rebuild-state.ts @@ -0,0 +1,45 @@ +// Memory Core plugin module owns persisted vector completeness state. +import type { DatabaseSync } from "node:sqlite"; +import { MEMORY_INDEX_META_TABLE } from "openclaw/plugin-sdk/memory-core-host-engine-storage"; + +const VECTOR_REBUILD_META_KEY = "memory_vector_rebuild_v1"; + +function vectorTableExists(db: DatabaseSync, tableName: string): boolean { + return Boolean( + db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(tableName), + ); +} + +export function markMemoryVectorIndexClean(db: DatabaseSync): void { + db.prepare( + `INSERT INTO ${MEMORY_INDEX_META_TABLE} (key, value) VALUES (?, 'clean') + ON CONFLICT(key) DO UPDATE SET value=excluded.value`, + ).run(VECTOR_REBUILD_META_KEY); +} + +export function markMemoryVectorRebuildRequired(db: DatabaseSync): void { + db.prepare( + `INSERT INTO ${MEMORY_INDEX_META_TABLE} (key, value) VALUES (?, '1') + ON CONFLICT(key) DO UPDATE SET value=excluded.value`, + ).run(VECTOR_REBUILD_META_KEY); +} + +export function requiresMemoryVectorRebuild(params: { + db: DatabaseSync; + vectorTable: string; + metaVectorDims?: number; + hasSemanticChunks: boolean; +}): boolean { + const row = params.db + .prepare(`SELECT value FROM ${MEMORY_INDEX_META_TABLE} WHERE key = ?`) + .get(VECTOR_REBUILD_META_KEY) as { value?: unknown } | undefined; + if (row?.value === "1") { + return true; + } + if (!vectorTableExists(params.db, params.vectorTable)) { + return Boolean(params.metaVectorDims && params.hasSemanticChunks); + } + // Existing releases had no completeness marker. Rebuild their vector table + // once rather than assuming it has neither missing nor orphaned rows. + return row?.value !== "clean"; +} diff --git a/extensions/memory-core/src/memory/manager.legacy-migration-cleanup.test.ts b/extensions/memory-core/src/memory/manager.legacy-migration-cleanup.test.ts new file mode 100644 index 000000000000..86e556e3dafb --- /dev/null +++ b/extensions/memory-core/src/memory/manager.legacy-migration-cleanup.test.ts @@ -0,0 +1,272 @@ +// Memory Core tests cover deleted-file cleanup after same-file legacy migration. +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/memory-core-host-engine-foundation"; +import { + ensureMemoryIndexSchema, + loadSqliteVecExtension, +} from "openclaw/plugin-sdk/memory-core-host-engine-storage"; +import { resolveOpenClawAgentSqlitePath } from "openclaw/plugin-sdk/sqlite-runtime"; +import { + closeOpenClawAgentDatabasesForTest, + closeOpenClawStateDatabaseForTest, +} from "openclaw/plugin-sdk/sqlite-runtime-testing"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import "./test-runtime-mocks.js"; +import { closeAllMemorySearchManagers, getMemorySearchManager } from "./index.js"; +import type { MemoryIndexManager } from "./manager.js"; + +const originalStateDir = process.env.OPENCLAW_STATE_DIR; + +describe("memory legacy migration cleanup", () => { + let fixtureRoot = ""; + let workspaceDir = ""; + let manager: MemoryIndexManager | undefined; + + beforeEach(async () => { + fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-memory-migration-cleanup-")); + workspaceDir = path.join(fixtureRoot, "workspace"); + await fs.mkdir(path.join(workspaceDir, "memory"), { recursive: true }); + Reflect.set(process.env, "OPENCLAW_STATE_DIR", path.join(fixtureRoot, "state")); + }); + + afterEach(async () => { + await manager?.close(); + manager = undefined; + await closeAllMemorySearchManagers(); + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + if (originalStateDir === undefined) { + Reflect.deleteProperty(process.env, "OPENCLAW_STATE_DIR"); + } else { + Reflect.set(process.env, "OPENCLAW_STATE_DIR", originalStateDir); + } + await fs.rm(fixtureRoot, { recursive: true, force: true }); + }); + + it("removes migrated chunks and FTS rows when the dirty source file is already deleted", async () => { + const dbPath = resolveOpenClawAgentSqlitePath({ agentId: "main" }); + await fs.mkdir(path.dirname(dbPath), { recursive: true }); + const seedDb = new DatabaseSync(dbPath, { allowExtension: true }); + let vectorExtensionPath: string | undefined; + try { + const loaded = await loadSqliteVecExtension({ db: seedDb }); + expect(loaded.ok, loaded.error).toBe(true); + vectorExtensionPath = loaded.extensionPath; + ensureMemoryIndexSchema({ db: seedDb, cacheEnabled: false, ftsEnabled: true }); + seedDb.exec(` + INSERT INTO memory_index_sources (path, source, hash, mtime, size) + VALUES + ('memory/deleted.md', 'memory', 'canonical-hash', 200, 20), + ('sessions/excluded.jsonl', 'sessions', '', 200, 20); + INSERT INTO memory_index_chunks VALUES ( + 'chunk-canonical', 'memory/deleted.md', 'memory', 1, 2, 'canonical-chunk-hash', + 'fts-only', 'obsolete saffronquasar', '[]', 200 + ); + INSERT INTO memory_index_chunks VALUES ( + 'chunk-ownerless', 'memory/ownerless.md', 'memory', 1, 2, 'ownerless-chunk-hash', + 'fts-only', 'obsolete ambercomet', '[]', 190 + ); + INSERT INTO memory_index_chunks_fts + (text, id, path, source, model, start_line, end_line) + VALUES + ( + 'obsolete saffronquasar', 'chunk-canonical', 'memory/deleted.md', + 'memory', 'fts-only', 1, 2 + ), + ( + 'obsolete ambercomet', 'chunk-ownerless', 'memory/ownerless.md', + 'memory', 'fts-only', 1, 2 + ); + CREATE VIRTUAL TABLE memory_index_chunks_vec USING vec0( + id TEXT PRIMARY KEY, + embedding FLOAT[3] + ); + INSERT INTO memory_index_chunks_vec VALUES ('chunk-canonical', '[1,0,0]'); + INSERT INTO memory_index_chunks_vec VALUES ('chunk-ownerless', '[0,1,0]'); + + CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL); + CREATE TABLE files ( + path TEXT PRIMARY KEY, + source TEXT NOT NULL DEFAULT 'memory', + hash TEXT NOT NULL, + mtime INTEGER NOT NULL, + size INTEGER NOT NULL + ); + CREATE TABLE chunks ( + id TEXT PRIMARY KEY, + path TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'memory', + start_line INTEGER NOT NULL, + end_line INTEGER NOT NULL, + hash TEXT NOT NULL, + model TEXT NOT NULL, + text TEXT NOT NULL, + embedding TEXT NOT NULL, + updated_at INTEGER NOT NULL + ); + INSERT INTO files VALUES ( + 'memory/deleted.md', 'memory', 'legacy-hash', 100, 10 + ); + INSERT INTO chunks VALUES ( + 'chunk-legacy-extra', 'memory/deleted.md', 'memory', 3, 4, 'legacy-chunk-hash', + 'fts-only', 'stale legacy tail', '[]', 100 + ); + `); + expect(seedDb.prepare("SELECT COUNT(*) AS count FROM memory_index_chunks_vec").get()).toEqual( + { count: 2 }, + ); + } finally { + seedDb.close(); + } + + const createConfig = (params: { + extensionPath?: string; + provider: "none" | "openai"; + vectorEnabled: boolean; + }) => + ({ + memory: { backend: "builtin" }, + agents: { + defaults: { + workspace: workspaceDir, + memorySearch: { + provider: params.provider, + model: params.provider === "none" ? "" : "text-embedding-3-small", + store: { + vector: { + enabled: params.vectorEnabled, + ...(params.extensionPath ? { extensionPath: params.extensionPath } : {}), + }, + }, + cache: { enabled: false }, + sync: { watch: false, onSessionStart: false, onSearch: false }, + query: { hybrid: { enabled: true } }, + }, + }, + list: [{ id: "main", default: true }], + }, + }) as OpenClawConfig; + const cfg = createConfig({ provider: "none", vectorEnabled: false }); + const result = await getMemorySearchManager({ cfg, agentId: "main" }); + if (!result.manager) { + throw new Error(result.error ?? "memory manager missing"); + } + manager = result.manager as unknown as MemoryIndexManager; + expect(manager.status().fts?.available).toBe(true); + expect(Reflect.get(manager, "sessionsFullRetryDirty")).toBe(false); + + const db = Reflect.get(manager, "db") as DatabaseSync; + expect( + db.prepare("SELECT hash FROM memory_index_sources WHERE path = 'memory/deleted.md'").get(), + ).toEqual({ hash: "" }); + expect( + db.prepare("SELECT hash FROM memory_index_sources WHERE path = 'memory/ownerless.md'").get(), + ).toEqual({ hash: "" }); + expect( + db + .prepare("SELECT COUNT(*) AS count FROM memory_index_chunks WHERE path = ?") + .get("memory/deleted.md"), + ).toEqual({ count: 1 }); + expect( + db + .prepare("SELECT COUNT(*) AS count FROM memory_index_chunks_fts WHERE path = ?") + .get("memory/deleted.md"), + ).toEqual({ count: 1 }); + expect( + db + .prepare("SELECT COUNT(*) AS count FROM memory_index_chunks_fts WHERE path = ?") + .get("memory/ownerless.md"), + ).toEqual({ count: 1 }); + + await ( + manager as unknown as { + syncMemoryFiles(params: { needsFullReindex: boolean }): Promise; + } + ).syncMemoryFiles({ needsFullReindex: false }); + + expect( + db + .prepare("SELECT COUNT(*) AS count FROM memory_index_sources WHERE path = ?") + .get("memory/deleted.md"), + ).toEqual({ count: 0 }); + expect( + db + .prepare("SELECT COUNT(*) AS count FROM memory_index_chunks WHERE path = ?") + .get("memory/deleted.md"), + ).toEqual({ count: 0 }); + expect( + db + .prepare("SELECT COUNT(*) AS count FROM memory_index_chunks_fts WHERE path = ?") + .get("memory/deleted.md"), + ).toEqual({ count: 0 }); + expect( + db + .prepare("SELECT COUNT(*) AS count FROM memory_index_sources WHERE path = ?") + .get("memory/ownerless.md"), + ).toEqual({ count: 0 }); + expect( + db + .prepare("SELECT COUNT(*) AS count FROM memory_index_chunks WHERE path = ?") + .get("memory/ownerless.md"), + ).toEqual({ count: 0 }); + expect( + db + .prepare("SELECT COUNT(*) AS count FROM memory_index_chunks_fts WHERE path = ?") + .get("memory/ownerless.md"), + ).toEqual({ count: 0 }); + // Cleanup ran while vectors were disabled. Keep the old table untouched and + // persist a rebuild marker; one-sided orphan pruning would still miss vector + // rows that should exist but were never written. + expect( + db + .prepare("SELECT value FROM memory_index_meta WHERE key = 'memory_vector_rebuild_v1'") + .get(), + ).toEqual({ value: "1" }); + const observerDb = new DatabaseSync(dbPath, { allowExtension: true }); + try { + const observerLoaded = await loadSqliteVecExtension({ + db: observerDb, + extensionPath: vectorExtensionPath, + }); + expect(observerLoaded.ok, observerLoaded.error).toBe(true); + expect( + observerDb.prepare("SELECT COUNT(*) AS count FROM memory_index_chunks_vec").get(), + ).toEqual({ count: 2 }); + } finally { + observerDb.close(); + } + await closeAllMemorySearchManagers(); + manager = undefined; + + const reloadResult = await getMemorySearchManager({ + cfg: createConfig({ + extensionPath: vectorExtensionPath, + provider: "openai", + vectorEnabled: true, + }), + agentId: "main", + }); + if (!reloadResult.manager) { + throw new Error(reloadResult.error ?? "reloaded memory manager missing"); + } + manager = reloadResult.manager as unknown as MemoryIndexManager; + const reloadedDb = Reflect.get(manager, "db") as DatabaseSync; + await expect( + ( + manager as unknown as { + loadVectorExtension(): Promise; + } + ).loadVectorExtension(), + ).resolves.toBe(false); + expect(reloadedDb.prepare("SELECT vec_version() AS version").get()).toEqual({ + version: expect.any(String), + }); + expect( + reloadedDb.prepare("SELECT COUNT(*) AS count FROM memory_index_chunks_vec").get(), + ).toEqual({ count: 2 }); + expect(Reflect.get(manager, "memoryFullRetryDirty")).toBe(true); + }); +}); diff --git a/extensions/memory-core/src/memory/manager.ts b/extensions/memory-core/src/memory/manager.ts index 2e31e9569145..795e719bb2a9 100644 --- a/extensions/memory-core/src/memory/manager.ts +++ b/extensions/memory-core/src/memory/manager.ts @@ -517,11 +517,29 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem this.ensureSessionListener(); this.ensureIntervalSync(); } - this.dirty = resolveInitialMemoryDirty({ - hasMemorySource: this.sources.has("memory"), - statusOnly: params.purpose === "status", - hasIndexedMeta: Boolean(meta), - }); + const invalidatedSources = new Set( + ( + this.db + .prepare("SELECT DISTINCT source FROM memory_index_sources WHERE hash = ''") + .all() as Array<{ source?: unknown }> + ).flatMap((row) => + row.source === "memory" || row.source === "sessions" ? [row.source] : [], + ), + ); + this.dirty = + resolveInitialMemoryDirty({ + hasMemorySource: this.sources.has("memory"), + statusOnly: params.purpose === "status", + hasIndexedMeta: Boolean(meta), + }) || + (this.sources.has("memory") && invalidatedSources.has("memory")); + if (this.sources.has("sessions") && invalidatedSources.has("sessions")) { + // Migration cannot map a durable session source path back to one live + // transcript file. Carry a full-session retry so unchanged and deleted + // transcripts both converge on the next startup/search sync. + this.sessionsDirty = true; + this.sessionsFullRetryDirty = true; + } this.batch = this.resolveBatchConfig(); if (!transient) { this.ensureSessionStartupCatchup(); diff --git a/packages/memory-host-sdk/src/host/memory-schema-fts.ts b/packages/memory-host-sdk/src/host/memory-schema-fts.ts new file mode 100644 index 000000000000..30190a6cf741 --- /dev/null +++ b/packages/memory-host-sdk/src/host/memory-schema-fts.ts @@ -0,0 +1,113 @@ +// Memory Host SDK module owns derived FTS schema and rebuild behavior. +import type { DatabaseSync } from "node:sqlite"; + +export const MEMORY_INDEX_SOURCES_TABLE = "memory_index_sources"; +export const MEMORY_INDEX_CHUNKS_TABLE = "memory_index_chunks"; +export const MEMORY_INDEX_FTS_TABLE = "memory_index_chunks_fts"; +export const MEMORY_INDEX_PATHS_FTS_TABLE = "memory_index_paths_fts"; + +/** Optional canonical triggers owned by the derived path FTS index. */ +export const MEMORY_PATH_FTS_TRIGGER_DEFINITIONS = [ + { + name: "memory_index_paths_fts_after_insert", + sql: ` + CREATE TRIGGER IF NOT EXISTS main.memory_index_paths_fts_after_insert + AFTER INSERT ON ${MEMORY_INDEX_SOURCES_TABLE} + BEGIN + INSERT INTO ${MEMORY_INDEX_PATHS_FTS_TABLE} (rowid, path, source) + VALUES (NEW.id, NEW.path, NEW.source); + END; + `, + }, + { + name: "memory_index_paths_fts_after_update", + sql: ` + CREATE TRIGGER IF NOT EXISTS main.memory_index_paths_fts_after_update + AFTER UPDATE OF id, path, source ON ${MEMORY_INDEX_SOURCES_TABLE} + BEGIN + DELETE FROM ${MEMORY_INDEX_PATHS_FTS_TABLE} + WHERE rowid = OLD.id; + INSERT INTO ${MEMORY_INDEX_PATHS_FTS_TABLE} (rowid, path, source) + VALUES (NEW.id, NEW.path, NEW.source); + END; + `, + }, + { + name: "memory_index_paths_fts_after_delete", + sql: ` + CREATE TRIGGER IF NOT EXISTS main.memory_index_paths_fts_after_delete + AFTER DELETE ON ${MEMORY_INDEX_SOURCES_TABLE} + BEGIN + DELETE FROM ${MEMORY_INDEX_PATHS_FTS_TABLE} + WHERE rowid = OLD.id; + END; + `, + }, +] as const; + +export function rebuildMemoryChunkFts(db: DatabaseSync, ftsTable: string): void { + db.exec(` + DELETE FROM ${ftsTable}; + INSERT INTO ${ftsTable} ( + text, id, path, source, model, start_line, end_line + ) + SELECT text, id, path, source, model, start_line, end_line + FROM ${MEMORY_INDEX_CHUNKS_TABLE}; + `); +} + +export function dropDisabledMemoryChunkFts( + db: DatabaseSync, + ftsTable: string, + enabled: boolean, +): void { + if (!enabled && ftsTable === MEMORY_INDEX_FTS_TABLE) { + // Body FTS has no maintenance triggers while disabled. Recreate it from + // canonical chunks on enable instead of retaining a partial derived index. + db.exec(`DROP TABLE IF EXISTS ${ftsTable}`); + } +} + +/** Drop the canonical source-to-path-FTS maintenance triggers. */ +export function dropMemoryPathFtsTriggers(db: DatabaseSync): void { + for (const trigger of MEMORY_PATH_FTS_TRIGGER_DEFINITIONS) { + db.exec(`DROP TRIGGER IF EXISTS main.${trigger.name}`); + } +} + +/** Install the canonical source-to-path-FTS maintenance triggers. */ +export function ensureMemoryPathFtsTriggers(db: DatabaseSync): void { + // The named integer source identity survives VACUUM and gives every + // FTS update/delete a direct rowid lookup instead of a virtual-table scan. + for (const trigger of MEMORY_PATH_FTS_TRIGGER_DEFINITIONS) { + db.exec(trigger.sql); + } +} + +export function ensureMemoryPathFtsSchema(params: { + db: DatabaseSync; + tokenizeClause: string; +}): void { + params.db.exec("SAVEPOINT ensure_memory_index_paths_fts"); + try { + params.db.exec(` + CREATE VIRTUAL TABLE IF NOT EXISTS ${MEMORY_INDEX_PATHS_FTS_TABLE} USING fts5( + path, + source UNINDEXED + ${params.tokenizeClause} + ); + -- The initial copy and trigger installation share this savepoint. Once + -- populated, the triggers own completeness; per-row FTS probes are too costly. + INSERT INTO ${MEMORY_INDEX_PATHS_FTS_TABLE} (rowid, path, source) + SELECT id, path, source + FROM ${MEMORY_INDEX_SOURCES_TABLE} + WHERE NOT EXISTS (SELECT 1 FROM ${MEMORY_INDEX_PATHS_FTS_TABLE} LIMIT 1); + `); + ensureMemoryPathFtsTriggers(params.db); + params.db.exec("RELEASE ensure_memory_index_paths_fts"); + } catch (err) { + params.db.exec("ROLLBACK TO ensure_memory_index_paths_fts"); + params.db.exec("RELEASE ensure_memory_index_paths_fts"); + throw err; + } +} diff --git a/packages/memory-host-sdk/src/host/memory-schema-migration.ts b/packages/memory-host-sdk/src/host/memory-schema-migration.ts new file mode 100644 index 000000000000..06baadb12067 --- /dev/null +++ b/packages/memory-host-sdk/src/host/memory-schema-migration.ts @@ -0,0 +1,28 @@ +// Memory Host SDK module owns temporary indexes used by legacy schema repair. +import type { DatabaseSync } from "node:sqlite"; + +// Same-identity values may diverge because canonical derived rows win. A row +// that cannot be represented canonically still aborts before legacy tables drop. +export function assertLegacyMemoryRowsCopied( + db: DatabaseSync, + query: string, + tableName: string, +): void { + const row = db.prepare(query).get() as { missing?: unknown } | undefined; + if (Number(row?.missing ?? 0) > 0) { + throw new Error( + `legacy memory ${tableName} rows could not be copied into canonical memory index rows`, + ); + } +} + +export function ensureLegacyMemoryMigrationIndexes(db: DatabaseSync, schema: string): void { + // Shipped legacy tables index chunk ids only, while ownership repair joins by + // path/source. These indexes disappear with the legacy tables after migration. + db.exec(` + CREATE INDEX IF NOT EXISTS ${schema}.memory_legacy_files_path_source_migration + ON files(path, source); + CREATE INDEX IF NOT EXISTS ${schema}.memory_legacy_chunks_path_source_migration + ON chunks(path, source); + `); +} diff --git a/packages/memory-host-sdk/src/host/memory-schema.legacy-migration.test.ts b/packages/memory-host-sdk/src/host/memory-schema.legacy-migration.test.ts new file mode 100644 index 000000000000..64d0953d3c23 --- /dev/null +++ b/packages/memory-host-sdk/src/host/memory-schema.legacy-migration.test.ts @@ -0,0 +1,640 @@ +// Same-file legacy migration tests cover conflict recovery and rollback boundaries. +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { describe, expect, it } from "vitest"; +import { ensureMemoryIndexSchema } from "./memory-schema.js"; + +describe("memory index same-file legacy migration", () => { + it("leaves unrelated generic tables untouched", () => { + const db = new DatabaseSync(":memory:"); + try { + db.exec(` + CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL, owner TEXT); + CREATE TABLE files ( + path TEXT PRIMARY KEY, + source TEXT NOT NULL, + hash TEXT NOT NULL, + mtime INTEGER NOT NULL, + size INTEGER NOT NULL + ); + CREATE TABLE chunks ( + id TEXT PRIMARY KEY, + path TEXT NOT NULL, + source TEXT NOT NULL, + start_line INTEGER NOT NULL, + end_line INTEGER NOT NULL, + hash TEXT NOT NULL, + model TEXT NOT NULL, + text TEXT NOT NULL, + embedding TEXT NOT NULL, + updated_at INTEGER NOT NULL + ); + `); + + ensureMemoryIndexSchema({ + db, + cacheEnabled: false, + ftsEnabled: false, + }); + + expect( + db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name IN ('meta', 'files', 'chunks') ORDER BY name", + ) + .all(), + ).toEqual([{ name: "chunks" }, { name: "files" }, { name: "meta" }]); + } finally { + db.close(); + } + }); + + it("recovers a partially migrated WAL index idempotently when legacy rows diverge", () => { + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-memory-diverged-")); + const dbPath = path.join(rootDir, "openclaw-agent.sqlite"); + const db = new DatabaseSync(dbPath); + try { + expect(db.prepare("PRAGMA journal_mode = WAL").get()).toEqual({ journal_mode: "wal" }); + ensureMemoryIndexSchema({ db, cacheEnabled: true, ftsEnabled: false }); + db.exec(` + INSERT INTO memory_index_meta VALUES ('memory_index_meta_v1', 'canonical'); + INSERT INTO memory_index_sources (path, source, hash, mtime, size) + VALUES ('doc.md', 'memory', 'new-hash', 200.0, 42); + INSERT INTO memory_index_chunks VALUES ( + 'chunk-new-1', 'doc.md', 'memory', 1, 10, 'new-chunk-hash', 'model', + 'current canonical body', '[1,2]', 200 + ); + INSERT INTO memory_embedding_cache VALUES ( + 'openai', 'model', 'key', 'new-chunk-hash', '[1,2]', 2, 200 + ); + `); + // This is the partial migration state seen in affected databases: canonical + // tables already contain current rows while the same file still has legacy tables. + db.exec(` + CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL); + CREATE TABLE files ( + path TEXT PRIMARY KEY, + source TEXT NOT NULL DEFAULT 'memory', + hash TEXT NOT NULL, + mtime INTEGER NOT NULL, + size INTEGER NOT NULL + ); + CREATE TABLE chunks ( + id TEXT PRIMARY KEY, + path TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'memory', + start_line INTEGER NOT NULL, + end_line INTEGER NOT NULL, + hash TEXT NOT NULL, + model TEXT NOT NULL, + text TEXT NOT NULL, + embedding TEXT NOT NULL, + updated_at INTEGER NOT NULL + ); + CREATE TABLE embedding_cache ( + provider TEXT NOT NULL, + model TEXT NOT NULL, + provider_key TEXT NOT NULL, + hash TEXT NOT NULL, + embedding TEXT NOT NULL, + dims INTEGER, + updated_at INTEGER NOT NULL, + PRIMARY KEY (provider, model, provider_key, hash) + ); + INSERT INTO meta VALUES ('memory_index_meta_v1', 'legacy'); + INSERT INTO meta VALUES ('legacy-only-key', 'imported'); + INSERT INTO files VALUES ('doc.md', 'memory', 'old-hash', 100, 40); + INSERT INTO files VALUES ('old-note.md', 'memory', 'old-note-hash', 90, 10); + INSERT INTO chunks VALUES ( + 'chunk-new-1', 'doc.md', 'memory', 1, 8, 'old-chunk-hash', 'model', + 'same-key stale legacy body', '[9,9]', 100 + ); + INSERT INTO chunks VALUES ( + 'chunk-old-7', 'doc.md', 'memory', 9, 12, 'old-tail-hash', 'model', + 'distinct stale legacy tail', '[8,8]', 100 + ); + INSERT INTO chunks VALUES ( + 'chunk-old-2', 'old-note.md', 'memory', 1, 5, 'note-hash', 'model', + 'note body', '[]', 90 + ); + INSERT INTO embedding_cache VALUES ( + 'openai', 'model', 'key', 'new-chunk-hash', '[9,9,9]', 3, 100 + ); + INSERT INTO embedding_cache VALUES ( + 'openai', 'model', 'legacy-key', 'legacy-hash', '[3,4]', 2, 90 + ); + `); + + const readCanonicalState = () => ({ + meta: db.prepare("SELECT key, value FROM memory_index_meta ORDER BY key").all(), + sources: db + .prepare("SELECT path, hash, size FROM memory_index_sources ORDER BY path") + .all(), + chunks: db.prepare("SELECT id, text FROM memory_index_chunks ORDER BY id").all(), + cache: db + .prepare( + "SELECT provider_key, hash, embedding, dims, updated_at FROM memory_embedding_cache ORDER BY provider_key", + ) + .all(), + }); + const expectedCanonicalState = { + meta: [ + { key: "legacy-only-key", value: "imported" }, + { key: "memory_index_meta_v1", value: "canonical" }, + ], + // The extra legacy chunk identity makes doc.md's canonical chunk set + // ambiguous. Its impossible hash forces a rebuild while preserving the + // source identity needed to clean up a file deleted before migration. + sources: [ + { path: "doc.md", hash: "", size: 42 }, + { path: "old-note.md", hash: "", size: 10 }, + ], + chunks: [ + { id: "chunk-new-1", text: "current canonical body" }, + { id: "chunk-old-2", text: "note body" }, + ], + cache: [ + { + provider_key: "key", + hash: "new-chunk-hash", + embedding: "[1,2]", + dims: 2, + updated_at: 200, + }, + { + provider_key: "legacy-key", + hash: "legacy-hash", + embedding: "[3,4]", + dims: 2, + updated_at: 90, + }, + ], + }; + const readLegacyTables = () => + db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name IN ('meta', 'files', 'chunks', 'embedding_cache') ORDER BY name", + ) + .all(); + + expect(() => + ensureMemoryIndexSchema({ db, cacheEnabled: true, ftsEnabled: false }), + ).not.toThrow(); + expect(readCanonicalState()).toEqual(expectedCanonicalState); + expect(readLegacyTables()).toEqual([]); + + expect(() => + ensureMemoryIndexSchema({ db, cacheEnabled: true, ftsEnabled: false }), + ).not.toThrow(); + expect(readCanonicalState()).toEqual(expectedCanonicalState); + expect(readLegacyTables()).toEqual([]); + } finally { + db.close(); + fs.rmSync(rootDir, { recursive: true, force: true }); + } + }); + + it("rebuilds canonical FTS, rejects legacy orphans, and adopts canonical orphans", () => { + const db = new DatabaseSync(":memory:"); + try { + ensureMemoryIndexSchema({ db, cacheEnabled: false, ftsEnabled: true }); + db.exec(` + INSERT INTO memory_index_sources (path, source, hash, mtime, size) + VALUES ('canonical.md', 'memory', 'canonical-hash', 200, 20); + INSERT INTO memory_index_chunks VALUES ( + 'chunk-canonical', 'canonical.md', 'memory', 1, 2, 'canonical-chunk-hash', + 'fts-only', 'canonical nebula', '[]', 200 + ); + INSERT INTO memory_index_chunks VALUES ( + 'chunk-canonical-ownerless', 'deleted.md', 'memory', 1, 2, 'orphan-chunk-hash', + 'fts-only', 'orphaned starlight', '[]', 190 + ); + INSERT INTO memory_index_chunks_fts + (text, id, path, source, model, start_line, end_line) + VALUES + ('canonical nebula', 'chunk-canonical', 'canonical.md', 'memory', 'fts-only', 1, 2), + ('orphaned starlight', 'chunk-canonical-ownerless', 'deleted.md', 'memory', 'fts-only', 1, 2); + + CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL); + CREATE TABLE files ( + path TEXT PRIMARY KEY, + source TEXT NOT NULL DEFAULT 'memory', + hash TEXT NOT NULL, + mtime INTEGER NOT NULL, + size INTEGER NOT NULL + ); + CREATE TABLE chunks ( + id TEXT PRIMARY KEY, + path TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'memory', + start_line INTEGER NOT NULL, + end_line INTEGER NOT NULL, + hash TEXT NOT NULL, + model TEXT NOT NULL, + text TEXT NOT NULL, + embedding TEXT NOT NULL, + updated_at INTEGER NOT NULL + ); + INSERT INTO files VALUES ('legacy.md', 'memory', 'legacy-hash', 100, 10); + INSERT INTO chunks VALUES ( + 'chunk-legacy', 'legacy.md', 'memory', 1, 2, 'legacy-chunk-hash', + 'fts-only', 'imported saffronquasar', '[]', 100 + ); + -- Shipped legacy cleanup could commit its source delete before deleting + -- chunks. This derived orphan must not become canonical/searchable. + INSERT INTO chunks VALUES ( + 'chunk-ownerless', 'deleted.md', 'memory', 1, 2, 'orphan-chunk-hash', + 'fts-only', 'orphaned ambercomet', '[]', 90 + ); + `); + + ensureMemoryIndexSchema({ db, cacheEnabled: false, ftsEnabled: true }); + + expect(db.prepare("SELECT id FROM memory_index_chunks ORDER BY id").all()).toEqual([ + { id: "chunk-canonical" }, + { id: "chunk-canonical-ownerless" }, + { id: "chunk-legacy" }, + ]); + expect(db.prepare("SELECT id FROM memory_index_chunks_fts ORDER BY id").all()).toEqual([ + { id: "chunk-canonical" }, + { id: "chunk-canonical-ownerless" }, + { id: "chunk-legacy" }, + ]); + expect( + db + .prepare( + "SELECT id FROM memory_index_chunks_fts WHERE memory_index_chunks_fts MATCH 'saffronquasar'", + ) + .all(), + ).toEqual([{ id: "chunk-legacy" }]); + expect( + db + .prepare( + "SELECT id FROM memory_index_chunks_fts WHERE memory_index_chunks_fts MATCH 'starlight'", + ) + .all(), + ).toEqual([{ id: "chunk-canonical-ownerless" }]); + expect( + db + .prepare( + "SELECT id FROM memory_index_chunks_fts WHERE memory_index_chunks_fts MATCH 'ambercomet'", + ) + .all(), + ).toEqual([]); + // Schema migration cannot write the runtime-owned sqlite-vec table. The + // dirty source guarantees normal sync republishes every derived index. + expect( + db.prepare("SELECT hash FROM memory_index_sources WHERE path = 'legacy.md'").get(), + ).toEqual({ hash: "" }); + expect( + db.prepare("SELECT hash FROM memory_index_sources WHERE path = 'deleted.md'").get(), + ).toEqual({ hash: "" }); + + ensureMemoryIndexSchema({ db, cacheEnabled: false, ftsEnabled: true }); + expect(db.prepare("SELECT COUNT(*) AS count FROM memory_index_chunks_fts").get()).toEqual({ + count: 3, + }); + } finally { + db.close(); + } + }); + + it("keeps canonical chunk sets coherent and invalidates ambiguous partial sources", () => { + const db = new DatabaseSync(":memory:"); + try { + ensureMemoryIndexSchema({ db, cacheEnabled: false, ftsEnabled: false }); + db.exec(` + -- doc.md is already chunk-owned: its stale legacy chunk must not ride along. + INSERT INTO memory_index_sources (path, source, hash, mtime, size) + VALUES ('doc.md', 'memory', 'doc-hash', 200.0, 42); + INSERT INTO memory_index_chunks VALUES ( + 'chunk-doc-canonical', 'doc.md', 'memory', 1, 10, 'doc-chunk-hash', 'model', + 'canonical body', '[]', 200 + ); + -- partial.md has only the first canonical chunk. Seeing a second legacy + -- identity makes completeness ambiguous, so the source must reindex. + INSERT INTO memory_index_sources (path, source, hash, mtime, size) + VALUES ('partial.md', 'memory', 'partial-hash', 175.0, 30); + INSERT INTO memory_index_chunks VALUES ( + 'chunk-partial-1', 'partial.md', 'memory', 1, 5, 'partial-1-hash', 'model', + 'canonical first half', '[]', 175 + ); + -- pending.md has a canonical source row but no chunks yet (indexing + -- interrupted before its chunks were written): its legacy chunk is the + -- only searchable content and must import instead of being stranded. + INSERT INTO memory_index_sources (path, source, hash, mtime, size) + VALUES ('pending.md', 'memory', 'pending-hash', 150.0, 20); + -- diverged.md is also chunkless, but its canonical metadata is newer; + -- pairing its stale legacy chunks with that hash would wedge stale text. + INSERT INTO memory_index_sources (path, source, hash, mtime, size) + VALUES ('diverged.md', 'memory', 'current-hash', 250.0, 30); + `); + db.exec(` + CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL); + CREATE TABLE files ( + path TEXT PRIMARY KEY, + source TEXT NOT NULL DEFAULT 'memory', + hash TEXT NOT NULL, + mtime INTEGER NOT NULL, + size INTEGER NOT NULL + ); + CREATE TABLE chunks ( + id TEXT PRIMARY KEY, + path TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'memory', + start_line INTEGER NOT NULL, + end_line INTEGER NOT NULL, + hash TEXT NOT NULL, + model TEXT NOT NULL, + text TEXT NOT NULL, + embedding TEXT NOT NULL, + updated_at INTEGER NOT NULL + ); + INSERT INTO files VALUES ('doc.md', 'memory', 'doc-hash', 200, 42); + INSERT INTO files VALUES ('partial.md', 'memory', 'partial-hash', 175, 30); + INSERT INTO files VALUES ('pending.md', 'memory', 'pending-hash', 150, 20); + INSERT INTO files VALUES ('diverged.md', 'memory', 'stale-hash', 50, 12); + INSERT INTO chunks VALUES ( + 'chunk-doc-legacy', 'doc.md', 'memory', 1, 8, 'stale-hash', 'model', + 'stale legacy body', '[]', 100 + ); + INSERT INTO chunks VALUES ( + 'chunk-partial-1', 'partial.md', 'memory', 1, 5, 'legacy-partial-1-hash', 'model', + 'legacy first half', '[]', 150 + ); + INSERT INTO chunks VALUES ( + 'chunk-partial-2', 'partial.md', 'memory', 6, 10, 'partial-2-hash', 'model', + 'legacy second half', '[]', 150 + ); + INSERT INTO chunks VALUES ( + 'chunk-pending-legacy', 'pending.md', 'memory', 1, 6, 'pending-chunk-hash', 'model', + 'only searchable content for pending', '[]', 150 + ); + INSERT INTO chunks VALUES ( + 'chunk-diverged-legacy', 'diverged.md', 'memory', 1, 4, 'diverged-chunk-hash', 'model', + 'stale diverged content', '[]', 50 + ); + `); + + ensureMemoryIndexSchema({ + db, + cacheEnabled: false, + ftsEnabled: false, + }); + + // doc.md keeps only its canonical chunk (stale legacy chunk excluded); + // pending.md, whose canonical source had no chunks, imports its legacy + // chunk so the file is not left silently unsearchable. + expect(db.prepare("SELECT id, text FROM memory_index_chunks ORDER BY id").all()).toEqual([ + { id: "chunk-doc-canonical", text: "canonical body" }, + { id: "chunk-partial-1", text: "canonical first half" }, + { id: "chunk-pending-legacy", text: "only searchable content for pending" }, + ]); + // An impossible hash prevents hash-based sync from skipping these rows, + // while retaining the identities needed for deleted-file cleanup. + expect( + db.prepare("SELECT hash FROM memory_index_sources WHERE path = 'doc.md'").get(), + ).toEqual({ hash: "" }); + expect( + db.prepare("SELECT hash FROM memory_index_sources WHERE path = 'partial.md'").get(), + ).toEqual({ hash: "" }); + expect( + db.prepare("SELECT hash FROM memory_index_sources WHERE path = 'diverged.md'").get(), + ).toEqual({ hash: "" }); + expect( + db.prepare("SELECT hash FROM memory_index_sources WHERE path = 'pending.md'").get(), + ).toEqual({ hash: "" }); + expect( + db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name IN ('meta', 'files', 'chunks')", + ) + .all(), + ).toEqual([]); + } finally { + db.close(); + } + }); + + it("keeps legacy tables when a chunk id belongs to a different canonical source", () => { + const db = new DatabaseSync(":memory:"); + try { + ensureMemoryIndexSchema({ db, cacheEnabled: false, ftsEnabled: false }); + db.exec(` + INSERT INTO memory_index_sources (path, source, hash, mtime, size) + VALUES ('canonical.md', 'memory', 'canonical-hash', 200, 20); + INSERT INTO memory_index_chunks VALUES ( + 'shared-id', 'canonical.md', 'memory', 1, 2, 'canonical-chunk-hash', 'model', + 'canonical body', '[]', 200 + ); + CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL); + CREATE TABLE files ( + path TEXT PRIMARY KEY, + source TEXT NOT NULL DEFAULT 'memory', + hash TEXT NOT NULL, + mtime INTEGER NOT NULL, + size INTEGER NOT NULL + ); + CREATE TABLE chunks ( + id TEXT PRIMARY KEY, + path TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'memory', + start_line INTEGER NOT NULL, + end_line INTEGER NOT NULL, + hash TEXT NOT NULL, + model TEXT NOT NULL, + text TEXT NOT NULL, + embedding TEXT NOT NULL, + updated_at INTEGER NOT NULL + ); + INSERT INTO files VALUES ('legacy.md', 'memory', 'legacy-hash', 100, 10); + INSERT INTO chunks VALUES ( + 'shared-id', 'legacy.md', 'memory', 1, 2, 'legacy-chunk-hash', 'model', + 'legacy body', '[]', 100 + ); + `); + + expect(() => ensureMemoryIndexSchema({ db, cacheEnabled: false, ftsEnabled: false })).toThrow( + "legacy memory chunks rows could not be copied", + ); + expect(db.prepare("SELECT path, text FROM memory_index_chunks").all()).toEqual([ + { path: "canonical.md", text: "canonical body" }, + ]); + expect( + db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name IN ('meta', 'files', 'chunks') ORDER BY name", + ) + .all(), + ).toEqual([{ name: "chunks" }, { name: "files" }, { name: "meta" }]); + } finally { + db.close(); + } + }); + + it("keeps legacy tables when legacy rows cannot be copied", () => { + const db = new DatabaseSync(":memory:"); + try { + db.exec(` + CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL); + CREATE TABLE files ( + path TEXT PRIMARY KEY, + source TEXT NOT NULL DEFAULT 'memory', + hash TEXT, + mtime INTEGER NOT NULL, + size INTEGER NOT NULL + ); + CREATE TABLE chunks ( + id TEXT PRIMARY KEY, + path TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'memory', + start_line INTEGER NOT NULL, + end_line INTEGER NOT NULL, + hash TEXT NOT NULL, + model TEXT NOT NULL, + text TEXT NOT NULL, + embedding TEXT NOT NULL, + updated_at INTEGER NOT NULL + ); + INSERT INTO files VALUES ('doc.md', 'memory', NULL, 1, 2); + INSERT INTO chunks VALUES ( + 'chunk-doc', 'doc.md', 'memory', 1, 2, 'chunk-hash', 'model', + 'body', '[]', 1 + ); + `); + + expect(() => + ensureMemoryIndexSchema({ + db, + cacheEnabled: false, + ftsEnabled: false, + }), + ).toThrow("legacy memory files rows could not be copied"); + expect(db.prepare("SELECT path FROM files").get()).toEqual({ path: "doc.md" }); + expect(db.prepare("SELECT COUNT(*) AS count FROM memory_index_sources").get()).toEqual({ + count: 0, + }); + expect(db.prepare("SELECT COUNT(*) AS count FROM memory_index_chunks").get()).toEqual({ + count: 0, + }); + expect( + db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name IN ('meta', 'files', 'chunks') ORDER BY name", + ) + .all(), + ).toEqual([{ name: "chunks" }, { name: "files" }, { name: "meta" }]); + } finally { + db.close(); + } + }); + + it("keeps legacy tables when legacy meta rows cannot be copied", () => { + const db = new DatabaseSync(":memory:"); + try { + db.exec(` + CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT); + CREATE TABLE files ( + path TEXT PRIMARY KEY, + source TEXT NOT NULL DEFAULT 'memory', + hash TEXT NOT NULL, + mtime INTEGER NOT NULL, + size INTEGER NOT NULL + ); + CREATE TABLE chunks ( + id TEXT PRIMARY KEY, + path TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'memory', + start_line INTEGER NOT NULL, + end_line INTEGER NOT NULL, + hash TEXT NOT NULL, + model TEXT NOT NULL, + text TEXT NOT NULL, + embedding TEXT NOT NULL, + updated_at INTEGER NOT NULL + ); + INSERT INTO meta VALUES ('broken-key', NULL); + `); + + expect(() => + ensureMemoryIndexSchema({ + db, + cacheEnabled: false, + ftsEnabled: false, + }), + ).toThrow("legacy memory meta rows could not be copied"); + expect( + db + .prepare("SELECT COUNT(*) AS count FROM memory_index_meta WHERE key = 'broken-key'") + .get(), + ).toEqual({ count: 0 }); + expect( + db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name IN ('meta', 'files', 'chunks') ORDER BY name", + ) + .all(), + ).toEqual([{ name: "chunks" }, { name: "files" }, { name: "meta" }]); + } finally { + db.close(); + } + }); + + it("keeps legacy tables when legacy chunk rows cannot be copied", () => { + const db = new DatabaseSync(":memory:"); + try { + db.exec(` + CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL); + CREATE TABLE files ( + path TEXT PRIMARY KEY, + source TEXT NOT NULL DEFAULT 'memory', + hash TEXT NOT NULL, + mtime INTEGER NOT NULL, + size INTEGER NOT NULL + ); + CREATE TABLE chunks ( + id TEXT PRIMARY KEY, + path TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'memory', + start_line INTEGER NOT NULL, + end_line INTEGER NOT NULL, + hash TEXT NOT NULL, + model TEXT NOT NULL, + text TEXT NOT NULL, + embedding TEXT, + updated_at INTEGER NOT NULL + ); + INSERT INTO files VALUES ('note.md', 'memory', 'note-hash', 90, 10); + -- Legacy-only source (canonical owns no chunks for it), so this chunk + -- must copy; a NULL embedding makes it uncopyable under STRICT and the + -- whole migration must abort with legacy tables retained. + INSERT INTO chunks VALUES ( + 'chunk-broken', 'note.md', 'memory', 1, 5, 'chunk-hash', 'model', + 'body', NULL, 90 + ); + `); + + expect(() => + ensureMemoryIndexSchema({ + db, + cacheEnabled: false, + ftsEnabled: false, + }), + ).toThrow("legacy memory chunks rows could not be copied"); + expect(db.prepare("SELECT COUNT(*) AS count FROM memory_index_chunks").get()).toEqual({ + count: 0, + }); + expect( + db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name IN ('meta', 'files', 'chunks') ORDER BY name", + ) + .all(), + ).toEqual([{ name: "chunks" }, { name: "files" }, { name: "meta" }]); + } finally { + db.close(); + } + }); +}); diff --git a/packages/memory-host-sdk/src/host/memory-schema.test.ts b/packages/memory-host-sdk/src/host/memory-schema.test.ts index f8d7430af4a3..7e4506ed5245 100644 --- a/packages/memory-host-sdk/src/host/memory-schema.test.ts +++ b/packages/memory-host-sdk/src/host/memory-schema.test.ts @@ -71,7 +71,7 @@ describe("memory index schema", () => { id: 1, path: "MEMORY.md", source: "memory", - hash: "file-hash", + hash: "", mtime: 10.75, size: 20, }, @@ -283,6 +283,47 @@ describe("memory index schema", () => { } }); + it("rebuilds body FTS after indexing while hybrid search is disabled", () => { + const db = new DatabaseSync(":memory:"); + try { + ensureMemoryIndexSchema({ db, cacheEnabled: false, ftsEnabled: true }); + db.exec(` + INSERT INTO memory_index_chunks VALUES ( + 'chunk-before', 'before.md', 'memory', 1, 1, 'before-hash', 'fts-only', + 'before body', '[]', 1 + ); + INSERT INTO memory_index_chunks_fts + (text, id, path, source, model, start_line, end_line) + VALUES ('before body', 'chunk-before', 'before.md', 'memory', 'fts-only', 1, 1); + `); + + ensureMemoryIndexSchema({ db, cacheEnabled: false, ftsEnabled: false }); + expect( + db + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'memory_index_chunks_fts'", + ) + .get(), + ).toBeUndefined(); + db.exec(` + INSERT INTO memory_index_chunks VALUES ( + 'chunk-disabled', 'disabled.md', 'memory', 1, 1, 'disabled-hash', 'fts-only', + 'disabled body', '[]', 2 + ); + `); + + expect( + ensureMemoryIndexSchema({ db, cacheEnabled: false, ftsEnabled: true }).ftsAvailable, + ).toBe(true); + expect(db.prepare("SELECT id, text FROM memory_index_chunks_fts ORDER BY id").all()).toEqual([ + { id: "chunk-before", text: "before body" }, + { id: "chunk-disabled", text: "disabled body" }, + ]); + } finally { + db.close(); + } + }); + it("backfills and maintains one path FTS row per source without changing body FTS", () => { const db = new DatabaseSync(":memory:"); try { @@ -835,93 +876,4 @@ describe("memory index schema", () => { db.close(); } }); - - it("leaves unrelated generic tables untouched", () => { - const db = new DatabaseSync(":memory:"); - try { - db.exec(` - CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL, owner TEXT); - CREATE TABLE files ( - path TEXT PRIMARY KEY, - source TEXT NOT NULL, - hash TEXT NOT NULL, - mtime INTEGER NOT NULL, - size INTEGER NOT NULL - ); - CREATE TABLE chunks ( - id TEXT PRIMARY KEY, - path TEXT NOT NULL, - source TEXT NOT NULL, - start_line INTEGER NOT NULL, - end_line INTEGER NOT NULL, - hash TEXT NOT NULL, - model TEXT NOT NULL, - text TEXT NOT NULL, - embedding TEXT NOT NULL, - updated_at INTEGER NOT NULL - ); - `); - - ensureMemoryIndexSchema({ - db, - cacheEnabled: false, - ftsEnabled: false, - }); - - expect( - db - .prepare( - "SELECT name FROM sqlite_master WHERE type = 'table' AND name IN ('meta', 'files', 'chunks') ORDER BY name", - ) - .all(), - ).toEqual([{ name: "chunks" }, { name: "files" }, { name: "meta" }]); - } finally { - db.close(); - } - }); - - it("keeps legacy tables when canonical rows conflict", () => { - const db = new DatabaseSync(":memory:"); - try { - db.exec(` - CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL); - CREATE TABLE files ( - path TEXT PRIMARY KEY, - source TEXT NOT NULL DEFAULT 'memory', - hash TEXT NOT NULL, - mtime INTEGER NOT NULL, - size INTEGER NOT NULL - ); - CREATE TABLE chunks ( - id TEXT PRIMARY KEY, - path TEXT NOT NULL, - source TEXT NOT NULL DEFAULT 'memory', - start_line INTEGER NOT NULL, - end_line INTEGER NOT NULL, - hash TEXT NOT NULL, - model TEXT NOT NULL, - text TEXT NOT NULL, - embedding TEXT NOT NULL, - updated_at INTEGER NOT NULL - ); - CREATE TABLE memory_index_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL); - INSERT INTO meta VALUES ('memory_index_meta_v1', 'legacy'); - INSERT INTO memory_index_meta VALUES ('memory_index_meta_v1', 'canonical'); - `); - - expect(() => - ensureMemoryIndexSchema({ - db, - cacheEnabled: false, - ftsEnabled: false, - }), - ).toThrow("legacy memory meta rows conflict"); - expect(db.prepare("SELECT value FROM meta").get()).toEqual({ value: "legacy" }); - expect(db.prepare("SELECT value FROM memory_index_meta").get()).toEqual({ - value: "canonical", - }); - } finally { - db.close(); - } - }); }); diff --git a/packages/memory-host-sdk/src/host/memory-schema.ts b/packages/memory-host-sdk/src/host/memory-schema.ts index a79b2e4a3835..61a329bf4d65 100644 --- a/packages/memory-host-sdk/src/host/memory-schema.ts +++ b/packages/memory-host-sdk/src/host/memory-schema.ts @@ -1,58 +1,40 @@ // Memory Host SDK module implements memory schema behavior. import type { DatabaseSync } from "node:sqlite"; import { formatErrorMessage } from "./error-utils.js"; +import { + dropDisabledMemoryChunkFts, + dropMemoryPathFtsTriggers, + ensureMemoryPathFtsSchema, + ensureMemoryPathFtsTriggers, + MEMORY_INDEX_CHUNKS_TABLE, + MEMORY_INDEX_FTS_TABLE, + MEMORY_INDEX_PATHS_FTS_TABLE, + MEMORY_INDEX_SOURCES_TABLE, + rebuildMemoryChunkFts, +} from "./memory-schema-fts.js"; +import { + assertLegacyMemoryRowsCopied, + ensureLegacyMemoryMigrationIndexes, +} from "./memory-schema-migration.js"; import { migrateSqliteSchemaToStrict } from "./openclaw-runtime-sqlite.js"; +export { + dropMemoryPathFtsTriggers, + ensureMemoryPathFtsTriggers, + MEMORY_INDEX_CHUNKS_TABLE, + MEMORY_INDEX_FTS_TABLE, + MEMORY_INDEX_PATHS_FTS_TABLE, + MEMORY_INDEX_SOURCES_TABLE, + MEMORY_PATH_FTS_TRIGGER_DEFINITIONS, +} from "./memory-schema-fts.js"; + // SQLite schema setup for builtin memory index, embedding cache, and FTS. export const MEMORY_INDEX_META_TABLE = "memory_index_meta"; -export const MEMORY_INDEX_SOURCES_TABLE = "memory_index_sources"; -export const MEMORY_INDEX_CHUNKS_TABLE = "memory_index_chunks"; export const MEMORY_EMBEDDING_CACHE_TABLE = "memory_embedding_cache"; export const MEMORY_INDEX_STATE_TABLE = "memory_index_state"; -export const MEMORY_INDEX_FTS_TABLE = "memory_index_chunks_fts"; -export const MEMORY_INDEX_PATHS_FTS_TABLE = "memory_index_paths_fts"; export const MEMORY_INDEX_VECTOR_TABLE = "memory_index_chunks_vec"; -/** Optional canonical triggers owned by the derived path FTS index. */ -export const MEMORY_PATH_FTS_TRIGGER_DEFINITIONS = [ - { - name: "memory_index_paths_fts_after_insert", - sql: ` - CREATE TRIGGER IF NOT EXISTS main.memory_index_paths_fts_after_insert - AFTER INSERT ON ${MEMORY_INDEX_SOURCES_TABLE} - BEGIN - INSERT INTO ${MEMORY_INDEX_PATHS_FTS_TABLE} (rowid, path, source) - VALUES (NEW.id, NEW.path, NEW.source); - END; - `, - }, - { - name: "memory_index_paths_fts_after_update", - sql: ` - CREATE TRIGGER IF NOT EXISTS main.memory_index_paths_fts_after_update - AFTER UPDATE OF id, path, source ON ${MEMORY_INDEX_SOURCES_TABLE} - BEGIN - DELETE FROM ${MEMORY_INDEX_PATHS_FTS_TABLE} - WHERE rowid = OLD.id; - INSERT INTO ${MEMORY_INDEX_PATHS_FTS_TABLE} (rowid, path, source) - VALUES (NEW.id, NEW.path, NEW.source); - END; - `, - }, - { - name: "memory_index_paths_fts_after_delete", - sql: ` - CREATE TRIGGER IF NOT EXISTS main.memory_index_paths_fts_after_delete - AFTER DELETE ON ${MEMORY_INDEX_SOURCES_TABLE} - BEGIN - DELETE FROM ${MEMORY_INDEX_PATHS_FTS_TABLE} - WHERE rowid = OLD.id; - END; - `, - }, -] as const; - const LEGACY_MEMORY_INDEX_TRIGGERS = [ "memory_files_revision_after_insert", "memory_files_revision_after_update", @@ -242,13 +224,6 @@ function tableExists(db: DatabaseSync, tableName: string): boolean { return row?.found === 1; } -function assertLegacyRowsCopied(db: DatabaseSync, query: string, tableName: string): void { - const row = db.prepare(query).get() as { missing?: unknown } | undefined; - if (Number(row?.missing ?? 0) > 0) { - throw new Error(`legacy memory ${tableName} rows conflict with canonical memory index rows`); - } -} - /** Upgrade canonical memory sources to stable integer identities. */ export function migrateMemoryIndexSourcesIdentity(db: DatabaseSync): void { if (!tableExists(db, MEMORY_INDEX_SOURCES_TABLE)) { @@ -359,63 +334,167 @@ function copyLegacyMemoryIndexRows( schema: string, preservedEmbeddingCacheTable?: string, ): void { + ensureLegacyMemoryMigrationIndexes(db, schema); + // Canonical-owned chunk sets stay intact; any extra legacy identity invalidates + // the source for rebuild. Chunkless sources import only when metadata matches. + // Keep invalidated rows for deleted-file cleanup; snapshot before inserts. db.exec(` - INSERT OR IGNORE INTO main.${MEMORY_INDEX_META_TABLE} (key, value) - SELECT key, value FROM ${schema}.meta; - - INSERT OR IGNORE INTO main.${MEMORY_INDEX_SOURCES_TABLE} (path, source, hash, mtime, size) - SELECT path, source, hash, mtime, size - FROM ${schema}.files; - - INSERT OR IGNORE INTO main.${MEMORY_INDEX_CHUNKS_TABLE} ( - id, path, source, start_line, end_line, hash, model, text, embedding, updated_at + CREATE TEMP TABLE legacy_import_chunk_excluded_sources AS + SELECT DISTINCT owned.path, owned.source, + CASE WHEN EXISTS ( + SELECT 1 FROM ${schema}.chunks AS legacy_chunk + WHERE legacy_chunk.path = owned.path AND legacy_chunk.source IS owned.source + AND NOT EXISTS ( + SELECT 1 FROM main.${MEMORY_INDEX_CHUNKS_TABLE} AS canonical_chunk + WHERE canonical_chunk.id = legacy_chunk.id + AND canonical_chunk.path IS legacy_chunk.path AND canonical_chunk.source IS legacy_chunk.source + ) + ) THEN 1 ELSE 0 END AS force_reindex + FROM main.${MEMORY_INDEX_CHUNKS_TABLE} AS owned + WHERE EXISTS ( + SELECT 1 FROM ${schema}.files AS legacy_file + WHERE legacy_file.path = owned.path AND legacy_file.source IS owned.source ) - SELECT id, path, source, start_line, end_line, hash, model, text, embedding, updated_at - FROM ${schema}.chunks; + UNION ALL + SELECT canonical.path, canonical.source, 1 AS force_reindex + FROM main.${MEMORY_INDEX_SOURCES_TABLE} AS canonical + JOIN ${schema}.files AS legacy + ON legacy.path = canonical.path AND legacy.source IS canonical.source + WHERE ( + canonical.hash IS NOT legacy.hash + OR canonical.mtime IS NOT legacy.mtime + OR canonical.size IS NOT legacy.size + ) + AND NOT EXISTS ( + SELECT 1 FROM main.${MEMORY_INDEX_CHUNKS_TABLE} AS chunk + WHERE chunk.path = canonical.path AND chunk.source IS canonical.source + ); + + CREATE TEMP TABLE legacy_import_dirty_sources AS + SELECT legacy.path, legacy.source + FROM ${schema}.files AS legacy + WHERE NOT EXISTS ( + SELECT 1 FROM main.${MEMORY_INDEX_SOURCES_TABLE} AS canonical + WHERE canonical.path = legacy.path AND canonical.source IS legacy.source + ) + UNION + SELECT legacy.path, legacy.source + FROM ${schema}.chunks AS legacy + WHERE EXISTS ( + SELECT 1 FROM ${schema}.files AS owner + WHERE owner.path = legacy.path AND owner.source IS legacy.source + ) + AND NOT EXISTS ( + SELECT 1 FROM temp.legacy_import_chunk_excluded_sources AS excluded + WHERE excluded.path = legacy.path AND excluded.source IS legacy.source + ) + AND NOT EXISTS ( + SELECT 1 FROM main.${MEMORY_INDEX_CHUNKS_TABLE} AS canonical + WHERE canonical.id = legacy.id + ) + UNION + SELECT excluded.path, excluded.source + FROM temp.legacy_import_chunk_excluded_sources AS excluded + WHERE excluded.force_reindex = 1; `); - assertLegacyRowsCopied( - db, - `SELECT COUNT(*) AS missing - FROM ${schema}.meta AS legacy - WHERE NOT EXISTS ( - SELECT 1 FROM main.${MEMORY_INDEX_META_TABLE} AS canonical - WHERE canonical.key = legacy.key AND canonical.value IS legacy.value - )`, - "meta", - ); - assertLegacyRowsCopied( - db, - `SELECT COUNT(*) AS missing - FROM ${schema}.files AS legacy - WHERE NOT EXISTS ( - SELECT 1 FROM main.${MEMORY_INDEX_SOURCES_TABLE} AS canonical - WHERE canonical.path = legacy.path - AND canonical.source IS legacy.source - AND canonical.hash IS legacy.hash - AND canonical.mtime IS legacy.mtime - AND canonical.size IS legacy.size - )`, - "files", - ); - assertLegacyRowsCopied( - db, - `SELECT COUNT(*) AS missing - FROM ${schema}.chunks AS legacy - WHERE NOT EXISTS ( - SELECT 1 FROM main.${MEMORY_INDEX_CHUNKS_TABLE} AS canonical - WHERE canonical.id = legacy.id - AND canonical.path IS legacy.path - AND canonical.source IS legacy.source - AND canonical.start_line IS legacy.start_line - AND canonical.end_line IS legacy.end_line - AND canonical.hash IS legacy.hash - AND canonical.model IS legacy.model - AND canonical.text IS legacy.text - AND canonical.embedding IS legacy.embedding - AND canonical.updated_at IS legacy.updated_at - )`, - "chunks", - ); + try { + db.exec(` + INSERT OR IGNORE INTO main.${MEMORY_INDEX_META_TABLE} (key, value) + SELECT key, value FROM ${schema}.meta; + + INSERT OR IGNORE INTO main.${MEMORY_INDEX_SOURCES_TABLE} (path, source, hash, mtime, size) + SELECT path, source, hash, mtime, size + FROM ${schema}.files; + + INSERT OR IGNORE INTO main.${MEMORY_INDEX_CHUNKS_TABLE} ( + id, path, source, start_line, end_line, hash, model, text, embedding, updated_at + ) + -- Chunks are derived from source rows. Shipped cleanup could leave an + -- ownerless legacy chunk, which must not become permanently searchable. + SELECT id, path, source, start_line, end_line, hash, model, text, embedding, updated_at + FROM ${schema}.chunks AS legacy + WHERE EXISTS ( + SELECT 1 FROM ${schema}.files AS owner + WHERE owner.path = legacy.path AND owner.source IS legacy.source + ) + AND NOT EXISTS ( + SELECT 1 FROM temp.legacy_import_chunk_excluded_sources AS excluded + WHERE excluded.path = legacy.path AND excluded.source IS legacy.source + ); + + -- Content hashes are SHA-256 hex, so an empty hash cannot match a file. + -- Imported sources or chunks may be absent from runtime-owned vector + -- indexes, while excluded sources need a canonical rebuild. Retaining the + -- dirty source lets sync rebuild every derived row or clean up a deleted file. + UPDATE main.${MEMORY_INDEX_SOURCES_TABLE} + SET hash = '' + WHERE EXISTS ( + SELECT 1 FROM temp.legacy_import_dirty_sources AS dirty + WHERE dirty.path = main.${MEMORY_INDEX_SOURCES_TABLE}.path + AND dirty.source IS main.${MEMORY_INDEX_SOURCES_TABLE}.source + ); + `); + assertLegacyMemoryRowsCopied( + db, + `SELECT COUNT(*) AS missing + FROM ${schema}.meta AS legacy + WHERE NOT EXISTS ( + SELECT 1 FROM main.${MEMORY_INDEX_META_TABLE} AS canonical + WHERE canonical.key = legacy.key + )`, + "meta", + ); + assertLegacyMemoryRowsCopied( + db, + `SELECT COUNT(*) AS missing + FROM ${schema}.files AS legacy + WHERE NOT EXISTS ( + SELECT 1 FROM main.${MEMORY_INDEX_SOURCES_TABLE} AS canonical + WHERE canonical.path = legacy.path + AND canonical.source IS legacy.source + ) + AND NOT EXISTS ( + SELECT 1 FROM temp.legacy_import_chunk_excluded_sources AS excluded + WHERE excluded.force_reindex = 1 + AND excluded.path = legacy.path + AND excluded.source IS legacy.source + )`, + "files", + ); + assertLegacyMemoryRowsCopied( + db, + `SELECT COUNT(*) AS missing + FROM ${schema}.chunks AS legacy + WHERE EXISTS ( + SELECT 1 FROM ${schema}.files AS owner + WHERE owner.path = legacy.path AND owner.source IS legacy.source + ) + AND NOT EXISTS ( + SELECT 1 FROM main.${MEMORY_INDEX_CHUNKS_TABLE} AS canonical + WHERE canonical.id = legacy.id + AND canonical.path IS legacy.path + AND canonical.source IS legacy.source + ) + AND NOT EXISTS ( + SELECT 1 FROM temp.legacy_import_chunk_excluded_sources AS excluded + WHERE excluded.path = legacy.path AND excluded.source IS legacy.source + )`, + "chunks", + ); + // Repair derived orphans only after authoritative copy assertions pass; + // otherwise a synthetic owner could mask an uncopyable legacy source row. + db.exec(` + INSERT OR IGNORE INTO main.${MEMORY_INDEX_SOURCES_TABLE} (path, source, hash, mtime, size) + SELECT DISTINCT orphan.path, orphan.source, '', 0, 0 FROM main.${MEMORY_INDEX_CHUNKS_TABLE} AS orphan + WHERE NOT EXISTS ( + SELECT 1 FROM main.${MEMORY_INDEX_SOURCES_TABLE} AS owner + WHERE owner.path = orphan.path AND owner.source IS orphan.source + ); + `); + } finally { + db.exec("DROP TABLE temp.legacy_import_dirty_sources"); + db.exec("DROP TABLE temp.legacy_import_chunk_excluded_sources"); + } if ( preservedEmbeddingCacheTable !== "embedding_cache" && hasLegacyEmbeddingCacheTable(db, schema) @@ -437,7 +516,7 @@ function copyLegacyMemoryIndexRows( SELECT provider, model, provider_key, hash, embedding, dims, updated_at FROM ${schema}.embedding_cache; `); - assertLegacyRowsCopied( + assertLegacyMemoryRowsCopied( db, `SELECT COUNT(*) AS missing FROM ${schema}.embedding_cache AS legacy @@ -447,9 +526,6 @@ function copyLegacyMemoryIndexRows( AND canonical.model = legacy.model AND canonical.provider_key = legacy.provider_key AND canonical.hash = legacy.hash - AND canonical.embedding IS legacy.embedding - AND canonical.dims IS legacy.dims - AND canonical.updated_at IS legacy.updated_at )`, "embedding_cache", ); @@ -459,6 +535,7 @@ function copyLegacyMemoryIndexRows( function migrateLegacyMemoryIndexTables( db: DatabaseSync, preservedEmbeddingCacheTable?: string, + ftsTable = MEMORY_INDEX_FTS_TABLE, ): void { if (!hasLegacyMemoryIndexTables(db)) { return; @@ -467,6 +544,13 @@ function migrateLegacyMemoryIndexTables( db.exec("SAVEPOINT migrate_legacy_memory_index_tables"); try { copyLegacyMemoryIndexRows(db, "main", preservedEmbeddingCacheTable); + // `chunks_fts` belongs to the legacy schema and is dropped below even when + // a deprecated caller also supplied that name as its preferred FTS table. + if (ftsTable !== "chunks_fts" && tableExists(db, ftsTable)) { + // FTS is derived from canonical chunks. Rebuild inside the migration + // savepoint so imported rows and removed stale rows publish atomically. + rebuildMemoryChunkFts(db, ftsTable); + } if (preservedEmbeddingCacheTable !== "embedding_cache" && hasLegacyEmbeddingCacheTable(db)) { db.exec("DROP TABLE embedding_cache"); } @@ -487,47 +571,6 @@ function migrateLegacyMemoryIndexTables( } } -/** Drop the canonical source-to-path-FTS maintenance triggers. */ -export function dropMemoryPathFtsTriggers(db: DatabaseSync): void { - for (const trigger of MEMORY_PATH_FTS_TRIGGER_DEFINITIONS) { - db.exec(`DROP TRIGGER IF EXISTS main.${trigger.name}`); - } -} - -/** Install the canonical source-to-path-FTS maintenance triggers. */ -export function ensureMemoryPathFtsTriggers(db: DatabaseSync): void { - // The named integer source identity survives VACUUM and gives every - // FTS update/delete a direct rowid lookup instead of a virtual-table scan. - for (const trigger of MEMORY_PATH_FTS_TRIGGER_DEFINITIONS) { - db.exec(trigger.sql); - } -} - -function ensureMemoryPathFtsSchema(params: { db: DatabaseSync; tokenizeClause: string }): void { - params.db.exec("SAVEPOINT ensure_memory_index_paths_fts"); - try { - params.db.exec(` - CREATE VIRTUAL TABLE IF NOT EXISTS ${MEMORY_INDEX_PATHS_FTS_TABLE} USING fts5( - path, - source UNINDEXED - ${params.tokenizeClause} - ); - -- The initial copy and trigger installation share this savepoint. Once - -- populated, the triggers own completeness; per-row FTS probes are too costly. - INSERT INTO ${MEMORY_INDEX_PATHS_FTS_TABLE} (rowid, path, source) - SELECT id, path, source - FROM ${MEMORY_INDEX_SOURCES_TABLE} - WHERE NOT EXISTS (SELECT 1 FROM ${MEMORY_INDEX_PATHS_FTS_TABLE} LIMIT 1); - `); - ensureMemoryPathFtsTriggers(params.db); - params.db.exec("RELEASE ensure_memory_index_paths_fts"); - } catch (err) { - params.db.exec("ROLLBACK TO ensure_memory_index_paths_fts"); - params.db.exec("RELEASE ensure_memory_index_paths_fts"); - throw err; - } -} - function buildMemoryIndexStrictSchema(params: { embeddingCacheTable: string; includeEmbeddingCache: boolean; @@ -646,7 +689,8 @@ export function ensureMemoryIndexSchema(params: { CREATE INDEX IF NOT EXISTS idx_memory_index_chunks_source ON ${MEMORY_INDEX_CHUNKS_TABLE}(source); `); - migrateLegacyMemoryIndexTables(params.db, params.embeddingCacheTable); + migrateLegacyMemoryIndexTables(params.db, params.embeddingCacheTable, ftsTable); + dropDisabledMemoryChunkFts(params.db, ftsTable, params.ftsEnabled); if (params.cacheEnabled) { const updatedAtIndex = embeddingCacheTable === MEMORY_EMBEDDING_CACHE_TABLE @@ -683,8 +727,8 @@ export function ensureMemoryIndexSchema(params: { ` end_line UNINDEXED\n` + `${tokenizeClause});`, ); - // The shipped generic-table migration and a later FTS enablement both - // create an empty derived table beside already-canonical chunk rows. + // A migration rebuilds an existing FTS table in its savepoint. If the + // table is new, this same empty-table bootstrap covers all canonical rows. params.db.exec(` INSERT INTO ${ftsTable} ( text, id, path, source, model, start_line, end_line diff --git a/src/scripts/test-projects.test.ts b/src/scripts/test-projects.test.ts index b6fd205c374c..20c80dc78e7d 100644 --- a/src/scripts/test-projects.test.ts +++ b/src/scripts/test-projects.test.ts @@ -657,6 +657,7 @@ describe("test-projects args", () => { includePatterns: [ "extensions/memory-core/src/memory/index.test.ts", "extensions/memory-core/src/memory/manager.fts-only-reindex.test.ts", + "extensions/memory-core/src/memory/manager.legacy-migration-cleanup.test.ts", "extensions/memory-core/src/memory/manager.reindex-recovery.test.ts", "extensions/memory-core/src/memory/manager.self-heal-missing-identity.test.ts", ],