ewilliams28/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-08-12 21:53:00 -06:00
Code Issues Packages Projects Releases Wiki Activity

fix(memory): recover from same-file legacy index divergence (#110216)

Browse Source 
* fix(memory): keep canonical rows when same-file legacy memory tables diverge

* fix(memory): keep canonical-owned sources' chunk sets coherent during legacy import

* fix(memory): import legacy chunks when canonical source has no chunks

The same-file legacy migration excluded a legacy chunk from import whenever the
canonical index already had a source row for its (path, source). That stranded a
file whose canonical source was registered but had no chunks yet (indexing
interrupted before chunks were written, or embedding pending/failed): the legacy
chunks were its only searchable content, and the matching source hash stops sync
from re-indexing, so the file went silently unsearchable.

Re-key the chunk-coherence exclusion on canonical chunk ownership instead:
snapshot the (path, source) pairs that already have canonical chunks before the
import and skip legacy chunks only for those. A source with a canonical row but
no chunks now imports its legacy chunks. The snapshot is taken pre-insert because
the exclusion predicate reads the chunks table the import writes to.

Add regressions: legacy chunks import for a chunk-less canonical source while a
chunk-owning source still drops its stale legacy chunk; and restore abort
coverage for the meta and chunks copy assertions (previously only files was
exercised).

* fix(memory): harden same-file legacy conflict recovery

* fix(memory): rebuild ambiguous partial legacy sources

* fix(memory): reconcile migrated derived indexes

* fix(memory): close migrated index ownership gaps

* test(memory): align migration expectations

* test(tooling): match routed test order

* test(memory): exercise vector reload cleanup

* test(memory): prove real vector reload cleanup

* fix(memory): make migrated indexes converge

---------

Co-authored-by: Serhii Leniv <leniv.tech@gmail.com>
Co-authored-by: VACInc <3279061+VACInc@users.noreply.github.com>
Co-authored-by: Peter Steinberger <steipete@gmail.com>
This commit is contained in:
Vito Cappello
2026-07-22 03:27:01 -04:00
committed by GitHub
parent 3c4a1ec905
commit 4ab686220f
18 changed files with 1529 additions and 284 deletions
Download Patch File Download Diff File Expand all files Collapse all files
extensions/memory-core/src/memory/index.test.ts
+69
View File
@@ -1828,6 +1828,33 @@ describe("memory index", () => {
expect(status.vector?.available).toBe(available);
});
it("rebuilds vector tables created before completeness markers", async () => {
const cfg = createCfg({ provider: "gemini", vectorEnabled: true });
const legacyManager = await getFreshManager(cfg);
const available = await legacyManager.probeVectorStoreAvailability?.();
if (!available) {
await legacyManager.close?.();
return;
}
const legacyDb = Reflect.get(legacyManager, "db") as DatabaseSync;
legacyDb.exec(`
CREATE VIRTUAL TABLE memory_index_chunks_vec USING vec0(
id TEXT PRIMARY KEY,
embedding FLOAT[3]
);
INSERT INTO memory_index_chunks_vec VALUES ('orphan-before-marker', '[1,0,0]');
`);
await legacyManager.close?.();
const manager = await getFreshManager(cfg);
try {
await expect(manager.probeVectorStoreAvailability?.()).resolves.toBe(false);
expect(Reflect.get(manager, "memoryFullRetryDirty")).toBe(true);
} finally {
await manager.close?.();
}
});
it("drops the shipped legacy vector table and schedules a full reindex", async () => {
const cfg = createCfg({ vectorEnabled: true });
const manager = await getPersistentManager(cfg);
@@ -1887,6 +1914,48 @@ describe("memory index", () => {
}
});
it("forces a rebuild after incremental writes while vectors are disabled", async () => {
const enabledCfg = createCfg({ provider: "gemini", vectorEnabled: true });
const initialManager = await getFreshManager(enabledCfg);
await initialManager.sync({ reason: "test", force: true });
await initialManager.close?.();
await fs.writeFile(
path.join(memoryDir, "2026-01-12.md"),
"# Updated\n\nvector writes were disabled for this update\n",
);
const disabledManager = await getFreshManager(
createCfg({ provider: "gemini", vectorEnabled: false }),
);
Reflect.set(disabledManager, "dirty", true);
await disabledManager.sync({ reason: "test" });
const disabledDb = Reflect.get(disabledManager, "db") as DatabaseSync;
expect(
disabledDb
.prepare("SELECT value FROM memory_index_meta WHERE key = 'memory_vector_rebuild_v1'")
.get(),
).toEqual({ value: "1" });
await disabledManager.close?.();
const reloadedManager = await getFreshManager(enabledCfg);
try {
await expect(reloadedManager.probeVectorStoreAvailability?.()).resolves.toBe(false);
expect(Reflect.get(reloadedManager, "memoryFullRetryDirty")).toBe(true);
expect(reloadedManager.status().dirty).toBe(true);
await reloadedManager.sync({ reason: "test" });
const rebuiltDb = Reflect.get(reloadedManager, "db") as DatabaseSync;
expect(
rebuiltDb
.prepare("SELECT value FROM memory_index_meta WHERE key = 'memory_vector_rebuild_v1'")
.get(),
).toEqual({ value: "clean" });
await expect(reloadedManager.probeVectorStoreAvailability?.()).resolves.toBe(true);
} finally {
await reloadedManager.close?.();
}
});
it("keeps empty vector indexes clean after vector store probing", async () => {
await fs.rm(path.join(memoryDir, "2026-01-12.md"));
const legacyCfg = createCfg({
extensions/memory-core/src/memory/manager-embedding-ops.ts
+5 -9
View File
@@ -704,15 +704,7 @@ export abstract class MemoryManagerEmbeddingOps extends MemoryManagerSyncOps {
}
private clearIndexedFileData(pathname: string, source: MemorySource): void {
if (this.vector.enabled) {
try {
this.db
.prepare(
`DELETE FROM ${VECTOR_TABLE} WHERE id IN (SELECT id FROM memory_index_chunks WHERE path = ? AND source = ?)`,
)
.run(pathname, source);
} catch {}
}
this.deleteVectorRowsForSource(pathname, source);
if (this.fts.enabled && this.fts.available) {
try {
deleteMemoryFtsRows({
@@ -761,6 +753,7 @@ export abstract class MemoryManagerEmbeddingOps extends MemoryManagerSyncOps {
vectorReady: boolean,
): void {
const now = Date.now();
const needsVectorRebuild = !vectorReady && embeddings.some((embedding) => embedding.length > 0);
runSqliteImmediateTransactionSync(this.db, () => {
this.clearIndexedFileData(entry.path, source);
for (const [i, chunk] of chunks.entries()) {
@@ -809,6 +802,9 @@ export abstract class MemoryManagerEmbeddingOps extends MemoryManagerSyncOps {
}
}
this.upsertFileRecord(entry, source);
if (needsVectorRebuild) {
this.markVectorRebuildRequired();
}
});
this.vectorDegradedWriteWarningShown = logMemoryVectorDegradedWrite({
vectorEnabled: this.vector.enabled,
extensions/memory-core/src/memory/manager-session-sync-ops.ts
+1 -1
View File
@@ -179,7 +179,7 @@ export abstract class MemoryManagerSessionSyncOps extends MemoryManagerWatchOps
protected async runSessionStartupCatchup(): Promise<string[]> {
const dirtyFiles = await this.markSessionStartupCatchupDirtyFiles();
if (dirtyFiles.length === 0 || this.closed) {
if ((dirtyFiles.length === 0 && !this.sessionsFullRetryDirty) || this.closed) {
return dirtyFiles;
}
void this.sync({ reason: "session-startup-catchup" }).catch((err: unknown) => {
extensions/memory-core/src/memory/manager-session-sync-state.test.ts
+8
View File
@@ -94,6 +94,12 @@ describe("memory session sync state", () => {
mtimeMs: 100.75,
size: 10,
},
{
absPath: "/tmp/sessions/invalidated.jsonl",
path: "sessions/invalidated.jsonl",
mtimeMs: 200,
size: 20,
},
{
absPath: "/tmp/sessions/newer.jsonl",
path: "sessions/newer.jsonl",
@@ -116,6 +122,7 @@ describe("memory session sync state", () => {
existingRows: [
{ path: "sessions/unchanged.jsonl", hash: "hash-unchanged", mtime: 100.75, size: 10 },
{ path: "sessions/sub-ms-newer.jsonl", hash: "hash-sub-ms", mtime: 100.25, size: 10 },
{ path: "sessions/invalidated.jsonl", hash: "", mtime: 200, size: 20 },
{ path: "sessions/newer.jsonl", hash: "hash-newer", mtime: 200, size: 20 },
{ path: "sessions/resized.jsonl", hash: "hash-resized", mtime: 300, size: 30 },
],
@@ -123,6 +130,7 @@ describe("memory session sync state", () => {
expect(dirtyFiles).toEqual([
"/tmp/sessions/sub-ms-newer.jsonl",
"/tmp/sessions/invalidated.jsonl",
"/tmp/sessions/newer.jsonl",
"/tmp/sessions/resized.jsonl",
"/tmp/sessions/missing.jsonl",
extensions/memory-core/src/memory/manager-session-sync-state.ts
+1 -1
View File
@@ -16,7 +16,7 @@ export function resolveMemorySessionStartupDirtyFiles(params: {
const dirtyFiles: string[] = [];
for (const file of params.files) {
const existing = indexedRows.get(file.path);
if (!existing) {
if (!existing || existing.hash === "") {
dirtyFiles.push(file.absPath);
continue;
}
extensions/memory-core/src/memory/manager-source-sync-ops.ts
+2 -24
View File
@@ -14,7 +14,6 @@ import {
buildFileEntry,
listMemoryFiles,
MEMORY_INDEX_FTS_TABLE,
MEMORY_INDEX_VECTOR_TABLE,
runWithConcurrency,
} from "openclaw/plugin-sdk/memory-core-host-engine-storage";
import { MemoryManagerSessionSyncOps } from "./manager-session-sync-ops.js";
@@ -30,7 +29,6 @@ import type {
MemorySyncProgressState,
} from "./manager-sync-base.js";
const VECTOR_TABLE = MEMORY_INDEX_VECTOR_TABLE;
const FTS_TABLE = MEMORY_INDEX_FTS_TABLE;
const SESSION_SYNC_YIELD_EVERY = 10;
const SOURCE_WIDE_SESSION_INDEX_FLUSH_FILES = 128;
@@ -60,12 +58,6 @@ export abstract class MemoryManagerSourceSyncOps extends MemoryManagerSessionSyn
const deleteChunksByPathAndSource = this.db.prepare(
`DELETE FROM memory_index_chunks WHERE path = ? AND source = ?`,
);
const deleteVectorRowsByPathAndSource =
this.vector.enabled && this.vector.available
? this.db.prepare(
`DELETE FROM ${VECTOR_TABLE} WHERE id IN (SELECT id FROM memory_index_chunks WHERE path = ? AND source = ?)`,
)
: null;
const deleteFtsRowsByPathAndSource =
this.fts.enabled && this.fts.available
? this.db.prepare(`DELETE FROM ${FTS_TABLE} WHERE path = ? AND source = ?`)
@@ -113,11 +105,7 @@ export abstract class MemoryManagerSourceSyncOps extends MemoryManagerSessionSyn
continue;
}
deleteFileByPathAndSource.run(stale.path, "memory");
if (deleteVectorRowsByPathAndSource) {
try {
deleteVectorRowsByPathAndSource.run(stale.path, "memory");
} catch {}
}
this.deleteVectorRowsForSource(stale.path, "memory");
deleteChunksByPathAndSource.run(stale.path, "memory");
if (deleteFtsRowsByPathAndSource) {
try {
@@ -190,12 +178,6 @@ export abstract class MemoryManagerSourceSyncOps extends MemoryManagerSessionSyn
const deleteChunksByPathAndSource = this.db.prepare(
`DELETE FROM memory_index_chunks WHERE path = ? AND source = ?`,
);
const deleteVectorRowsByPathAndSource =
this.vector.enabled && this.vector.available
? this.db.prepare(
`DELETE FROM ${VECTOR_TABLE} WHERE id IN (SELECT id FROM memory_index_chunks WHERE path = ? AND source = ?)`,
)
: null;
const deleteFtsRowsByPathAndSource =
this.fts.enabled && this.fts.available
? this.db.prepare(`DELETE FROM ${FTS_TABLE} WHERE path = ? AND source = ?`)
@@ -254,11 +236,7 @@ export abstract class MemoryManagerSourceSyncOps extends MemoryManagerSessionSyn
const yieldAfterSessionFile = createSessionSyncYield(files.length);
const deleteIndexedSessionPath = (memoryPath: string) => {
deleteFileByPathAndSource.run(memoryPath, "sessions");
if (deleteVectorRowsByPathAndSource) {
try {
deleteVectorRowsByPathAndSource.run(memoryPath, "sessions");
} catch {}
}
this.deleteVectorRowsForSource(memoryPath, "sessions");
deleteChunksByPathAndSource.run(memoryPath, "sessions");
if (deleteFtsRowsByPathAndSource) {
try {
extensions/memory-core/src/memory/manager-sync-base.ts
+62
View File
@@ -43,6 +43,10 @@ import {
type MemoryIndexMeta,
type MemoryIndexProviderIdentity,
} from "./manager-reindex-state.js";
import {
markMemoryVectorRebuildRequired,
requiresMemoryVectorRebuild,
} from "./manager-vector-rebuild-state.js";
import type { MemoryWatchSettleQueue } from "./watch-settle.js";
export type MemorySyncProgressState = {
@@ -488,6 +492,10 @@ export abstract class MemoryManagerSyncBase {
}
private async loadVectorExtension(): Promise<boolean> {
if (this.vector.available === true && this.hasVectorRebuildMarker()) {
this.markConfiguredSourcesForFullReindex();
return false;
}
if (this.vector.available !== null) {
return this.vector.available;
}
@@ -505,6 +513,13 @@ export abstract class MemoryManagerSyncBase {
}
this.vector.extensionPath = loaded.extensionPath;
this.vector.available = true;
if (this.hasVectorRebuildMarker()) {
// A skipped vector write/delete can leave both missing and extra rows.
// Refuse partial KNN results and let the normal shadow reindex rebuild all
// configured sources before this manager treats vectors as ready.
this.markConfiguredSourcesForFullReindex();
return false;
}
if (this.dropLegacyVectorTable()) {
// A broad dirty sync can skip unchanged files whose source hashes were
// migrated. Force the next sync to republish the derived vector rows.
@@ -521,6 +536,53 @@ export abstract class MemoryManagerSyncBase {
}
}
protected deleteVectorRowsForSource(pathname: string, source: MemorySource): void {
if (!memoryTableExists(this.db, VECTOR_TABLE)) {
return;
}
if (!this.vector.enabled || this.vector.available !== true) {
this.markVectorRebuildRequired();
return;
}
try {
this.db
.prepare(
`DELETE FROM ${VECTOR_TABLE} WHERE id IN (
SELECT id FROM memory_index_chunks WHERE path = ? AND source = ?
)`,
)
.run(pathname, source);
} catch {
this.markVectorRebuildRequired();
}
}
protected markVectorRebuildRequired(): void {
markMemoryVectorRebuildRequired(this.db);
}
private hasVectorRebuildMarker(): boolean {
return requiresMemoryVectorRebuild({
db: this.db,
vectorTable: VECTOR_TABLE,
metaVectorDims: this.readMeta()?.vectorDims,
hasSemanticChunks: this.hasSemanticChunks(),
});
}
private markConfiguredSourcesForFullReindex(): void {
// This flag selects the shadow-reindex path even for a sessions-only index;
// the rebuild itself still filters work through the configured sources.
this.memoryFullRetryDirty = true;
if (this.sources.has("memory")) {
this.dirty = true;
}
if (this.sources.has("sessions")) {
this.sessionsDirty = true;
this.sessionsFullRetryDirty = true;
}
}
private ensureVectorTable(dimensions: number): void {
if (this.vector.dims === dimensions && memoryTableExists(this.db, VECTOR_TABLE)) {
return;
extensions/memory-core/src/memory/manager-sync-ops.startup-catchup.test.ts
+13
View File
@@ -212,6 +212,11 @@ class SessionStartupCatchupHarness extends MemoryManagerSyncOps {
return this.sessionsDirty;
}
markFullSessionRetry(): void {
this.sessionsDirty = true;
this.sessionsFullRetryDirty = true;
}
startTranscriptListener(): void {
this.ensureSessionListener();
}
@@ -389,6 +394,14 @@ describe("session startup catch-up", () => {
expect(harness.syncCalls).toEqual([{ reason: "session-startup-catchup" }]);
});
it("schedules a full retry when invalidated sessions no longer exist", async () => {
const harness = new SessionStartupCatchupHarness([]);
harness.markFullSessionRetry();
await expect(harness.catchUp()).resolves.toEqual([]);
expect(harness.syncCalls).toEqual([{ reason: "session-startup-catchup" }]);
});
it("retries transient session transcript reads during session indexing", async () => {
const session = await writeSessionFile("thread.jsonl.deleted.2026-02-16T22-27-33.000Z");
const harness = new SessionStartupCatchupHarness([]);
extensions/memory-core/src/memory/manager-sync-ops.ts
+8 -2
View File
@@ -40,6 +40,7 @@ import {
markMemoryTargetArchiveFilesDirty,
runMemoryTargetedSessionSync,
} from "./manager-targeted-sync.js";
import { markMemoryVectorIndexClean } from "./manager-vector-rebuild-state.js";
export type { MemoryIndexWorkItem } from "./manager-sync-base.js";
@@ -444,9 +445,9 @@ export abstract class MemoryManagerSyncOps extends MemoryManagerSourceSyncOps {
}
}
if (!shouldSyncMemory) {
this.dirty = false;
this.clearMemoryRetryState();
}
const vectorIndexComplete = this.vector.available === true;
const nextMeta: MemoryIndexMeta = {
model: this.provider?.model ?? "fts-only",
provider: this.provider?.id ?? "none",
@@ -487,6 +488,11 @@ export abstract class MemoryManagerSyncOps extends MemoryManagerSourceSyncOps {
});
this.db = originalDb;
if (vectorIndexComplete) {
// Publish completeness only after the shadow tables committed. A crash
// before this point leaves the rebuild marker conservative and retryable.
markMemoryVectorIndexClean(originalDb);
}
this.resetVectorState();
this.fts.available = nextFtsState.available;
this.fts.loadError = nextFtsState.loadError;
extensions/memory-core/src/memory/manager-vector-rebuild-state.ts
+45
View File
@@ -0,0 +1,45 @@
// Memory Core plugin module owns persisted vector completeness state.
import type { DatabaseSync } from "node:sqlite";
import { MEMORY_INDEX_META_TABLE } from "openclaw/plugin-sdk/memory-core-host-engine-storage";
const VECTOR_REBUILD_META_KEY = "memory_vector_rebuild_v1";
function vectorTableExists(db: DatabaseSync, tableName: string): boolean {
return Boolean(
db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(tableName),
);
}
export function markMemoryVectorIndexClean(db: DatabaseSync): void {
db.prepare(
`INSERT INTO ${MEMORY_INDEX_META_TABLE} (key, value) VALUES (?, 'clean')
ON CONFLICT(key) DO UPDATE SET value=excluded.value`,
).run(VECTOR_REBUILD_META_KEY);
}
export function markMemoryVectorRebuildRequired(db: DatabaseSync): void {
db.prepare(
`INSERT INTO ${MEMORY_INDEX_META_TABLE} (key, value) VALUES (?, '1')
ON CONFLICT(key) DO UPDATE SET value=excluded.value`,
).run(VECTOR_REBUILD_META_KEY);
}
export function requiresMemoryVectorRebuild(params: {
db: DatabaseSync;
vectorTable: string;
metaVectorDims?: number;
hasSemanticChunks: boolean;
}): boolean {
const row = params.db
.prepare(`SELECT value FROM ${MEMORY_INDEX_META_TABLE} WHERE key = ?`)
.get(VECTOR_REBUILD_META_KEY) as { value?: unknown } | undefined;
if (row?.value === "1") {
return true;
}
if (!vectorTableExists(params.db, params.vectorTable)) {
return Boolean(params.metaVectorDims && params.hasSemanticChunks);
}
// Existing releases had no completeness marker. Rebuild their vector table
// once rather than assuming it has neither missing nor orphaned rows.
return row?.value !== "clean";
}
extensions/memory-core/src/memory/manager.legacy-migration-cleanup.test.ts
+272
View File
@@ -0,0 +1,272 @@
// Memory Core tests cover deleted-file cleanup after same-file legacy migration.
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { DatabaseSync } from "node:sqlite";
import type { OpenClawConfig } from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
import {
ensureMemoryIndexSchema,
loadSqliteVecExtension,
} from "openclaw/plugin-sdk/memory-core-host-engine-storage";
import { resolveOpenClawAgentSqlitePath } from "openclaw/plugin-sdk/sqlite-runtime";
import {
closeOpenClawAgentDatabasesForTest,
closeOpenClawStateDatabaseForTest,
} from "openclaw/plugin-sdk/sqlite-runtime-testing";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import "./test-runtime-mocks.js";
import { closeAllMemorySearchManagers, getMemorySearchManager } from "./index.js";
import type { MemoryIndexManager } from "./manager.js";
const originalStateDir = process.env.OPENCLAW_STATE_DIR;
describe("memory legacy migration cleanup", () => {
let fixtureRoot = "";
let workspaceDir = "";
let manager: MemoryIndexManager | undefined;
beforeEach(async () => {
fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-memory-migration-cleanup-"));
workspaceDir = path.join(fixtureRoot, "workspace");
await fs.mkdir(path.join(workspaceDir, "memory"), { recursive: true });
Reflect.set(process.env, "OPENCLAW_STATE_DIR", path.join(fixtureRoot, "state"));
});
afterEach(async () => {
await manager?.close();
manager = undefined;
await closeAllMemorySearchManagers();
closeOpenClawAgentDatabasesForTest();
closeOpenClawStateDatabaseForTest();
if (originalStateDir === undefined) {
Reflect.deleteProperty(process.env, "OPENCLAW_STATE_DIR");
} else {
Reflect.set(process.env, "OPENCLAW_STATE_DIR", originalStateDir);
}
await fs.rm(fixtureRoot, { recursive: true, force: true });
});
it("removes migrated chunks and FTS rows when the dirty source file is already deleted", async () => {
const dbPath = resolveOpenClawAgentSqlitePath({ agentId: "main" });
await fs.mkdir(path.dirname(dbPath), { recursive: true });
const seedDb = new DatabaseSync(dbPath, { allowExtension: true });
let vectorExtensionPath: string | undefined;
try {
const loaded = await loadSqliteVecExtension({ db: seedDb });
expect(loaded.ok, loaded.error).toBe(true);
vectorExtensionPath = loaded.extensionPath;
ensureMemoryIndexSchema({ db: seedDb, cacheEnabled: false, ftsEnabled: true });
seedDb.exec(`
INSERT INTO memory_index_sources (path, source, hash, mtime, size)
VALUES
('memory/deleted.md', 'memory', 'canonical-hash', 200, 20),
('sessions/excluded.jsonl', 'sessions', '', 200, 20);
INSERT INTO memory_index_chunks VALUES (
'chunk-canonical', 'memory/deleted.md', 'memory', 1, 2, 'canonical-chunk-hash',
'fts-only', 'obsolete saffronquasar', '[]', 200
);
INSERT INTO memory_index_chunks VALUES (
'chunk-ownerless', 'memory/ownerless.md', 'memory', 1, 2, 'ownerless-chunk-hash',
'fts-only', 'obsolete ambercomet', '[]', 190
);
INSERT INTO memory_index_chunks_fts
(text, id, path, source, model, start_line, end_line)
VALUES
(
'obsolete saffronquasar', 'chunk-canonical', 'memory/deleted.md',
'memory', 'fts-only', 1, 2
),
(
'obsolete ambercomet', 'chunk-ownerless', 'memory/ownerless.md',
'memory', 'fts-only', 1, 2
);
CREATE VIRTUAL TABLE memory_index_chunks_vec USING vec0(
id TEXT PRIMARY KEY,
embedding FLOAT[3]
);
INSERT INTO memory_index_chunks_vec VALUES ('chunk-canonical', '[1,0,0]');
INSERT INTO memory_index_chunks_vec VALUES ('chunk-ownerless', '[0,1,0]');
CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
CREATE TABLE files (
path TEXT PRIMARY KEY,
source TEXT NOT NULL DEFAULT 'memory',
hash TEXT NOT NULL,
mtime INTEGER NOT NULL,
size INTEGER NOT NULL
);
CREATE TABLE chunks (
id TEXT PRIMARY KEY,
path TEXT NOT NULL,
source TEXT NOT NULL DEFAULT 'memory',
start_line INTEGER NOT NULL,
end_line INTEGER NOT NULL,
hash TEXT NOT NULL,
model TEXT NOT NULL,
text TEXT NOT NULL,
embedding TEXT NOT NULL,
updated_at INTEGER NOT NULL
);
INSERT INTO files VALUES (
'memory/deleted.md', 'memory', 'legacy-hash', 100, 10
);
INSERT INTO chunks VALUES (
'chunk-legacy-extra', 'memory/deleted.md', 'memory', 3, 4, 'legacy-chunk-hash',
'fts-only', 'stale legacy tail', '[]', 100
);
`);
expect(seedDb.prepare("SELECT COUNT(*) AS count FROM memory_index_chunks_vec").get()).toEqual(
{ count: 2 },
);
} finally {
seedDb.close();
}
const createConfig = (params: {
extensionPath?: string;
provider: "none" | "openai";
vectorEnabled: boolean;
}) =>
({
memory: { backend: "builtin" },
agents: {
defaults: {
workspace: workspaceDir,
memorySearch: {
provider: params.provider,
model: params.provider === "none" ? "" : "text-embedding-3-small",
store: {
vector: {
enabled: params.vectorEnabled,
...(params.extensionPath ? { extensionPath: params.extensionPath } : {}),
},
},
cache: { enabled: false },
sync: { watch: false, onSessionStart: false, onSearch: false },
query: { hybrid: { enabled: true } },
},
},
list: [{ id: "main", default: true }],
},
}) as OpenClawConfig;
const cfg = createConfig({ provider: "none", vectorEnabled: false });
const result = await getMemorySearchManager({ cfg, agentId: "main" });
if (!result.manager) {
throw new Error(result.error ?? "memory manager missing");
}
manager = result.manager as unknown as MemoryIndexManager;
expect(manager.status().fts?.available).toBe(true);
expect(Reflect.get(manager, "sessionsFullRetryDirty")).toBe(false);
const db = Reflect.get(manager, "db") as DatabaseSync;
expect(
db.prepare("SELECT hash FROM memory_index_sources WHERE path = 'memory/deleted.md'").get(),
).toEqual({ hash: "" });
expect(
db.prepare("SELECT hash FROM memory_index_sources WHERE path = 'memory/ownerless.md'").get(),
).toEqual({ hash: "" });
expect(
db
.prepare("SELECT COUNT(*) AS count FROM memory_index_chunks WHERE path = ?")
.get("memory/deleted.md"),
).toEqual({ count: 1 });
expect(
db
.prepare("SELECT COUNT(*) AS count FROM memory_index_chunks_fts WHERE path = ?")
.get("memory/deleted.md"),
).toEqual({ count: 1 });
expect(
db
.prepare("SELECT COUNT(*) AS count FROM memory_index_chunks_fts WHERE path = ?")
.get("memory/ownerless.md"),
).toEqual({ count: 1 });
await (
manager as unknown as {
syncMemoryFiles(params: { needsFullReindex: boolean }): Promise<unknown>;
}
).syncMemoryFiles({ needsFullReindex: false });
expect(
db
.prepare("SELECT COUNT(*) AS count FROM memory_index_sources WHERE path = ?")
.get("memory/deleted.md"),
).toEqual({ count: 0 });
expect(
db
.prepare("SELECT COUNT(*) AS count FROM memory_index_chunks WHERE path = ?")
.get("memory/deleted.md"),
).toEqual({ count: 0 });
expect(
db
.prepare("SELECT COUNT(*) AS count FROM memory_index_chunks_fts WHERE path = ?")
.get("memory/deleted.md"),
).toEqual({ count: 0 });
expect(
db
.prepare("SELECT COUNT(*) AS count FROM memory_index_sources WHERE path = ?")
.get("memory/ownerless.md"),
).toEqual({ count: 0 });
expect(
db
.prepare("SELECT COUNT(*) AS count FROM memory_index_chunks WHERE path = ?")
.get("memory/ownerless.md"),
).toEqual({ count: 0 });
expect(
db
.prepare("SELECT COUNT(*) AS count FROM memory_index_chunks_fts WHERE path = ?")
.get("memory/ownerless.md"),
).toEqual({ count: 0 });
// Cleanup ran while vectors were disabled. Keep the old table untouched and
// persist a rebuild marker; one-sided orphan pruning would still miss vector
// rows that should exist but were never written.
expect(
db
.prepare("SELECT value FROM memory_index_meta WHERE key = 'memory_vector_rebuild_v1'")
.get(),
).toEqual({ value: "1" });
const observerDb = new DatabaseSync(dbPath, { allowExtension: true });
try {
const observerLoaded = await loadSqliteVecExtension({
db: observerDb,
extensionPath: vectorExtensionPath,
});
expect(observerLoaded.ok, observerLoaded.error).toBe(true);
expect(
observerDb.prepare("SELECT COUNT(*) AS count FROM memory_index_chunks_vec").get(),
).toEqual({ count: 2 });
} finally {
observerDb.close();
}
await closeAllMemorySearchManagers();
manager = undefined;
const reloadResult = await getMemorySearchManager({
cfg: createConfig({
extensionPath: vectorExtensionPath,
provider: "openai",
vectorEnabled: true,
}),
agentId: "main",
});
if (!reloadResult.manager) {
throw new Error(reloadResult.error ?? "reloaded memory manager missing");
}
manager = reloadResult.manager as unknown as MemoryIndexManager;
const reloadedDb = Reflect.get(manager, "db") as DatabaseSync;
await expect(
(
manager as unknown as {
loadVectorExtension(): Promise<boolean>;
}
).loadVectorExtension(),
).resolves.toBe(false);
expect(reloadedDb.prepare("SELECT vec_version() AS version").get()).toEqual({
version: expect.any(String),
});
expect(
reloadedDb.prepare("SELECT COUNT(*) AS count FROM memory_index_chunks_vec").get(),
).toEqual({ count: 2 });
expect(Reflect.get(manager, "memoryFullRetryDirty")).toBe(true);
});
});
extensions/memory-core/src/memory/manager.ts
+23 -5
View File
@@ -517,11 +517,29 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem
this.ensureSessionListener();
this.ensureIntervalSync();
}
this.dirty = resolveInitialMemoryDirty({
hasMemorySource: this.sources.has("memory"),
statusOnly: params.purpose === "status",
hasIndexedMeta: Boolean(meta),
});
const invalidatedSources = new Set(
(
this.db
.prepare("SELECT DISTINCT source FROM memory_index_sources WHERE hash = ''")
.all() as Array<{ source?: unknown }>
).flatMap((row) =>
row.source === "memory" || row.source === "sessions" ? [row.source] : [],
),
);
this.dirty =
resolveInitialMemoryDirty({
hasMemorySource: this.sources.has("memory"),
statusOnly: params.purpose === "status",
hasIndexedMeta: Boolean(meta),
}) ||
(this.sources.has("memory") && invalidatedSources.has("memory"));
if (this.sources.has("sessions") && invalidatedSources.has("sessions")) {
// Migration cannot map a durable session source path back to one live
// transcript file. Carry a full-session retry so unchanged and deleted
// transcripts both converge on the next startup/search sync.
this.sessionsDirty = true;
this.sessionsFullRetryDirty = true;
}
this.batch = this.resolveBatchConfig();
if (!transient) {
this.ensureSessionStartupCatchup();
packages/memory-host-sdk/src/host/memory-schema-fts.ts
+113
View File
@@ -0,0 +1,113 @@
// Memory Host SDK module owns derived FTS schema and rebuild behavior.
import type { DatabaseSync } from "node:sqlite";
export const MEMORY_INDEX_SOURCES_TABLE = "memory_index_sources";
export const MEMORY_INDEX_CHUNKS_TABLE = "memory_index_chunks";
export const MEMORY_INDEX_FTS_TABLE = "memory_index_chunks_fts";
export const MEMORY_INDEX_PATHS_FTS_TABLE = "memory_index_paths_fts";
/** Optional canonical triggers owned by the derived path FTS index. */
export const MEMORY_PATH_FTS_TRIGGER_DEFINITIONS = [
{
name: "memory_index_paths_fts_after_insert",
sql: `
CREATE TRIGGER IF NOT EXISTS main.memory_index_paths_fts_after_insert
AFTER INSERT ON ${MEMORY_INDEX_SOURCES_TABLE}
BEGIN
INSERT INTO ${MEMORY_INDEX_PATHS_FTS_TABLE} (rowid, path, source)
VALUES (NEW.id, NEW.path, NEW.source);
END;
`,
},
{
name: "memory_index_paths_fts_after_update",
sql: `
CREATE TRIGGER IF NOT EXISTS main.memory_index_paths_fts_after_update
AFTER UPDATE OF id, path, source ON ${MEMORY_INDEX_SOURCES_TABLE}
BEGIN
DELETE FROM ${MEMORY_INDEX_PATHS_FTS_TABLE}
WHERE rowid = OLD.id;
INSERT INTO ${MEMORY_INDEX_PATHS_FTS_TABLE} (rowid, path, source)
VALUES (NEW.id, NEW.path, NEW.source);
END;
`,
},
{
name: "memory_index_paths_fts_after_delete",
sql: `
CREATE TRIGGER IF NOT EXISTS main.memory_index_paths_fts_after_delete
AFTER DELETE ON ${MEMORY_INDEX_SOURCES_TABLE}
BEGIN
DELETE FROM ${MEMORY_INDEX_PATHS_FTS_TABLE}
WHERE rowid = OLD.id;
END;
`,
},
] as const;
export function rebuildMemoryChunkFts(db: DatabaseSync, ftsTable: string): void {
db.exec(`
DELETE FROM ${ftsTable};
INSERT INTO ${ftsTable} (
text, id, path, source, model, start_line, end_line
)
SELECT text, id, path, source, model, start_line, end_line
FROM ${MEMORY_INDEX_CHUNKS_TABLE};
`);
}
export function dropDisabledMemoryChunkFts(
db: DatabaseSync,
ftsTable: string,
enabled: boolean,
): void {
if (!enabled && ftsTable === MEMORY_INDEX_FTS_TABLE) {
// Body FTS has no maintenance triggers while disabled. Recreate it from
// canonical chunks on enable instead of retaining a partial derived index.
db.exec(`DROP TABLE IF EXISTS ${ftsTable}`);
}
}
/** Drop the canonical source-to-path-FTS maintenance triggers. */
export function dropMemoryPathFtsTriggers(db: DatabaseSync): void {
for (const trigger of MEMORY_PATH_FTS_TRIGGER_DEFINITIONS) {
db.exec(`DROP TRIGGER IF EXISTS main.${trigger.name}`);
}
}
/** Install the canonical source-to-path-FTS maintenance triggers. */
export function ensureMemoryPathFtsTriggers(db: DatabaseSync): void {
// The named integer source identity survives VACUUM and gives every
// FTS update/delete a direct rowid lookup instead of a virtual-table scan.
for (const trigger of MEMORY_PATH_FTS_TRIGGER_DEFINITIONS) {
db.exec(trigger.sql);
}
}
export function ensureMemoryPathFtsSchema(params: {
db: DatabaseSync;
tokenizeClause: string;
}): void {
params.db.exec("SAVEPOINT ensure_memory_index_paths_fts");
try {
params.db.exec(`
CREATE VIRTUAL TABLE IF NOT EXISTS ${MEMORY_INDEX_PATHS_FTS_TABLE} USING fts5(
path,
source UNINDEXED
${params.tokenizeClause}
);
-- The initial copy and trigger installation share this savepoint. Once
-- populated, the triggers own completeness; per-row FTS probes are too costly.
INSERT INTO ${MEMORY_INDEX_PATHS_FTS_TABLE} (rowid, path, source)
SELECT id, path, source
FROM ${MEMORY_INDEX_SOURCES_TABLE}
WHERE NOT EXISTS (SELECT 1 FROM ${MEMORY_INDEX_PATHS_FTS_TABLE} LIMIT 1);
`);
ensureMemoryPathFtsTriggers(params.db);
params.db.exec("RELEASE ensure_memory_index_paths_fts");
} catch (err) {
params.db.exec("ROLLBACK TO ensure_memory_index_paths_fts");
params.db.exec("RELEASE ensure_memory_index_paths_fts");
throw err;
}
}
packages/memory-host-sdk/src/host/memory-schema-migration.ts
+28
View File
@@ -0,0 +1,28 @@
// Memory Host SDK module owns temporary indexes used by legacy schema repair.
import type { DatabaseSync } from "node:sqlite";
// Same-identity values may diverge because canonical derived rows win. A row
// that cannot be represented canonically still aborts before legacy tables drop.
export function assertLegacyMemoryRowsCopied(
db: DatabaseSync,
query: string,
tableName: string,
): void {
const row = db.prepare(query).get() as { missing?: unknown } | undefined;
if (Number(row?.missing ?? 0) > 0) {
throw new Error(
`legacy memory ${tableName} rows could not be copied into canonical memory index rows`,
);
}
}
export function ensureLegacyMemoryMigrationIndexes(db: DatabaseSync, schema: string): void {
// Shipped legacy tables index chunk ids only, while ownership repair joins by
// path/source. These indexes disappear with the legacy tables after migration.
db.exec(`
CREATE INDEX IF NOT EXISTS ${schema}.memory_legacy_files_path_source_migration
ON files(path, source);
CREATE INDEX IF NOT EXISTS ${schema}.memory_legacy_chunks_path_source_migration
ON chunks(path, source);
`);
}
packages/memory-host-sdk/src/host/memory-schema.legacy-migration.test.ts
+640
View File
@@ -0,0 +1,640 @@
// Same-file legacy migration tests cover conflict recovery and rollback boundaries.
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { DatabaseSync } from "node:sqlite";
import { describe, expect, it } from "vitest";
import { ensureMemoryIndexSchema } from "./memory-schema.js";
describe("memory index same-file legacy migration", () => {
it("leaves unrelated generic tables untouched", () => {
const db = new DatabaseSync(":memory:");
try {
db.exec(`
CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL, owner TEXT);
CREATE TABLE files (
path TEXT PRIMARY KEY,
source TEXT NOT NULL,
hash TEXT NOT NULL,
mtime INTEGER NOT NULL,
size INTEGER NOT NULL
);
CREATE TABLE chunks (
id TEXT PRIMARY KEY,
path TEXT NOT NULL,
source TEXT NOT NULL,
start_line INTEGER NOT NULL,
end_line INTEGER NOT NULL,
hash TEXT NOT NULL,
model TEXT NOT NULL,
text TEXT NOT NULL,
embedding TEXT NOT NULL,
updated_at INTEGER NOT NULL
);
`);
ensureMemoryIndexSchema({
db,
cacheEnabled: false,
ftsEnabled: false,
});
expect(
db
.prepare(
"SELECT name FROM sqlite_master WHERE type = 'table' AND name IN ('meta', 'files', 'chunks') ORDER BY name",
)
.all(),
).toEqual([{ name: "chunks" }, { name: "files" }, { name: "meta" }]);
} finally {
db.close();
}
});
it("recovers a partially migrated WAL index idempotently when legacy rows diverge", () => {
const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-memory-diverged-"));
const dbPath = path.join(rootDir, "openclaw-agent.sqlite");
const db = new DatabaseSync(dbPath);
try {
expect(db.prepare("PRAGMA journal_mode = WAL").get()).toEqual({ journal_mode: "wal" });
ensureMemoryIndexSchema({ db, cacheEnabled: true, ftsEnabled: false });
db.exec(`
INSERT INTO memory_index_meta VALUES ('memory_index_meta_v1', 'canonical');
INSERT INTO memory_index_sources (path, source, hash, mtime, size)
VALUES ('doc.md', 'memory', 'new-hash', 200.0, 42);
INSERT INTO memory_index_chunks VALUES (
'chunk-new-1', 'doc.md', 'memory', 1, 10, 'new-chunk-hash', 'model',
'current canonical body', '[1,2]', 200
);
INSERT INTO memory_embedding_cache VALUES (
'openai', 'model', 'key', 'new-chunk-hash', '[1,2]', 2, 200
);
`);
// This is the partial migration state seen in affected databases: canonical
// tables already contain current rows while the same file still has legacy tables.
db.exec(`
CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
CREATE TABLE files (
path TEXT PRIMARY KEY,
source TEXT NOT NULL DEFAULT 'memory',
hash TEXT NOT NULL,
mtime INTEGER NOT NULL,
size INTEGER NOT NULL
);
CREATE TABLE chunks (
id TEXT PRIMARY KEY,
path TEXT NOT NULL,
source TEXT NOT NULL DEFAULT 'memory',
start_line INTEGER NOT NULL,
end_line INTEGER NOT NULL,
hash TEXT NOT NULL,
model TEXT NOT NULL,
text TEXT NOT NULL,
embedding TEXT NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE TABLE embedding_cache (
provider TEXT NOT NULL,
model TEXT NOT NULL,
provider_key TEXT NOT NULL,
hash TEXT NOT NULL,
embedding TEXT NOT NULL,
dims INTEGER,
updated_at INTEGER NOT NULL,
PRIMARY KEY (provider, model, provider_key, hash)
);
INSERT INTO meta VALUES ('memory_index_meta_v1', 'legacy');
INSERT INTO meta VALUES ('legacy-only-key', 'imported');
INSERT INTO files VALUES ('doc.md', 'memory', 'old-hash', 100, 40);
INSERT INTO files VALUES ('old-note.md', 'memory', 'old-note-hash', 90, 10);
INSERT INTO chunks VALUES (
'chunk-new-1', 'doc.md', 'memory', 1, 8, 'old-chunk-hash', 'model',
'same-key stale legacy body', '[9,9]', 100
);
INSERT INTO chunks VALUES (
'chunk-old-7', 'doc.md', 'memory', 9, 12, 'old-tail-hash', 'model',
'distinct stale legacy tail', '[8,8]', 100
);
INSERT INTO chunks VALUES (
'chunk-old-2', 'old-note.md', 'memory', 1, 5, 'note-hash', 'model',
'note body', '[]', 90
);
INSERT INTO embedding_cache VALUES (
'openai', 'model', 'key', 'new-chunk-hash', '[9,9,9]', 3, 100
);
INSERT INTO embedding_cache VALUES (
'openai', 'model', 'legacy-key', 'legacy-hash', '[3,4]', 2, 90
);
`);
const readCanonicalState = () => ({
meta: db.prepare("SELECT key, value FROM memory_index_meta ORDER BY key").all(),
sources: db
.prepare("SELECT path, hash, size FROM memory_index_sources ORDER BY path")
.all(),
chunks: db.prepare("SELECT id, text FROM memory_index_chunks ORDER BY id").all(),
cache: db
.prepare(
"SELECT provider_key, hash, embedding, dims, updated_at FROM memory_embedding_cache ORDER BY provider_key",
)
.all(),
});
const expectedCanonicalState = {
meta: [
{ key: "legacy-only-key", value: "imported" },
{ key: "memory_index_meta_v1", value: "canonical" },
],
// The extra legacy chunk identity makes doc.md's canonical chunk set
// ambiguous. Its impossible hash forces a rebuild while preserving the
// source identity needed to clean up a file deleted before migration.
sources: [
{ path: "doc.md", hash: "", size: 42 },
{ path: "old-note.md", hash: "", size: 10 },
],
chunks: [
{ id: "chunk-new-1", text: "current canonical body" },
{ id: "chunk-old-2", text: "note body" },
],
cache: [
{
provider_key: "key",
hash: "new-chunk-hash",
embedding: "[1,2]",
dims: 2,
updated_at: 200,
},
{
provider_key: "legacy-key",
hash: "legacy-hash",
embedding: "[3,4]",
dims: 2,
updated_at: 90,
},
],
};
const readLegacyTables = () =>
db
.prepare(
"SELECT name FROM sqlite_master WHERE type = 'table' AND name IN ('meta', 'files', 'chunks', 'embedding_cache') ORDER BY name",
)
.all();
expect(() =>
ensureMemoryIndexSchema({ db, cacheEnabled: true, ftsEnabled: false }),
).not.toThrow();
expect(readCanonicalState()).toEqual(expectedCanonicalState);
expect(readLegacyTables()).toEqual([]);
expect(() =>
ensureMemoryIndexSchema({ db, cacheEnabled: true, ftsEnabled: false }),
).not.toThrow();
expect(readCanonicalState()).toEqual(expectedCanonicalState);
expect(readLegacyTables()).toEqual([]);
} finally {
db.close();
fs.rmSync(rootDir, { recursive: true, force: true });
}
});
it("rebuilds canonical FTS, rejects legacy orphans, and adopts canonical orphans", () => {
const db = new DatabaseSync(":memory:");
try {
ensureMemoryIndexSchema({ db, cacheEnabled: false, ftsEnabled: true });
db.exec(`
INSERT INTO memory_index_sources (path, source, hash, mtime, size)
VALUES ('canonical.md', 'memory', 'canonical-hash', 200, 20);
INSERT INTO memory_index_chunks VALUES (
'chunk-canonical', 'canonical.md', 'memory', 1, 2, 'canonical-chunk-hash',
'fts-only', 'canonical nebula', '[]', 200
);
INSERT INTO memory_index_chunks VALUES (
'chunk-canonical-ownerless', 'deleted.md', 'memory', 1, 2, 'orphan-chunk-hash',
'fts-only', 'orphaned starlight', '[]', 190
);
INSERT INTO memory_index_chunks_fts
(text, id, path, source, model, start_line, end_line)
VALUES
('canonical nebula', 'chunk-canonical', 'canonical.md', 'memory', 'fts-only', 1, 2),
('orphaned starlight', 'chunk-canonical-ownerless', 'deleted.md', 'memory', 'fts-only', 1, 2);
CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
CREATE TABLE files (
path TEXT PRIMARY KEY,
source TEXT NOT NULL DEFAULT 'memory',
hash TEXT NOT NULL,
mtime INTEGER NOT NULL,
size INTEGER NOT NULL
);
CREATE TABLE chunks (
id TEXT PRIMARY KEY,
path TEXT NOT NULL,
source TEXT NOT NULL DEFAULT 'memory',
start_line INTEGER NOT NULL,
end_line INTEGER NOT NULL,
hash TEXT NOT NULL,
model TEXT NOT NULL,
text TEXT NOT NULL,
embedding TEXT NOT NULL,
updated_at INTEGER NOT NULL
);
INSERT INTO files VALUES ('legacy.md', 'memory', 'legacy-hash', 100, 10);
INSERT INTO chunks VALUES (
'chunk-legacy', 'legacy.md', 'memory', 1, 2, 'legacy-chunk-hash',
'fts-only', 'imported saffronquasar', '[]', 100
);
-- Shipped legacy cleanup could commit its source delete before deleting
-- chunks. This derived orphan must not become canonical/searchable.
INSERT INTO chunks VALUES (
'chunk-ownerless', 'deleted.md', 'memory', 1, 2, 'orphan-chunk-hash',
'fts-only', 'orphaned ambercomet', '[]', 90
);
`);
ensureMemoryIndexSchema({ db, cacheEnabled: false, ftsEnabled: true });
expect(db.prepare("SELECT id FROM memory_index_chunks ORDER BY id").all()).toEqual([
{ id: "chunk-canonical" },
{ id: "chunk-canonical-ownerless" },
{ id: "chunk-legacy" },
]);
expect(db.prepare("SELECT id FROM memory_index_chunks_fts ORDER BY id").all()).toEqual([
{ id: "chunk-canonical" },
{ id: "chunk-canonical-ownerless" },
{ id: "chunk-legacy" },
]);
expect(
db
.prepare(
"SELECT id FROM memory_index_chunks_fts WHERE memory_index_chunks_fts MATCH 'saffronquasar'",
)
.all(),
).toEqual([{ id: "chunk-legacy" }]);
expect(
db
.prepare(
"SELECT id FROM memory_index_chunks_fts WHERE memory_index_chunks_fts MATCH 'starlight'",
)
.all(),
).toEqual([{ id: "chunk-canonical-ownerless" }]);
expect(
db
.prepare(
"SELECT id FROM memory_index_chunks_fts WHERE memory_index_chunks_fts MATCH 'ambercomet'",
)
.all(),
).toEqual([]);
// Schema migration cannot write the runtime-owned sqlite-vec table. The
// dirty source guarantees normal sync republishes every derived index.
expect(
db.prepare("SELECT hash FROM memory_index_sources WHERE path = 'legacy.md'").get(),
).toEqual({ hash: "" });
expect(
db.prepare("SELECT hash FROM memory_index_sources WHERE path = 'deleted.md'").get(),
).toEqual({ hash: "" });
ensureMemoryIndexSchema({ db, cacheEnabled: false, ftsEnabled: true });
expect(db.prepare("SELECT COUNT(*) AS count FROM memory_index_chunks_fts").get()).toEqual({
count: 3,
});
} finally {
db.close();
}
});
it("keeps canonical chunk sets coherent and invalidates ambiguous partial sources", () => {
const db = new DatabaseSync(":memory:");
try {
ensureMemoryIndexSchema({ db, cacheEnabled: false, ftsEnabled: false });
db.exec(`
-- doc.md is already chunk-owned: its stale legacy chunk must not ride along.
INSERT INTO memory_index_sources (path, source, hash, mtime, size)
VALUES ('doc.md', 'memory', 'doc-hash', 200.0, 42);
INSERT INTO memory_index_chunks VALUES (
'chunk-doc-canonical', 'doc.md', 'memory', 1, 10, 'doc-chunk-hash', 'model',
'canonical body', '[]', 200
);
-- partial.md has only the first canonical chunk. Seeing a second legacy
-- identity makes completeness ambiguous, so the source must reindex.
INSERT INTO memory_index_sources (path, source, hash, mtime, size)
VALUES ('partial.md', 'memory', 'partial-hash', 175.0, 30);
INSERT INTO memory_index_chunks VALUES (
'chunk-partial-1', 'partial.md', 'memory', 1, 5, 'partial-1-hash', 'model',
'canonical first half', '[]', 175
);
-- pending.md has a canonical source row but no chunks yet (indexing
-- interrupted before its chunks were written): its legacy chunk is the
-- only searchable content and must import instead of being stranded.
INSERT INTO memory_index_sources (path, source, hash, mtime, size)
VALUES ('pending.md', 'memory', 'pending-hash', 150.0, 20);
-- diverged.md is also chunkless, but its canonical metadata is newer;
-- pairing its stale legacy chunks with that hash would wedge stale text.
INSERT INTO memory_index_sources (path, source, hash, mtime, size)
VALUES ('diverged.md', 'memory', 'current-hash', 250.0, 30);
`);
db.exec(`
CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
CREATE TABLE files (
path TEXT PRIMARY KEY,
source TEXT NOT NULL DEFAULT 'memory',
hash TEXT NOT NULL,
mtime INTEGER NOT NULL,
size INTEGER NOT NULL
);
CREATE TABLE chunks (
id TEXT PRIMARY KEY,
path TEXT NOT NULL,
source TEXT NOT NULL DEFAULT 'memory',
start_line INTEGER NOT NULL,
end_line INTEGER NOT NULL,
hash TEXT NOT NULL,
model TEXT NOT NULL,
text TEXT NOT NULL,
embedding TEXT NOT NULL,
updated_at INTEGER NOT NULL
);
INSERT INTO files VALUES ('doc.md', 'memory', 'doc-hash', 200, 42);
INSERT INTO files VALUES ('partial.md', 'memory', 'partial-hash', 175, 30);
INSERT INTO files VALUES ('pending.md', 'memory', 'pending-hash', 150, 20);
INSERT INTO files VALUES ('diverged.md', 'memory', 'stale-hash', 50, 12);
INSERT INTO chunks VALUES (
'chunk-doc-legacy', 'doc.md', 'memory', 1, 8, 'stale-hash', 'model',
'stale legacy body', '[]', 100
);
INSERT INTO chunks VALUES (
'chunk-partial-1', 'partial.md', 'memory', 1, 5, 'legacy-partial-1-hash', 'model',
'legacy first half', '[]', 150
);
INSERT INTO chunks VALUES (
'chunk-partial-2', 'partial.md', 'memory', 6, 10, 'partial-2-hash', 'model',
'legacy second half', '[]', 150
);
INSERT INTO chunks VALUES (
'chunk-pending-legacy', 'pending.md', 'memory', 1, 6, 'pending-chunk-hash', 'model',
'only searchable content for pending', '[]', 150
);
INSERT INTO chunks VALUES (
'chunk-diverged-legacy', 'diverged.md', 'memory', 1, 4, 'diverged-chunk-hash', 'model',
'stale diverged content', '[]', 50
);
`);
ensureMemoryIndexSchema({
db,
cacheEnabled: false,
ftsEnabled: false,
});
// doc.md keeps only its canonical chunk (stale legacy chunk excluded);
// pending.md, whose canonical source had no chunks, imports its legacy
// chunk so the file is not left silently unsearchable.
expect(db.prepare("SELECT id, text FROM memory_index_chunks ORDER BY id").all()).toEqual([
{ id: "chunk-doc-canonical", text: "canonical body" },
{ id: "chunk-partial-1", text: "canonical first half" },
{ id: "chunk-pending-legacy", text: "only searchable content for pending" },
]);
// An impossible hash prevents hash-based sync from skipping these rows,
// while retaining the identities needed for deleted-file cleanup.
expect(
db.prepare("SELECT hash FROM memory_index_sources WHERE path = 'doc.md'").get(),
).toEqual({ hash: "" });
expect(
db.prepare("SELECT hash FROM memory_index_sources WHERE path = 'partial.md'").get(),
).toEqual({ hash: "" });
expect(
db.prepare("SELECT hash FROM memory_index_sources WHERE path = 'diverged.md'").get(),
).toEqual({ hash: "" });
expect(
db.prepare("SELECT hash FROM memory_index_sources WHERE path = 'pending.md'").get(),
).toEqual({ hash: "" });
expect(
db
.prepare(
"SELECT name FROM sqlite_master WHERE type = 'table' AND name IN ('meta', 'files', 'chunks')",
)
.all(),
).toEqual([]);
} finally {
db.close();
}
});
it("keeps legacy tables when a chunk id belongs to a different canonical source", () => {
const db = new DatabaseSync(":memory:");
try {
ensureMemoryIndexSchema({ db, cacheEnabled: false, ftsEnabled: false });
db.exec(`
INSERT INTO memory_index_sources (path, source, hash, mtime, size)
VALUES ('canonical.md', 'memory', 'canonical-hash', 200, 20);
INSERT INTO memory_index_chunks VALUES (
'shared-id', 'canonical.md', 'memory', 1, 2, 'canonical-chunk-hash', 'model',
'canonical body', '[]', 200
);
CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
CREATE TABLE files (
path TEXT PRIMARY KEY,
source TEXT NOT NULL DEFAULT 'memory',
hash TEXT NOT NULL,
mtime INTEGER NOT NULL,
size INTEGER NOT NULL
);
CREATE TABLE chunks (
id TEXT PRIMARY KEY,
path TEXT NOT NULL,
source TEXT NOT NULL DEFAULT 'memory',
start_line INTEGER NOT NULL,
end_line INTEGER NOT NULL,
hash TEXT NOT NULL,
model TEXT NOT NULL,
text TEXT NOT NULL,
embedding TEXT NOT NULL,
updated_at INTEGER NOT NULL
);
INSERT INTO files VALUES ('legacy.md', 'memory', 'legacy-hash', 100, 10);
INSERT INTO chunks VALUES (
'shared-id', 'legacy.md', 'memory', 1, 2, 'legacy-chunk-hash', 'model',
'legacy body', '[]', 100
);
`);
expect(() => ensureMemoryIndexSchema({ db, cacheEnabled: false, ftsEnabled: false })).toThrow(
"legacy memory chunks rows could not be copied",
);
expect(db.prepare("SELECT path, text FROM memory_index_chunks").all()).toEqual([
{ path: "canonical.md", text: "canonical body" },
]);
expect(
db
.prepare(
"SELECT name FROM sqlite_master WHERE type = 'table' AND name IN ('meta', 'files', 'chunks') ORDER BY name",
)
.all(),
).toEqual([{ name: "chunks" }, { name: "files" }, { name: "meta" }]);
} finally {
db.close();
}
});
it("keeps legacy tables when legacy rows cannot be copied", () => {
const db = new DatabaseSync(":memory:");
try {
db.exec(`
CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
CREATE TABLE files (
path TEXT PRIMARY KEY,
source TEXT NOT NULL DEFAULT 'memory',
hash TEXT,
mtime INTEGER NOT NULL,
size INTEGER NOT NULL
);
CREATE TABLE chunks (
id TEXT PRIMARY KEY,
path TEXT NOT NULL,
source TEXT NOT NULL DEFAULT 'memory',
start_line INTEGER NOT NULL,
end_line INTEGER NOT NULL,
hash TEXT NOT NULL,
model TEXT NOT NULL,
text TEXT NOT NULL,
embedding TEXT NOT NULL,
updated_at INTEGER NOT NULL
);
INSERT INTO files VALUES ('doc.md', 'memory', NULL, 1, 2);
INSERT INTO chunks VALUES (
'chunk-doc', 'doc.md', 'memory', 1, 2, 'chunk-hash', 'model',
'body', '[]', 1
);
`);
expect(() =>
ensureMemoryIndexSchema({
db,
cacheEnabled: false,
ftsEnabled: false,
}),
).toThrow("legacy memory files rows could not be copied");
expect(db.prepare("SELECT path FROM files").get()).toEqual({ path: "doc.md" });
expect(db.prepare("SELECT COUNT(*) AS count FROM memory_index_sources").get()).toEqual({
count: 0,
});
expect(db.prepare("SELECT COUNT(*) AS count FROM memory_index_chunks").get()).toEqual({
count: 0,
});
expect(
db
.prepare(
"SELECT name FROM sqlite_master WHERE type = 'table' AND name IN ('meta', 'files', 'chunks') ORDER BY name",
)
.all(),
).toEqual([{ name: "chunks" }, { name: "files" }, { name: "meta" }]);
} finally {
db.close();
}
});
it("keeps legacy tables when legacy meta rows cannot be copied", () => {
const db = new DatabaseSync(":memory:");
try {
db.exec(`
CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT);
CREATE TABLE files (
path TEXT PRIMARY KEY,
source TEXT NOT NULL DEFAULT 'memory',
hash TEXT NOT NULL,
mtime INTEGER NOT NULL,
size INTEGER NOT NULL
);
CREATE TABLE chunks (
id TEXT PRIMARY KEY,
path TEXT NOT NULL,
source TEXT NOT NULL DEFAULT 'memory',
start_line INTEGER NOT NULL,
end_line INTEGER NOT NULL,
hash TEXT NOT NULL,
model TEXT NOT NULL,
text TEXT NOT NULL,
embedding TEXT NOT NULL,
updated_at INTEGER NOT NULL
);
INSERT INTO meta VALUES ('broken-key', NULL);
`);
expect(() =>
ensureMemoryIndexSchema({
db,
cacheEnabled: false,
ftsEnabled: false,
}),
).toThrow("legacy memory meta rows could not be copied");
expect(
db
.prepare("SELECT COUNT(*) AS count FROM memory_index_meta WHERE key = 'broken-key'")
.get(),
).toEqual({ count: 0 });
expect(
db
.prepare(
"SELECT name FROM sqlite_master WHERE type = 'table' AND name IN ('meta', 'files', 'chunks') ORDER BY name",
)
.all(),
).toEqual([{ name: "chunks" }, { name: "files" }, { name: "meta" }]);
} finally {
db.close();
}
});
it("keeps legacy tables when legacy chunk rows cannot be copied", () => {
const db = new DatabaseSync(":memory:");
try {
db.exec(`
CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
CREATE TABLE files (
path TEXT PRIMARY KEY,
source TEXT NOT NULL DEFAULT 'memory',
hash TEXT NOT NULL,
mtime INTEGER NOT NULL,
size INTEGER NOT NULL
);
CREATE TABLE chunks (
id TEXT PRIMARY KEY,
path TEXT NOT NULL,
source TEXT NOT NULL DEFAULT 'memory',
start_line INTEGER NOT NULL,
end_line INTEGER NOT NULL,
hash TEXT NOT NULL,
model TEXT NOT NULL,
text TEXT NOT NULL,
embedding TEXT,
updated_at INTEGER NOT NULL
);
INSERT INTO files VALUES ('note.md', 'memory', 'note-hash', 90, 10);
-- Legacy-only source (canonical owns no chunks for it), so this chunk
-- must copy; a NULL embedding makes it uncopyable under STRICT and the
-- whole migration must abort with legacy tables retained.
INSERT INTO chunks VALUES (
'chunk-broken', 'note.md', 'memory', 1, 5, 'chunk-hash', 'model',
'body', NULL, 90
);
`);
expect(() =>
ensureMemoryIndexSchema({
db,
cacheEnabled: false,
ftsEnabled: false,
}),
).toThrow("legacy memory chunks rows could not be copied");
expect(db.prepare("SELECT COUNT(*) AS count FROM memory_index_chunks").get()).toEqual({
count: 0,
});
expect(
db
.prepare(
"SELECT name FROM sqlite_master WHERE type = 'table' AND name IN ('meta', 'files', 'chunks') ORDER BY name",
)
.all(),
).toEqual([{ name: "chunks" }, { name: "files" }, { name: "meta" }]);
} finally {
db.close();
}
});
});
packages/memory-host-sdk/src/host/memory-schema.test.ts
+42 -90
View File
@@ -71,7 +71,7 @@ describe("memory index schema", () => {
id: 1,
path: "MEMORY.md",
source: "memory",
hash: "file-hash",
hash: "",
mtime: 10.75,
size: 20,
},
@@ -283,6 +283,47 @@ describe("memory index schema", () => {
}
});
it("rebuilds body FTS after indexing while hybrid search is disabled", () => {
const db = new DatabaseSync(":memory:");
try {
ensureMemoryIndexSchema({ db, cacheEnabled: false, ftsEnabled: true });
db.exec(`
INSERT INTO memory_index_chunks VALUES (
'chunk-before', 'before.md', 'memory', 1, 1, 'before-hash', 'fts-only',
'before body', '[]', 1
);
INSERT INTO memory_index_chunks_fts
(text, id, path, source, model, start_line, end_line)
VALUES ('before body', 'chunk-before', 'before.md', 'memory', 'fts-only', 1, 1);
`);
ensureMemoryIndexSchema({ db, cacheEnabled: false, ftsEnabled: false });
expect(
db
.prepare(
"SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'memory_index_chunks_fts'",
)
.get(),
).toBeUndefined();
db.exec(`
INSERT INTO memory_index_chunks VALUES (
'chunk-disabled', 'disabled.md', 'memory', 1, 1, 'disabled-hash', 'fts-only',
'disabled body', '[]', 2
);
`);
expect(
ensureMemoryIndexSchema({ db, cacheEnabled: false, ftsEnabled: true }).ftsAvailable,
).toBe(true);
expect(db.prepare("SELECT id, text FROM memory_index_chunks_fts ORDER BY id").all()).toEqual([
{ id: "chunk-before", text: "before body" },
{ id: "chunk-disabled", text: "disabled body" },
]);
} finally {
db.close();
}
});
it("backfills and maintains one path FTS row per source without changing body FTS", () => {
const db = new DatabaseSync(":memory:");
try {
@@ -835,93 +876,4 @@ describe("memory index schema", () => {
db.close();
}
});
it("leaves unrelated generic tables untouched", () => {
const db = new DatabaseSync(":memory:");
try {
db.exec(`
CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL, owner TEXT);
CREATE TABLE files (
path TEXT PRIMARY KEY,
source TEXT NOT NULL,
hash TEXT NOT NULL,
mtime INTEGER NOT NULL,
size INTEGER NOT NULL
);
CREATE TABLE chunks (
id TEXT PRIMARY KEY,
path TEXT NOT NULL,
source TEXT NOT NULL,
start_line INTEGER NOT NULL,
end_line INTEGER NOT NULL,
hash TEXT NOT NULL,
model TEXT NOT NULL,
text TEXT NOT NULL,
embedding TEXT NOT NULL,
updated_at INTEGER NOT NULL
);
`);
ensureMemoryIndexSchema({
db,
cacheEnabled: false,
ftsEnabled: false,
});
expect(
db
.prepare(
"SELECT name FROM sqlite_master WHERE type = 'table' AND name IN ('meta', 'files', 'chunks') ORDER BY name",
)
.all(),
).toEqual([{ name: "chunks" }, { name: "files" }, { name: "meta" }]);
} finally {
db.close();
}
});
it("keeps legacy tables when canonical rows conflict", () => {
const db = new DatabaseSync(":memory:");
try {
db.exec(`
CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
CREATE TABLE files (
path TEXT PRIMARY KEY,
source TEXT NOT NULL DEFAULT 'memory',
hash TEXT NOT NULL,
mtime INTEGER NOT NULL,
size INTEGER NOT NULL
);
CREATE TABLE chunks (
id TEXT PRIMARY KEY,
path TEXT NOT NULL,
source TEXT NOT NULL DEFAULT 'memory',
start_line INTEGER NOT NULL,
end_line INTEGER NOT NULL,
hash TEXT NOT NULL,
model TEXT NOT NULL,
text TEXT NOT NULL,
embedding TEXT NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE TABLE memory_index_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL);
INSERT INTO meta VALUES ('memory_index_meta_v1', 'legacy');
INSERT INTO memory_index_meta VALUES ('memory_index_meta_v1', 'canonical');
`);
expect(() =>
ensureMemoryIndexSchema({
db,
cacheEnabled: false,
ftsEnabled: false,
}),
).toThrow("legacy memory meta rows conflict");
expect(db.prepare("SELECT value FROM meta").get()).toEqual({ value: "legacy" });
expect(db.prepare("SELECT value FROM memory_index_meta").get()).toEqual({
value: "canonical",
});
} finally {
db.close();
}
});
});
packages/memory-host-sdk/src/host/memory-schema.ts
+196 -152
View File
@@ -1,58 +1,40 @@
// Memory Host SDK module implements memory schema behavior.
import type { DatabaseSync } from "node:sqlite";
import { formatErrorMessage } from "./error-utils.js";
import {
dropDisabledMemoryChunkFts,
dropMemoryPathFtsTriggers,
ensureMemoryPathFtsSchema,
ensureMemoryPathFtsTriggers,
MEMORY_INDEX_CHUNKS_TABLE,
MEMORY_INDEX_FTS_TABLE,
MEMORY_INDEX_PATHS_FTS_TABLE,
MEMORY_INDEX_SOURCES_TABLE,
rebuildMemoryChunkFts,
} from "./memory-schema-fts.js";
import {
assertLegacyMemoryRowsCopied,
ensureLegacyMemoryMigrationIndexes,
} from "./memory-schema-migration.js";
import { migrateSqliteSchemaToStrict } from "./openclaw-runtime-sqlite.js";
export {
dropMemoryPathFtsTriggers,
ensureMemoryPathFtsTriggers,
MEMORY_INDEX_CHUNKS_TABLE,
MEMORY_INDEX_FTS_TABLE,
MEMORY_INDEX_PATHS_FTS_TABLE,
MEMORY_INDEX_SOURCES_TABLE,
MEMORY_PATH_FTS_TRIGGER_DEFINITIONS,
} from "./memory-schema-fts.js";
// SQLite schema setup for builtin memory index, embedding cache, and FTS.
export const MEMORY_INDEX_META_TABLE = "memory_index_meta";
export const MEMORY_INDEX_SOURCES_TABLE = "memory_index_sources";
export const MEMORY_INDEX_CHUNKS_TABLE = "memory_index_chunks";
export const MEMORY_EMBEDDING_CACHE_TABLE = "memory_embedding_cache";
export const MEMORY_INDEX_STATE_TABLE = "memory_index_state";
export const MEMORY_INDEX_FTS_TABLE = "memory_index_chunks_fts";
export const MEMORY_INDEX_PATHS_FTS_TABLE = "memory_index_paths_fts";
export const MEMORY_INDEX_VECTOR_TABLE = "memory_index_chunks_vec";
/** Optional canonical triggers owned by the derived path FTS index. */
export const MEMORY_PATH_FTS_TRIGGER_DEFINITIONS = [
{
name: "memory_index_paths_fts_after_insert",
sql: `
CREATE TRIGGER IF NOT EXISTS main.memory_index_paths_fts_after_insert
AFTER INSERT ON ${MEMORY_INDEX_SOURCES_TABLE}
BEGIN
INSERT INTO ${MEMORY_INDEX_PATHS_FTS_TABLE} (rowid, path, source)
VALUES (NEW.id, NEW.path, NEW.source);
END;
`,
},
{
name: "memory_index_paths_fts_after_update",
sql: `
CREATE TRIGGER IF NOT EXISTS main.memory_index_paths_fts_after_update
AFTER UPDATE OF id, path, source ON ${MEMORY_INDEX_SOURCES_TABLE}
BEGIN
DELETE FROM ${MEMORY_INDEX_PATHS_FTS_TABLE}
WHERE rowid = OLD.id;
INSERT INTO ${MEMORY_INDEX_PATHS_FTS_TABLE} (rowid, path, source)
VALUES (NEW.id, NEW.path, NEW.source);
END;
`,
},
{
name: "memory_index_paths_fts_after_delete",
sql: `
CREATE TRIGGER IF NOT EXISTS main.memory_index_paths_fts_after_delete
AFTER DELETE ON ${MEMORY_INDEX_SOURCES_TABLE}
BEGIN
DELETE FROM ${MEMORY_INDEX_PATHS_FTS_TABLE}
WHERE rowid = OLD.id;
END;
`,
},
] as const;
const LEGACY_MEMORY_INDEX_TRIGGERS = [
"memory_files_revision_after_insert",
"memory_files_revision_after_update",
@@ -242,13 +224,6 @@ function tableExists(db: DatabaseSync, tableName: string): boolean {
return row?.found === 1;
}
function assertLegacyRowsCopied(db: DatabaseSync, query: string, tableName: string): void {
const row = db.prepare(query).get() as { missing?: unknown } | undefined;
if (Number(row?.missing ?? 0) > 0) {
throw new Error(`legacy memory ${tableName} rows conflict with canonical memory index rows`);
}
}
/** Upgrade canonical memory sources to stable integer identities. */
export function migrateMemoryIndexSourcesIdentity(db: DatabaseSync): void {
if (!tableExists(db, MEMORY_INDEX_SOURCES_TABLE)) {
@@ -359,63 +334,167 @@ function copyLegacyMemoryIndexRows(
schema: string,
preservedEmbeddingCacheTable?: string,
): void {
ensureLegacyMemoryMigrationIndexes(db, schema);
// Canonical-owned chunk sets stay intact; any extra legacy identity invalidates
// the source for rebuild. Chunkless sources import only when metadata matches.
// Keep invalidated rows for deleted-file cleanup; snapshot before inserts.
db.exec(`
INSERT OR IGNORE INTO main.${MEMORY_INDEX_META_TABLE} (key, value)
SELECT key, value FROM ${schema}.meta;
INSERT OR IGNORE INTO main.${MEMORY_INDEX_SOURCES_TABLE} (path, source, hash, mtime, size)
SELECT path, source, hash, mtime, size
FROM ${schema}.files;
INSERT OR IGNORE INTO main.${MEMORY_INDEX_CHUNKS_TABLE} (
id, path, source, start_line, end_line, hash, model, text, embedding, updated_at
CREATE TEMP TABLE legacy_import_chunk_excluded_sources AS
SELECT DISTINCT owned.path, owned.source,
CASE WHEN EXISTS (
SELECT 1 FROM ${schema}.chunks AS legacy_chunk
WHERE legacy_chunk.path = owned.path AND legacy_chunk.source IS owned.source
AND NOT EXISTS (
SELECT 1 FROM main.${MEMORY_INDEX_CHUNKS_TABLE} AS canonical_chunk
WHERE canonical_chunk.id = legacy_chunk.id
AND canonical_chunk.path IS legacy_chunk.path AND canonical_chunk.source IS legacy_chunk.source
)
) THEN 1 ELSE 0 END AS force_reindex
FROM main.${MEMORY_INDEX_CHUNKS_TABLE} AS owned
WHERE EXISTS (
SELECT 1 FROM ${schema}.files AS legacy_file
WHERE legacy_file.path = owned.path AND legacy_file.source IS owned.source
)
SELECT id, path, source, start_line, end_line, hash, model, text, embedding, updated_at
FROM ${schema}.chunks;
UNION ALL
SELECT canonical.path, canonical.source, 1 AS force_reindex
FROM main.${MEMORY_INDEX_SOURCES_TABLE} AS canonical
JOIN ${schema}.files AS legacy
ON legacy.path = canonical.path AND legacy.source IS canonical.source
WHERE (
canonical.hash IS NOT legacy.hash
OR canonical.mtime IS NOT legacy.mtime
OR canonical.size IS NOT legacy.size
)
AND NOT EXISTS (
SELECT 1 FROM main.${MEMORY_INDEX_CHUNKS_TABLE} AS chunk
WHERE chunk.path = canonical.path AND chunk.source IS canonical.source
);
CREATE TEMP TABLE legacy_import_dirty_sources AS
SELECT legacy.path, legacy.source
FROM ${schema}.files AS legacy
WHERE NOT EXISTS (
SELECT 1 FROM main.${MEMORY_INDEX_SOURCES_TABLE} AS canonical
WHERE canonical.path = legacy.path AND canonical.source IS legacy.source
)
UNION
SELECT legacy.path, legacy.source
FROM ${schema}.chunks AS legacy
WHERE EXISTS (
SELECT 1 FROM ${schema}.files AS owner
WHERE owner.path = legacy.path AND owner.source IS legacy.source
)
AND NOT EXISTS (
SELECT 1 FROM temp.legacy_import_chunk_excluded_sources AS excluded
WHERE excluded.path = legacy.path AND excluded.source IS legacy.source
)
AND NOT EXISTS (
SELECT 1 FROM main.${MEMORY_INDEX_CHUNKS_TABLE} AS canonical
WHERE canonical.id = legacy.id
)
UNION
SELECT excluded.path, excluded.source
FROM temp.legacy_import_chunk_excluded_sources AS excluded
WHERE excluded.force_reindex = 1;
`);
assertLegacyRowsCopied(
db,
`SELECT COUNT(*) AS missing
FROM ${schema}.meta AS legacy
WHERE NOT EXISTS (
SELECT 1 FROM main.${MEMORY_INDEX_META_TABLE} AS canonical
WHERE canonical.key = legacy.key AND canonical.value IS legacy.value
)`,
"meta",
);
assertLegacyRowsCopied(
db,
`SELECT COUNT(*) AS missing
FROM ${schema}.files AS legacy
WHERE NOT EXISTS (
SELECT 1 FROM main.${MEMORY_INDEX_SOURCES_TABLE} AS canonical
WHERE canonical.path = legacy.path
AND canonical.source IS legacy.source
AND canonical.hash IS legacy.hash
AND canonical.mtime IS legacy.mtime
AND canonical.size IS legacy.size
)`,
"files",
);
assertLegacyRowsCopied(
db,
`SELECT COUNT(*) AS missing
FROM ${schema}.chunks AS legacy
WHERE NOT EXISTS (
SELECT 1 FROM main.${MEMORY_INDEX_CHUNKS_TABLE} AS canonical
WHERE canonical.id = legacy.id
AND canonical.path IS legacy.path
AND canonical.source IS legacy.source
AND canonical.start_line IS legacy.start_line
AND canonical.end_line IS legacy.end_line
AND canonical.hash IS legacy.hash
AND canonical.model IS legacy.model
AND canonical.text IS legacy.text
AND canonical.embedding IS legacy.embedding
AND canonical.updated_at IS legacy.updated_at
)`,
"chunks",
);
try {
db.exec(`
INSERT OR IGNORE INTO main.${MEMORY_INDEX_META_TABLE} (key, value)
SELECT key, value FROM ${schema}.meta;
INSERT OR IGNORE INTO main.${MEMORY_INDEX_SOURCES_TABLE} (path, source, hash, mtime, size)
SELECT path, source, hash, mtime, size
FROM ${schema}.files;
INSERT OR IGNORE INTO main.${MEMORY_INDEX_CHUNKS_TABLE} (
id, path, source, start_line, end_line, hash, model, text, embedding, updated_at
)
-- Chunks are derived from source rows. Shipped cleanup could leave an
-- ownerless legacy chunk, which must not become permanently searchable.
SELECT id, path, source, start_line, end_line, hash, model, text, embedding, updated_at
FROM ${schema}.chunks AS legacy
WHERE EXISTS (
SELECT 1 FROM ${schema}.files AS owner
WHERE owner.path = legacy.path AND owner.source IS legacy.source
)
AND NOT EXISTS (
SELECT 1 FROM temp.legacy_import_chunk_excluded_sources AS excluded
WHERE excluded.path = legacy.path AND excluded.source IS legacy.source
);
-- Content hashes are SHA-256 hex, so an empty hash cannot match a file.
-- Imported sources or chunks may be absent from runtime-owned vector
-- indexes, while excluded sources need a canonical rebuild. Retaining the
-- dirty source lets sync rebuild every derived row or clean up a deleted file.
UPDATE main.${MEMORY_INDEX_SOURCES_TABLE}
SET hash = ''
WHERE EXISTS (
SELECT 1 FROM temp.legacy_import_dirty_sources AS dirty
WHERE dirty.path = main.${MEMORY_INDEX_SOURCES_TABLE}.path
AND dirty.source IS main.${MEMORY_INDEX_SOURCES_TABLE}.source
);
`);
assertLegacyMemoryRowsCopied(
db,
`SELECT COUNT(*) AS missing
FROM ${schema}.meta AS legacy
WHERE NOT EXISTS (
SELECT 1 FROM main.${MEMORY_INDEX_META_TABLE} AS canonical
WHERE canonical.key = legacy.key
)`,
"meta",
);
assertLegacyMemoryRowsCopied(
db,
`SELECT COUNT(*) AS missing
FROM ${schema}.files AS legacy
WHERE NOT EXISTS (
SELECT 1 FROM main.${MEMORY_INDEX_SOURCES_TABLE} AS canonical
WHERE canonical.path = legacy.path
AND canonical.source IS legacy.source
)
AND NOT EXISTS (
SELECT 1 FROM temp.legacy_import_chunk_excluded_sources AS excluded
WHERE excluded.force_reindex = 1
AND excluded.path = legacy.path
AND excluded.source IS legacy.source
)`,
"files",
);
assertLegacyMemoryRowsCopied(
db,
`SELECT COUNT(*) AS missing
FROM ${schema}.chunks AS legacy
WHERE EXISTS (
SELECT 1 FROM ${schema}.files AS owner
WHERE owner.path = legacy.path AND owner.source IS legacy.source
)
AND NOT EXISTS (
SELECT 1 FROM main.${MEMORY_INDEX_CHUNKS_TABLE} AS canonical
WHERE canonical.id = legacy.id
AND canonical.path IS legacy.path
AND canonical.source IS legacy.source
)
AND NOT EXISTS (
SELECT 1 FROM temp.legacy_import_chunk_excluded_sources AS excluded
WHERE excluded.path = legacy.path AND excluded.source IS legacy.source
)`,
"chunks",
);
// Repair derived orphans only after authoritative copy assertions pass;
// otherwise a synthetic owner could mask an uncopyable legacy source row.
db.exec(`
INSERT OR IGNORE INTO main.${MEMORY_INDEX_SOURCES_TABLE} (path, source, hash, mtime, size)
SELECT DISTINCT orphan.path, orphan.source, '', 0, 0 FROM main.${MEMORY_INDEX_CHUNKS_TABLE} AS orphan
WHERE NOT EXISTS (
SELECT 1 FROM main.${MEMORY_INDEX_SOURCES_TABLE} AS owner
WHERE owner.path = orphan.path AND owner.source IS orphan.source
);
`);
} finally {
db.exec("DROP TABLE temp.legacy_import_dirty_sources");
db.exec("DROP TABLE temp.legacy_import_chunk_excluded_sources");
}
if (
preservedEmbeddingCacheTable !== "embedding_cache" &&
hasLegacyEmbeddingCacheTable(db, schema)
@@ -437,7 +516,7 @@ function copyLegacyMemoryIndexRows(
SELECT provider, model, provider_key, hash, embedding, dims, updated_at
FROM ${schema}.embedding_cache;
`);
assertLegacyRowsCopied(
assertLegacyMemoryRowsCopied(
db,
`SELECT COUNT(*) AS missing
FROM ${schema}.embedding_cache AS legacy
@@ -447,9 +526,6 @@ function copyLegacyMemoryIndexRows(
AND canonical.model = legacy.model
AND canonical.provider_key = legacy.provider_key
AND canonical.hash = legacy.hash
AND canonical.embedding IS legacy.embedding
AND canonical.dims IS legacy.dims
AND canonical.updated_at IS legacy.updated_at
)`,
"embedding_cache",
);
@@ -459,6 +535,7 @@ function copyLegacyMemoryIndexRows(
function migrateLegacyMemoryIndexTables(
db: DatabaseSync,
preservedEmbeddingCacheTable?: string,
ftsTable = MEMORY_INDEX_FTS_TABLE,
): void {
if (!hasLegacyMemoryIndexTables(db)) {
return;
@@ -467,6 +544,13 @@ function migrateLegacyMemoryIndexTables(
db.exec("SAVEPOINT migrate_legacy_memory_index_tables");
try {
copyLegacyMemoryIndexRows(db, "main", preservedEmbeddingCacheTable);
// `chunks_fts` belongs to the legacy schema and is dropped below even when
// a deprecated caller also supplied that name as its preferred FTS table.
if (ftsTable !== "chunks_fts" && tableExists(db, ftsTable)) {
// FTS is derived from canonical chunks. Rebuild inside the migration
// savepoint so imported rows and removed stale rows publish atomically.
rebuildMemoryChunkFts(db, ftsTable);
}
if (preservedEmbeddingCacheTable !== "embedding_cache" && hasLegacyEmbeddingCacheTable(db)) {
db.exec("DROP TABLE embedding_cache");
}
@@ -487,47 +571,6 @@ function migrateLegacyMemoryIndexTables(
}
}
/** Drop the canonical source-to-path-FTS maintenance triggers. */
export function dropMemoryPathFtsTriggers(db: DatabaseSync): void {
for (const trigger of MEMORY_PATH_FTS_TRIGGER_DEFINITIONS) {
db.exec(`DROP TRIGGER IF EXISTS main.${trigger.name}`);
}
}
/** Install the canonical source-to-path-FTS maintenance triggers. */
export function ensureMemoryPathFtsTriggers(db: DatabaseSync): void {
// The named integer source identity survives VACUUM and gives every
// FTS update/delete a direct rowid lookup instead of a virtual-table scan.
for (const trigger of MEMORY_PATH_FTS_TRIGGER_DEFINITIONS) {
db.exec(trigger.sql);
}
}
function ensureMemoryPathFtsSchema(params: { db: DatabaseSync; tokenizeClause: string }): void {
params.db.exec("SAVEPOINT ensure_memory_index_paths_fts");
try {
params.db.exec(`
CREATE VIRTUAL TABLE IF NOT EXISTS ${MEMORY_INDEX_PATHS_FTS_TABLE} USING fts5(
path,
source UNINDEXED
${params.tokenizeClause}
);
-- The initial copy and trigger installation share this savepoint. Once
-- populated, the triggers own completeness; per-row FTS probes are too costly.
INSERT INTO ${MEMORY_INDEX_PATHS_FTS_TABLE} (rowid, path, source)
SELECT id, path, source
FROM ${MEMORY_INDEX_SOURCES_TABLE}
WHERE NOT EXISTS (SELECT 1 FROM ${MEMORY_INDEX_PATHS_FTS_TABLE} LIMIT 1);
`);
ensureMemoryPathFtsTriggers(params.db);
params.db.exec("RELEASE ensure_memory_index_paths_fts");
} catch (err) {
params.db.exec("ROLLBACK TO ensure_memory_index_paths_fts");
params.db.exec("RELEASE ensure_memory_index_paths_fts");
throw err;
}
}
function buildMemoryIndexStrictSchema(params: {
embeddingCacheTable: string;
includeEmbeddingCache: boolean;
@@ -646,7 +689,8 @@ export function ensureMemoryIndexSchema(params: {
CREATE INDEX IF NOT EXISTS idx_memory_index_chunks_source
ON ${MEMORY_INDEX_CHUNKS_TABLE}(source);
`);
migrateLegacyMemoryIndexTables(params.db, params.embeddingCacheTable);
migrateLegacyMemoryIndexTables(params.db, params.embeddingCacheTable, ftsTable);
dropDisabledMemoryChunkFts(params.db, ftsTable, params.ftsEnabled);
if (params.cacheEnabled) {
const updatedAtIndex =
embeddingCacheTable === MEMORY_EMBEDDING_CACHE_TABLE
@@ -683,8 +727,8 @@ export function ensureMemoryIndexSchema(params: {
` end_line UNINDEXED\n` +
`${tokenizeClause});`,
);
// The shipped generic-table migration and a later FTS enablement both
// create an empty derived table beside already-canonical chunk rows.
// A migration rebuilds an existing FTS table in its savepoint. If the
// table is new, this same empty-table bootstrap covers all canonical rows.
params.db.exec(`
INSERT INTO ${ftsTable} (
text, id, path, source, model, start_line, end_line
src/scripts/test-projects.test.ts
+1
View File
@@ -657,6 +657,7 @@ describe("test-projects args", () => {
includePatterns: [
"extensions/memory-core/src/memory/index.test.ts",
"extensions/memory-core/src/memory/manager.fts-only-reindex.test.ts",
"extensions/memory-core/src/memory/manager.legacy-migration-cleanup.test.ts",
"extensions/memory-core/src/memory/manager.reindex-recovery.test.ts",
"extensions/memory-core/src/memory/manager.self-heal-missing-identity.test.ts",
],