From 7e42dae61199d0eddcb556b8242e29c2c6454e7d Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 12 Aug 2026 02:28:20 -0700 Subject: [PATCH] refactor(memory-core): split index manager along the inheritance chain (#122540) * refactor(memory-core): split index manager along the inheritance chain * fix(memory): restore the manager lazy boundary * refactor(memory-core): delete dead manager plumbing * chore(lint): ratchet max-lines baseline after memory manager split * fix(memory): keep the manager out of the runtime facade * test(memory): cover transitive manager facade imports --- config/max-lines-baseline.txt | 1 - extensions/memory-core/runtime-api.ts | 2 +- .../memory-core/src/memory/index.test.ts | 2436 +---------------- extensions/memory-core/src/memory/index.ts | 1 - .../src/memory/manager-async-state.test.ts | 71 + .../memory/manager-keyword-retrieval.test.ts | 972 +++++++ .../src/memory/manager-keyword-retrieval.ts | 405 +++ ...anager-provider-lifecycle-fallback.test.ts | 775 ++++++ .../manager-provider-lifecycle-leases.test.ts | 927 +++++++ .../memory/manager-provider-lifecycle.test.ts | 869 ++++++ .../src/memory/manager-provider-lifecycle.ts | 607 ++++ ...test.ts => manager-provider-state.test.ts} | 6 +- .../src/memory/manager-registry.test.ts | 809 ++++++ .../src/memory/manager-registry.ts | 256 ++ .../manager-search-orchestration.test.ts | 848 ++++++ .../memory/manager-search-orchestration.ts | 495 ++++ ...est.ts => manager-session-reindex.test.ts} | 0 ...e.test.ts => manager-vector-write.test.ts} | 0 .../src/memory/manager.async-search.test.ts | 143 - .../memory/manager.reindex-recovery.test.ts | 2 +- extensions/memory-core/src/memory/manager.ts | 1886 +------------ .../src/memory/manager.watcher-config.test.ts | 7 +- .../src/memory/search-manager.test.ts | 2 +- .../memory-core/src/memory/search-manager.ts | 1 - .../src/memory/test-manager-helpers.ts | 2 +- .../src/host/read-file-manager-compat.test.ts | 0 src/scripts/test-projects.test.ts | 6 + .../check-runtime-sidecar-loaders.test.ts | 67 + 28 files changed, 7217 insertions(+), 4379 deletions(-) create mode 100644 extensions/memory-core/src/memory/manager-async-state.test.ts create mode 100644 extensions/memory-core/src/memory/manager-keyword-retrieval.test.ts create mode 100644 extensions/memory-core/src/memory/manager-keyword-retrieval.ts create mode 100644 extensions/memory-core/src/memory/manager-provider-lifecycle-fallback.test.ts create mode 100644 extensions/memory-core/src/memory/manager-provider-lifecycle-leases.test.ts create mode 100644 extensions/memory-core/src/memory/manager-provider-lifecycle.test.ts create mode 100644 extensions/memory-core/src/memory/manager-provider-lifecycle.ts rename extensions/memory-core/src/memory/{manager.mistral-provider.test.ts => manager-provider-state.test.ts} (97%) create mode 100644 extensions/memory-core/src/memory/manager-registry.test.ts create mode 100644 extensions/memory-core/src/memory/manager-registry.ts create mode 100644 extensions/memory-core/src/memory/manager-search-orchestration.test.ts create mode 100644 extensions/memory-core/src/memory/manager-search-orchestration.ts rename extensions/memory-core/src/memory/{manager.session-reindex.test.ts => manager-session-reindex.test.ts} (100%) rename extensions/memory-core/src/memory/{manager.vector-dedupe.test.ts => manager-vector-write.test.ts} (100%) delete mode 100644 extensions/memory-core/src/memory/manager.async-search.test.ts rename extensions/memory-core/src/memory/manager.read-file.test.ts => packages/memory-host-sdk/src/host/read-file-manager-compat.test.ts (100%) diff --git a/config/max-lines-baseline.txt b/config/max-lines-baseline.txt index 55506e75e08b..56d35e6496a5 100644 --- a/config/max-lines-baseline.txt +++ b/config/max-lines-baseline.txt @@ -150,7 +150,6 @@ extensions/memory-core/src/memory/index.test.ts extensions/memory-core/src/memory/manager-embedding-ops.ts extensions/memory-core/src/memory/manager-search.test.ts extensions/memory-core/src/memory/manager-search.ts -extensions/memory-core/src/memory/manager.ts extensions/memory-core/src/rem-evidence.ts extensions/memory-core/src/short-term-promotion.test.ts extensions/memory-core/src/tools.test.ts diff --git a/extensions/memory-core/runtime-api.ts b/extensions/memory-core/runtime-api.ts index c3fe81dc7afb..a4d1479ca6b5 100644 --- a/extensions/memory-core/runtime-api.ts +++ b/extensions/memory-core/runtime-api.ts @@ -1,5 +1,5 @@ // Memory Core API module exposes the plugin public contract. -export { getMemorySearchManager, MemoryIndexManager } from "./src/memory/index.js"; +export { getMemorySearchManager } from "./src/memory/index.js"; export { memoryRuntime } from "./src/runtime-provider.js"; export { DEFAULT_LOCAL_MODEL, diff --git a/extensions/memory-core/src/memory/index.test.ts b/extensions/memory-core/src/memory/index.test.ts index 2e44219b54c5..0ffa607b5970 100644 --- a/extensions/memory-core/src/memory/index.test.ts +++ b/extensions/memory-core/src/memory/index.test.ts @@ -27,14 +27,9 @@ import { resetMemoryCoreDreamingStateForTests, } from "../test-helpers.js"; import "./test-runtime-mocks.js"; -import type { MemoryIndexManager } from "./index.js"; import { closeAllMemorySearchManagers, getMemorySearchManager } from "./index.js"; import type { MemoryIndexMeta } from "./manager-reindex-state.js"; -import { - closeAllMemoryIndexManagers, - closeMemoryIndexManagersForAgent, - MemoryIndexManager as RuntimeMemoryIndexManager, -} from "./manager.js"; +import type { MemoryIndexManager } from "./manager.js"; import { isolateMemoryManagerTestConfig } from "./test-config-helpers.js"; // This suite performs real sqlite/media indexing and can exceed the global @@ -63,6 +58,7 @@ let providerCloseGate: Promise | null = null; let providerInitGate: Promise | null = null; let providerCalls: Array<{ provider?: string; model?: string; outputDimensionality?: number }> = []; let forceNoProvider = false; + const originalMemoryIndexStateDir = process.env.OPENCLAW_STATE_DIR; const identityAliasFixture = vi.hoisted(() => ({ @@ -71,14 +67,6 @@ const identityAliasFixture = vi.hoisted(() => ({ cacheModel: "/fixture/cache/default-model.gguf", })); -function createLocalWorkerExitError(): Error { - return Object.assign(new Error("Local embedding worker exited unexpectedly (exit code 134)"), { - code: "LOCAL_EMBEDDING_WORKER_EXITED", - reason: "exit", - exitCode: 134, - }); -} - function setMemoryIndexStateDir(stateDir: string): void { Reflect.set(process.env, "OPENCLAW_STATE_DIR", stateDir); } @@ -519,8 +507,9 @@ describe("memory index", () => { cfg: TestCfg, purpose?: "default" | "status" | "cli", ): Promise { - const { getRequiredMemoryIndexManager } = await import("./test-manager-helpers.js"); - return await getRequiredMemoryIndexManager({ cfg, agentId: "main", purpose }); + const manager = requireManager(await getMemorySearchManager({ cfg, agentId: "main", purpose })); + managersForCleanup.add(manager); + return manager; } function rewritePersistedProviderIdentity(manager: MemoryIndexManager, model: string): void { @@ -550,23 +539,6 @@ describe("memory index", () => { ).run(model, providerKey, identityAliasFixture.provider); } - async function expectHybridKeywordSearchFindsMemory(cfg: TestCfg) { - const manager = await getFreshManager(cfg); - try { - const status = manager.status(); - if (!status.fts?.available) { - return; - } - - await manager.sync({ reason: "test" }); - const results = await manager.search("zebra"); - expect(results.length).toBeGreaterThan(0); - expect(results[0]?.path).toContain("memory/2026-01-12.md"); - } finally { - await manager.close?.(); - } - } - it("does not prepare vector deletes after in-place reset drops a missing vector table", async () => { const cfg = createCfg({ vectorEnabled: true, @@ -2495,390 +2467,6 @@ describe("memory index", () => { expect(providerCloseCalls).toBe(1); }); - it("waits for scoped manager close before initializing a replacement", async () => { - let releaseProviderClose: () => void = () => {}; - providerCloseGate = new Promise((resolve) => { - releaseProviderClose = resolve; - }); - const cfg = createCfg({ - hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, - }); - const first = requireManager(await getMemorySearchManager({ cfg, agentId: "main" })); - managersForCleanup.add(first); - await first.probeEmbeddingAvailability(); - const closePromise = closeMemoryIndexManagersForAgent({ cfg, agentId: "main" }); - const callsBeforeReplacement = providerCalls.length; - const secondPromise = getMemorySearchManager({ cfg, agentId: "main" }).then((result) => - requireManager(result), - ); - const concurrentSecondPromise = getMemorySearchManager({ cfg, agentId: "main" }).then( - (result) => requireManager(result), - ); - const secondProbe = secondPromise.then(async (manager) => { - await manager.probeEmbeddingAvailability(); - }); - let secondSettled = false; - void secondPromise.then( - () => { - secondSettled = true; - }, - () => { - secondSettled = true; - }, - ); - try { - await vi.waitFor(() => { - expect(providerCloseCalls).toBe(1); - }); - await Promise.resolve(); - expect(secondSettled).toBe(false); - expect(providerCalls).toHaveLength(callsBeforeReplacement); - } finally { - releaseProviderClose(); - providerCloseGate = null; - } - await closePromise; - const second = await secondPromise; - const concurrentSecond = await concurrentSecondPromise; - await secondProbe; - managersForCleanup.add(second); - expect(second === first).toBe(false); - expect(concurrentSecond).toBe(second); - - const third = requireManager(await getMemorySearchManager({ cfg, agentId: "main" })); - managersForCleanup.add(third); - expect(third).toBe(second); - }); - - it("does not reuse a cached manager after direct close starts", async () => { - let releaseProviderClose: () => void = () => {}; - providerCloseGate = new Promise((resolve) => { - releaseProviderClose = resolve; - }); - const cfg = createCfg({ - hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, - }); - const first = requireManager(await getMemorySearchManager({ cfg, agentId: "main" })); - managersForCleanup.add(first); - await first.probeEmbeddingAvailability(); - - const closePromise = first.close(); - const replacementPromise = getMemorySearchManager({ cfg, agentId: "main" }).then((result) => - requireManager(result), - ); - let replacementSettled = false; - void replacementPromise.then( - () => { - replacementSettled = true; - }, - () => { - replacementSettled = true; - }, - ); - try { - await vi.waitFor(() => expect(providerCloseCalls).toBe(1)); - await Promise.resolve(); - expect(replacementSettled).toBe(false); - } finally { - releaseProviderClose(); - providerCloseGate = null; - } - - await closePromise; - const replacement = await replacementPromise; - managersForCleanup.add(replacement); - expect(replacement === first).toBe(false); - }); - - it("serializes concurrent acquisitions with different cache identities", async () => { - const firstCfg = createCfg({ - model: "first-model", - hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, - }); - const first = requireManager(await getMemorySearchManager({ cfg: firstCfg, agentId: "main" })); - managersForCleanup.add(first); - await first.probeEmbeddingAvailability(); - let releaseProviderClose: () => void = () => {}; - providerCloseGate = new Promise((resolve) => { - releaseProviderClose = resolve; - }); - - const secondPromise = getMemorySearchManager({ - cfg: createCfg({ model: "second-model" }), - agentId: "main", - }).then((result) => requireManager(result)); - await vi.waitFor(() => expect(providerCloseCalls).toBe(1)); - const thirdPromise = getMemorySearchManager({ - cfg: createCfg({ model: "third-model" }), - agentId: "main", - }).then((result) => requireManager(result)); - try { - await Promise.resolve(); - expect(providerCalls).toHaveLength(1); - } finally { - releaseProviderClose(); - providerCloseGate = null; - } - - const [second, third] = await Promise.all([secondPromise, thirdPromise]); - managersForCleanup.add(second); - managersForCleanup.add(third); - expect(second === first).toBe(false); - expect(third === second).toBe(false); - expect((second as unknown as { closed: boolean }).closed).toBe(true); - expect((third as unknown as { closed: boolean }).closed).toBe(false); - }); - - it("canonicalizes agent ids before builtin manager acquisition", async () => { - const cfg = createCfg({ model: "canonical-model" }); - const first = await RuntimeMemoryIndexManager.get({ cfg, agentId: "Main-Agent" }); - const second = await RuntimeMemoryIndexManager.get({ cfg, agentId: "main-agent" }); - if (!first || !second) { - throw new Error("Expected canonical memory index managers"); - } - managersForCleanup.add(first); - managersForCleanup.add(second); - expect(second).toBe(first); - }); - - it("retires the prior builtin manager when an agent workspace changes", async () => { - const firstCfg = createCfg({ model: "workspace-model" }); - const secondCfg = createCfg({ model: "workspace-model" }); - if (!firstCfg.agents?.defaults || !secondCfg.agents?.defaults) { - throw new Error("Expected agent defaults"); - } - firstCfg.agents.defaults.workspace = path.join(fixtureRoot, "workspace-a"); - secondCfg.agents.defaults.workspace = path.join(fixtureRoot, "workspace-b"); - - const first = await RuntimeMemoryIndexManager.get({ cfg: firstCfg, agentId: "main" }); - const second = await RuntimeMemoryIndexManager.get({ cfg: secondCfg, agentId: "main" }); - if (!first || !second) { - throw new Error("Expected workspace memory index managers"); - } - managersForCleanup.add(first); - managersForCleanup.add(second); - expect(second === first).toBe(false); - expect((first as unknown as { closed: boolean }).closed).toBe(true); - }); - - it("does not block another agent while one scope retires its manager", async () => { - const firstCfg = createCfg({ - model: "first-model", - hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, - }); - const first = requireManager(await getMemorySearchManager({ cfg: firstCfg, agentId: "main" })); - managersForCleanup.add(first); - await first.probeEmbeddingAvailability(); - let releaseProviderClose: () => void = () => {}; - providerCloseGate = new Promise((resolve) => { - releaseProviderClose = resolve; - }); - - const replacementPromise = getMemorySearchManager({ - cfg: createCfg({ model: "second-model" }), - agentId: "main", - }); - await vi.waitFor(() => expect(providerCloseCalls).toBe(1)); - const otherAgentPromise = getMemorySearchManager({ - cfg: createCfg({ model: "other-model" }), - agentId: "other", - }); - let otherAgentSettled = false; - void otherAgentPromise.then( - () => { - otherAgentSettled = true; - }, - () => { - otherAgentSettled = true; - }, - ); - try { - await vi.waitFor(() => expect(otherAgentSettled).toBe(true)); - } finally { - releaseProviderClose(); - providerCloseGate = null; - } - - const otherAgent = requireManager(await otherAgentPromise); - const replacement = requireManager(await replacementPromise); - managersForCleanup.add(otherAgent); - managersForCleanup.add(replacement); - expect((otherAgent as unknown as { closed: boolean }).closed).toBe(false); - }); - - it("global teardown waits for an admitted builtin manager replacement", async () => { - const first = await RuntimeMemoryIndexManager.get({ - cfg: createCfg({ model: "first-model" }), - agentId: "main", - }); - if (!first) { - throw new Error("Expected first memory index manager"); - } - managersForCleanup.add(first); - await first.probeEmbeddingAvailability(); - let releaseProviderClose: () => void = () => {}; - providerCloseGate = new Promise((resolve) => { - releaseProviderClose = resolve; - }); - - const replacementPromise = RuntimeMemoryIndexManager.get({ - cfg: createCfg({ model: "second-model" }), - agentId: "main", - }); - await vi.waitFor(() => expect(providerCloseCalls).toBe(1)); - const globalClosePromise = closeAllMemoryIndexManagers(); - let globalCloseSettled = false; - void globalClosePromise.then( - () => { - globalCloseSettled = true; - }, - () => { - globalCloseSettled = true; - }, - ); - try { - await Promise.resolve(); - expect(globalCloseSettled).toBe(false); - } finally { - releaseProviderClose(); - providerCloseGate = null; - } - - const replacement = await replacementPromise; - await globalClosePromise; - if (!replacement) { - throw new Error("Expected replacement memory index manager"); - } - managersForCleanup.add(replacement); - expect((replacement as unknown as { closed: boolean }).closed).toBe(true); - }); - - it("retains a failed scoped close owner until provider retirement succeeds", async () => { - const cfg = createCfg({ - hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, - }); - const first = requireManager(await getMemorySearchManager({ cfg, agentId: "main" })); - managersForCleanup.add(first); - await first.probeEmbeddingAvailability(); - providerCloseFailuresRemaining = 2; - - await expect(closeMemoryIndexManagersForAgent({ cfg, agentId: "main" })).rejects.toThrow( - "provider close failed", - ); - expect(providerCloseCalls).toBe(2); - - let releaseProviderClose: () => void = () => {}; - providerCloseGate = new Promise((resolve) => { - releaseProviderClose = resolve; - }); - const callsBeforeReplacement = providerCalls.length; - const replacementPromise = getMemorySearchManager({ cfg, agentId: "main" }).then((result) => - requireManager(result), - ); - try { - await vi.waitFor(() => expect(providerCloseCalls).toBe(3)); - expect(providerCalls).toHaveLength(callsBeforeReplacement); - } finally { - releaseProviderClose(); - providerCloseGate = null; - } - - const replacement = await replacementPromise; - managersForCleanup.add(replacement); - expect(replacement === first).toBe(false); - }); - - it("retains a failed global close owner until provider retirement succeeds", async () => { - const cfg = createCfg({ - hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, - }); - const first = requireManager(await getMemorySearchManager({ cfg, agentId: "main" })); - managersForCleanup.add(first); - await first.probeEmbeddingAvailability(); - providerCloseFailuresRemaining = 2; - providerCloseFailure = undefined; - - let globalCloseRejected = false; - await closeAllMemorySearchManagers().then( - () => {}, - () => { - globalCloseRejected = true; - }, - ); - expect(globalCloseRejected).toBe(true); - expect(providerCloseCalls).toBe(2); - - let releaseProviderClose: () => void = () => {}; - providerCloseGate = new Promise((resolve) => { - releaseProviderClose = resolve; - }); - const callsBeforeReplacement = providerCalls.length; - const replacementPromise = getMemorySearchManager({ cfg, agentId: "main" }).then((result) => - requireManager(result), - ); - let concurrentGlobalClose: Promise = Promise.resolve(); - try { - await vi.waitFor(() => expect(providerCloseCalls).toBe(3)); - expect(providerCalls).toHaveLength(callsBeforeReplacement); - concurrentGlobalClose = closeAllMemorySearchManagers(); - } finally { - releaseProviderClose(); - providerCloseGate = null; - } - - const replacement = await replacementPromise; - await concurrentGlobalClose; - managersForCleanup.add(replacement); - expect(replacement === first).toBe(false); - expect((replacement as unknown as { closed: boolean }).closed).toBe(false); - }); - - it("does not reuse memory index managers across local-service hosts", async () => { - const cfg = createCfg({}); - const firstAcquire = vi.fn(async () => undefined); - const secondAcquire = vi.fn(async () => undefined); - const first = requireManager( - await getMemorySearchManager({ - cfg, - agentId: "main", - acquireLocalService: firstAcquire, - }), - ); - managersForCleanup.add(first); - - const second = requireManager( - await getMemorySearchManager({ - cfg, - agentId: "main", - acquireLocalService: secondAcquire, - }), - ); - managersForCleanup.add(second); - const secondAgain = requireManager( - await getMemorySearchManager({ - cfg, - agentId: "main", - acquireLocalService: secondAcquire, - }), - ); - - expect(Object.is(second, first)).toBe(false); - expect(Object.is(secondAgain, second)).toBe(true); - }); - - it("retries embedding provider close before releasing the manager", async () => { - providerCloseFailuresRemaining = 1; - const cfg = createCfg({ - hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, - }); - const manager = await getFreshManager(cfg); - - await manager.probeEmbeddingAvailability(); - await manager.close(); - - expect(providerCloseCalls).toBe(2); - }); - it("indexes multimodal files only from extra paths", async () => { const mediaDir = path.join(workspaceDir, "media-memory"); await fs.mkdir(mediaDir, { recursive: true }); @@ -2915,190 +2503,6 @@ describe("memory index", () => { expect(audioResults.some((result) => result.path.endsWith("meeting.wav"))).toBe(true); }); - it("finds keyword matches via hybrid search when query embedding is zero", async () => { - await expectHybridKeywordSearchFindsMemory( - createCfg({ - hybrid: { enabled: true, vectorWeight: 0, textWeight: 1 }, - }), - ); - }); - - it("retries transient query embedding transport failures during search", async () => { - const cfg = createCfg({ - hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, - }); - const manager = await getPersistentManager(cfg); - await manager.sync({ reason: "test" }); - - let queryCalls = 0; - ( - manager as unknown as { - provider: { - id: string; - model: string; - embedQuery: (text: string) => Promise; - embedBatch: (texts: string[]) => Promise; - close: () => Promise; - }; - waitForEmbeddingRetry: (delayMs: number, action: string) => Promise; - } - ).provider = { - id: "mock", - model: "mock-embed", - embedQuery: async () => { - queryCalls += 1; - if (queryCalls === 1) { - throw new Error("TypeError: fetch failed | other side closed"); - } - return [1, 0, 0, 0]; - }, - embedBatch: async (texts: string[]) => texts.map(() => [1, 0, 0, 0]), - close: async () => {}, - }; - ( - manager as unknown as { - waitForEmbeddingRetry: (delayMs: number, action: string) => Promise; - } - ).waitForEmbeddingRetry = async () => {}; - - const results = await manager.search("alpha"); - - expect(queryCalls).toBe(2); - expect(results.some((result) => result.path.endsWith("memory/2026-01-12.md"))).toBe(true); - }); - - it("fails search after bounded query embedding retries are exhausted", async () => { - const cfg = createCfg({ - hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, - }); - const manager = await getPersistentManager(cfg); - await manager.sync({ reason: "test" }); - - let queryCalls = 0; - ( - manager as unknown as { - provider: { - id: string; - model: string; - embedQuery: (text: string) => Promise; - embedBatch: (texts: string[]) => Promise; - close: () => Promise; - }; - } - ).provider = { - id: "mock", - model: "mock-embed", - embedQuery: async () => { - queryCalls += 1; - throw new Error("TypeError: fetch failed | other side closed"); - }, - embedBatch: async (texts: string[]) => texts.map(() => [1, 0, 0, 0]), - close: async () => {}, - }; - ( - manager as unknown as { - waitForEmbeddingRetry: (delayMs: number, action: string) => Promise; - } - ).waitForEmbeddingRetry = async () => {}; - - await expect(manager.search("alpha")).rejects.toThrow("fetch failed"); - expect(queryCalls).toBe(3); - }); - - it("preserves keyword-only hybrid hits when minScore exceeds text weight", async () => { - await expectHybridKeywordSearchFindsMemory( - createCfg({ - minScore: 0.35, - hybrid: { enabled: true, vectorWeight: 0.7, textWeight: 0.3 }, - }), - ); - }); - - it("supplements thin strict FTS results for conversational queries", async () => { - const cases = [ - { - query: "that thing we discussed about the API", - strictFile: "strict-english.md", - strictText: "That thing we discussed about the API belongs in the first draft.", - recallFile: "recall-english.md", - recallText: "API authentication uses short-lived OAuth tokens.", - }, - { - query: "ayer hablamos sobre estrategia de despliegue", - strictFile: "strict-spanish.md", - strictText: "Ayer hablamos sobre estrategia de despliegue para la primera region.", - recallFile: "recall-spanish.md", - recallText: "La estrategia de despliegue requiere una ventana de mantenimiento.", - }, - ] as const; - for (const entry of cases) { - await fs.writeFile(path.join(memoryDir, entry.strictFile), entry.strictText); - await fs.writeFile(path.join(memoryDir, entry.recallFile), entry.recallText); - } - - const manager = await getPersistentManager( - createCfg({ - minScore: 0, - hybrid: { enabled: true, vectorWeight: 0.7, textWeight: 0.3 }, - }), - ); - await manager.sync({ reason: "test" }); - const provider = Reflect.get(manager, "provider") as { - embedQuery: (text: string) => Promise; - }; - const embedQuerySpy = vi.spyOn(provider, "embedQuery"); - - for (const entry of cases) { - const results = await manager.search(entry.query, { maxResults: 6 }); - expect(results.some((result) => result.path.endsWith(`memory/${entry.recallFile}`))).toBe( - true, - ); - } - expect(embedQuerySpy).toHaveBeenCalledTimes(cases.length); - }); - - it("bounds per-keyword FTS fallback in provider-backed hybrid search", async () => { - const cfg = createCfg({ - minScore: 0.35, - hybrid: { enabled: true, vectorWeight: 0.7, textWeight: 0.3 }, - }); - const manager = await getPersistentManager(cfg); - await manager.sync({ reason: "test" }); - - const db = ( - manager as unknown as { - db: { - prepare: (sql: string) => unknown; - }; - } - ).db; - const originalPrepare = db.prepare.bind(db); - let ftsSelects = 0; - const prepareSpy = vi.spyOn(db, "prepare").mockImplementation((sql: string) => { - if ( - sql.includes("FROM memory_index_chunks_fts") && - sql.includes("WHERE memory_index_chunks_fts MATCH ?") - ) { - ftsSelects += 1; - } - return originalPrepare(sql); - }); - - try { - const results = await manager.search( - "zebra project router gateway session transcript approval command owner workspace token budget retry queue", - { maxResults: 5 }, - ); - - expect(results.length).toBeGreaterThan(0); - expect(results[0]?.path).toContain("memory/2026-01-12.md"); - expect(ftsSelects).toBeGreaterThan(1); - expect(ftsSelects).toBeLessThanOrEqual(7); - } finally { - prepareSpy.mockRestore(); - } - }); - it("reports vector availability after probe", async () => { const cfg = createCfg({ vectorEnabled: true }); const manager = await getPersistentManager(cfg); @@ -3304,1129 +2708,6 @@ describe("memory index", () => { } }); - it("caches embedding probe readiness across transient status managers", async () => { - const cfg = createCfg({}); - const first = requireManager( - await getMemorySearchManager({ cfg, agentId: "main", purpose: "status" }), - ); - managersForCleanup.add(first); - - await expect(first.probeEmbeddingAvailability()).resolves.toEqual({ ok: true }); - expect(embedBatchCalls).toBe(1); - await first.close(); - - const second = requireManager( - await getMemorySearchManager({ cfg, agentId: "main", purpose: "status" }), - ); - managersForCleanup.add(second); - - const cachedBeforeProbe = second.getCachedEmbeddingAvailability?.(); - expect(cachedBeforeProbe?.ok).toBe(true); - expect(cachedBeforeProbe?.checked).toBe(true); - expect(cachedBeforeProbe?.cached).toBe(true); - expect(cachedBeforeProbe?.checkedAtMs).toBeTypeOf("number"); - expect(cachedBeforeProbe?.cacheExpiresAtMs).toBeTypeOf("number"); - if ( - typeof cachedBeforeProbe?.checkedAtMs === "number" && - typeof cachedBeforeProbe.cacheExpiresAtMs === "number" - ) { - expect(cachedBeforeProbe.cacheExpiresAtMs - cachedBeforeProbe.checkedAtMs).toBe(30_000); - } - await expect(second.probeEmbeddingAvailability()).resolves.toStrictEqual({ - ok: true, - checked: true, - cached: true, - checkedAtMs: cachedBeforeProbe?.checkedAtMs, - cacheExpiresAtMs: cachedBeforeProbe?.cacheExpiresAtMs, - }); - expect(embedBatchCalls).toBe(1); - - const cached = second.getCachedEmbeddingAvailability?.(); - expect((cached?.cacheExpiresAtMs ?? 0) - (cached?.checkedAtMs ?? 0)).toBe(30_000); - }); - - it("clears cached embedding probe readiness when local embeddings degrade", async () => { - const cfg = createCfg({}); - const manager = await getPersistentManager(cfg); - - await expect(manager.probeEmbeddingAvailability()).resolves.toEqual({ ok: true }); - expect(manager.getCachedEmbeddingAvailability()?.ok).toBe(true); - ( - manager as unknown as { - provider: { - id: string; - model: string; - embedQuery: (text: string) => Promise; - embedBatch: (texts: string[]) => Promise; - close: () => Promise; - }; - } - ).provider = { - id: "local", - model: "local-model", - embedQuery: async () => [1, 0], - embedBatch: async (texts: string[]) => texts.map(() => [1, 0]), - close: async () => {}, - }; - - ( - manager as unknown as { - markLocalEmbeddingProviderDegraded: (err: unknown) => void; - } - ).markLocalEmbeddingProviderDegraded(createLocalWorkerExitError()); - - expect(manager.getCachedEmbeddingAvailability()).toBeNull(); - await expect(manager.probeEmbeddingAvailability()).resolves.toMatchObject({ - ok: false, - error: expect.stringContaining("Local embeddings degraded"), - }); - }); - - it("waits for degraded provider shutdown before fallback initialization", async () => { - const cfg = createCfg({ fallback: "fallback-provider" }); - const manager = await getPersistentManager(cfg); - await manager.sync({ reason: "test" }); - - let releaseProviderClose: () => void = () => {}; - providerCloseGate = new Promise((resolve) => { - releaseProviderClose = resolve; - }); - const fields = manager as unknown as { - provider: { - id: string; - model: string; - embedQuery: (text: string) => Promise; - embedBatch: (texts: string[]) => Promise; - close: () => Promise; - } | null; - markLocalEmbeddingProviderDegraded: (err: unknown) => void; - activateFallbackProvider: (reason: string) => Promise; - withTimeout: (promise: Promise, timeoutMs: number, message: string) => Promise; - }; - if (!fields.provider) { - throw new Error("Expected a test embedding provider"); - } - fields.provider.id = "local"; - fields.markLocalEmbeddingProviderDegraded(createLocalWorkerExitError()); - await vi.waitFor(() => expect(providerCloseCalls).toBe(1)); - - const callsBeforeFallback = providerCalls.length; - const fallbackPromise = fields.activateFallbackProvider("local worker exited"); - try { - await Promise.resolve(); - expect(providerCalls).toHaveLength(callsBeforeFallback); - } finally { - releaseProviderClose(); - providerCloseGate = null; - await fallbackPromise; - } - expect(providerCalls.slice(callsBeforeFallback).map((call) => call.provider)).toEqual([ - "fallback-provider", - ]); - }); - - it("retries failed provider retirement before fallback initialization", async () => { - const cfg = createCfg({ fallback: "fallback-provider" }); - const manager = await getPersistentManager(cfg); - await manager.sync({ reason: "test" }); - providerCloseFailuresRemaining = 1; - const fields = manager as unknown as { - activateFallbackProvider: (reason: string) => Promise; - }; - const callsBeforeFallback = providerCalls.length; - - await expect(fields.activateFallbackProvider("provider failed")).rejects.toThrow( - "provider close failed", - ); - expect(providerCalls).toHaveLength(callsBeforeFallback); - - await expect(fields.activateFallbackProvider("provider failed")).resolves.toBe(true); - expect(providerCloseCalls).toBe(2); - expect(providerCalls.slice(callsBeforeFallback).map((call) => call.provider)).toEqual([ - "fallback-provider", - ]); - }); - - it("waits for provider shutdown before retry initialization", async () => { - const cfg = createCfg({ provider: "openai" }); - const manager = await getPersistentManager(cfg); - await manager.sync({ reason: "test" }); - - let releaseProviderClose: () => void = () => {}; - providerCloseGate = new Promise((resolve) => { - releaseProviderClose = resolve; - }); - ( - manager as unknown as { - resetProviderInitializationForRetry: () => void; - } - ).resetProviderInitializationForRetry(); - await vi.waitFor(() => expect(providerCloseCalls).toBe(1)); - - const callsBeforeProbe = providerCalls.length; - const probePromise = manager.probeEmbeddingAvailability(); - try { - await Promise.resolve(); - expect(providerCalls).toHaveLength(callsBeforeProbe); - } finally { - releaseProviderClose(); - providerCloseGate = null; - await probePromise; - } - expect(providerCalls.slice(callsBeforeProbe).map((call) => call.provider)).toEqual(["openai"]); - }); - - it("waits for active provider shutdown before fallback initialization", async () => { - const cfg = createCfg({ - provider: "openai", - fallback: "fallback-provider", - hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, - }); - const manager = await getPersistentManager(cfg); - await manager.sync({ reason: "test" }); - - let releaseProviderClose: () => void = () => {}; - providerCloseGate = new Promise((resolve) => { - releaseProviderClose = resolve; - }); - const fields = manager as unknown as { - provider: { - embedQuery: (text: string) => Promise; - } | null; - }; - if (!fields.provider) { - throw new Error("Expected a test embedding provider"); - } - fields.provider.embedQuery = async () => { - throw new Error("embedding provider failed"); - }; - - const callsBeforeSearch = providerCalls.length; - const searchPromise = manager.search("alpha"); - let concurrentSearch: ReturnType = Promise.resolve([]); - try { - await vi.waitFor(() => expect(providerCloseCalls).toBe(1)); - concurrentSearch = manager.search("zebra"); - let concurrentSettled = false; - void concurrentSearch.then( - () => { - concurrentSettled = true; - }, - () => { - concurrentSettled = true; - }, - ); - await Promise.resolve(); - expect(concurrentSettled).toBe(false); - expect(providerCalls).toHaveLength(callsBeforeSearch); - } finally { - releaseProviderClose(); - providerCloseGate = null; - await Promise.allSettled([searchPromise, concurrentSearch]); - } - expect(providerCalls.slice(callsBeforeSearch).map((call) => call.provider)).toEqual([ - "fallback-provider", - ]); - await expect(concurrentSearch).resolves.toBeDefined(); - }); - - it("leases the indexing provider generation through chunk publication", async () => { - const manager = await getFreshManager( - createCfg({ - provider: "openai", - fallback: "fallback-provider", - cacheEnabled: true, - hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, - }), - "cli", - ); - managersForCleanup.add(manager); - const fields = manager as unknown as { - provider: { - id: string; - model: string; - embedBatch: (texts: string[]) => Promise; - } | null; - providerKey: string; - computeProviderKey: () => string; - ensureProviderInitialized: () => Promise; - markLocalEmbeddingProviderDegraded: (err: unknown) => void; - activateFallbackProvider: (reason: string) => Promise; - withTimeout: (promise: Promise, timeoutMs: number, message: string) => Promise; - indexFile: ( - entry: { - path: string; - absPath: string; - mtimeMs: number; - size: number; - hash: string; - content: string; - }, - options: { source: "memory"; content: string }, - ) => Promise; - ensureVectorReady: (dimensions?: number) => Promise; - db: { - prepare: (sql: string) => { - get: ( - ...params: unknown[] - ) => { model?: string; provider?: string; provider_key?: string } | undefined; - }; - }; - }; - await fields.ensureProviderInitialized(); - if (!fields.provider) { - throw new Error("Expected a test embedding provider"); - } - const indexedProvider = fields.provider; - indexedProvider.id = "local"; - fields.providerKey = fields.computeProviderKey(); - const indexedProviderKey = fields.providerKey; - const firstContent = "# Log\nFirst memory line indexed during provider fallback."; - const secondContent = "# Log\nSecond memory line indexed during provider fallback."; - - let releaseFirstEmbedding: () => void = () => {}; - let releaseSecondEmbedding: () => void = () => {}; - let markFirstEmbeddingStarted: () => void = () => {}; - let markSecondEmbeddingStarted: () => void = () => {}; - const firstEmbeddingGate = new Promise((resolve) => { - releaseFirstEmbedding = resolve; - }); - const secondEmbeddingGate = new Promise((resolve) => { - releaseSecondEmbedding = resolve; - }); - const firstEmbeddingStarted = new Promise((resolve) => { - markFirstEmbeddingStarted = resolve; - }); - const secondEmbeddingStarted = new Promise((resolve) => { - markSecondEmbeddingStarted = resolve; - }); - indexedProvider.embedBatch = async (texts) => { - if (texts.some((text) => text.includes("First"))) { - markFirstEmbeddingStarted(); - await firstEmbeddingGate; - } else { - markSecondEmbeddingStarted(); - await secondEmbeddingGate; - } - return texts.map(() => [1, 0, 0, 0]); - }; - let releasePublication: () => void = () => {}; - let markPublicationStarted: () => void = () => {}; - const publicationGate = new Promise((resolve) => { - releasePublication = resolve; - }); - const publicationStarted = new Promise((resolve) => { - markPublicationStarted = resolve; - }); - const ensureVectorReady = fields.ensureVectorReady.bind(manager); - let publicationCalls = 0; - fields.ensureVectorReady = async (dimensions) => { - publicationCalls += 1; - if (publicationCalls === 1) { - return await ensureVectorReady(dimensions); - } - markPublicationStarted(); - await publicationGate; - return await ensureVectorReady(dimensions); - }; - - const callsBeforeFallback = providerCalls.length; - const firstIndexPromise = fields.indexFile( - { - path: "memory/generation-race-first.md", - absPath: path.join(memoryDir, "generation-race-first.md"), - mtimeMs: Date.now(), - size: Buffer.byteLength(firstContent), - hash: hashText(firstContent), - content: firstContent, - }, - { source: "memory", content: firstContent }, - ); - const secondIndexPromise = fields.indexFile( - { - path: "memory/generation-race-second.md", - absPath: path.join(memoryDir, "generation-race-second.md"), - mtimeMs: Date.now(), - size: Buffer.byteLength(secondContent), - hash: hashText(secondContent), - content: secondContent, - }, - { source: "memory", content: secondContent }, - ); - let fallbackPromise: Promise | null = null; - try { - await fields.withTimeout( - Promise.all([firstEmbeddingStarted, secondEmbeddingStarted]), - 5_000, - "concurrent embeddings did not start", - ); - fields.markLocalEmbeddingProviderDegraded(createLocalWorkerExitError()); - await vi.waitFor(() => expect(fields.provider).toBeNull()); - fallbackPromise = fields.activateFallbackProvider("local worker exited"); - releaseFirstEmbedding(); - await firstIndexPromise; - expect(providerCloseCalls).toBe(0); - expect(providerCalls).toHaveLength(callsBeforeFallback); - - releaseSecondEmbedding(); - await fields.withTimeout(publicationStarted, 5_000, "publication did not start"); - expect(providerCloseCalls).toBe(0); - expect(providerCalls).toHaveLength(callsBeforeFallback); - - releasePublication(); - await secondIndexPromise; - await expect(fallbackPromise).resolves.toBe(true); - } finally { - releaseFirstEmbedding(); - releaseSecondEmbedding(); - releasePublication(); - await Promise.allSettled([ - firstIndexPromise, - secondIndexPromise, - ...(fallbackPromise ? [fallbackPromise] : []), - ]); - } - - expect(providerCalls.slice(callsBeforeFallback).map((call) => call.provider)).toEqual([ - "fallback-provider", - ]); - expect( - fields.db - .prepare("SELECT model FROM memory_index_chunks WHERE path = ?") - .get("memory/generation-race-second.md")?.model, - ).toBe(indexedProvider.model); - expect( - fields.db - .prepare("SELECT provider, model, provider_key FROM memory_embedding_cache LIMIT 1") - .get(), - ).toEqual({ - provider: indexedProvider.id, - model: indexedProvider.model, - provider_key: indexedProviderKey, - }); - }); - - it("keeps an active FTS-only generation stable while fallback activates", async () => { - const manager = await getFreshManager( - createCfg({ provider: "openai", fallback: "fallback-provider" }), - "cli", - ); - managersForCleanup.add(manager); - type IndexEntry = { - path: string; - absPath: string; - mtimeMs: number; - size: number; - hash: string; - content: string; - }; - const fields = manager as unknown as { - provider: { id: string } | null; - providerKey: string; - computeProviderKey: () => string; - ensureProviderInitialized: () => Promise; - markLocalEmbeddingProviderDegraded: (err: unknown) => void; - activateFallbackProvider: (reason: string) => Promise; - beginSyncProviderGeneration: () => void; - endSyncProviderGeneration: () => void; - indexFile: ( - entry: IndexEntry, - options: { source: "memory"; content: string }, - ) => Promise; - db: { - prepare: (sql: string) => { - get: (...params: unknown[]) => { model?: string } | undefined; - }; - }; - }; - await fields.ensureProviderInitialized(); - if (!fields.provider) { - throw new Error("Expected a test embedding provider"); - } - fields.provider.id = "local"; - fields.providerKey = fields.computeProviderKey(); - fields.markLocalEmbeddingProviderDegraded(createLocalWorkerExitError()); - await vi.waitFor(() => { - expect(fields.provider).toBeNull(); - expect(providerCloseCalls).toBe(1); - }); - - const createEntry = (name: string): IndexEntry => { - const content = `# Log\n${name} FTS-only generation.`; - return { - path: `memory/${name}.md`, - absPath: path.join(memoryDir, `${name}.md`), - mtimeMs: Date.now(), - size: Buffer.byteLength(content), - hash: hashText(content), - content, - }; - }; - const first = createEntry("fts-first"); - const second = createEntry("fts-second"); - - fields.beginSyncProviderGeneration(); - try { - await fields.indexFile(first, { source: "memory", content: first.content }); - await expect(fields.activateFallbackProvider("local worker exited")).resolves.toBe(true); - await fields.indexFile(second, { source: "memory", content: second.content }); - } finally { - fields.endSyncProviderGeneration(); - } - - expect( - fields.db.prepare("SELECT model FROM memory_index_chunks WHERE path = ?").get(first.path) - ?.model, - ).toBe("fts-only"); - expect( - fields.db.prepare("SELECT model FROM memory_index_chunks WHERE path = ?").get(second.path) - ?.model, - ).toBe("fts-only"); - }); - - it("waits for admitted provider users before retirement", async () => { - const cfg = createCfg({ provider: "openai" }); - const manager = await getPersistentManager(cfg); - await manager.sync({ reason: "test" }); - const fields = manager as unknown as { - provider: { - embedQuery: (text: string) => Promise; - } | null; - embedQueryWithRetry: (text: string) => Promise; - retireCurrentProvider: () => Promise; - }; - if (!fields.provider) { - throw new Error("Expected a test embedding provider"); - } - let releaseFirstQuery: () => void = () => {}; - let markFirstQueryStarted: () => void = () => {}; - const firstQueryGate = new Promise((resolve) => { - releaseFirstQuery = resolve; - }); - const firstQueryStarted = new Promise((resolve) => { - markFirstQueryStarted = resolve; - }); - fields.provider.embedQuery = async () => { - markFirstQueryStarted(); - await firstQueryGate; - return [1, 0, 0, 0]; - }; - - const queryPromise = fields.embedQueryWithRetry("alpha"); - await firstQueryStarted; - const retirementPromise = fields.retireCurrentProvider(); - let retirementSettled = false; - void retirementPromise.then( - () => { - retirementSettled = true; - }, - () => { - retirementSettled = true; - }, - ); - try { - await Promise.resolve(); - expect(retirementSettled).toBe(false); - expect(providerCloseCalls).toBe(0); - } finally { - releaseFirstQuery(); - } - - await expect(queryPromise).resolves.toEqual([1, 0, 0, 0]); - await retirementPromise; - expect(providerCloseCalls).toBe(1); - }); - - it("uses the leased provider runtime after retirement starts", async () => { - const manager = await getPersistentManager(createCfg({ provider: "openai" })); - type QueryProvider = { - embedQuery: (text: string, options?: { signal?: AbortSignal }) => Promise; - }; - const fields = manager as unknown as { - provider: QueryProvider | null; - providerRuntime?: { inlineQueryTimeoutMs?: number }; - acquireProviderUse: (provider: QueryProvider) => () => void; - retireCurrentProvider: () => Promise; - embedQueryWithRetry: ( - text: string, - signal: AbortSignal | undefined, - provider: QueryProvider, - markDegraded: boolean, - providerRuntime: { inlineQueryTimeoutMs?: number }, - ) => Promise; - }; - await manager.probeEmbeddingAvailability(); - const provider = fields.provider; - if (!provider) { - throw new Error("Expected a test embedding provider"); - } - const providerRuntime = { inlineQueryTimeoutMs: 10 }; - fields.providerRuntime = providerRuntime; - provider.embedQuery = async (_text, options) => - await new Promise((resolve, reject) => { - const timer = setTimeout(() => resolve([1, 0, 0, 0]), 100); - options?.signal?.addEventListener( - "abort", - () => { - clearTimeout(timer); - const reason = options.signal?.reason; - reject(reason instanceof Error ? reason : new Error("embedding aborted")); - }, - { once: true }, - ); - }); - - const releaseProvider = fields.acquireProviderUse(provider); - const retirementPromise = fields.retireCurrentProvider(); - try { - await vi.waitFor(() => expect(fields.provider).toBeNull()); - await expect( - fields.embedQueryWithRetry("alpha", undefined, provider, false, providerRuntime), - ).rejects.toThrow("timed out"); - expect(providerCloseCalls).toBe(0); - } finally { - releaseProvider(); - } - - await retirementPromise; - expect(providerCloseCalls).toBe(1); - }); - - it("waits for an admitted search before manager teardown", async () => { - const manager = await getPersistentManager(createCfg({ provider: "openai" })); - await manager.sync({ reason: "test" }); - const fields = manager as unknown as { - searchVector: () => Promise; - closing: boolean; - closed: boolean; - }; - let releaseVectorSearch: () => void = () => {}; - let markVectorSearchStarted: () => void = () => {}; - const vectorSearchGate = new Promise((resolve) => { - releaseVectorSearch = resolve; - }); - const vectorSearchStarted = new Promise((resolve) => { - markVectorSearchStarted = resolve; - }); - fields.searchVector = async () => { - markVectorSearchStarted(); - await vectorSearchGate; - return []; - }; - - const searchPromise = manager.search("alpha"); - await vectorSearchStarted; - const closePromise = manager.close(); - let closeSettled = false; - void closePromise.then( - () => { - closeSettled = true; - }, - () => { - closeSettled = true; - }, - ); - try { - await Promise.resolve(); - expect(closeSettled).toBe(false); - expect(fields.closing).toBe(true); - expect(fields.closed).toBe(false); - expect(providerCloseCalls).toBe(0); - } finally { - releaseVectorSearch(); - } - - await expect(searchPromise).resolves.toBeDefined(); - await closePromise; - expect(providerCloseCalls).toBe(1); - }); - - it("waits for an admitted vector probe before manager teardown", async () => { - const manager = await getPersistentManager(createCfg({ provider: "openai" })); - const fields = manager as unknown as { - ensureVectorReady: () => Promise; - }; - let releaseProbe: () => void = () => {}; - let markProbeStarted: () => void = () => {}; - const probeGate = new Promise((resolve) => { - releaseProbe = resolve; - }); - const probeStarted = new Promise((resolve) => { - markProbeStarted = resolve; - }); - fields.ensureVectorReady = async () => { - markProbeStarted(); - await probeGate; - return true; - }; - - const probePromise = manager.probeVectorAvailability(); - await probeStarted; - const closePromise = manager.close(); - let closeSettled = false; - void closePromise.then( - () => { - closeSettled = true; - }, - () => { - closeSettled = true; - }, - ); - try { - await Promise.resolve(); - expect(closeSettled).toBe(false); - expect(providerCloseCalls).toBe(0); - } finally { - releaseProbe(); - } - - await expect(probePromise).resolves.toBe(true); - await closePromise; - expect(providerCloseCalls).toBe(1); - }); - - it("fails closed when fallback initialization fails for an explicit provider", async () => { - const cfg = createCfg({ - provider: "openai", - fallback: "fallback-provider", - hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, - }); - const manager = await getPersistentManager(cfg); - await manager.sync({ reason: "test" }); - const fields = manager as unknown as { - provider: { - embedQuery: (text: string) => Promise; - } | null; - }; - if (!fields.provider) { - throw new Error("Expected a test embedding provider"); - } - fields.provider.embedQuery = async () => { - throw new Error("embedding provider failed"); - }; - providerCreationFailure = "fallback-provider"; - - await expect(manager.search("alpha")).rejects.toThrow( - /Memory search unavailable: embedding provider "openai" is configured but unavailable\./, - ); - - providerCreationFailure = null; - await expect(manager.search("alpha")).resolves.toBeDefined(); - }); - - it("retries the optional primary after fallback initialization fails", async () => { - const cfg = createCfg({ - fallback: "fallback-provider", - hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, - }); - const manager = await getPersistentManager(cfg); - await manager.sync({ reason: "test" }); - const fields = manager as unknown as { - provider: { - id: string; - embedQuery: (text: string) => Promise; - } | null; - }; - if (!fields.provider) { - throw new Error("Expected a test embedding provider"); - } - fields.provider.embedQuery = async () => { - throw new Error("embedding provider failed"); - }; - providerCreationFailure = "fallback-provider"; - const callsBeforeSearch = providerCalls.length; - - await expect(manager.search("alpha")).resolves.toBeDefined(); - - providerCreationFailure = null; - await expect(manager.search("alpha")).resolves.toBeDefined(); - expect(providerCalls.slice(callsBeforeSearch).map((call) => call.provider)).toEqual([ - "fallback-provider", - "openai", - ]); - expect(fields.provider?.id).toBe("mock"); - }); - - it("fails closed and retries a required primary after a null fallback result", async () => { - const cfg = createCfg({ - provider: "openai", - fallback: "fallback-provider", - hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, - }); - const manager = await getPersistentManager(cfg); - await manager.sync({ reason: "test" }); - const fields = manager as unknown as { - provider: { embedQuery: (text: string) => Promise } | null; - }; - if (!fields.provider) { - throw new Error("Expected a test embedding provider"); - } - fields.provider.embedQuery = async () => { - throw new Error("embedding provider failed"); - }; - providerNullResult = "fallback-provider"; - - await expect(manager.search("alpha")).rejects.toThrow( - /Memory search unavailable: embedding provider "openai" is configured but unavailable\./, - ); - - providerNullResult = null; - await expect(manager.search("alpha")).resolves.toBeDefined(); - }); - - it("retries an optional primary after a null fallback result", async () => { - const cfg = createCfg({ - fallback: "fallback-provider", - hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, - }); - const manager = await getPersistentManager(cfg); - await manager.sync({ reason: "test" }); - const fields = manager as unknown as { - provider: { id: string; embedQuery: (text: string) => Promise } | null; - }; - if (!fields.provider) { - throw new Error("Expected a test embedding provider"); - } - fields.provider.embedQuery = async () => { - throw new Error("embedding provider failed"); - }; - providerNullResult = "fallback-provider"; - - await expect(manager.search("alpha")).resolves.toBeDefined(); - - providerNullResult = null; - await expect(manager.search("alpha")).resolves.toBeDefined(); - expect(fields.provider?.id).toBe("mock"); - }); - - it("keeps concurrent optional searches in FTS mode when shared fallback fails", async () => { - const cfg = createCfg({ - fallback: "fallback-provider", - hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, - }); - const manager = await getPersistentManager(cfg); - await manager.sync({ reason: "test" }); - const fields = manager as unknown as { - provider: { - embedQuery: (text: string) => Promise; - } | null; - ensureProviderInitialized: () => Promise; - }; - if (!fields.provider) { - throw new Error("Expected a test embedding provider"); - } - fields.provider.embedQuery = async () => { - throw new Error("embedding provider failed"); - }; - const ensureProviderInitialized = fields.ensureProviderInitialized.bind(manager); - let providerInitializationCalls = 0; - fields.ensureProviderInitialized = async () => { - providerInitializationCalls += 1; - await ensureProviderInitialized(); - }; - providerCreationFailure = "fallback-provider"; - let releaseProviderInit: () => void = () => {}; - providerInitGate = new Promise((resolve) => { - releaseProviderInit = resolve; - }); - - const callsBeforeSearch = providerCalls.length; - const firstSearch = manager.search("alpha"); - await vi.waitFor(() => - expect(providerCalls.some((call) => call.provider === "fallback-provider")).toBe(true), - ); - const initializationCallsBeforeSecondSearch = providerInitializationCalls; - const secondSearch = manager.search("zebra"); - let secondSettled = false; - void secondSearch.then( - () => { - secondSettled = true; - }, - () => { - secondSettled = true; - }, - ); - try { - await vi.waitFor(() => - expect(providerInitializationCalls).toBeGreaterThan(initializationCallsBeforeSecondSearch), - ); - expect(secondSettled).toBe(false); - releaseProviderInit(); - const results = await Promise.all([firstSearch, secondSearch]); - expect(results.every((result) => result.length > 0)).toBe(true); - expect( - providerCalls - .slice(callsBeforeSearch) - .filter((call) => call.provider === "fallback-provider"), - ).toHaveLength(1); - } finally { - providerInitGate = null; - releaseProviderInit(); - await Promise.allSettled([firstSearch, secondSearch]); - } - }); - - it("does not activate fallback during search when index identity is already mismatched", async () => { - const cfg = createCfg({ - fallback: "fallback-provider", - hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, - }); - const manager = await getPersistentManager(cfg); - - await manager.sync({ reason: "test" }); - const callsBeforeSearch = providerCalls.length; - ( - manager as unknown as { - provider: { - id: string; - model: string; - embedQuery: () => Promise; - embedBatch: (texts: string[]) => Promise; - close: () => Promise; - }; - } - ).provider = { - id: "local", - model: "mock-embed", - embedQuery: async () => { - throw createLocalWorkerExitError(); - }, - embedBatch: async (texts: string[]) => texts.map(() => [1, 0, 0, 0]), - close: async () => {}, - }; - - const results = await manager.search("alpha"); - - expect(results).toStrictEqual([]); - expect(providerCalls.slice(callsBeforeSearch)).toStrictEqual([]); - expect( - ( - manager as unknown as { - provider: { id: string } | null; - } - ).provider?.id, - ).toBe("local"); - }); - - it("rebuilds with fallback provider during explicit identity repair", async () => { - const oldCfg = createCfg({ - model: "old-embed", - }); - const oldManager = await getFreshManager(oldCfg); - await oldManager.sync({ reason: "test", force: true }); - await oldManager.close?.(); - - const cfg = createCfg({ - model: "new-embed", - fallback: "fallback-provider", - }); - const manager = await getFreshManager(cfg); - try { - expect(manager.status().dirty).toBe(true); - const fields = manager as unknown as { - providerInitialized: boolean; - provider: { - id: string; - model: string; - embedQuery: (text: string) => Promise; - embedBatch: (texts: string[]) => Promise; - close: () => Promise; - }; - }; - fields.providerInitialized = true; - fields.provider = { - id: "mock", - model: "new-embed", - embedQuery: async () => { - throw createLocalWorkerExitError(); - }, - embedBatch: async () => { - throw createLocalWorkerExitError(); - }, - close: async () => {}, - }; - - await manager.sync({ reason: "cli" }); - - expect(manager.status().dirty).toBe(false); - expect(manager.status().provider).toBe("fallback-provider"); - expect(manager.status().model).toBe("fallback-provider-embed"); - expect(manager.status().custom?.indexIdentity).toEqual({ status: "valid" }); - await expect(manager.search("alpha")).resolves.not.toStrictEqual([]); - } finally { - await manager.close?.(); - } - }); - - it("reinitializes the configured provider after probe-time local degradation", async () => { - const cfg = createCfg({ - fallback: "fallback-provider", - hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, - }); - const manager = await getPersistentManager(cfg); - - await manager.sync({ reason: "test" }); - ( - manager as unknown as { - provider: { - id: string; - model: string; - embedQuery: () => Promise; - embedBatch: () => Promise; - close: () => Promise; - }; - } - ).provider = { - id: "local", - model: "mock-embed", - embedQuery: async () => { - throw createLocalWorkerExitError(); - }, - embedBatch: async () => { - throw createLocalWorkerExitError(); - }, - close: async () => {}, - }; - const callsBeforeSearch = providerCalls.length; - - await expect(manager.probeEmbeddingAvailability()).resolves.toMatchObject({ - ok: false, - error: expect.stringContaining("Local embedding worker exited"), - }); - - const results = await manager.search("alpha"); - - expect(results.length).toBeGreaterThan(0); - expect(providerCalls.slice(callsBeforeSearch).map((call) => call.provider)).toContain("openai"); - expect( - ( - manager as unknown as { - provider: { id: string } | null; - } - ).provider?.id, - ).toBe("mock"); - }); - - it("clears identity dirty after status resolves the indexed fallback provider", async () => { - const indexedCfg = createCfg({ - provider: "fallback-provider", - model: "new-embed", - }); - const indexedManager = await getFreshManager(indexedCfg); - await indexedManager.sync({ reason: "test", force: true }); - await indexedManager.close?.(); - - const cfg = createCfg({ - fallback: "fallback-provider", - model: "new-embed", - }); - const { getRequiredMemoryIndexManager } = await import("./test-manager-helpers.js"); - const manager = await getRequiredMemoryIndexManager({ - cfg, - agentId: "main", - purpose: "status", - }); - try { - expect(manager.status().dirty).toBe(true); - - const fields = manager as unknown as { - provider: { - id: string; - model: string; - embedQuery: (text: string) => Promise; - embedBatch: (texts: string[]) => Promise; - close: () => Promise; - }; - providerInitialized: boolean; - providerRuntime: { - id: string; - cacheKeyData: Record; - }; - providerKey: string; - computeProviderKey: () => string; - }; - fields.provider = { - id: "fallback-provider", - model: "new-embed", - embedQuery: async () => [1, 0, 0, 0], - embedBatch: async (texts) => texts.map(() => [1, 0, 0, 0]), - close: async () => {}, - }; - fields.providerRuntime = { - id: "fallback-provider", - cacheKeyData: { - provider: "fallback-provider", - baseUrl: "https://generativelanguage.googleapis.com/v1beta", - model: "new-embed", - headers: [], - }, - }; - fields.providerInitialized = true; - fields.providerKey = fields.computeProviderKey(); - - expect(manager.status().dirty).toBe(false); - expect(manager.status().custom?.indexIdentity).toEqual({ status: "valid" }); - } finally { - await manager.close?.(); - } - }); - - it("exposes already-created local runtime facts without probing embeddings", async () => { - const cfg = createCfg({}); - const { getRequiredMemoryIndexManager } = await import("./test-manager-helpers.js"); - const manager = await getRequiredMemoryIndexManager({ - cfg, - agentId: "main", - purpose: "status", - }); - try { - const getRuntimeFacts = vi.fn(() => ({ - engine: "llama.cpp" as const, - state: "ready" as const, - backend: "cuda" as const, - buildType: "prebuilt" as const, - deviceNames: ["NVIDIA Test GPU"], - offload: { - supported: true, - offloadedLayers: 24, - totalLayers: 24, - }, - context: { - requestedSize: 4096, - }, - })); - const provider = { - id: "local", - model: "test-model.gguf", - embedQuery: vi.fn(async () => [1, 0, 0, 0]), - embedBatch: vi.fn(async (texts: string[]) => texts.map(() => [1, 0, 0, 0])), - }; - Object.defineProperty(provider, Symbol.for("openclaw.localEmbeddingRuntimeFacts"), { - value: getRuntimeFacts, - }); - const fields = manager as unknown as { - provider: typeof provider | null; - }; - fields.provider = provider; - - expect(manager.status().custom?.llamaCppRuntime).toMatchObject({ - state: "ready", - backend: "cuda", - deviceNames: ["NVIDIA Test GPU"], - offload: { - offloadedLayers: 24, - totalLayers: 24, - }, - context: { - requestedSize: 4096, - }, - }); - expect(getRuntimeFacts).toHaveBeenCalledTimes(1); - } finally { - await manager.close?.(); - } - }); - it("keeps metadata after unchanged in-place force reindex", async () => { const cfg = createCfg({}); const manager = await getFreshManager(cfg); @@ -4457,640 +2738,6 @@ describe("memory index", () => { expect(embedBatchCalls).toBe(beforeCalls); }); - it("builds FTS index and returns search results when no embedding provider is available", async () => { - forceNoProvider = true; - - const cfg = createCfg({ - provider: "none", - minScore: 0.35, - hybrid: { enabled: true }, - }); - const result = await getMemorySearchManager({ cfg, agentId: "main" }); - const manager = requireManager(result); - managersForCleanup.add(manager); - resetManagerForTest(manager); - if (!manager.status().fts?.available) { - return; - } - - await fs.writeFile( - path.join(memoryDir, "2026-01-12.md"), - "# Log\nAlpha memory line.\nZebra memory line.", - ); - await manager.sync({ reason: "test" }); - - const status = manager.status(); - expect(status.chunks).toBeGreaterThan(0); - expect(embedBatchCalls).toBe(0); - - const results = await manager.search("Alpha"); - expect(results.length).toBeGreaterThan(0); - expect(results[0]?.snippet).toMatch(/Alpha/i); - - const noResults = await manager.search("nonexistent_xyz_keyword"); - expect(noResults.length).toBe(0); - }); - - it("ranks an exact path stem ahead of a body match before applying the result limit", async () => { - forceNoProvider = true; - const cfg = createCfg({ - provider: "none", - minScore: 0.35, - hybrid: { enabled: true }, - }); - const result = await getMemorySearchManager({ cfg, agentId: "main" }); - const manager = requireManager(result); - managersForCleanup.add(manager); - resetManagerForTest(manager); - if (!manager.status().fts?.available) { - return; - } - - await fs.writeFile(path.join(memoryDir, "project-lantern.md"), "Unrelated exact-path body."); - await fs.writeFile( - path.join(memoryDir, "body-match.md"), - "Project lantern project lantern project lantern.", - ); - await manager.sync({ reason: "test" }); - - const results = await manager.search("project-lantern", { maxResults: 1 }); - expect(results).toHaveLength(1); - expect(results[0]?.path).toContain("memory/project-lantern.md"); - expect(results[0]?.score).toBe(1); - }); - - it("does not let fallback-term filenames consume the candidate cap", async () => { - forceNoProvider = true; - const cfg = createCfg({ - provider: "none", - minScore: 0, - hybrid: { enabled: true }, - }); - const result = await getMemorySearchManager({ cfg, agentId: "main" }); - const manager = requireManager(result); - managersForCleanup.add(manager); - resetManagerForTest(manager); - if (!manager.status().fts?.available) { - return; - } - - for (let index = 0; index < 5; index += 1) { - const duplicateDir = path.join(memoryDir, `alpha-${index}`); - await fs.mkdir(duplicateDir, { recursive: true }); - await fs.writeFile(path.join(duplicateDir, "alpha.md"), "Unrelated path-only candidate."); - } - await fs.writeFile( - path.join(memoryDir, "body-match.md"), - "Alpha alpha alpha alpha alpha strongest fallback body match.", - ); - await manager.sync({ reason: "test" }); - - const results = await manager.search("alpha gamma", { maxResults: 1, minScore: 0 }); - expect(results).toHaveLength(1); - expect(results[0]?.path).toContain("memory/body-match.md"); - }); - - it("preserves fallback body boosts through hybrid weighting", async () => { - const cfg = createCfg({ - minScore: 0, - hybrid: { enabled: true, vectorWeight: 0, textWeight: 1 }, - }); - const manager = await getPersistentManager(cfg); - type HybridKeywordHit = { - id: string; - path: string; - startLine: number; - endLine: number; - score: number; - snippet: string; - source: "memory"; - textScore: number; - pathScore: number; - exactPathSpecificity: 0; - }; - const internal = manager as unknown as { - mergeHybridResults: (params: { - query: string; - vector: []; - keyword: HybridKeywordHit[]; - vectorWeight: number; - textWeight: number; - }) => Promise>; - }; - - const results = await internal.mergeHybridResults({ - query: "alpha gamma", - vector: [], - keyword: [ - { - id: "body", - path: "memory/body.md", - startLine: 1, - endLine: 2, - score: 0.9, - snippet: "body", - source: "memory", - textScore: 0.1, - pathScore: 0, - exactPathSpecificity: 0, - }, - { - id: "path", - path: "memory/alpha.md", - startLine: 1, - endLine: 2, - score: 0.5, - snippet: "path", - source: "memory", - textScore: 0, - pathScore: 0.5, - exactPathSpecificity: 0, - }, - ], - vectorWeight: 0, - textWeight: 1, - }); - - expect(results.map((entry) => entry.path)).toEqual(["memory/body.md", "memory/alpha.md"]); - expect(results[0]).toMatchObject({ score: 0.9, textScore: 0.1 }); - }); - - it("bounds the merged six-term fallback candidate set", async () => { - forceNoProvider = true; - const cfg = createCfg({ - minScore: 0, - hybrid: { enabled: true }, - }); - const manager = await getPersistentManager(cfg); - const terms = ["alpha", "beta", "gamma", "delta", "epsilon", "zeta"]; - for (const term of terms) { - for (let index = 0; index < 5; index += 1) { - await fs.writeFile(path.join(memoryDir, `${term}-${index}.md`), `${term} body ${index}`); - } - } - await manager.sync({ reason: "test" }); - - const internal = manager as unknown as { - searchKeywordWithFallback: ( - query: string, - limit: number, - options: { boostFallbackRanking?: boolean }, - sources: Array<"memory">, - ) => Promise>; - }; - const candidates = await internal.searchKeywordWithFallback( - terms.join(" "), - 4, - { boostFallbackRanking: true }, - ["memory"], - ); - - expect(candidates).toHaveLength(4); - expect(candidates.every((entry) => entry.exactPathSpecificity === 0)).toBe(true); - }); - - it("counts exact candidate headroom by distinct path instead of chunk", async () => { - const manager = await getPersistentManager(createCfg({ hybrid: { enabled: true } })); - type TestKeywordHit = { - id: string; - path: string; - source: "memory"; - startLine: number; - endLine: number; - score: number; - textScore: number; - pathScore: number; - exactPathSpecificity: 2; - snippet: string; - }; - const sharedPath = "memory/000/foo.md"; - const bodyHits: TestKeywordHit[] = Array.from({ length: 4 }, (_, index) => ({ - id: `body-${index}`, - path: sharedPath, - source: "memory", - startLine: index + 2, - endLine: index + 2, - score: 1 - index / 100, - textScore: 1 - index / 100, - pathScore: 0, - exactPathSpecificity: 2, - snippet: `body ${index}`, - })); - const pathHits: TestKeywordHit[] = Array.from({ length: 200 }, (_, index) => ({ - id: `path-${index}`, - path: `memory/${index.toString().padStart(3, "0")}/foo.md`, - source: "memory", - startLine: 1, - endLine: 1, - score: 1, - textScore: 0, - pathScore: 0, - exactPathSpecificity: 2, - snippet: `path ${index}`, - })); - const internal = manager as unknown as { - limitKeywordSearchHits: (hits: TestKeywordHit[], nonExactLimit: number) => TestKeywordHit[]; - }; - - const limited = internal.limitKeywordSearchHits(bodyHits.concat(pathHits), 4); - const paths = new Set(limited.map((entry) => entry.path)); - - expect(limited).toHaveLength(204); - expect(paths.size).toBe(200); - expect(paths.has("memory/199/foo.md")).toBe(true); - }); - - it("uses body relevance within the same exact basename tier in FTS-only mode", async () => { - forceNoProvider = true; - const cfg = createCfg({ - provider: "none", - minScore: 0, - hybrid: { enabled: true }, - }); - const result = await getMemorySearchManager({ cfg, agentId: "main" }); - const manager = requireManager(result); - managersForCleanup.add(manager); - resetManagerForTest(manager); - if (!manager.status().fts?.available) { - return; - } - - const weakDir = path.join(memoryDir, "a"); - const strongDir = path.join(memoryDir, "z"); - await fs.mkdir(weakDir, { recursive: true }); - await fs.mkdir(strongDir, { recursive: true }); - await fs.writeFile(path.join(weakDir, "foo.md"), "Unrelated weak body."); - await fs.writeFile(path.join(strongDir, "foo.md"), "foo md foo md foo md strong body"); - await manager.sync({ reason: "test" }); - - const results = await manager.search("foo.md", { maxResults: 1, minScore: 0 }); - expect(results).toHaveLength(1); - expect(results[0]?.path).toContain("memory/z/foo.md"); - expect(results[0]?.score).toBe(1); - }); - - it("returns exact basename candidates with fixed FTS ranking", async () => { - forceNoProvider = true; - const staleDir = path.join(fixtureRoot, "decay-a-stale"); - const freshDir = path.join(fixtureRoot, "decay-z-fresh"); - await fs.mkdir(staleDir, { recursive: true }); - await fs.mkdir(freshDir, { recursive: true }); - const staleFooPath = path.join(staleDir, "foo.md"); - const freshFooPath = path.join(freshDir, "foo.md"); - const staleBarPath = path.join(staleDir, "bar.md"); - await fs.writeFile(staleFooPath, "Unrelated stale candidate."); - await fs.writeFile(freshFooPath, "Unrelated fresh candidate."); - await fs.writeFile(staleBarPath, "bar md bar md bar md strongest stale body"); - await fs.writeFile(path.join(freshDir, "bar.md"), "bar md fresh body"); - const staleMtime = new Date(Date.now() - 90 * 24 * 60 * 60_000); - await Promise.all([ - fs.utimes(staleFooPath, staleMtime, staleMtime), - fs.utimes(staleBarPath, staleMtime, staleMtime), - ]); - const cfg = createCfg({ - provider: "none", - extraPaths: [staleDir, freshDir], - minScore: 0, - }); - const result = await getMemorySearchManager({ cfg, agentId: "main" }); - const manager = requireManager(result); - managersForCleanup.add(manager); - resetManagerForTest(manager); - if (!manager.status().fts?.available) { - return; - } - await manager.sync({ reason: "test" }); - - for (const basename of ["foo.md", "bar.md"]) { - const results = await manager.search(basename, { maxResults: 1, minScore: 0 }); - expect(results).toHaveLength(1); - expect(results[0]?.score).toBe(1); - } - }); - - it("applies the fixed FTS candidate cap to exact paths", async () => { - forceNoProvider = true; - const staleMtime = new Date(Date.now() - 90 * 24 * 60 * 60_000); - const extraPaths: string[] = []; - for (let index = 0; index < 5; index += 1) { - const suffix = index === 4 ? "z-fresh" : `a-stale-${index}`; - const extraDir = path.join(fixtureRoot, `decay-cap-${suffix}`); - const filePath = path.join(extraDir, "foo.md"); - await fs.mkdir(extraDir, { recursive: true }); - const body = index < 4 ? "foo md stale content candidate." : "Unrelated fresh candidate."; - await fs.writeFile(filePath, body); - if (index < 4) { - await fs.utimes(filePath, staleMtime, staleMtime); - } - extraPaths.push(extraDir); - } - const cfg = createCfg({ - provider: "none", - extraPaths, - minScore: 0, - }); - const result = await getMemorySearchManager({ cfg, agentId: "main" }); - const manager = requireManager(result); - managersForCleanup.add(manager); - resetManagerForTest(manager); - if (!manager.status().fts?.available) { - return; - } - await manager.sync({ reason: "test" }); - - const results = await manager.search("foo.md", { maxResults: 1, minScore: 0 }); - expect(results).toHaveLength(1); - expect(results[0]?.score).toBe(1); - }); - - it("applies the fixed hybrid candidate cap", async () => { - const staleMtime = new Date(Date.now() - 90 * 24 * 60 * 60_000); - const extraPaths: string[] = []; - for (let index = 0; index < 5; index += 1) { - const suffix = index === 4 ? "z-fresh" : `a-stale-${index}`; - const extraDir = path.join(fixtureRoot, `hybrid-decay-cap-${suffix}`); - const filePath = path.join(extraDir, "alpha.md"); - await fs.mkdir(extraDir, { recursive: true }); - const body = index === 4 ? "Alpha beta lower-similarity candidate." : "Alpha candidate."; - await fs.writeFile(filePath, body); - if (index < 4) { - await fs.utimes(filePath, staleMtime, staleMtime); - } - extraPaths.push(extraDir); - } - const cfg = createCfg({ - extraPaths, - minScore: 0, - }); - const manager = await getPersistentManager(cfg); - await manager.sync({ reason: "test" }); - - const results = await manager.search("alpha.md", { maxResults: 1, minScore: 0 }); - expect(results).toHaveLength(1); - expect(results[0]?.score).toBe(1); - }); - - it("keeps fixed hybrid ranking when search degrades to keyword-only", async () => { - const staleMtime = new Date(Date.now() - 90 * 24 * 60 * 60_000); - const extraPaths: string[] = []; - for (let index = 0; index < 5; index += 1) { - const suffix = index === 4 ? "z-fresh" : `a-stale-${index}`; - const extraDir = path.join(fixtureRoot, `degraded-decay-cap-${suffix}`); - const filePath = path.join(extraDir, "beta.md"); - await fs.mkdir(extraDir, { recursive: true }); - await fs.writeFile(filePath, "Beta equal content candidate."); - if (index < 4) { - await fs.utimes(filePath, staleMtime, staleMtime); - } - extraPaths.push(extraDir); - } - const cfg = createCfg({ - extraPaths, - fallback: "none", - minScore: 0, - }); - const manager = await getPersistentManager(cfg); - await manager.sync({ reason: "test" }); - const degraded = manager as unknown as { - provider: { - id: string; - model: string; - embedQuery: () => Promise; - embedBatch: (texts: string[]) => Promise; - close: () => Promise; - } | null; - markLocalEmbeddingProviderDegraded: (err: unknown) => void; - }; - const provider = degraded.provider; - if (!provider) { - throw new Error("Expected a test embedding provider"); - } - provider.embedQuery = async () => { - throw createLocalWorkerExitError(); - }; - degraded.markLocalEmbeddingProviderDegraded = () => { - degraded.provider = null; - }; - - const results = await manager.search("beta.md", { maxResults: 1, minScore: 0 }); - expect(results).toHaveLength(1); - expect(results[0]?.score).toBe(1); - }); - - it("keeps body relevance for an exact basename beyond the exact candidate cap", async () => { - forceNoProvider = true; - const cfg = createCfg({ - provider: "none", - minScore: 0, - hybrid: { enabled: true }, - }); - const result = await getMemorySearchManager({ cfg, agentId: "main" }); - const manager = requireManager(result); - managersForCleanup.add(manager); - resetManagerForTest(manager); - if (!manager.status().fts?.available) { - return; - } - - const duplicatesDir = path.join(memoryDir, "readme-dupes"); - for (let index = 0; index < 205; index += 1) { - const duplicateDir = path.join(duplicatesDir, `a-${index.toString().padStart(3, "0")}`); - await fs.mkdir(duplicateDir, { recursive: true }); - await fs.writeFile(path.join(duplicateDir, "README.md"), "Unrelated weak body."); - } - const strongDir = path.join(duplicatesDir, "z-strong"); - await fs.mkdir(strongDir, { recursive: true }); - await fs.writeFile( - path.join(strongDir, "README.md"), - "README md README md README md strongest body match.", - ); - await fs.writeFile( - path.join(memoryDir, "readme-body-only.md"), - "README md body-only candidate.", - ); - await fs.writeFile(path.join(memoryDir, "README.md.notes"), "Unrelated partial path."); - await manager.sync({ reason: "test" }); - - const results = await manager.search("README.md", { maxResults: 1, minScore: 0 }); - expect(results).toHaveLength(1); - expect(results[0]?.path).toContain("memory/readme-dupes/z-strong/README.md"); - expect(results[0]?.score).toBe(1); - - const internal = manager as unknown as { - searchKeyword: ( - query: string, - limit: number, - options: { boostFallbackRanking?: boolean }, - sources: Array<"memory">, - ) => Promise>; - }; - const candidates = await internal.searchKeyword( - "README.md", - 4, - { boostFallbackRanking: true }, - ["memory"], - ); - const exactCandidates = candidates.filter((entry) => entry.exactPathSpecificity > 0); - const exactPathCount = new Set(exactCandidates.map((entry) => `${entry.source}:${entry.path}`)) - .size; - const nonExactCount = candidates.length - exactCandidates.length; - expect(exactPathCount).toBe(200); - expect(exactCandidates.length).toBeLessThanOrEqual(204); - expect(nonExactCount).toBeGreaterThan(0); - expect(nonExactCount).toBeLessThanOrEqual(4); - expect(candidates.length).toBeLessThanOrEqual(208); - }); - - it("keeps boosted score ordering for non-exact FTS-only body matches", async () => { - forceNoProvider = true; - const cfg = createCfg({ - provider: "none", - minScore: 0, - hybrid: { enabled: true }, - }); - const result = await getMemorySearchManager({ cfg, agentId: "main" }); - const manager = requireManager(result); - managersForCleanup.add(manager); - resetManagerForTest(manager); - if (!manager.status().fts?.available) { - return; - } - - await fs.writeFile( - path.join(memoryDir, "project-memory-notes.md"), - "Project memory notes covering workspace context and retrieval behavior.", - ); - await fs.writeFile(path.join(memoryDir, "notes.md"), "Project memory context."); - await manager.sync({ reason: "test" }); - - const results = await manager.search("project memory context", { - maxResults: 1, - minScore: 0, - }); - expect(results).toHaveLength(1); - expect(results[0]?.path).toContain("memory/project-memory-notes.md"); - expect(results[0]?.score).toBeLessThanOrEqual(1); - }); - - it("keeps an exact dated path ahead in FTS-only mode", async () => { - forceNoProvider = true; - const cfg = createCfg({ - provider: "none", - minScore: 0.35, - }); - const result = await getMemorySearchManager({ cfg, agentId: "main" }); - const manager = requireManager(result); - managersForCleanup.add(manager); - resetManagerForTest(manager); - if (!manager.status().fts?.available) { - return; - } - - await fs.writeFile(path.join(memoryDir, "2020-01-01.md"), "Unrelated exact-path body."); - await fs.writeFile(path.join(memoryDir, "body-match.md"), "2020 01 01 2020 01 01 2020 01 01"); - await manager.sync({ reason: "test" }); - - const results = await manager.search("2020-01-01", { maxResults: 1 }); - expect(results).toHaveLength(1); - expect(results[0]?.path).toContain("memory/2020-01-01.md"); - expect(results[0]?.score).toBe(1); - }); - - it("fails fast instead of searching FTS when an explicit provider is unavailable", async () => { - forceNoProvider = true; - - const cfg = createCfg({ - provider: "openai", - minScore: 0.35, - hybrid: { enabled: true }, - }); - const manager = await getFreshManager(cfg); - try { - await expect(manager.search("Alpha")).rejects.toThrow( - /Memory search unavailable: embedding provider "openai" is configured but unavailable\.[\s\S]*agentId=main purpose=default[\s\S]*registeredMemoryEmbeddingProviders=none/, - ); - await expect(manager.sync({ reason: "test" })).rejects.toThrow( - /Memory sync unavailable: embedding provider "openai" is configured but unavailable\./, - ); - forceNoProvider = false; - await manager.sync({ reason: "test", force: true }); - const results = await manager.search("Alpha"); - expect(results.length).toBeGreaterThan(0); - } finally { - await manager.close?.(); - } - }); - - it("fails fast instead of returning FTS when an explicit provider is lost at runtime", async () => { - const cfg = createCfg({ - provider: "openai", - minScore: 0.35, - hybrid: { enabled: true }, - }); - const manager = await getFreshManager(cfg); - try { - await manager.sync({ reason: "test", force: true }); - ( - manager as unknown as { - provider: null; - } - ).provider = null; - - await expect(manager.search("Alpha")).rejects.toThrow( - /Memory search unavailable: embedding provider "openai" is configured but unavailable\./, - ); - } finally { - await manager.close?.(); - } - }); - it("prefers exact session transcript hits in FTS-only mode", async () => { - try { - const manager = await getFtsSessionManager({ - stateDirName: ".state-session-ranking", - }); - if (!manager) { - return; - } - - const memoryPath = path.join(workspaceDir, "MEMORY.md"); - await fs.writeFile(memoryPath, "Project Nebula stale codename: ORBIT-9.\n", "utf8"); - const staleAt = new Date("2020-01-01T00:00:00.000Z"); - await fs.utimes(memoryPath, staleAt, staleAt); - - const now = Date.parse("2026-04-07T15:25:04.113Z"); - await seedMemoryIndexSessionTranscript({ - sessionId: "session-ranking", - messages: [ - { - role: "user", - timestamp: new Date(now - 30_000).toISOString(), - content: "What is the current Project Nebula codename?", - }, - { - role: "assistant", - timestamp: new Date(now).toISOString(), - content: "The current Project Nebula codename is ORBIT-10.", - }, - ], - }); - - await manager.sync({ reason: "test", force: true }); - const results = await manager.search("current Project Nebula codename ORBIT-10", { - minScore: 0, - maxResults: 3, - }); - - expect(results[0]?.source).toBe("sessions"); - expect(results[0]?.snippet).toContain("ORBIT-10"); - expect(results[0]?.provenance).toMatchObject({ - originClass: "untrusted", - sessionKind: "interactive", - }); - } finally { - restoreMemoryIndexStateDir(); - } - }); - it("preserves trusted per-line provenance through session indexing", async () => { try { const manager = await getFtsSessionManager({ @@ -5129,79 +2776,6 @@ describe("memory index", () => { } }); - it("bootstraps an empty index on first search so session transcript hits are available", async () => { - try { - const manager = await getFtsSessionManager({ - stateDirName: ".state-session-bootstrap", - }); - if (!manager) { - return; - } - - await seedMemoryIndexSessionTranscript({ - sessionId: "session-bootstrap", - messages: [ - { - role: "assistant", - timestamp: "2026-04-07T15:25:04.113Z", - content: "The current Project Nebula codename is ORBIT-10.", - }, - ], - }); - - const results = await manager.search("current Project Nebula codename ORBIT-10", { - minScore: 0, - maxResults: 3, - }); - - expect(results[0]?.source).toBe("sessions"); - expect(results[0]?.snippet).toContain("ORBIT-10"); - } finally { - restoreMemoryIndexStateDir(); - } - }); - it("keeps remember-only session transcripts out of ordinary manager searches", async () => { - forceNoProvider = true; - setMemoryIndexStateDir(path.join(workspaceDir, ".state-remember-search-sources")); - try { - const cfg = createCfg({ - provider: "none", - rememberAcrossConversations: true, - minScore: 0, - hybrid: { enabled: true, vectorWeight: 0.7, textWeight: 0.3 }, - }); - const manager = await getFreshManager(cfg); - managersForCleanup.add(manager); - if (!manager.status().fts?.available) { - return; - } - - await seedMemoryIndexSessionTranscript({ - sessionId: "remember-only", - messages: [ - { - role: "assistant", - timestamp: "2026-04-07T15:25:04.113Z", - content: "Recall-only canary is NEBULA-47.", - }, - ], - }); - - await manager.sync({ reason: "test", force: true }); - - await expect( - manager.search("Recall-only canary NEBULA-47", { minScore: 0 }), - ).resolves.toEqual([]); - const trustedResults = await manager.search("Recall-only canary NEBULA-47", { - minScore: 0, - sources: ["sessions"], - }); - expect(trustedResults[0]?.source).toBe("sessions"); - } finally { - restoreMemoryIndexStateDir(); - } - }); - it("status-purpose manager detects unindexed session transcripts as dirty", async () => { // Regression test for #97814: plain openclaw memory status (purpose: status) // must report dirty=true when session files exist without index rows. diff --git a/extensions/memory-core/src/memory/index.ts b/extensions/memory-core/src/memory/index.ts index cd0d43c784b8..0dbb7986cedf 100644 --- a/extensions/memory-core/src/memory/index.ts +++ b/extensions/memory-core/src/memory/index.ts @@ -1,5 +1,4 @@ // Memory Core plugin entrypoint registers its OpenClaw integration. -export { MemoryIndexManager } from "./manager.js"; export { closeAllMemorySearchManagers, closeMemorySearchManager, diff --git a/extensions/memory-core/src/memory/manager-async-state.test.ts b/extensions/memory-core/src/memory/manager-async-state.test.ts new file mode 100644 index 000000000000..27c92956b9b3 --- /dev/null +++ b/extensions/memory-core/src/memory/manager-async-state.test.ts @@ -0,0 +1,71 @@ +// Memory Core tests cover asynchronous manager state helpers. +import { describe, expect, it, vi } from "vitest"; +import { awaitPendingManagerWork, startAsyncSearchSync } from "./manager-async-state.js"; + +describe("memory manager async state", () => { + it("waits for in-flight search sync during close", async () => { + let releaseSync = () => {}; + const pendingSync = new Promise((resolve) => { + releaseSync = () => resolve(); + }); + + let closed = false; + const closePromise = awaitPendingManagerWork({ pendingSync }).then(() => { + closed = true; + }); + + await Promise.resolve(); + expect(closed).toBe(false); + + releaseSync(); + await closePromise; + }); + + it("reports pending sync failures during close", async () => { + const onError = vi.fn(); + const syncError = new Error("sync failed"); + + await awaitPendingManagerWork({ + pendingSync: Promise.reject(syncError), + onError, + }); + + expect(onError).toHaveBeenCalledWith(syncError); + }); + + it("reports pending provider initialization failures during close", async () => { + const onError = vi.fn(); + const providerError = new Error("provider init failed"); + + await awaitPendingManagerWork({ + pendingProviderInit: Promise.reject(providerError), + onError, + }); + + expect(onError).toHaveBeenCalledWith(providerError); + }); + + it("does not report errors for completed pending close work", async () => { + const onError = vi.fn(); + + await awaitPendingManagerWork({ + pendingSync: Promise.resolve(), + pendingProviderInit: Promise.resolve(), + onError, + }); + + expect(onError).not.toHaveBeenCalled(); + }); + + it("skips background search sync when search-triggered sync is disabled", async () => { + const syncMock = vi.fn(async () => {}); + await startAsyncSearchSync({ + enabled: false, + dirty: true, + sessionsDirty: false, + sync: syncMock, + onError: vi.fn(), + }); + expect(syncMock).not.toHaveBeenCalled(); + }); +}); diff --git a/extensions/memory-core/src/memory/manager-keyword-retrieval.test.ts b/extensions/memory-core/src/memory/manager-keyword-retrieval.test.ts new file mode 100644 index 000000000000..666961e93da2 --- /dev/null +++ b/extensions/memory-core/src/memory/manager-keyword-retrieval.test.ts @@ -0,0 +1,972 @@ +// Memory Core tests cover manager keyword retrieval behavior. +import { mkdirSync, rmSync } from "node:fs"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { clearMemoryEmbeddingProviders as clearRegistry } from "openclaw/plugin-sdk/memory-core-host-engine-embeddings"; +import { resolveSessionTranscriptsDirForAgent } from "openclaw/plugin-sdk/memory-core-host-runtime-core"; +import { upsertSessionEntry } from "openclaw/plugin-sdk/session-store-runtime"; +import { appendSessionTranscriptMessageByIdentity } from "openclaw/plugin-sdk/session-transcript-runtime"; +import { + closeOpenClawAgentDatabasesForTest, + closeOpenClawStateDatabaseForTest, +} from "openclaw/plugin-sdk/sqlite-runtime-testing"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { + configureMemoryCoreDreamingStateForTests, + resetMemoryCoreDreamingStateForTests, +} from "../test-helpers.js"; +import "./test-runtime-mocks.js"; +import { closeAllMemorySearchManagers, getMemorySearchManager } from "./index.js"; +import type { MemoryIndexManager } from "./manager.js"; +import { isolateMemoryManagerTestConfig } from "./test-config-helpers.js"; + +// This suite performs real sqlite/media indexing and can exceed the global +// timeout when it shares a packed CI extension shard. +vi.setConfig({ testTimeout: 240_000 }); + +afterAll(() => { + vi.resetConfig(); +}); + +let embedBatchCalls = 0; +let embeddedBatchTexts: string[] = []; +let embedBatchInputCalls = 0; +let providerRuntimeBatchCalls: string[][] = []; +let providerRuntimeBatchGate: Promise | null = null; +let providerRuntimeBatchErrors: unknown[] = []; +let providerRuntimeBatchFailuresRemaining = 0; +let providerRuntimeActiveBatchCalls = 0; +let providerRuntimeMaxActiveBatchCalls = 0; +let providerCloseCalls = 0; +let providerCloseFailuresRemaining = 0; +let providerCloseFailure: unknown = new Error("provider close failed"); +let providerCreationFailure: string | null = null; +let providerNullResult: string | null = null; +let providerCloseGate: Promise | null = null; +let providerInitGate: Promise | null = null; +let providerCalls: Array<{ provider?: string; model?: string; outputDimensionality?: number }> = []; +let forceNoProvider = false; + +const originalMemoryIndexStateDir = process.env.OPENCLAW_STATE_DIR; + +const identityAliasFixture = vi.hoisted(() => ({ + provider: "identity-alias-test", + canonicalModel: "hf:fixture/default-model.gguf", + cacheModel: "/fixture/cache/default-model.gguf", +})); + +function createLocalWorkerExitError(): Error { + return Object.assign(new Error("Local embedding worker exited unexpectedly (exit code 134)"), { + code: "LOCAL_EMBEDDING_WORKER_EXITED", + reason: "exit", + exitCode: 134, + }); +} + +function setMemoryIndexStateDir(stateDir: string): void { + Reflect.set(process.env, "OPENCLAW_STATE_DIR", stateDir); +} + +function restoreMemoryIndexStateDir(): void { + if (originalMemoryIndexStateDir === undefined) { + Reflect.deleteProperty(process.env, "OPENCLAW_STATE_DIR"); + } else { + Reflect.set(process.env, "OPENCLAW_STATE_DIR", originalMemoryIndexStateDir); + } +} + +vi.mock("./embeddings.js", async (importOriginal) => { + const actual = await importOriginal(); + const embedText = (text: string) => { + const lower = text.toLowerCase(); + const alpha = lower.split("alpha").length - 1; + const beta = lower.split("beta").length - 1; + const image = lower.split("image").length - 1; + const audio = lower.split("audio").length - 1; + return [alpha, beta, image, audio]; + }; + return { + ...actual, + resolveEmbeddingProviderFallbackModel: (providerId: string, fallbackSourceModel: string) => + providerId === "gemini" || providerId === "fallback-provider" + ? `${providerId}-embed` + : fallbackSourceModel, + resolveEmbeddingProviderAdapterId: ( + providerId: string, + config?: { + models?: { + providers?: Record; + }; + }, + ) => config?.models?.providers?.[providerId]?.api ?? providerId, + resolveEmbeddingProviderAdapterTransport: (providerId: string) => + providerId === "local" ? "local" : "remote", + resolveEmbeddingProviderIndexIdentity: (options: { provider?: string; model?: string }) => + options.provider === identityAliasFixture.provider + ? { + provider: { + id: identityAliasFixture.provider, + model: identityAliasFixture.canonicalModel, + }, + cacheKeyData: { + provider: identityAliasFixture.provider, + model: identityAliasFixture.canonicalModel, + }, + aliases: [ + { + model: identityAliasFixture.cacheModel, + cacheKeyData: { + provider: identityAliasFixture.provider, + model: identityAliasFixture.cacheModel, + }, + }, + ], + } + : undefined, + createEmbeddingProvider: async (options: { + provider?: string; + model?: string; + outputDimensionality?: number; + }) => { + providerCalls.push({ + provider: options.provider, + model: options.model, + outputDimensionality: options.outputDimensionality, + }); + await providerInitGate; + if (options.provider === providerCreationFailure) { + throw new Error(`provider creation failed: ${options.provider}`); + } + if (options.provider === providerNullResult) { + return { + provider: null, + requestedProvider: options.provider, + providerUnavailableReason: `provider unavailable: ${options.provider}`, + }; + } + if (forceNoProvider) { + return { + provider: null, + requestedProvider: options.provider ?? "auto", + providerUnavailableReason: "No API key found for provider", + }; + } + const providerId = + options.provider === "gemini" || + options.provider === "fallback-provider" || + options.provider === "batch-test" || + options.provider === "batch-wide-test" || + options.provider === identityAliasFixture.provider || + options.provider === "ollama" + ? options.provider + : "mock"; + const requestedModel = options.model ?? "mock-embed"; + const model = + providerId === identityAliasFixture.provider && + (requestedModel === identityAliasFixture.canonicalModel || + requestedModel === identityAliasFixture.cacheModel) + ? identityAliasFixture.canonicalModel + : requestedModel; + return { + requestedProvider: options.provider ?? "openai", + provider: { + id: providerId, + model, + close: async () => { + providerCloseCalls += 1; + await providerCloseGate; + if (providerCloseFailuresRemaining > 0) { + providerCloseFailuresRemaining -= 1; + throw providerCloseFailure; + } + }, + embedQuery: async (text: string) => embedText(text), + embedBatch: async (texts: string[]) => { + embedBatchCalls += 1; + embeddedBatchTexts.push(...texts); + return texts.map(embedText); + }, + ...(providerId === "gemini" || providerId === "fallback-provider" + ? { + embedBatchInputs: async ( + inputs: Array<{ + text: string; + parts?: Array< + | { type: "text"; text: string } + | { type: "inline-data"; mimeType: string; data: string } + >; + }>, + ) => { + embedBatchInputCalls += 1; + return inputs.map((input) => { + const inlineData = input.parts?.find((part) => part.type === "inline-data"); + if (inlineData?.type === "inline-data" && inlineData.data.length > 9000) { + throw new Error("payload too large"); + } + const mimeType = + inlineData?.type === "inline-data" ? inlineData.mimeType : undefined; + if (mimeType?.startsWith("image/")) { + return [0, 0, 1, 0]; + } + if (mimeType?.startsWith("audio/")) { + return [0, 0, 0, 1]; + } + return embedText(input.text); + }); + }, + } + : {}), + }, + ...(providerId === identityAliasFixture.provider + ? { + runtime: { + id: providerId, + cacheKeyData: { + provider: providerId, + model: identityAliasFixture.canonicalModel, + }, + indexIdentityAliases: [ + { + model: identityAliasFixture.cacheModel, + cacheKeyData: { + provider: providerId, + model: identityAliasFixture.cacheModel, + }, + }, + ], + }, + } + : providerId === "batch-test" || providerId === "batch-wide-test" + ? { + runtime: { + id: providerId, + ...(providerId === "batch-wide-test" ? { sourceWideBatchEmbed: true } : {}), + batchEmbed: async (batch: { chunks: Array<{ text: string }> }) => { + providerRuntimeActiveBatchCalls += 1; + providerRuntimeMaxActiveBatchCalls = Math.max( + providerRuntimeMaxActiveBatchCalls, + providerRuntimeActiveBatchCalls, + ); + try { + await providerRuntimeBatchGate; + providerRuntimeBatchCalls.push(batch.chunks.map((chunk) => chunk.text)); + if (providerRuntimeBatchErrors.length > 0) { + throw providerRuntimeBatchErrors.shift(); + } + if (providerRuntimeBatchFailuresRemaining > 0) { + providerRuntimeBatchFailuresRemaining -= 1; + throw new Error("provider runtime batch failed"); + } + return batch.chunks.map((chunk) => embedText(chunk.text)); + } finally { + providerRuntimeActiveBatchCalls -= 1; + } + }, + }, + } + : providerId === "gemini" || providerId === "fallback-provider" + ? { + runtime: { + id: providerId, + cacheKeyData: { + provider: providerId, + baseUrl: "https://generativelanguage.googleapis.com/v1beta", + model, + outputDimensionality: options.outputDimensionality, + headers: [], + }, + }, + } + : {}), + }; + }, + }; +}); + +describe("memory index", () => { + let fixtureRoot = ""; + let workspaceDir = ""; + let memoryDir = ""; + + const managersForCleanup = new Set(); + + beforeAll(async () => { + fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-mem-fixtures-")); + workspaceDir = path.join(fixtureRoot, "workspace"); + memoryDir = path.join(workspaceDir, "memory"); + }); + + afterAll(async () => { + await Promise.all(Array.from(managersForCleanup).map((manager) => manager.close())); + await fs.rm(fixtureRoot, { recursive: true, force: true }); + }); + + afterEach(async () => { + vi.useRealTimers(); + await Promise.all(Array.from(managersForCleanup).map((manager) => manager.close())); + await closeAllMemorySearchManagers(); + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + resetMemoryCoreDreamingStateForTests(); + clearRegistry(); + managersForCleanup.clear(); + restoreMemoryIndexStateDir(); + }); + + beforeEach(async () => { + vi.useRealTimers(); + clearRegistry(); + embedBatchCalls = 0; + embeddedBatchTexts = []; + embedBatchInputCalls = 0; + providerRuntimeBatchCalls = []; + providerRuntimeBatchGate = null; + providerRuntimeBatchErrors = []; + providerRuntimeBatchFailuresRemaining = 0; + providerRuntimeActiveBatchCalls = 0; + providerRuntimeMaxActiveBatchCalls = 0; + providerCloseCalls = 0; + providerCloseFailuresRemaining = 0; + providerCloseFailure = new Error("provider close failed"); + providerCreationFailure = null; + providerNullResult = null; + providerCloseGate = null; + providerInitGate = null; + providerCalls = []; + forceNoProvider = false; + + rmSync(workspaceDir, { recursive: true, force: true }); + mkdirSync(memoryDir, { recursive: true }); + setMemoryIndexStateDir(path.join(workspaceDir, ".state-memory-index")); + await configureMemoryCoreDreamingStateForTests(); + await fs.writeFile( + path.join(memoryDir, "2026-01-12.md"), + "# Log\nAlpha memory line.\nZebra memory line.", + ); + }); + + function resetManagerForTest(manager: MemoryIndexManager) { + // These tests reuse managers for performance. Clear the index + embedding + // cache to keep each test fully isolated. + const db = ( + manager as unknown as { + db: { + exec: (sql: string) => void; + prepare: (sql: string) => { get: (name: string) => { name?: string } | undefined }; + }; + } + ).db; + for (const table of [ + "memory_index_sources", + "memory_index_chunks", + "memory_embedding_cache", + "memory_index_chunks_fts", + "memory_index_chunks_vec", + ]) { + const existingTable = db + .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?") + .get(table); + if (existingTable?.name === table) { + db.exec(`DELETE FROM ${table}`); + } + } + (manager as unknown as { dirty: boolean }).dirty = true; + (manager as unknown as { sessionsDirty: boolean }).sessionsDirty = false; + (manager as unknown as { sessionsDirtyFiles: Set }).sessionsDirtyFiles.clear(); + } + + type TestCfg = Parameters[0]["cfg"]; + + function createCfg(params: { + extraPaths?: string[]; + sources?: Array<"memory" | "sessions">; + sessionMemory?: boolean; + rememberAcrossConversations?: boolean; + provider?: string; + fallback?: "none" | "gemini" | "fallback-provider"; + providerAliases?: NonNullable["providers"]>; + batchEnabled?: boolean; + model?: string; + outputDimensionality?: number; + multimodal?: { + enabled?: boolean; + modalities?: Array<"image" | "audio" | "all">; + maxFileBytes?: number; + }; + vectorEnabled?: boolean; + cacheEnabled?: boolean; + minScore?: number; + onSearch?: boolean; + hybrid?: { + enabled: boolean; + vectorWeight?: number; + textWeight?: number; + temporalDecay?: { enabled: boolean }; + }; + }): TestCfg { + return isolateMemoryManagerTestConfig({ + memory: { + search: { + ...(params.provider !== undefined ? { provider: params.provider } : {}), + model: params.model ?? "mock-embed", + fallback: params.fallback, + outputDimensionality: params.outputDimensionality, + store: { + vector: params.vectorEnabled !== undefined ? { enabled: params.vectorEnabled } : {}, + }, + remote: params.batchEnabled + ? { + batch: { enabled: true }, + } + : undefined, + query: { minScore: params.minScore ?? 0 }, + cache: params.cacheEnabled ? { enabled: true } : undefined, + extraPaths: params.extraPaths, + multimodal: params.multimodal, + sources: params.sources, + rememberAcrossConversations: + params.rememberAcrossConversations ?? params.sessionMemory ?? false, + }, + }, + + agents: { + defaults: { + workspace: workspaceDir, + }, + list: [{ id: "main", default: true }], + }, + models: params.providerAliases ? { providers: params.providerAliases } : undefined, + }); + } + + async function seedMemoryIndexSessionTranscript(params: { + messages: Array<{ + content: string; + role: "assistant" | "user"; + senderIsOwner?: boolean; + timestamp: number | string; + }>; + sessionId: string; + sessionKey?: string; + }): Promise { + const sessionsDir = resolveSessionTranscriptsDirForAgent("main"); + const storePath = path.join(sessionsDir, "sessions.json"); + const sessionKey = params.sessionKey ?? `agent:main:memory:${params.sessionId}`; + // Message timestamps are behavioral inputs; entry freshness only keeps the + // fixture out of real session-retention maintenance as wall time advances. + const updatedAt = Date.now(); + await fs.mkdir(sessionsDir, { recursive: true }); + await upsertSessionEntry({ + agentId: "main", + sessionKey, + storePath, + entry: { + sessionId: params.sessionId, + updatedAt, + }, + }); + for (const message of params.messages) { + await appendSessionTranscriptMessageByIdentity({ + agentId: "main", + sessionId: params.sessionId, + sessionKey, + storePath, + message: { + role: message.role, + timestamp: message.timestamp, + content: [{ type: "text", text: message.content }], + ...(message.senderIsOwner ? { __openclaw: { senderIsOwner: true } } : {}), + }, + }); + } + } + + function requireManager( + result: Awaited>, + missingMessage = "manager missing", + ): MemoryIndexManager { + if (!result.manager) { + throw new Error(missingMessage); + } + return result.manager as unknown as MemoryIndexManager; + } + + async function getPersistentManager(cfg: TestCfg): Promise { + const result = await getMemorySearchManager({ cfg, agentId: "main" }); + const manager = requireManager(result); + managersForCleanup.add(manager); + resetManagerForTest(manager); + return manager; + } + + async function getFtsSessionManager(params: { + stateDirName: string; + }): Promise { + forceNoProvider = true; + setMemoryIndexStateDir(path.join(workspaceDir, params.stateDirName)); + const cfg = createCfg({ + provider: "none", + sources: ["memory", "sessions"], + sessionMemory: true, + minScore: 0, + hybrid: { enabled: true, vectorWeight: 0.7, textWeight: 0.3 }, + }); + const result = await getMemorySearchManager({ cfg, agentId: "main" }); + const manager = requireManager(result); + managersForCleanup.add(manager); + resetManagerForTest(manager); + return manager.status().fts?.available ? manager : null; + } + + it("builds FTS index and returns search results when no embedding provider is available", async () => { + forceNoProvider = true; + + const cfg = createCfg({ + provider: "none", + minScore: 0.35, + hybrid: { enabled: true }, + }); + const result = await getMemorySearchManager({ cfg, agentId: "main" }); + const manager = requireManager(result); + managersForCleanup.add(manager); + resetManagerForTest(manager); + if (!manager.status().fts?.available) { + return; + } + + await fs.writeFile( + path.join(memoryDir, "2026-01-12.md"), + "# Log\nAlpha memory line.\nZebra memory line.", + ); + await manager.sync({ reason: "test" }); + + const status = manager.status(); + expect(status.chunks).toBeGreaterThan(0); + expect(embedBatchCalls).toBe(0); + + const results = await manager.search("Alpha"); + expect(results.length).toBeGreaterThan(0); + expect(results[0]?.snippet).toMatch(/Alpha/i); + + const noResults = await manager.search("nonexistent_xyz_keyword"); + expect(noResults.length).toBe(0); + }); + + it("ranks an exact path stem ahead of a body match before applying the result limit", async () => { + forceNoProvider = true; + const cfg = createCfg({ + provider: "none", + minScore: 0.35, + hybrid: { enabled: true }, + }); + const result = await getMemorySearchManager({ cfg, agentId: "main" }); + const manager = requireManager(result); + managersForCleanup.add(manager); + resetManagerForTest(manager); + if (!manager.status().fts?.available) { + return; + } + + await fs.writeFile(path.join(memoryDir, "project-lantern.md"), "Unrelated exact-path body."); + await fs.writeFile( + path.join(memoryDir, "body-match.md"), + "Project lantern project lantern project lantern.", + ); + await manager.sync({ reason: "test" }); + + const results = await manager.search("project-lantern", { maxResults: 1 }); + expect(results).toHaveLength(1); + expect(results[0]?.path).toContain("memory/project-lantern.md"); + expect(results[0]?.score).toBe(1); + }); + + it("does not let fallback-term filenames consume the candidate cap", async () => { + forceNoProvider = true; + const cfg = createCfg({ + provider: "none", + minScore: 0, + hybrid: { enabled: true }, + }); + const result = await getMemorySearchManager({ cfg, agentId: "main" }); + const manager = requireManager(result); + managersForCleanup.add(manager); + resetManagerForTest(manager); + if (!manager.status().fts?.available) { + return; + } + + for (let index = 0; index < 5; index += 1) { + const duplicateDir = path.join(memoryDir, `alpha-${index}`); + await fs.mkdir(duplicateDir, { recursive: true }); + await fs.writeFile(path.join(duplicateDir, "alpha.md"), "Unrelated path-only candidate."); + } + await fs.writeFile( + path.join(memoryDir, "body-match.md"), + "Alpha alpha alpha alpha alpha strongest fallback body match.", + ); + await manager.sync({ reason: "test" }); + + const results = await manager.search("alpha gamma", { maxResults: 1, minScore: 0 }); + expect(results).toHaveLength(1); + expect(results[0]?.path).toContain("memory/body-match.md"); + }); + + it("bounds the merged six-term fallback candidate set", async () => { + forceNoProvider = true; + const manager = await getPersistentManager( + createCfg({ provider: "none", minScore: 0, hybrid: { enabled: true } }), + ); + const terms = ["alpha", "beta", "gamma", "delta", "epsilon", "zeta"]; + for (const term of terms) { + for (let index = 0; index < 5; index += 1) { + await fs.writeFile(path.join(memoryDir, `${term}-${index}.md`), `${term} body ${index}`); + } + } + await manager.sync({ reason: "test" }); + + const results = await manager.search(terms.join(" "), { maxResults: 4, minScore: 0 }); + + expect(results).toHaveLength(4); + expect(new Set(results.map((entry) => entry.path)).size).toBe(4); + }); + + it("counts exact candidate headroom by distinct path instead of chunk", async () => { + forceNoProvider = true; + const manager = await getPersistentManager( + createCfg({ provider: "none", minScore: 0, hybrid: { enabled: true } }), + ); + for (let index = 0; index < 200; index += 1) { + const dir = path.join(memoryDir, index.toString().padStart(3, "0")); + await fs.mkdir(dir, { recursive: true }); + await fs.writeFile(path.join(dir, "foo.md"), `foo body ${index}`); + } + await manager.sync({ reason: "test" }); + + const results = await manager.search("foo.md", { maxResults: 204, minScore: 0 }); + + expect(results).toHaveLength(200); + expect(new Set(results.map((entry) => entry.path)).size).toBe(200); + expect(results.some((entry) => entry.path === "memory/199/foo.md")).toBe(true); + }); + + it("uses body relevance within the same exact basename tier in FTS-only mode", async () => { + forceNoProvider = true; + const cfg = createCfg({ + provider: "none", + minScore: 0, + hybrid: { enabled: true }, + }); + const result = await getMemorySearchManager({ cfg, agentId: "main" }); + const manager = requireManager(result); + managersForCleanup.add(manager); + resetManagerForTest(manager); + if (!manager.status().fts?.available) { + return; + } + + const weakDir = path.join(memoryDir, "a"); + const strongDir = path.join(memoryDir, "z"); + await fs.mkdir(weakDir, { recursive: true }); + await fs.mkdir(strongDir, { recursive: true }); + await fs.writeFile(path.join(weakDir, "foo.md"), "Unrelated weak body."); + await fs.writeFile(path.join(strongDir, "foo.md"), "foo md foo md foo md strong body"); + await manager.sync({ reason: "test" }); + + const results = await manager.search("foo.md", { maxResults: 1, minScore: 0 }); + expect(results).toHaveLength(1); + expect(results[0]?.path).toContain("memory/z/foo.md"); + expect(results[0]?.score).toBe(1); + }); + + it("returns exact basename candidates with fixed FTS ranking", async () => { + forceNoProvider = true; + const staleDir = path.join(fixtureRoot, "decay-a-stale"); + const freshDir = path.join(fixtureRoot, "decay-z-fresh"); + await fs.mkdir(staleDir, { recursive: true }); + await fs.mkdir(freshDir, { recursive: true }); + const staleFooPath = path.join(staleDir, "foo.md"); + const freshFooPath = path.join(freshDir, "foo.md"); + const staleBarPath = path.join(staleDir, "bar.md"); + await fs.writeFile(staleFooPath, "Unrelated stale candidate."); + await fs.writeFile(freshFooPath, "Unrelated fresh candidate."); + await fs.writeFile(staleBarPath, "bar md bar md bar md strongest stale body"); + await fs.writeFile(path.join(freshDir, "bar.md"), "bar md fresh body"); + const staleMtime = new Date(Date.now() - 90 * 24 * 60 * 60_000); + await Promise.all([ + fs.utimes(staleFooPath, staleMtime, staleMtime), + fs.utimes(staleBarPath, staleMtime, staleMtime), + ]); + const cfg = createCfg({ + provider: "none", + extraPaths: [staleDir, freshDir], + minScore: 0, + }); + const result = await getMemorySearchManager({ cfg, agentId: "main" }); + const manager = requireManager(result); + managersForCleanup.add(manager); + resetManagerForTest(manager); + if (!manager.status().fts?.available) { + return; + } + await manager.sync({ reason: "test" }); + + for (const basename of ["foo.md", "bar.md"]) { + const results = await manager.search(basename, { maxResults: 1, minScore: 0 }); + expect(results).toHaveLength(1); + expect(results[0]?.score).toBe(1); + } + }); + + it("applies the fixed FTS candidate cap to exact paths", async () => { + forceNoProvider = true; + const staleMtime = new Date(Date.now() - 90 * 24 * 60 * 60_000); + const extraPaths: string[] = []; + for (let index = 0; index < 5; index += 1) { + const suffix = index === 4 ? "z-fresh" : `a-stale-${index}`; + const extraDir = path.join(fixtureRoot, `decay-cap-${suffix}`); + const filePath = path.join(extraDir, "foo.md"); + await fs.mkdir(extraDir, { recursive: true }); + const body = index < 4 ? "foo md stale content candidate." : "Unrelated fresh candidate."; + await fs.writeFile(filePath, body); + if (index < 4) { + await fs.utimes(filePath, staleMtime, staleMtime); + } + extraPaths.push(extraDir); + } + const cfg = createCfg({ + provider: "none", + extraPaths, + minScore: 0, + }); + const result = await getMemorySearchManager({ cfg, agentId: "main" }); + const manager = requireManager(result); + managersForCleanup.add(manager); + resetManagerForTest(manager); + if (!manager.status().fts?.available) { + return; + } + await manager.sync({ reason: "test" }); + + const results = await manager.search("foo.md", { maxResults: 1, minScore: 0 }); + expect(results).toHaveLength(1); + expect(results[0]?.score).toBe(1); + }); + + it("applies the fixed hybrid candidate cap", async () => { + const staleMtime = new Date(Date.now() - 90 * 24 * 60 * 60_000); + const extraPaths: string[] = []; + for (let index = 0; index < 5; index += 1) { + const suffix = index === 4 ? "z-fresh" : `a-stale-${index}`; + const extraDir = path.join(fixtureRoot, `hybrid-decay-cap-${suffix}`); + const filePath = path.join(extraDir, "alpha.md"); + await fs.mkdir(extraDir, { recursive: true }); + const body = index === 4 ? "Alpha beta lower-similarity candidate." : "Alpha candidate."; + await fs.writeFile(filePath, body); + if (index < 4) { + await fs.utimes(filePath, staleMtime, staleMtime); + } + extraPaths.push(extraDir); + } + const cfg = createCfg({ + extraPaths, + minScore: 0, + }); + const manager = await getPersistentManager(cfg); + await manager.sync({ reason: "test" }); + + const results = await manager.search("alpha.md", { maxResults: 1, minScore: 0 }); + expect(results).toHaveLength(1); + expect(results[0]?.score).toBe(1); + }); + + it("keeps fixed hybrid ranking when search degrades to keyword-only", async () => { + const staleMtime = new Date(Date.now() - 90 * 24 * 60 * 60_000); + const extraPaths: string[] = []; + for (let index = 0; index < 5; index += 1) { + const suffix = index === 4 ? "z-fresh" : `a-stale-${index}`; + const extraDir = path.join(fixtureRoot, `degraded-decay-cap-${suffix}`); + const filePath = path.join(extraDir, "beta.md"); + await fs.mkdir(extraDir, { recursive: true }); + await fs.writeFile(filePath, "Beta equal content candidate."); + if (index < 4) { + await fs.utimes(filePath, staleMtime, staleMtime); + } + extraPaths.push(extraDir); + } + const cfg = createCfg({ + extraPaths, + fallback: "none", + minScore: 0, + }); + const manager = await getPersistentManager(cfg); + await manager.sync({ reason: "test" }); + const degraded = manager as unknown as { + provider: { + id: string; + model: string; + embedQuery: () => Promise; + embedBatch: (texts: string[]) => Promise; + close: () => Promise; + } | null; + markLocalEmbeddingProviderDegraded: (err: unknown) => void; + }; + const provider = degraded.provider; + if (!provider) { + throw new Error("Expected a test embedding provider"); + } + provider.embedQuery = async () => { + throw createLocalWorkerExitError(); + }; + degraded.markLocalEmbeddingProviderDegraded = () => { + degraded.provider = null; + }; + + const results = await manager.search("beta.md", { maxResults: 1, minScore: 0 }); + expect(results).toHaveLength(1); + expect(results[0]?.score).toBe(1); + }); + + it("keeps body relevance for an exact basename beyond the exact candidate cap", async () => { + forceNoProvider = true; + const cfg = createCfg({ + provider: "none", + minScore: 0, + hybrid: { enabled: true }, + }); + const result = await getMemorySearchManager({ cfg, agentId: "main" }); + const manager = requireManager(result); + managersForCleanup.add(manager); + resetManagerForTest(manager); + if (!manager.status().fts?.available) { + return; + } + + const duplicatesDir = path.join(memoryDir, "readme-dupes"); + for (let index = 0; index < 205; index += 1) { + const duplicateDir = path.join(duplicatesDir, `a-${index.toString().padStart(3, "0")}`); + await fs.mkdir(duplicateDir, { recursive: true }); + await fs.writeFile(path.join(duplicateDir, "README.md"), "Unrelated weak body."); + } + const strongDir = path.join(duplicatesDir, "z-strong"); + await fs.mkdir(strongDir, { recursive: true }); + await fs.writeFile( + path.join(strongDir, "README.md"), + "README md README md README md strongest body match.", + ); + await fs.writeFile( + path.join(memoryDir, "readme-body-only.md"), + "README md body-only candidate.", + ); + await fs.writeFile(path.join(memoryDir, "README.md.notes"), "Unrelated partial path."); + await manager.sync({ reason: "test" }); + + const results = await manager.search("README.md", { maxResults: 1, minScore: 0 }); + expect(results).toHaveLength(1); + expect(results[0]?.path).toContain("memory/readme-dupes/z-strong/README.md"); + expect(results[0]?.score).toBe(1); + }); + + it("keeps boosted score ordering for non-exact FTS-only body matches", async () => { + forceNoProvider = true; + const cfg = createCfg({ + provider: "none", + minScore: 0, + hybrid: { enabled: true }, + }); + const result = await getMemorySearchManager({ cfg, agentId: "main" }); + const manager = requireManager(result); + managersForCleanup.add(manager); + resetManagerForTest(manager); + if (!manager.status().fts?.available) { + return; + } + + await fs.writeFile( + path.join(memoryDir, "project-memory-notes.md"), + "Project memory notes covering workspace context and retrieval behavior.", + ); + await fs.writeFile(path.join(memoryDir, "notes.md"), "Project memory context."); + await manager.sync({ reason: "test" }); + + const results = await manager.search("project memory context", { + maxResults: 1, + minScore: 0, + }); + expect(results).toHaveLength(1); + expect(results[0]?.path).toContain("memory/project-memory-notes.md"); + expect(results[0]?.score).toBeLessThanOrEqual(1); + }); + + it("keeps an exact dated path ahead in FTS-only mode", async () => { + forceNoProvider = true; + const cfg = createCfg({ + provider: "none", + minScore: 0.35, + }); + const result = await getMemorySearchManager({ cfg, agentId: "main" }); + const manager = requireManager(result); + managersForCleanup.add(manager); + resetManagerForTest(manager); + if (!manager.status().fts?.available) { + return; + } + + await fs.writeFile(path.join(memoryDir, "2020-01-01.md"), "Unrelated exact-path body."); + await fs.writeFile(path.join(memoryDir, "body-match.md"), "2020 01 01 2020 01 01 2020 01 01"); + await manager.sync({ reason: "test" }); + + const results = await manager.search("2020-01-01", { maxResults: 1 }); + expect(results).toHaveLength(1); + expect(results[0]?.path).toContain("memory/2020-01-01.md"); + expect(results[0]?.score).toBe(1); + }); + + it("prefers exact session transcript hits in FTS-only mode", async () => { + try { + const manager = await getFtsSessionManager({ + stateDirName: ".state-session-ranking", + }); + if (!manager) { + return; + } + + const memoryPath = path.join(workspaceDir, "MEMORY.md"); + await fs.writeFile(memoryPath, "Project Nebula stale codename: ORBIT-9.\n", "utf8"); + const staleAt = new Date("2020-01-01T00:00:00.000Z"); + await fs.utimes(memoryPath, staleAt, staleAt); + + const now = Date.parse("2026-04-07T15:25:04.113Z"); + await seedMemoryIndexSessionTranscript({ + sessionId: "session-ranking", + messages: [ + { + role: "user", + timestamp: new Date(now - 30_000).toISOString(), + content: "What is the current Project Nebula codename?", + }, + { + role: "assistant", + timestamp: new Date(now).toISOString(), + content: "The current Project Nebula codename is ORBIT-10.", + }, + ], + }); + + await manager.sync({ reason: "test", force: true }); + const results = await manager.search("current Project Nebula codename ORBIT-10", { + minScore: 0, + maxResults: 3, + }); + + expect(results[0]?.source).toBe("sessions"); + expect(results[0]?.snippet).toContain("ORBIT-10"); + expect(results[0]?.provenance).toMatchObject({ + originClass: "untrusted", + sessionKind: "interactive", + }); + } finally { + restoreMemoryIndexStateDir(); + } + }); +}); diff --git a/extensions/memory-core/src/memory/manager-keyword-retrieval.ts b/extensions/memory-core/src/memory/manager-keyword-retrieval.ts new file mode 100644 index 000000000000..a1ac9f752526 --- /dev/null +++ b/extensions/memory-core/src/memory/manager-keyword-retrieval.ts @@ -0,0 +1,405 @@ +// Memory Core plugin module owns keyword retrieval and ranking. +import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; +import { createSubsystemLogger } from "openclaw/plugin-sdk/memory-core-host-engine-foundation"; +import { extractKeywords } from "openclaw/plugin-sdk/memory-core-host-engine-sessions"; +import { + readCuratedProjectMemoryCandidates, + readCuratedMemoryTriggerCandidates, + readMemoryRecallMetadata, + MEMORY_INDEX_FTS_TABLE, + MEMORY_INDEX_PATHS_FTS_TABLE, + type MemorySearchResult, + type MemorySource, +} from "openclaw/plugin-sdk/memory-core-host-engine-storage"; +import { bm25RankToScore, buildFtsQuery, scoreExactPathTieForTemporalDecay } from "./hybrid.js"; +import { applyImportanceMultiplier } from "./importance.js"; +import { MemoryProviderLifecycle } from "./manager-provider-lifecycle.js"; +import { + resolveExactPathSpecificity, + searchKeyword, + searchPathKeyword, + type ExactPathSpecificity, +} from "./manager-search.js"; +import { applyProjectRanking } from "./project-ranking.js"; +import { applyTemporalDecayToHybridResults } from "./temporal-decay.js"; + +const SNIPPET_MAX_CHARS = 700; +const FTS_TABLE = MEMORY_INDEX_FTS_TABLE; +const PATH_FTS_TABLE = MEMORY_INDEX_PATHS_FTS_TABLE; +const KEYWORD_FALLBACK_SEARCH_TERM_LIMIT = 6; +const EXACT_PATH_CANDIDATE_LIMIT = 200; +const log = createSubsystemLogger("memory"); + +export type KeywordSearchHit = MemorySearchResult & { + id: string; + textScore: number; + pathScore: number; + exactPathSpecificity: ExactPathSpecificity; +}; + +function compareKeywordSearchHits( + a: KeywordSearchHit, + b: KeywordSearchHit, + preferExactBody = true, +): number { + const specificityDelta = b.exactPathSpecificity - a.exactPathSpecificity; + if (specificityDelta !== 0) { + return specificityDelta; + } + if (preferExactBody && a.exactPathSpecificity > 0) { + const bodyPresenceDelta = Number(b.textScore > 0) - Number(a.textScore > 0); + if (bodyPresenceDelta !== 0) { + return bodyPresenceDelta; + } + } + // Score carries body relevance plus any configured decay. Exact tiers ignore + // path BM25 because specificity already owns path precedence. + const relevanceDelta = b.score - a.score; + if (relevanceDelta !== 0) { + return relevanceDelta; + } + const textDelta = b.textScore - a.textScore; + if (textDelta !== 0) { + return textDelta; + } + if (a.exactPathSpecificity === 0) { + const pathDelta = b.pathScore - a.pathScore; + if (pathDelta !== 0) { + return pathDelta; + } + } + return a.path.localeCompare(b.path) || a.startLine - b.startLine || a.id.localeCompare(b.id); +} + +export abstract class MemoryKeywordRetrieval extends MemoryProviderLifecycle { + private selectScoredResults( + results: T[], + maxResults: number, + minScore: number, + relaxedMinScore = minScore, + ): T[] { + const strict = results.filter((entry) => entry.score >= minScore); + if (strict.length > 0) { + return strict.slice(0, maxResults); + } + return results.filter((entry) => entry.score >= relaxedMinScore).slice(0, maxResults); + } + + async listTriggerCandidates(opts?: { + limit?: number; + activeProjectKeys?: string[]; + }): Promise { + const limit = Math.max(1, Math.min(512, Math.floor(opts?.limit ?? 512))); + return this.toCuratedMemorySearchResults( + readCuratedMemoryTriggerCandidates(this.db, limit, opts?.activeProjectKeys), + ); + } + + async listCuratedProjectCandidates(opts: { + activeProjectKeys: string[]; + limit?: number; + }): Promise { + const limit = Math.max(1, Math.min(512, Math.floor(opts.limit ?? 48))); + return this.toCuratedMemorySearchResults( + readCuratedProjectMemoryCandidates(this.db, limit, opts.activeProjectKeys), + ); + } + + private toCuratedMemorySearchResults( + rows: ReturnType, + ): MemorySearchResult[] { + return rows.map((row) => { + const result: MemorySearchResult = { + path: row.path, + startLine: row.start_line, + endLine: row.end_line, + score: 0, + snippet: row.text, + source: "memory", + }; + if (typeof row.importance === "number") { + result.importance = row.importance; + } + if (typeof row.triggers === "string" && row.triggers.trim()) { + result.triggers = row.triggers.trim(); + } + if (typeof row.project_key === "string" && row.project_key.trim()) { + result.projectKey = row.project_key.trim(); + } + return result; + }); + } + + private rankKeywordOnlyResults( + results: KeywordSearchHit[], + preferExactBody = true, + ): KeywordSearchHit[] { + return results + .toSorted((left, right) => compareKeywordSearchHits(left, right, preferExactBody)) + .map((entry) => + entry.exactPathSpecificity > 0 ? Object.assign(entry, { score: 1 }) : entry, + ); + } + + protected async finalizeKeywordOnlyResults(params: { + results: KeywordSearchHit[]; + temporalDecay?: { enabled: boolean; halfLifeDays: number }; + maxResults: number; + minScore: number; + activeProjectKeys?: readonly string[]; + }): Promise { + const appliesTemporalDecay = params.temporalDecay?.enabled === true; + const decayInputs = appliesTemporalDecay + ? params.results.map((entry) => { + if (entry.exactPathSpecificity === 0) { + return entry; + } + const contentScore = entry.textScore > 0 ? entry.score : 0; + return { ...entry, score: scoreExactPathTieForTemporalDecay(contentScore) }; + }) + : params.results; + const decayed = await applyTemporalDecayToHybridResults({ + results: decayInputs, + temporalDecay: params.temporalDecay, + workspaceDir: this.workspaceDir, + }); + const ranked = applyProjectRanking( + this.rankKeywordOnlyResults(applyImportanceMultiplier(decayed), !appliesTemporalDecay), + params.activeProjectKeys, + ); + return this.toMemorySearchResults( + this.selectScoredResults(ranked, params.maxResults, params.minScore, 0), + ); + } + + protected attachRecallMetadata(results: T[]): T[] { + if (results.length === 0) { + return results; + } + const metadataById = readMemoryRecallMetadata( + this.db, + results.map((entry) => entry.id), + ); + return results.map((entry) => { + const row = metadataById.get(entry.id); + return { + ...entry, + ...(typeof row?.importance === "number" ? { importance: row.importance } : {}), + ...(typeof row?.triggers === "string" && row.triggers.trim() + ? { triggers: row.triggers.trim() } + : {}), + ...(typeof row?.project_key === "string" && row.project_key.trim() + ? { projectKey: row.project_key.trim() } + : {}), + }; + }); + } + + private async searchKeyword( + query: string, + limit: number, + options?: { + boostFallbackRanking?: boolean; + exactPathQuery?: string; + rankingQuery?: string; + }, + sourceFilterList?: MemorySource[], + ): Promise { + if (!this.fts.enabled || !this.fts.available) { + return []; + } + const bodySearch = searchKeyword({ + db: this.db, + ftsTable: FTS_TABLE, + query, + ftsTokenizer: this.settings.store.fts.tokenizer, + limit, + snippetMaxChars: SNIPPET_MAX_CHARS, + sourceFilter: this.buildSourceFilter(undefined, sourceFilterList), + buildFtsQuery, + bm25RankToScore, + boostFallbackRanking: options?.boostFallbackRanking, + rankingQuery: options?.rankingQuery, + }).catch((err: unknown) => { + log.warn(`memory search: body keyword query failed: ${formatErrorMessage(err)}`); + return []; + }); + const exactPathQuery = options?.exactPathQuery ?? query; + const pathSearch = searchPathKeyword({ + db: this.db, + pathFtsTable: PATH_FTS_TABLE, + query, + exactPathQuery, + exactPathLimit: EXACT_PATH_CANDIDATE_LIMIT, + ftsTokenizer: this.settings.store.fts.tokenizer, + limit, + snippetMaxChars: SNIPPET_MAX_CHARS, + sourceFilter: this.buildSourceFilter(PATH_FTS_TABLE, sourceFilterList), + buildFtsQuery, + bm25RankToScore, + }).catch((err: unknown) => { + log.warn(`memory search: path keyword query failed: ${formatErrorMessage(err)}`); + return []; + }); + const [bodyResults, pathResults] = await Promise.all([bodySearch, pathSearch]); + const merged = this.mergeKeywordSearchHits( + [ + bodyResults.map((entry) => + Object.assign(entry, { + exactPathSpecificity: resolveExactPathSpecificity(exactPathQuery, entry.path), + pathScore: 0, + }), + ), + pathResults, + ], + exactPathQuery, + ); + return this.attachRecallMetadata(this.limitKeywordSearchHits(merged, limit)); + } + + protected async searchKeywordWithFallback( + query: string, + limit: number, + options: { boostFallbackRanking?: boolean } | undefined, + sourceFilterList: MemorySource[], + ): Promise { + const fullQueryResults = await this.searchKeyword( + query, + limit, + options, + sourceFilterList, + ).catch(() => []); + const nonExactResults = fullQueryResults.filter((result) => result.exactPathSpecificity === 0); + if (nonExactResults.length >= limit) { + return fullQueryResults; + } + + // Supplement thin candidate pools for conversational queries, but cap the + // extra FTS probes so long prompts cannot fan out into unbounded sqlite work. + const fallbackTerms = this.resolveKeywordFallbackTerms(query); + if (fallbackTerms.length === 0) { + return fullQueryResults; + } + const strictFtsQuery = buildFtsQuery(query)?.toLowerCase(); + const keywordFtsQuery = buildFtsQuery(fallbackTerms.join(" "))?.toLowerCase(); + if (fullQueryResults.length > 0 && strictFtsQuery === keywordFtsQuery) { + // Expansion did not normalize this already-matching keyword query; OR + // probes can only weaken its strict relevance before importance ranking. + return fullQueryResults; + } + + const resultSets = await Promise.all( + fallbackTerms.map((term) => + this.searchKeyword( + term, + limit, + { ...options, exactPathQuery: query, rankingQuery: query }, + sourceFilterList, + ).catch(() => []), + ), + ); + return this.limitKeywordSearchHits( + this.mergeKeywordSearchHits([fullQueryResults, ...resultSets], query), + limit, + ); + } + + private resolveKeywordFallbackTerms(query: string): string[] { + const normalizedQuery = query.trim().toLowerCase(); + const keywords = extractKeywords(query, { + ftsTokenizer: this.settings.store.fts.tokenizer, + }).filter((term) => term !== normalizedQuery); + return keywords.slice(0, KEYWORD_FALLBACK_SEARCH_TERM_LIMIT); + } + + private mergeKeywordSearchHits( + resultSets: KeywordSearchHit[][], + exactPathQuery?: string, + ): KeywordSearchHit[] { + const seenIds = new Map(); + for (const results of resultSets) { + for (const result of results) { + const existing = seenIds.get(result.id); + if (!existing) { + seenIds.set(result.id, result); + continue; + } + const existingHasBody = existing.textScore > 0; + const resultHasBody = result.textScore > 0; + const existingBodyScore = existingHasBody ? existing.score : 0; + const resultBodyScore = resultHasBody ? result.score : 0; + existing.textScore = Math.max(existing.textScore, result.textScore); + existing.pathScore = Math.max(existing.pathScore, result.pathScore); + existing.exactPathSpecificity = Math.max( + existing.exactPathSpecificity, + result.exactPathSpecificity, + ) as ExactPathSpecificity; + const bodyScore = Math.max(existingBodyScore, resultBodyScore); + existing.score = bodyScore > 0 ? bodyScore : existing.pathScore; + // Path hits project the first chunk; keep a real body-match snippet + // authoritative when both retrieval surfaces find the same document. + if ( + (resultHasBody && !existingHasBody) || + (resultHasBody === existingHasBody && result.snippet.length > existing.snippet.length) + ) { + existing.snippet = result.snippet; + } + } + } + const merged = [...seenIds.values()]; + if (exactPathQuery !== undefined) { + // Fallback terms broaden lexical recall, but only the original user query + // can claim exact path, basename, or stem precedence. + for (const result of merged) { + result.exactPathSpecificity = resolveExactPathSpecificity(exactPathQuery, result.path); + } + } + for (const result of merged) { + if (result.textScore === 0) { + // A uniform exact-only baseline lets temporal decay order otherwise + // equivalent filename hits without reusing incomparable path BM25. + result.score = result.exactPathSpecificity > 0 ? 1 : result.pathScore; + } + } + return merged.toSorted(compareKeywordSearchHits); + } + + private limitKeywordSearchHits( + results: KeywordSearchHit[], + nonExactLimit: number, + ): KeywordSearchHit[] { + const ranked = results.toSorted(compareKeywordSearchHits); + const exactBody = ranked + .filter((entry) => entry.exactPathSpecificity > 0 && entry.textScore > 0) + .slice(0, nonExactLimit); + const exactPathOnly = ranked.filter( + (entry) => entry.exactPathSpecificity > 0 && entry.textScore === 0, + ); + const boundedExact = exactBody.concat(exactPathOnly).toSorted(compareKeywordSearchHits); + const selectedPathKeys = new Set(); + for (const entry of boundedExact) { + selectedPathKeys.add(`${entry.source}:${entry.path}`); + if (selectedPathKeys.size === EXACT_PATH_CANDIDATE_LIMIT) { + break; + } + } + const exact = boundedExact.filter((entry) => + selectedPathKeys.has(`${entry.source}:${entry.path}`), + ); + const nonExact = ranked + .filter((entry) => entry.exactPathSpecificity === 0) + .slice(0, nonExactLimit); + return exact.concat(nonExact); + } + + protected toMemorySearchResults(results: KeywordSearchHit[]): MemorySearchResult[] { + return results.map( + ({ + id: _id, + pathScore: _pathScore, + exactPathSpecificity: _exactPathSpecificity, + ...result + }) => result, + ); + } +} diff --git a/extensions/memory-core/src/memory/manager-provider-lifecycle-fallback.test.ts b/extensions/memory-core/src/memory/manager-provider-lifecycle-fallback.test.ts new file mode 100644 index 000000000000..e1200ac406f8 --- /dev/null +++ b/extensions/memory-core/src/memory/manager-provider-lifecycle-fallback.test.ts @@ -0,0 +1,775 @@ +// Memory Core tests cover manager provider lifecycle fallback behavior. +import { mkdirSync, rmSync } from "node:fs"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { clearMemoryEmbeddingProviders as clearRegistry } from "openclaw/plugin-sdk/memory-core-host-engine-embeddings"; +import { + closeOpenClawAgentDatabasesForTest, + closeOpenClawStateDatabaseForTest, +} from "openclaw/plugin-sdk/sqlite-runtime-testing"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { + configureMemoryCoreDreamingStateForTests, + resetMemoryCoreDreamingStateForTests, +} from "../test-helpers.js"; +import "./test-runtime-mocks.js"; +import { closeAllMemorySearchManagers, getMemorySearchManager } from "./index.js"; +import type { MemoryIndexManager } from "./manager.js"; +import { isolateMemoryManagerTestConfig } from "./test-config-helpers.js"; + +// This suite performs real sqlite/media indexing and can exceed the global +// timeout when it shares a packed CI extension shard. +vi.setConfig({ testTimeout: 240_000 }); + +afterAll(() => { + vi.resetConfig(); +}); + +let embedBatchCalls = 0; +let embeddedBatchTexts: string[] = []; +let embedBatchInputCalls = 0; +let providerRuntimeBatchCalls: string[][] = []; +let providerRuntimeBatchGate: Promise | null = null; +let providerRuntimeBatchErrors: unknown[] = []; +let providerRuntimeBatchFailuresRemaining = 0; +let providerRuntimeActiveBatchCalls = 0; +let providerRuntimeMaxActiveBatchCalls = 0; +let providerCloseCalls = 0; +let providerCloseFailuresRemaining = 0; +let providerCloseFailure: unknown = new Error("provider close failed"); +let providerCreationFailure: string | null = null; +let providerNullResult: string | null = null; +let providerCloseGate: Promise | null = null; +let providerInitGate: Promise | null = null; +let providerCalls: Array<{ provider?: string; model?: string; outputDimensionality?: number }> = []; +let forceNoProvider = false; + +const originalMemoryIndexStateDir = process.env.OPENCLAW_STATE_DIR; + +const identityAliasFixture = vi.hoisted(() => ({ + provider: "identity-alias-test", + canonicalModel: "hf:fixture/default-model.gguf", + cacheModel: "/fixture/cache/default-model.gguf", +})); + +function createLocalWorkerExitError(): Error { + return Object.assign(new Error("Local embedding worker exited unexpectedly (exit code 134)"), { + code: "LOCAL_EMBEDDING_WORKER_EXITED", + reason: "exit", + exitCode: 134, + }); +} + +function setMemoryIndexStateDir(stateDir: string): void { + Reflect.set(process.env, "OPENCLAW_STATE_DIR", stateDir); +} + +function restoreMemoryIndexStateDir(): void { + if (originalMemoryIndexStateDir === undefined) { + Reflect.deleteProperty(process.env, "OPENCLAW_STATE_DIR"); + } else { + Reflect.set(process.env, "OPENCLAW_STATE_DIR", originalMemoryIndexStateDir); + } +} + +vi.mock("./embeddings.js", async (importOriginal) => { + const actual = await importOriginal(); + const embedText = (text: string) => { + const lower = text.toLowerCase(); + const alpha = lower.split("alpha").length - 1; + const beta = lower.split("beta").length - 1; + const image = lower.split("image").length - 1; + const audio = lower.split("audio").length - 1; + return [alpha, beta, image, audio]; + }; + return { + ...actual, + resolveEmbeddingProviderFallbackModel: (providerId: string, fallbackSourceModel: string) => + providerId === "gemini" || providerId === "fallback-provider" + ? `${providerId}-embed` + : fallbackSourceModel, + resolveEmbeddingProviderAdapterId: ( + providerId: string, + config?: { + models?: { + providers?: Record; + }; + }, + ) => config?.models?.providers?.[providerId]?.api ?? providerId, + resolveEmbeddingProviderAdapterTransport: (providerId: string) => + providerId === "local" ? "local" : "remote", + resolveEmbeddingProviderIndexIdentity: (options: { provider?: string; model?: string }) => + options.provider === identityAliasFixture.provider + ? { + provider: { + id: identityAliasFixture.provider, + model: identityAliasFixture.canonicalModel, + }, + cacheKeyData: { + provider: identityAliasFixture.provider, + model: identityAliasFixture.canonicalModel, + }, + aliases: [ + { + model: identityAliasFixture.cacheModel, + cacheKeyData: { + provider: identityAliasFixture.provider, + model: identityAliasFixture.cacheModel, + }, + }, + ], + } + : undefined, + createEmbeddingProvider: async (options: { + provider?: string; + model?: string; + outputDimensionality?: number; + }) => { + providerCalls.push({ + provider: options.provider, + model: options.model, + outputDimensionality: options.outputDimensionality, + }); + await providerInitGate; + if (options.provider === providerCreationFailure) { + throw new Error(`provider creation failed: ${options.provider}`); + } + if (options.provider === providerNullResult) { + return { + provider: null, + requestedProvider: options.provider, + providerUnavailableReason: `provider unavailable: ${options.provider}`, + }; + } + if (forceNoProvider) { + return { + provider: null, + requestedProvider: options.provider ?? "auto", + providerUnavailableReason: "No API key found for provider", + }; + } + const providerId = + options.provider === "gemini" || + options.provider === "fallback-provider" || + options.provider === "batch-test" || + options.provider === "batch-wide-test" || + options.provider === identityAliasFixture.provider || + options.provider === "ollama" + ? options.provider + : "mock"; + const requestedModel = options.model ?? "mock-embed"; + const model = + providerId === identityAliasFixture.provider && + (requestedModel === identityAliasFixture.canonicalModel || + requestedModel === identityAliasFixture.cacheModel) + ? identityAliasFixture.canonicalModel + : requestedModel; + return { + requestedProvider: options.provider ?? "openai", + provider: { + id: providerId, + model, + close: async () => { + providerCloseCalls += 1; + await providerCloseGate; + if (providerCloseFailuresRemaining > 0) { + providerCloseFailuresRemaining -= 1; + throw providerCloseFailure; + } + }, + embedQuery: async (text: string) => embedText(text), + embedBatch: async (texts: string[]) => { + embedBatchCalls += 1; + embeddedBatchTexts.push(...texts); + return texts.map(embedText); + }, + ...(providerId === "gemini" || providerId === "fallback-provider" + ? { + embedBatchInputs: async ( + inputs: Array<{ + text: string; + parts?: Array< + | { type: "text"; text: string } + | { type: "inline-data"; mimeType: string; data: string } + >; + }>, + ) => { + embedBatchInputCalls += 1; + return inputs.map((input) => { + const inlineData = input.parts?.find((part) => part.type === "inline-data"); + if (inlineData?.type === "inline-data" && inlineData.data.length > 9000) { + throw new Error("payload too large"); + } + const mimeType = + inlineData?.type === "inline-data" ? inlineData.mimeType : undefined; + if (mimeType?.startsWith("image/")) { + return [0, 0, 1, 0]; + } + if (mimeType?.startsWith("audio/")) { + return [0, 0, 0, 1]; + } + return embedText(input.text); + }); + }, + } + : {}), + }, + ...(providerId === identityAliasFixture.provider + ? { + runtime: { + id: providerId, + cacheKeyData: { + provider: providerId, + model: identityAliasFixture.canonicalModel, + }, + indexIdentityAliases: [ + { + model: identityAliasFixture.cacheModel, + cacheKeyData: { + provider: providerId, + model: identityAliasFixture.cacheModel, + }, + }, + ], + }, + } + : providerId === "batch-test" || providerId === "batch-wide-test" + ? { + runtime: { + id: providerId, + ...(providerId === "batch-wide-test" ? { sourceWideBatchEmbed: true } : {}), + batchEmbed: async (batch: { chunks: Array<{ text: string }> }) => { + providerRuntimeActiveBatchCalls += 1; + providerRuntimeMaxActiveBatchCalls = Math.max( + providerRuntimeMaxActiveBatchCalls, + providerRuntimeActiveBatchCalls, + ); + try { + await providerRuntimeBatchGate; + providerRuntimeBatchCalls.push(batch.chunks.map((chunk) => chunk.text)); + if (providerRuntimeBatchErrors.length > 0) { + throw providerRuntimeBatchErrors.shift(); + } + if (providerRuntimeBatchFailuresRemaining > 0) { + providerRuntimeBatchFailuresRemaining -= 1; + throw new Error("provider runtime batch failed"); + } + return batch.chunks.map((chunk) => embedText(chunk.text)); + } finally { + providerRuntimeActiveBatchCalls -= 1; + } + }, + }, + } + : providerId === "gemini" || providerId === "fallback-provider" + ? { + runtime: { + id: providerId, + cacheKeyData: { + provider: providerId, + baseUrl: "https://generativelanguage.googleapis.com/v1beta", + model, + outputDimensionality: options.outputDimensionality, + headers: [], + }, + }, + } + : {}), + }; + }, + }; +}); + +describe("memory index", () => { + let fixtureRoot = ""; + let workspaceDir = ""; + let memoryDir = ""; + + const managersForCleanup = new Set(); + + beforeAll(async () => { + fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-mem-fixtures-")); + workspaceDir = path.join(fixtureRoot, "workspace"); + memoryDir = path.join(workspaceDir, "memory"); + }); + + afterAll(async () => { + await Promise.all(Array.from(managersForCleanup).map((manager) => manager.close())); + await fs.rm(fixtureRoot, { recursive: true, force: true }); + }); + + afterEach(async () => { + vi.useRealTimers(); + await Promise.all(Array.from(managersForCleanup).map((manager) => manager.close())); + await closeAllMemorySearchManagers(); + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + resetMemoryCoreDreamingStateForTests(); + clearRegistry(); + managersForCleanup.clear(); + restoreMemoryIndexStateDir(); + }); + + beforeEach(async () => { + vi.useRealTimers(); + clearRegistry(); + embedBatchCalls = 0; + embeddedBatchTexts = []; + embedBatchInputCalls = 0; + providerRuntimeBatchCalls = []; + providerRuntimeBatchGate = null; + providerRuntimeBatchErrors = []; + providerRuntimeBatchFailuresRemaining = 0; + providerRuntimeActiveBatchCalls = 0; + providerRuntimeMaxActiveBatchCalls = 0; + providerCloseCalls = 0; + providerCloseFailuresRemaining = 0; + providerCloseFailure = new Error("provider close failed"); + providerCreationFailure = null; + providerNullResult = null; + providerCloseGate = null; + providerInitGate = null; + providerCalls = []; + forceNoProvider = false; + + rmSync(workspaceDir, { recursive: true, force: true }); + mkdirSync(memoryDir, { recursive: true }); + setMemoryIndexStateDir(path.join(workspaceDir, ".state-memory-index")); + await configureMemoryCoreDreamingStateForTests(); + await fs.writeFile( + path.join(memoryDir, "2026-01-12.md"), + "# Log\nAlpha memory line.\nZebra memory line.", + ); + }); + + function resetManagerForTest(manager: MemoryIndexManager) { + // These tests reuse managers for performance. Clear the index + embedding + // cache to keep each test fully isolated. + const db = ( + manager as unknown as { + db: { + exec: (sql: string) => void; + prepare: (sql: string) => { get: (name: string) => { name?: string } | undefined }; + }; + } + ).db; + for (const table of [ + "memory_index_sources", + "memory_index_chunks", + "memory_embedding_cache", + "memory_index_chunks_fts", + "memory_index_chunks_vec", + ]) { + const existingTable = db + .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?") + .get(table); + if (existingTable?.name === table) { + db.exec(`DELETE FROM ${table}`); + } + } + (manager as unknown as { dirty: boolean }).dirty = true; + (manager as unknown as { sessionsDirty: boolean }).sessionsDirty = false; + (manager as unknown as { sessionsDirtyFiles: Set }).sessionsDirtyFiles.clear(); + } + + type TestCfg = Parameters[0]["cfg"]; + + function createCfg(params: { + extraPaths?: string[]; + sources?: Array<"memory" | "sessions">; + sessionMemory?: boolean; + rememberAcrossConversations?: boolean; + provider?: string; + fallback?: "none" | "gemini" | "fallback-provider"; + providerAliases?: NonNullable["providers"]>; + batchEnabled?: boolean; + model?: string; + outputDimensionality?: number; + multimodal?: { + enabled?: boolean; + modalities?: Array<"image" | "audio" | "all">; + maxFileBytes?: number; + }; + vectorEnabled?: boolean; + cacheEnabled?: boolean; + minScore?: number; + onSearch?: boolean; + hybrid?: { + enabled: boolean; + vectorWeight?: number; + textWeight?: number; + temporalDecay?: { enabled: boolean }; + }; + }): TestCfg { + return isolateMemoryManagerTestConfig({ + memory: { + search: { + ...(params.provider !== undefined ? { provider: params.provider } : {}), + model: params.model ?? "mock-embed", + fallback: params.fallback, + outputDimensionality: params.outputDimensionality, + store: { + vector: params.vectorEnabled !== undefined ? { enabled: params.vectorEnabled } : {}, + }, + remote: params.batchEnabled + ? { + batch: { enabled: true }, + } + : undefined, + query: { minScore: params.minScore ?? 0 }, + cache: params.cacheEnabled ? { enabled: true } : undefined, + extraPaths: params.extraPaths, + multimodal: params.multimodal, + sources: params.sources, + rememberAcrossConversations: + params.rememberAcrossConversations ?? params.sessionMemory ?? false, + }, + }, + + agents: { + defaults: { + workspace: workspaceDir, + }, + list: [{ id: "main", default: true }], + }, + models: params.providerAliases ? { providers: params.providerAliases } : undefined, + }); + } + + function requireManager( + result: Awaited>, + missingMessage = "manager missing", + ): MemoryIndexManager { + if (!result.manager) { + throw new Error(missingMessage); + } + return result.manager as unknown as MemoryIndexManager; + } + + async function getPersistentManager(cfg: TestCfg): Promise { + const result = await getMemorySearchManager({ cfg, agentId: "main" }); + const manager = requireManager(result); + managersForCleanup.add(manager); + resetManagerForTest(manager); + return manager; + } + + async function getFreshManager( + cfg: TestCfg, + purpose?: "default" | "status" | "cli", + ): Promise { + const manager = requireManager(await getMemorySearchManager({ cfg, agentId: "main", purpose })); + managersForCleanup.add(manager); + return manager; + } + + it("does not activate fallback during search when index identity is already mismatched", async () => { + const cfg = createCfg({ + fallback: "fallback-provider", + hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, + }); + const manager = await getPersistentManager(cfg); + + await manager.sync({ reason: "test" }); + const callsBeforeSearch = providerCalls.length; + ( + manager as unknown as { + provider: { + id: string; + model: string; + embedQuery: () => Promise; + embedBatch: (texts: string[]) => Promise; + close: () => Promise; + }; + } + ).provider = { + id: "local", + model: "mock-embed", + embedQuery: async () => { + throw createLocalWorkerExitError(); + }, + embedBatch: async (texts: string[]) => texts.map(() => [1, 0, 0, 0]), + close: async () => {}, + }; + + const results = await manager.search("alpha"); + + expect(results).toStrictEqual([]); + expect(providerCalls.slice(callsBeforeSearch)).toStrictEqual([]); + expect( + ( + manager as unknown as { + provider: { id: string } | null; + } + ).provider?.id, + ).toBe("local"); + }); + + it("rebuilds with fallback provider during explicit identity repair", async () => { + const oldCfg = createCfg({ + model: "old-embed", + }); + const oldManager = await getFreshManager(oldCfg); + await oldManager.sync({ reason: "test", force: true }); + await oldManager.close?.(); + + const cfg = createCfg({ + model: "new-embed", + fallback: "fallback-provider", + }); + const manager = await getFreshManager(cfg); + try { + expect(manager.status().dirty).toBe(true); + const fields = manager as unknown as { + providerInitialized: boolean; + provider: { + id: string; + model: string; + embedQuery: (text: string) => Promise; + embedBatch: (texts: string[]) => Promise; + close: () => Promise; + }; + }; + fields.providerInitialized = true; + fields.provider = { + id: "mock", + model: "new-embed", + embedQuery: async () => { + throw createLocalWorkerExitError(); + }, + embedBatch: async () => { + throw createLocalWorkerExitError(); + }, + close: async () => {}, + }; + + await manager.sync({ reason: "cli" }); + + expect(manager.status().dirty).toBe(false); + expect(manager.status().provider).toBe("fallback-provider"); + expect(manager.status().model).toBe("fallback-provider-embed"); + expect(manager.status().custom?.indexIdentity).toEqual({ status: "valid" }); + await expect(manager.search("alpha")).resolves.not.toStrictEqual([]); + } finally { + await manager.close?.(); + } + }); + + it("reinitializes the configured provider after probe-time local degradation", async () => { + const cfg = createCfg({ + fallback: "fallback-provider", + hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, + }); + const manager = await getPersistentManager(cfg); + + await manager.sync({ reason: "test" }); + ( + manager as unknown as { + provider: { + id: string; + model: string; + embedQuery: () => Promise; + embedBatch: () => Promise; + close: () => Promise; + }; + } + ).provider = { + id: "local", + model: "mock-embed", + embedQuery: async () => { + throw createLocalWorkerExitError(); + }, + embedBatch: async () => { + throw createLocalWorkerExitError(); + }, + close: async () => {}, + }; + const callsBeforeSearch = providerCalls.length; + + await expect(manager.probeEmbeddingAvailability()).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining("Local embedding worker exited"), + }); + + const results = await manager.search("alpha"); + + expect(results.length).toBeGreaterThan(0); + expect(providerCalls.slice(callsBeforeSearch).map((call) => call.provider)).toContain("openai"); + expect( + ( + manager as unknown as { + provider: { id: string } | null; + } + ).provider?.id, + ).toBe("mock"); + }); + + it("clears identity dirty after status resolves the indexed fallback provider", async () => { + const indexedCfg = createCfg({ + provider: "fallback-provider", + model: "new-embed", + }); + const indexedManager = await getFreshManager(indexedCfg); + await indexedManager.sync({ reason: "test", force: true }); + await indexedManager.close?.(); + + const cfg = createCfg({ + fallback: "fallback-provider", + model: "new-embed", + }); + const { getRequiredMemoryIndexManager } = await import("./test-manager-helpers.js"); + const manager = await getRequiredMemoryIndexManager({ + cfg, + agentId: "main", + purpose: "status", + }); + try { + expect(manager.status().dirty).toBe(true); + + const fields = manager as unknown as { + provider: { + id: string; + model: string; + embedQuery: (text: string) => Promise; + embedBatch: (texts: string[]) => Promise; + close: () => Promise; + }; + providerInitialized: boolean; + providerRuntime: { + id: string; + cacheKeyData: Record; + }; + providerKey: string; + computeProviderKey: () => string; + }; + fields.provider = { + id: "fallback-provider", + model: "new-embed", + embedQuery: async () => [1, 0, 0, 0], + embedBatch: async (texts) => texts.map(() => [1, 0, 0, 0]), + close: async () => {}, + }; + fields.providerRuntime = { + id: "fallback-provider", + cacheKeyData: { + provider: "fallback-provider", + baseUrl: "https://generativelanguage.googleapis.com/v1beta", + model: "new-embed", + headers: [], + }, + }; + fields.providerInitialized = true; + fields.providerKey = fields.computeProviderKey(); + + expect(manager.status().dirty).toBe(false); + expect(manager.status().custom?.indexIdentity).toEqual({ status: "valid" }); + } finally { + await manager.close?.(); + } + }); + + it("exposes already-created local runtime facts without probing embeddings", async () => { + const cfg = createCfg({}); + const { getRequiredMemoryIndexManager } = await import("./test-manager-helpers.js"); + const manager = await getRequiredMemoryIndexManager({ + cfg, + agentId: "main", + purpose: "status", + }); + try { + const getRuntimeFacts = vi.fn(() => ({ + engine: "llama.cpp" as const, + state: "ready" as const, + backend: "cuda" as const, + buildType: "prebuilt" as const, + deviceNames: ["NVIDIA Test GPU"], + offload: { + supported: true, + offloadedLayers: 24, + totalLayers: 24, + }, + context: { + requestedSize: 4096, + }, + })); + const provider = { + id: "local", + model: "test-model.gguf", + embedQuery: vi.fn(async () => [1, 0, 0, 0]), + embedBatch: vi.fn(async (texts: string[]) => texts.map(() => [1, 0, 0, 0])), + }; + Object.defineProperty(provider, Symbol.for("openclaw.localEmbeddingRuntimeFacts"), { + value: getRuntimeFacts, + }); + const fields = manager as unknown as { + provider: typeof provider | null; + }; + fields.provider = provider; + + expect(manager.status().custom?.llamaCppRuntime).toMatchObject({ + state: "ready", + backend: "cuda", + deviceNames: ["NVIDIA Test GPU"], + offload: { + offloadedLayers: 24, + totalLayers: 24, + }, + context: { + requestedSize: 4096, + }, + }); + expect(getRuntimeFacts).toHaveBeenCalledTimes(1); + } finally { + await manager.close?.(); + } + }); + + it("fails fast instead of searching FTS when an explicit provider is unavailable", async () => { + forceNoProvider = true; + + const cfg = createCfg({ + provider: "openai", + minScore: 0.35, + hybrid: { enabled: true }, + }); + const manager = await getFreshManager(cfg); + try { + await expect(manager.search("Alpha")).rejects.toThrow( + /Memory search unavailable: embedding provider "openai" is configured but unavailable\.[\s\S]*agentId=main purpose=default[\s\S]*registeredMemoryEmbeddingProviders=none/, + ); + await expect(manager.sync({ reason: "test" })).rejects.toThrow( + /Memory sync unavailable: embedding provider "openai" is configured but unavailable\./, + ); + forceNoProvider = false; + await manager.sync({ reason: "test", force: true }); + const results = await manager.search("Alpha"); + expect(results.length).toBeGreaterThan(0); + } finally { + await manager.close?.(); + } + }); + + it("fails fast instead of returning FTS when an explicit provider is lost at runtime", async () => { + const cfg = createCfg({ + provider: "openai", + minScore: 0.35, + hybrid: { enabled: true }, + }); + const manager = await getFreshManager(cfg); + try { + await manager.sync({ reason: "test", force: true }); + ( + manager as unknown as { + provider: null; + } + ).provider = null; + + await expect(manager.search("Alpha")).rejects.toThrow( + /Memory search unavailable: embedding provider "openai" is configured but unavailable\./, + ); + } finally { + await manager.close?.(); + } + }); +}); diff --git a/extensions/memory-core/src/memory/manager-provider-lifecycle-leases.test.ts b/extensions/memory-core/src/memory/manager-provider-lifecycle-leases.test.ts new file mode 100644 index 000000000000..1391f617ae12 --- /dev/null +++ b/extensions/memory-core/src/memory/manager-provider-lifecycle-leases.test.ts @@ -0,0 +1,927 @@ +// Memory Core tests cover manager provider lifecycle lease behavior. +import { mkdirSync, rmSync } from "node:fs"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { clearMemoryEmbeddingProviders as clearRegistry } from "openclaw/plugin-sdk/memory-core-host-engine-embeddings"; +import { hashText } from "openclaw/plugin-sdk/memory-core-host-engine-storage"; +import { + closeOpenClawAgentDatabasesForTest, + closeOpenClawStateDatabaseForTest, +} from "openclaw/plugin-sdk/sqlite-runtime-testing"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { + configureMemoryCoreDreamingStateForTests, + resetMemoryCoreDreamingStateForTests, +} from "../test-helpers.js"; +import "./test-runtime-mocks.js"; +import { closeAllMemorySearchManagers, getMemorySearchManager } from "./index.js"; +import type { MemoryIndexManager } from "./manager.js"; +import { isolateMemoryManagerTestConfig } from "./test-config-helpers.js"; + +// This suite performs real sqlite/media indexing and can exceed the global +// timeout when it shares a packed CI extension shard. +vi.setConfig({ testTimeout: 240_000 }); + +afterAll(() => { + vi.resetConfig(); +}); + +let embedBatchCalls = 0; +let embeddedBatchTexts: string[] = []; +let embedBatchInputCalls = 0; +let providerRuntimeBatchCalls: string[][] = []; +let providerRuntimeBatchGate: Promise | null = null; +let providerRuntimeBatchErrors: unknown[] = []; +let providerRuntimeBatchFailuresRemaining = 0; +let providerRuntimeActiveBatchCalls = 0; +let providerRuntimeMaxActiveBatchCalls = 0; +let providerCloseCalls = 0; +let providerCloseFailuresRemaining = 0; +let providerCloseFailure: unknown = new Error("provider close failed"); +let providerCreationFailure: string | null = null; +let providerNullResult: string | null = null; +let providerCloseGate: Promise | null = null; +let providerInitGate: Promise | null = null; +let providerCalls: Array<{ provider?: string; model?: string; outputDimensionality?: number }> = []; +let forceNoProvider = false; + +const originalMemoryIndexStateDir = process.env.OPENCLAW_STATE_DIR; + +const identityAliasFixture = vi.hoisted(() => ({ + provider: "identity-alias-test", + canonicalModel: "hf:fixture/default-model.gguf", + cacheModel: "/fixture/cache/default-model.gguf", +})); + +function createLocalWorkerExitError(): Error { + return Object.assign(new Error("Local embedding worker exited unexpectedly (exit code 134)"), { + code: "LOCAL_EMBEDDING_WORKER_EXITED", + reason: "exit", + exitCode: 134, + }); +} + +function setMemoryIndexStateDir(stateDir: string): void { + Reflect.set(process.env, "OPENCLAW_STATE_DIR", stateDir); +} + +function restoreMemoryIndexStateDir(): void { + if (originalMemoryIndexStateDir === undefined) { + Reflect.deleteProperty(process.env, "OPENCLAW_STATE_DIR"); + } else { + Reflect.set(process.env, "OPENCLAW_STATE_DIR", originalMemoryIndexStateDir); + } +} + +vi.mock("./embeddings.js", async (importOriginal) => { + const actual = await importOriginal(); + const embedText = (text: string) => { + const lower = text.toLowerCase(); + const alpha = lower.split("alpha").length - 1; + const beta = lower.split("beta").length - 1; + const image = lower.split("image").length - 1; + const audio = lower.split("audio").length - 1; + return [alpha, beta, image, audio]; + }; + return { + ...actual, + resolveEmbeddingProviderFallbackModel: (providerId: string, fallbackSourceModel: string) => + providerId === "gemini" || providerId === "fallback-provider" + ? `${providerId}-embed` + : fallbackSourceModel, + resolveEmbeddingProviderAdapterId: ( + providerId: string, + config?: { + models?: { + providers?: Record; + }; + }, + ) => config?.models?.providers?.[providerId]?.api ?? providerId, + resolveEmbeddingProviderAdapterTransport: (providerId: string) => + providerId === "local" ? "local" : "remote", + resolveEmbeddingProviderIndexIdentity: (options: { provider?: string; model?: string }) => + options.provider === identityAliasFixture.provider + ? { + provider: { + id: identityAliasFixture.provider, + model: identityAliasFixture.canonicalModel, + }, + cacheKeyData: { + provider: identityAliasFixture.provider, + model: identityAliasFixture.canonicalModel, + }, + aliases: [ + { + model: identityAliasFixture.cacheModel, + cacheKeyData: { + provider: identityAliasFixture.provider, + model: identityAliasFixture.cacheModel, + }, + }, + ], + } + : undefined, + createEmbeddingProvider: async (options: { + provider?: string; + model?: string; + outputDimensionality?: number; + }) => { + providerCalls.push({ + provider: options.provider, + model: options.model, + outputDimensionality: options.outputDimensionality, + }); + await providerInitGate; + if (options.provider === providerCreationFailure) { + throw new Error(`provider creation failed: ${options.provider}`); + } + if (options.provider === providerNullResult) { + return { + provider: null, + requestedProvider: options.provider, + providerUnavailableReason: `provider unavailable: ${options.provider}`, + }; + } + if (forceNoProvider) { + return { + provider: null, + requestedProvider: options.provider ?? "auto", + providerUnavailableReason: "No API key found for provider", + }; + } + const providerId = + options.provider === "gemini" || + options.provider === "fallback-provider" || + options.provider === "batch-test" || + options.provider === "batch-wide-test" || + options.provider === identityAliasFixture.provider || + options.provider === "ollama" + ? options.provider + : "mock"; + const requestedModel = options.model ?? "mock-embed"; + const model = + providerId === identityAliasFixture.provider && + (requestedModel === identityAliasFixture.canonicalModel || + requestedModel === identityAliasFixture.cacheModel) + ? identityAliasFixture.canonicalModel + : requestedModel; + return { + requestedProvider: options.provider ?? "openai", + provider: { + id: providerId, + model, + close: async () => { + providerCloseCalls += 1; + await providerCloseGate; + if (providerCloseFailuresRemaining > 0) { + providerCloseFailuresRemaining -= 1; + throw providerCloseFailure; + } + }, + embedQuery: async (text: string) => embedText(text), + embedBatch: async (texts: string[]) => { + embedBatchCalls += 1; + embeddedBatchTexts.push(...texts); + return texts.map(embedText); + }, + ...(providerId === "gemini" || providerId === "fallback-provider" + ? { + embedBatchInputs: async ( + inputs: Array<{ + text: string; + parts?: Array< + | { type: "text"; text: string } + | { type: "inline-data"; mimeType: string; data: string } + >; + }>, + ) => { + embedBatchInputCalls += 1; + return inputs.map((input) => { + const inlineData = input.parts?.find((part) => part.type === "inline-data"); + if (inlineData?.type === "inline-data" && inlineData.data.length > 9000) { + throw new Error("payload too large"); + } + const mimeType = + inlineData?.type === "inline-data" ? inlineData.mimeType : undefined; + if (mimeType?.startsWith("image/")) { + return [0, 0, 1, 0]; + } + if (mimeType?.startsWith("audio/")) { + return [0, 0, 0, 1]; + } + return embedText(input.text); + }); + }, + } + : {}), + }, + ...(providerId === identityAliasFixture.provider + ? { + runtime: { + id: providerId, + cacheKeyData: { + provider: providerId, + model: identityAliasFixture.canonicalModel, + }, + indexIdentityAliases: [ + { + model: identityAliasFixture.cacheModel, + cacheKeyData: { + provider: providerId, + model: identityAliasFixture.cacheModel, + }, + }, + ], + }, + } + : providerId === "batch-test" || providerId === "batch-wide-test" + ? { + runtime: { + id: providerId, + ...(providerId === "batch-wide-test" ? { sourceWideBatchEmbed: true } : {}), + batchEmbed: async (batch: { chunks: Array<{ text: string }> }) => { + providerRuntimeActiveBatchCalls += 1; + providerRuntimeMaxActiveBatchCalls = Math.max( + providerRuntimeMaxActiveBatchCalls, + providerRuntimeActiveBatchCalls, + ); + try { + await providerRuntimeBatchGate; + providerRuntimeBatchCalls.push(batch.chunks.map((chunk) => chunk.text)); + if (providerRuntimeBatchErrors.length > 0) { + throw providerRuntimeBatchErrors.shift(); + } + if (providerRuntimeBatchFailuresRemaining > 0) { + providerRuntimeBatchFailuresRemaining -= 1; + throw new Error("provider runtime batch failed"); + } + return batch.chunks.map((chunk) => embedText(chunk.text)); + } finally { + providerRuntimeActiveBatchCalls -= 1; + } + }, + }, + } + : providerId === "gemini" || providerId === "fallback-provider" + ? { + runtime: { + id: providerId, + cacheKeyData: { + provider: providerId, + baseUrl: "https://generativelanguage.googleapis.com/v1beta", + model, + outputDimensionality: options.outputDimensionality, + headers: [], + }, + }, + } + : {}), + }; + }, + }; +}); + +describe("memory index", () => { + let fixtureRoot = ""; + let workspaceDir = ""; + let memoryDir = ""; + + const managersForCleanup = new Set(); + + beforeAll(async () => { + fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-mem-fixtures-")); + workspaceDir = path.join(fixtureRoot, "workspace"); + memoryDir = path.join(workspaceDir, "memory"); + }); + + afterAll(async () => { + await Promise.all(Array.from(managersForCleanup).map((manager) => manager.close())); + await fs.rm(fixtureRoot, { recursive: true, force: true }); + }); + + afterEach(async () => { + vi.useRealTimers(); + await Promise.all(Array.from(managersForCleanup).map((manager) => manager.close())); + await closeAllMemorySearchManagers(); + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + resetMemoryCoreDreamingStateForTests(); + clearRegistry(); + managersForCleanup.clear(); + restoreMemoryIndexStateDir(); + }); + + beforeEach(async () => { + vi.useRealTimers(); + clearRegistry(); + embedBatchCalls = 0; + embeddedBatchTexts = []; + embedBatchInputCalls = 0; + providerRuntimeBatchCalls = []; + providerRuntimeBatchGate = null; + providerRuntimeBatchErrors = []; + providerRuntimeBatchFailuresRemaining = 0; + providerRuntimeActiveBatchCalls = 0; + providerRuntimeMaxActiveBatchCalls = 0; + providerCloseCalls = 0; + providerCloseFailuresRemaining = 0; + providerCloseFailure = new Error("provider close failed"); + providerCreationFailure = null; + providerNullResult = null; + providerCloseGate = null; + providerInitGate = null; + providerCalls = []; + forceNoProvider = false; + + rmSync(workspaceDir, { recursive: true, force: true }); + mkdirSync(memoryDir, { recursive: true }); + setMemoryIndexStateDir(path.join(workspaceDir, ".state-memory-index")); + await configureMemoryCoreDreamingStateForTests(); + await fs.writeFile( + path.join(memoryDir, "2026-01-12.md"), + "# Log\nAlpha memory line.\nZebra memory line.", + ); + }); + + function resetManagerForTest(manager: MemoryIndexManager) { + // These tests reuse managers for performance. Clear the index + embedding + // cache to keep each test fully isolated. + const db = ( + manager as unknown as { + db: { + exec: (sql: string) => void; + prepare: (sql: string) => { get: (name: string) => { name?: string } | undefined }; + }; + } + ).db; + for (const table of [ + "memory_index_sources", + "memory_index_chunks", + "memory_embedding_cache", + "memory_index_chunks_fts", + "memory_index_chunks_vec", + ]) { + const existingTable = db + .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?") + .get(table); + if (existingTable?.name === table) { + db.exec(`DELETE FROM ${table}`); + } + } + (manager as unknown as { dirty: boolean }).dirty = true; + (manager as unknown as { sessionsDirty: boolean }).sessionsDirty = false; + (manager as unknown as { sessionsDirtyFiles: Set }).sessionsDirtyFiles.clear(); + } + + type TestCfg = Parameters[0]["cfg"]; + + function createCfg(params: { + extraPaths?: string[]; + sources?: Array<"memory" | "sessions">; + sessionMemory?: boolean; + rememberAcrossConversations?: boolean; + provider?: string; + fallback?: "none" | "gemini" | "fallback-provider"; + providerAliases?: NonNullable["providers"]>; + batchEnabled?: boolean; + model?: string; + outputDimensionality?: number; + multimodal?: { + enabled?: boolean; + modalities?: Array<"image" | "audio" | "all">; + maxFileBytes?: number; + }; + vectorEnabled?: boolean; + cacheEnabled?: boolean; + minScore?: number; + onSearch?: boolean; + hybrid?: { + enabled: boolean; + vectorWeight?: number; + textWeight?: number; + temporalDecay?: { enabled: boolean }; + }; + }): TestCfg { + return isolateMemoryManagerTestConfig({ + memory: { + search: { + ...(params.provider !== undefined ? { provider: params.provider } : {}), + model: params.model ?? "mock-embed", + fallback: params.fallback, + outputDimensionality: params.outputDimensionality, + store: { + vector: params.vectorEnabled !== undefined ? { enabled: params.vectorEnabled } : {}, + }, + remote: params.batchEnabled + ? { + batch: { enabled: true }, + } + : undefined, + query: { minScore: params.minScore ?? 0 }, + cache: params.cacheEnabled ? { enabled: true } : undefined, + extraPaths: params.extraPaths, + multimodal: params.multimodal, + sources: params.sources, + rememberAcrossConversations: + params.rememberAcrossConversations ?? params.sessionMemory ?? false, + }, + }, + + agents: { + defaults: { + workspace: workspaceDir, + }, + list: [{ id: "main", default: true }], + }, + models: params.providerAliases ? { providers: params.providerAliases } : undefined, + }); + } + + function requireManager( + result: Awaited>, + missingMessage = "manager missing", + ): MemoryIndexManager { + if (!result.manager) { + throw new Error(missingMessage); + } + return result.manager as unknown as MemoryIndexManager; + } + + async function getPersistentManager(cfg: TestCfg): Promise { + const result = await getMemorySearchManager({ cfg, agentId: "main" }); + const manager = requireManager(result); + managersForCleanup.add(manager); + resetManagerForTest(manager); + return manager; + } + + async function getFreshManager( + cfg: TestCfg, + purpose?: "default" | "status" | "cli", + ): Promise { + const manager = requireManager(await getMemorySearchManager({ cfg, agentId: "main", purpose })); + managersForCleanup.add(manager); + return manager; + } + + it("keeps an active FTS-only generation stable while fallback activates", async () => { + const manager = await getFreshManager( + createCfg({ provider: "openai", fallback: "fallback-provider" }), + "cli", + ); + managersForCleanup.add(manager); + type IndexEntry = { + path: string; + absPath: string; + mtimeMs: number; + size: number; + hash: string; + content: string; + }; + const fields = manager as unknown as { + provider: { id: string } | null; + providerKey: string; + computeProviderKey: () => string; + ensureProviderInitialized: () => Promise; + markLocalEmbeddingProviderDegraded: (err: unknown) => void; + activateFallbackProvider: (reason: string) => Promise; + beginSyncProviderGeneration: () => void; + endSyncProviderGeneration: () => void; + indexFile: ( + entry: IndexEntry, + options: { source: "memory"; content: string }, + ) => Promise; + db: { + prepare: (sql: string) => { + get: (...params: unknown[]) => { model?: string } | undefined; + }; + }; + }; + await fields.ensureProviderInitialized(); + if (!fields.provider) { + throw new Error("Expected a test embedding provider"); + } + fields.provider.id = "local"; + fields.providerKey = fields.computeProviderKey(); + fields.markLocalEmbeddingProviderDegraded(createLocalWorkerExitError()); + await vi.waitFor(() => { + expect(fields.provider).toBeNull(); + expect(providerCloseCalls).toBe(1); + }); + + const createEntry = (name: string): IndexEntry => { + const content = `# Log\n${name} FTS-only generation.`; + return { + path: `memory/${name}.md`, + absPath: path.join(memoryDir, `${name}.md`), + mtimeMs: Date.now(), + size: Buffer.byteLength(content), + hash: hashText(content), + content, + }; + }; + const first = createEntry("fts-first"); + const second = createEntry("fts-second"); + + fields.beginSyncProviderGeneration(); + try { + await fields.indexFile(first, { source: "memory", content: first.content }); + await expect(fields.activateFallbackProvider("local worker exited")).resolves.toBe(true); + await fields.indexFile(second, { source: "memory", content: second.content }); + } finally { + fields.endSyncProviderGeneration(); + } + + expect( + fields.db.prepare("SELECT model FROM memory_index_chunks WHERE path = ?").get(first.path) + ?.model, + ).toBe("fts-only"); + expect( + fields.db.prepare("SELECT model FROM memory_index_chunks WHERE path = ?").get(second.path) + ?.model, + ).toBe("fts-only"); + }); + + it("waits for admitted provider users before retirement", async () => { + const cfg = createCfg({ provider: "openai" }); + const manager = await getPersistentManager(cfg); + await manager.sync({ reason: "test" }); + const fields = manager as unknown as { + provider: { + embedQuery: (text: string) => Promise; + } | null; + embedQueryWithRetry: (text: string) => Promise; + retireCurrentProvider: () => Promise; + }; + if (!fields.provider) { + throw new Error("Expected a test embedding provider"); + } + let releaseFirstQuery: () => void = () => {}; + let markFirstQueryStarted: () => void = () => {}; + const firstQueryGate = new Promise((resolve) => { + releaseFirstQuery = resolve; + }); + const firstQueryStarted = new Promise((resolve) => { + markFirstQueryStarted = resolve; + }); + fields.provider.embedQuery = async () => { + markFirstQueryStarted(); + await firstQueryGate; + return [1, 0, 0, 0]; + }; + + const queryPromise = fields.embedQueryWithRetry("alpha"); + await firstQueryStarted; + const retirementPromise = fields.retireCurrentProvider(); + let retirementSettled = false; + void retirementPromise.then( + () => { + retirementSettled = true; + }, + () => { + retirementSettled = true; + }, + ); + try { + await Promise.resolve(); + expect(retirementSettled).toBe(false); + expect(providerCloseCalls).toBe(0); + } finally { + releaseFirstQuery(); + } + + await expect(queryPromise).resolves.toEqual([1, 0, 0, 0]); + await retirementPromise; + expect(providerCloseCalls).toBe(1); + }); + + it("uses the leased provider runtime after retirement starts", async () => { + const manager = await getPersistentManager(createCfg({ provider: "openai" })); + type QueryProvider = { + embedQuery: (text: string, options?: { signal?: AbortSignal }) => Promise; + }; + const fields = manager as unknown as { + provider: QueryProvider | null; + providerRuntime?: { inlineQueryTimeoutMs?: number }; + acquireProviderUse: (provider: QueryProvider) => () => void; + retireCurrentProvider: () => Promise; + embedQueryWithRetry: ( + text: string, + signal: AbortSignal | undefined, + provider: QueryProvider, + markDegraded: boolean, + providerRuntime: { inlineQueryTimeoutMs?: number }, + ) => Promise; + }; + await manager.probeEmbeddingAvailability(); + const provider = fields.provider; + if (!provider) { + throw new Error("Expected a test embedding provider"); + } + const providerRuntime = { inlineQueryTimeoutMs: 10 }; + fields.providerRuntime = providerRuntime; + provider.embedQuery = async (_text, options) => + await new Promise((resolve, reject) => { + const timer = setTimeout(() => resolve([1, 0, 0, 0]), 100); + options?.signal?.addEventListener( + "abort", + () => { + clearTimeout(timer); + const reason = options.signal?.reason; + reject(reason instanceof Error ? reason : new Error("embedding aborted")); + }, + { once: true }, + ); + }); + + const releaseProvider = fields.acquireProviderUse(provider); + const retirementPromise = fields.retireCurrentProvider(); + try { + await vi.waitFor(() => expect(fields.provider).toBeNull()); + await expect( + fields.embedQueryWithRetry("alpha", undefined, provider, false, providerRuntime), + ).rejects.toThrow("timed out"); + expect(providerCloseCalls).toBe(0); + } finally { + releaseProvider(); + } + + await retirementPromise; + expect(providerCloseCalls).toBe(1); + }); + + it("waits for an admitted search before manager teardown", async () => { + const manager = await getPersistentManager(createCfg({ provider: "openai" })); + await manager.sync({ reason: "test" }); + const fields = manager as unknown as { + searchVector: () => Promise; + closing: boolean; + closed: boolean; + }; + let releaseVectorSearch: () => void = () => {}; + let markVectorSearchStarted: () => void = () => {}; + const vectorSearchGate = new Promise((resolve) => { + releaseVectorSearch = resolve; + }); + const vectorSearchStarted = new Promise((resolve) => { + markVectorSearchStarted = resolve; + }); + fields.searchVector = async () => { + markVectorSearchStarted(); + await vectorSearchGate; + return []; + }; + + const searchPromise = manager.search("alpha"); + await vectorSearchStarted; + const closePromise = manager.close(); + let closeSettled = false; + void closePromise.then( + () => { + closeSettled = true; + }, + () => { + closeSettled = true; + }, + ); + try { + await Promise.resolve(); + expect(closeSettled).toBe(false); + expect(fields.closing).toBe(true); + expect(fields.closed).toBe(false); + expect(providerCloseCalls).toBe(0); + } finally { + releaseVectorSearch(); + } + + await expect(searchPromise).resolves.toBeDefined(); + await closePromise; + expect(providerCloseCalls).toBe(1); + }); + + it("waits for an admitted vector probe before manager teardown", async () => { + const manager = await getPersistentManager(createCfg({ provider: "openai" })); + const fields = manager as unknown as { + ensureVectorReady: () => Promise; + }; + let releaseProbe: () => void = () => {}; + let markProbeStarted: () => void = () => {}; + const probeGate = new Promise((resolve) => { + releaseProbe = resolve; + }); + const probeStarted = new Promise((resolve) => { + markProbeStarted = resolve; + }); + fields.ensureVectorReady = async () => { + markProbeStarted(); + await probeGate; + return true; + }; + + const probePromise = manager.probeVectorAvailability(); + await probeStarted; + const closePromise = manager.close(); + let closeSettled = false; + void closePromise.then( + () => { + closeSettled = true; + }, + () => { + closeSettled = true; + }, + ); + try { + await Promise.resolve(); + expect(closeSettled).toBe(false); + expect(providerCloseCalls).toBe(0); + } finally { + releaseProbe(); + } + + await expect(probePromise).resolves.toBe(true); + await closePromise; + expect(providerCloseCalls).toBe(1); + }); + + it("fails closed when fallback initialization fails for an explicit provider", async () => { + const cfg = createCfg({ + provider: "openai", + fallback: "fallback-provider", + hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, + }); + const manager = await getPersistentManager(cfg); + await manager.sync({ reason: "test" }); + const fields = manager as unknown as { + provider: { + embedQuery: (text: string) => Promise; + } | null; + }; + if (!fields.provider) { + throw new Error("Expected a test embedding provider"); + } + fields.provider.embedQuery = async () => { + throw new Error("embedding provider failed"); + }; + providerCreationFailure = "fallback-provider"; + + await expect(manager.search("alpha")).rejects.toThrow( + /Memory search unavailable: embedding provider "openai" is configured but unavailable\./, + ); + + providerCreationFailure = null; + await expect(manager.search("alpha")).resolves.toBeDefined(); + }); + + it("retries the optional primary after fallback initialization fails", async () => { + const cfg = createCfg({ + fallback: "fallback-provider", + hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, + }); + const manager = await getPersistentManager(cfg); + await manager.sync({ reason: "test" }); + const fields = manager as unknown as { + provider: { + id: string; + embedQuery: (text: string) => Promise; + } | null; + }; + if (!fields.provider) { + throw new Error("Expected a test embedding provider"); + } + fields.provider.embedQuery = async () => { + throw new Error("embedding provider failed"); + }; + providerCreationFailure = "fallback-provider"; + const callsBeforeSearch = providerCalls.length; + + await expect(manager.search("alpha")).resolves.toBeDefined(); + + providerCreationFailure = null; + await expect(manager.search("alpha")).resolves.toBeDefined(); + expect(providerCalls.slice(callsBeforeSearch).map((call) => call.provider)).toEqual([ + "fallback-provider", + "openai", + ]); + expect(fields.provider?.id).toBe("mock"); + }); + + it("fails closed and retries a required primary after a null fallback result", async () => { + const cfg = createCfg({ + provider: "openai", + fallback: "fallback-provider", + hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, + }); + const manager = await getPersistentManager(cfg); + await manager.sync({ reason: "test" }); + const fields = manager as unknown as { + provider: { embedQuery: (text: string) => Promise } | null; + }; + if (!fields.provider) { + throw new Error("Expected a test embedding provider"); + } + fields.provider.embedQuery = async () => { + throw new Error("embedding provider failed"); + }; + providerNullResult = "fallback-provider"; + + await expect(manager.search("alpha")).rejects.toThrow( + /Memory search unavailable: embedding provider "openai" is configured but unavailable\./, + ); + + providerNullResult = null; + await expect(manager.search("alpha")).resolves.toBeDefined(); + }); + + it("retries an optional primary after a null fallback result", async () => { + const cfg = createCfg({ + fallback: "fallback-provider", + hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, + }); + const manager = await getPersistentManager(cfg); + await manager.sync({ reason: "test" }); + const fields = manager as unknown as { + provider: { id: string; embedQuery: (text: string) => Promise } | null; + }; + if (!fields.provider) { + throw new Error("Expected a test embedding provider"); + } + fields.provider.embedQuery = async () => { + throw new Error("embedding provider failed"); + }; + providerNullResult = "fallback-provider"; + + await expect(manager.search("alpha")).resolves.toBeDefined(); + + providerNullResult = null; + await expect(manager.search("alpha")).resolves.toBeDefined(); + expect(fields.provider?.id).toBe("mock"); + }); + + it("keeps concurrent optional searches in FTS mode when shared fallback fails", async () => { + const cfg = createCfg({ + fallback: "fallback-provider", + hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, + }); + const manager = await getPersistentManager(cfg); + await manager.sync({ reason: "test" }); + const fields = manager as unknown as { + provider: { + embedQuery: (text: string) => Promise; + } | null; + ensureProviderInitialized: () => Promise; + }; + if (!fields.provider) { + throw new Error("Expected a test embedding provider"); + } + fields.provider.embedQuery = async () => { + throw new Error("embedding provider failed"); + }; + const ensureProviderInitialized = fields.ensureProviderInitialized.bind(manager); + let providerInitializationCalls = 0; + fields.ensureProviderInitialized = async () => { + providerInitializationCalls += 1; + await ensureProviderInitialized(); + }; + providerCreationFailure = "fallback-provider"; + let releaseProviderInit: () => void = () => {}; + providerInitGate = new Promise((resolve) => { + releaseProviderInit = resolve; + }); + + const callsBeforeSearch = providerCalls.length; + const firstSearch = manager.search("alpha"); + await vi.waitFor(() => + expect(providerCalls.some((call) => call.provider === "fallback-provider")).toBe(true), + ); + const initializationCallsBeforeSecondSearch = providerInitializationCalls; + const secondSearch = manager.search("zebra"); + let secondSettled = false; + void secondSearch.then( + () => { + secondSettled = true; + }, + () => { + secondSettled = true; + }, + ); + try { + await vi.waitFor(() => + expect(providerInitializationCalls).toBeGreaterThan(initializationCallsBeforeSecondSearch), + ); + expect(secondSettled).toBe(false); + releaseProviderInit(); + const results = await Promise.all([firstSearch, secondSearch]); + expect(results.every((result) => result.length > 0)).toBe(true); + expect( + providerCalls + .slice(callsBeforeSearch) + .filter((call) => call.provider === "fallback-provider"), + ).toHaveLength(1); + } finally { + providerInitGate = null; + releaseProviderInit(); + await Promise.allSettled([firstSearch, secondSearch]); + } + }); +}); diff --git a/extensions/memory-core/src/memory/manager-provider-lifecycle.test.ts b/extensions/memory-core/src/memory/manager-provider-lifecycle.test.ts new file mode 100644 index 000000000000..a9633ebe96f2 --- /dev/null +++ b/extensions/memory-core/src/memory/manager-provider-lifecycle.test.ts @@ -0,0 +1,869 @@ +// Memory Core tests cover manager provider lifecycle availability behavior. +import { mkdirSync, rmSync } from "node:fs"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { clearMemoryEmbeddingProviders as clearRegistry } from "openclaw/plugin-sdk/memory-core-host-engine-embeddings"; +import { hashText } from "openclaw/plugin-sdk/memory-core-host-engine-storage"; +import { + closeOpenClawAgentDatabasesForTest, + closeOpenClawStateDatabaseForTest, +} from "openclaw/plugin-sdk/sqlite-runtime-testing"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { + configureMemoryCoreDreamingStateForTests, + resetMemoryCoreDreamingStateForTests, +} from "../test-helpers.js"; +import "./test-runtime-mocks.js"; +import { closeAllMemorySearchManagers, getMemorySearchManager } from "./index.js"; +import type { MemoryIndexManager } from "./manager.js"; +import { isolateMemoryManagerTestConfig } from "./test-config-helpers.js"; + +// This suite performs real sqlite/media indexing and can exceed the global +// timeout when it shares a packed CI extension shard. +vi.setConfig({ testTimeout: 240_000 }); + +afterAll(() => { + vi.resetConfig(); +}); + +let embedBatchCalls = 0; +let embeddedBatchTexts: string[] = []; +let embedBatchInputCalls = 0; +let providerRuntimeBatchCalls: string[][] = []; +let providerRuntimeBatchGate: Promise | null = null; +let providerRuntimeBatchErrors: unknown[] = []; +let providerRuntimeBatchFailuresRemaining = 0; +let providerRuntimeActiveBatchCalls = 0; +let providerRuntimeMaxActiveBatchCalls = 0; +let providerCloseCalls = 0; +let providerCloseFailuresRemaining = 0; +let providerCloseFailure: unknown = new Error("provider close failed"); +let providerCreationFailure: string | null = null; +let providerNullResult: string | null = null; +let providerCloseGate: Promise | null = null; +let providerInitGate: Promise | null = null; +let providerCalls: Array<{ provider?: string; model?: string; outputDimensionality?: number }> = []; +let forceNoProvider = false; + +const originalMemoryIndexStateDir = process.env.OPENCLAW_STATE_DIR; + +const identityAliasFixture = vi.hoisted(() => ({ + provider: "identity-alias-test", + canonicalModel: "hf:fixture/default-model.gguf", + cacheModel: "/fixture/cache/default-model.gguf", +})); + +function createLocalWorkerExitError(): Error { + return Object.assign(new Error("Local embedding worker exited unexpectedly (exit code 134)"), { + code: "LOCAL_EMBEDDING_WORKER_EXITED", + reason: "exit", + exitCode: 134, + }); +} + +function setMemoryIndexStateDir(stateDir: string): void { + Reflect.set(process.env, "OPENCLAW_STATE_DIR", stateDir); +} + +function restoreMemoryIndexStateDir(): void { + if (originalMemoryIndexStateDir === undefined) { + Reflect.deleteProperty(process.env, "OPENCLAW_STATE_DIR"); + } else { + Reflect.set(process.env, "OPENCLAW_STATE_DIR", originalMemoryIndexStateDir); + } +} + +vi.mock("./embeddings.js", async (importOriginal) => { + const actual = await importOriginal(); + const embedText = (text: string) => { + const lower = text.toLowerCase(); + const alpha = lower.split("alpha").length - 1; + const beta = lower.split("beta").length - 1; + const image = lower.split("image").length - 1; + const audio = lower.split("audio").length - 1; + return [alpha, beta, image, audio]; + }; + return { + ...actual, + resolveEmbeddingProviderFallbackModel: (providerId: string, fallbackSourceModel: string) => + providerId === "gemini" || providerId === "fallback-provider" + ? `${providerId}-embed` + : fallbackSourceModel, + resolveEmbeddingProviderAdapterId: ( + providerId: string, + config?: { + models?: { + providers?: Record; + }; + }, + ) => config?.models?.providers?.[providerId]?.api ?? providerId, + resolveEmbeddingProviderAdapterTransport: (providerId: string) => + providerId === "local" ? "local" : "remote", + resolveEmbeddingProviderIndexIdentity: (options: { provider?: string; model?: string }) => + options.provider === identityAliasFixture.provider + ? { + provider: { + id: identityAliasFixture.provider, + model: identityAliasFixture.canonicalModel, + }, + cacheKeyData: { + provider: identityAliasFixture.provider, + model: identityAliasFixture.canonicalModel, + }, + aliases: [ + { + model: identityAliasFixture.cacheModel, + cacheKeyData: { + provider: identityAliasFixture.provider, + model: identityAliasFixture.cacheModel, + }, + }, + ], + } + : undefined, + createEmbeddingProvider: async (options: { + provider?: string; + model?: string; + outputDimensionality?: number; + }) => { + providerCalls.push({ + provider: options.provider, + model: options.model, + outputDimensionality: options.outputDimensionality, + }); + await providerInitGate; + if (options.provider === providerCreationFailure) { + throw new Error(`provider creation failed: ${options.provider}`); + } + if (options.provider === providerNullResult) { + return { + provider: null, + requestedProvider: options.provider, + providerUnavailableReason: `provider unavailable: ${options.provider}`, + }; + } + if (forceNoProvider) { + return { + provider: null, + requestedProvider: options.provider ?? "auto", + providerUnavailableReason: "No API key found for provider", + }; + } + const providerId = + options.provider === "gemini" || + options.provider === "fallback-provider" || + options.provider === "batch-test" || + options.provider === "batch-wide-test" || + options.provider === identityAliasFixture.provider || + options.provider === "ollama" + ? options.provider + : "mock"; + const requestedModel = options.model ?? "mock-embed"; + const model = + providerId === identityAliasFixture.provider && + (requestedModel === identityAliasFixture.canonicalModel || + requestedModel === identityAliasFixture.cacheModel) + ? identityAliasFixture.canonicalModel + : requestedModel; + return { + requestedProvider: options.provider ?? "openai", + provider: { + id: providerId, + model, + close: async () => { + providerCloseCalls += 1; + await providerCloseGate; + if (providerCloseFailuresRemaining > 0) { + providerCloseFailuresRemaining -= 1; + throw providerCloseFailure; + } + }, + embedQuery: async (text: string) => embedText(text), + embedBatch: async (texts: string[]) => { + embedBatchCalls += 1; + embeddedBatchTexts.push(...texts); + return texts.map(embedText); + }, + ...(providerId === "gemini" || providerId === "fallback-provider" + ? { + embedBatchInputs: async ( + inputs: Array<{ + text: string; + parts?: Array< + | { type: "text"; text: string } + | { type: "inline-data"; mimeType: string; data: string } + >; + }>, + ) => { + embedBatchInputCalls += 1; + return inputs.map((input) => { + const inlineData = input.parts?.find((part) => part.type === "inline-data"); + if (inlineData?.type === "inline-data" && inlineData.data.length > 9000) { + throw new Error("payload too large"); + } + const mimeType = + inlineData?.type === "inline-data" ? inlineData.mimeType : undefined; + if (mimeType?.startsWith("image/")) { + return [0, 0, 1, 0]; + } + if (mimeType?.startsWith("audio/")) { + return [0, 0, 0, 1]; + } + return embedText(input.text); + }); + }, + } + : {}), + }, + ...(providerId === identityAliasFixture.provider + ? { + runtime: { + id: providerId, + cacheKeyData: { + provider: providerId, + model: identityAliasFixture.canonicalModel, + }, + indexIdentityAliases: [ + { + model: identityAliasFixture.cacheModel, + cacheKeyData: { + provider: providerId, + model: identityAliasFixture.cacheModel, + }, + }, + ], + }, + } + : providerId === "batch-test" || providerId === "batch-wide-test" + ? { + runtime: { + id: providerId, + ...(providerId === "batch-wide-test" ? { sourceWideBatchEmbed: true } : {}), + batchEmbed: async (batch: { chunks: Array<{ text: string }> }) => { + providerRuntimeActiveBatchCalls += 1; + providerRuntimeMaxActiveBatchCalls = Math.max( + providerRuntimeMaxActiveBatchCalls, + providerRuntimeActiveBatchCalls, + ); + try { + await providerRuntimeBatchGate; + providerRuntimeBatchCalls.push(batch.chunks.map((chunk) => chunk.text)); + if (providerRuntimeBatchErrors.length > 0) { + throw providerRuntimeBatchErrors.shift(); + } + if (providerRuntimeBatchFailuresRemaining > 0) { + providerRuntimeBatchFailuresRemaining -= 1; + throw new Error("provider runtime batch failed"); + } + return batch.chunks.map((chunk) => embedText(chunk.text)); + } finally { + providerRuntimeActiveBatchCalls -= 1; + } + }, + }, + } + : providerId === "gemini" || providerId === "fallback-provider" + ? { + runtime: { + id: providerId, + cacheKeyData: { + provider: providerId, + baseUrl: "https://generativelanguage.googleapis.com/v1beta", + model, + outputDimensionality: options.outputDimensionality, + headers: [], + }, + }, + } + : {}), + }; + }, + }; +}); + +describe("memory index", () => { + let fixtureRoot = ""; + let workspaceDir = ""; + let memoryDir = ""; + + const managersForCleanup = new Set(); + + beforeAll(async () => { + fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-mem-fixtures-")); + workspaceDir = path.join(fixtureRoot, "workspace"); + memoryDir = path.join(workspaceDir, "memory"); + }); + + afterAll(async () => { + await Promise.all(Array.from(managersForCleanup).map((manager) => manager.close())); + await fs.rm(fixtureRoot, { recursive: true, force: true }); + }); + + afterEach(async () => { + vi.useRealTimers(); + await Promise.all(Array.from(managersForCleanup).map((manager) => manager.close())); + await closeAllMemorySearchManagers(); + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + resetMemoryCoreDreamingStateForTests(); + clearRegistry(); + managersForCleanup.clear(); + restoreMemoryIndexStateDir(); + }); + + beforeEach(async () => { + vi.useRealTimers(); + clearRegistry(); + embedBatchCalls = 0; + embeddedBatchTexts = []; + embedBatchInputCalls = 0; + providerRuntimeBatchCalls = []; + providerRuntimeBatchGate = null; + providerRuntimeBatchErrors = []; + providerRuntimeBatchFailuresRemaining = 0; + providerRuntimeActiveBatchCalls = 0; + providerRuntimeMaxActiveBatchCalls = 0; + providerCloseCalls = 0; + providerCloseFailuresRemaining = 0; + providerCloseFailure = new Error("provider close failed"); + providerCreationFailure = null; + providerNullResult = null; + providerCloseGate = null; + providerInitGate = null; + providerCalls = []; + forceNoProvider = false; + + rmSync(workspaceDir, { recursive: true, force: true }); + mkdirSync(memoryDir, { recursive: true }); + setMemoryIndexStateDir(path.join(workspaceDir, ".state-memory-index")); + await configureMemoryCoreDreamingStateForTests(); + await fs.writeFile( + path.join(memoryDir, "2026-01-12.md"), + "# Log\nAlpha memory line.\nZebra memory line.", + ); + }); + + function resetManagerForTest(manager: MemoryIndexManager) { + // These tests reuse managers for performance. Clear the index + embedding + // cache to keep each test fully isolated. + const db = ( + manager as unknown as { + db: { + exec: (sql: string) => void; + prepare: (sql: string) => { get: (name: string) => { name?: string } | undefined }; + }; + } + ).db; + for (const table of [ + "memory_index_sources", + "memory_index_chunks", + "memory_embedding_cache", + "memory_index_chunks_fts", + "memory_index_chunks_vec", + ]) { + const existingTable = db + .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?") + .get(table); + if (existingTable?.name === table) { + db.exec(`DELETE FROM ${table}`); + } + } + (manager as unknown as { dirty: boolean }).dirty = true; + (manager as unknown as { sessionsDirty: boolean }).sessionsDirty = false; + (manager as unknown as { sessionsDirtyFiles: Set }).sessionsDirtyFiles.clear(); + } + + type TestCfg = Parameters[0]["cfg"]; + + function createCfg(params: { + extraPaths?: string[]; + sources?: Array<"memory" | "sessions">; + sessionMemory?: boolean; + rememberAcrossConversations?: boolean; + provider?: string; + fallback?: "none" | "gemini" | "fallback-provider"; + providerAliases?: NonNullable["providers"]>; + batchEnabled?: boolean; + model?: string; + outputDimensionality?: number; + multimodal?: { + enabled?: boolean; + modalities?: Array<"image" | "audio" | "all">; + maxFileBytes?: number; + }; + vectorEnabled?: boolean; + cacheEnabled?: boolean; + minScore?: number; + onSearch?: boolean; + hybrid?: { + enabled: boolean; + vectorWeight?: number; + textWeight?: number; + temporalDecay?: { enabled: boolean }; + }; + }): TestCfg { + return isolateMemoryManagerTestConfig({ + memory: { + search: { + ...(params.provider !== undefined ? { provider: params.provider } : {}), + model: params.model ?? "mock-embed", + fallback: params.fallback, + outputDimensionality: params.outputDimensionality, + store: { + vector: params.vectorEnabled !== undefined ? { enabled: params.vectorEnabled } : {}, + }, + remote: params.batchEnabled + ? { + batch: { enabled: true }, + } + : undefined, + query: { minScore: params.minScore ?? 0 }, + cache: params.cacheEnabled ? { enabled: true } : undefined, + extraPaths: params.extraPaths, + multimodal: params.multimodal, + sources: params.sources, + rememberAcrossConversations: + params.rememberAcrossConversations ?? params.sessionMemory ?? false, + }, + }, + + agents: { + defaults: { + workspace: workspaceDir, + }, + list: [{ id: "main", default: true }], + }, + models: params.providerAliases ? { providers: params.providerAliases } : undefined, + }); + } + + function requireManager( + result: Awaited>, + missingMessage = "manager missing", + ): MemoryIndexManager { + if (!result.manager) { + throw new Error(missingMessage); + } + return result.manager as unknown as MemoryIndexManager; + } + + async function getPersistentManager(cfg: TestCfg): Promise { + const result = await getMemorySearchManager({ cfg, agentId: "main" }); + const manager = requireManager(result); + managersForCleanup.add(manager); + resetManagerForTest(manager); + return manager; + } + + async function getFreshManager( + cfg: TestCfg, + purpose?: "default" | "status" | "cli", + ): Promise { + const manager = requireManager(await getMemorySearchManager({ cfg, agentId: "main", purpose })); + managersForCleanup.add(manager); + return manager; + } + + it("caches embedding probe readiness across transient status managers", async () => { + const cfg = createCfg({}); + const first = requireManager( + await getMemorySearchManager({ cfg, agentId: "main", purpose: "status" }), + ); + managersForCleanup.add(first); + + await expect(first.probeEmbeddingAvailability()).resolves.toEqual({ ok: true }); + expect(embedBatchCalls).toBe(1); + await first.close(); + + const second = requireManager( + await getMemorySearchManager({ cfg, agentId: "main", purpose: "status" }), + ); + managersForCleanup.add(second); + + const cachedBeforeProbe = second.getCachedEmbeddingAvailability?.(); + expect(cachedBeforeProbe?.ok).toBe(true); + expect(cachedBeforeProbe?.checked).toBe(true); + expect(cachedBeforeProbe?.cached).toBe(true); + expect(cachedBeforeProbe?.checkedAtMs).toBeTypeOf("number"); + expect(cachedBeforeProbe?.cacheExpiresAtMs).toBeTypeOf("number"); + if ( + typeof cachedBeforeProbe?.checkedAtMs === "number" && + typeof cachedBeforeProbe.cacheExpiresAtMs === "number" + ) { + expect(cachedBeforeProbe.cacheExpiresAtMs - cachedBeforeProbe.checkedAtMs).toBe(30_000); + } + await expect(second.probeEmbeddingAvailability()).resolves.toStrictEqual({ + ok: true, + checked: true, + cached: true, + checkedAtMs: cachedBeforeProbe?.checkedAtMs, + cacheExpiresAtMs: cachedBeforeProbe?.cacheExpiresAtMs, + }); + expect(embedBatchCalls).toBe(1); + + const cached = second.getCachedEmbeddingAvailability?.(); + expect((cached?.cacheExpiresAtMs ?? 0) - (cached?.checkedAtMs ?? 0)).toBe(30_000); + }); + + it("clears cached embedding probe readiness when local embeddings degrade", async () => { + const cfg = createCfg({}); + const manager = await getPersistentManager(cfg); + + await expect(manager.probeEmbeddingAvailability()).resolves.toEqual({ ok: true }); + expect(manager.getCachedEmbeddingAvailability()?.ok).toBe(true); + ( + manager as unknown as { + provider: { + id: string; + model: string; + embedQuery: (text: string) => Promise; + embedBatch: (texts: string[]) => Promise; + close: () => Promise; + }; + } + ).provider = { + id: "local", + model: "local-model", + embedQuery: async () => [1, 0], + embedBatch: async (texts: string[]) => texts.map(() => [1, 0]), + close: async () => {}, + }; + + ( + manager as unknown as { + markLocalEmbeddingProviderDegraded: (err: unknown) => void; + } + ).markLocalEmbeddingProviderDegraded(createLocalWorkerExitError()); + + expect(manager.getCachedEmbeddingAvailability()).toBeNull(); + await expect(manager.probeEmbeddingAvailability()).resolves.toMatchObject({ + ok: false, + error: expect.stringContaining("Local embeddings degraded"), + }); + }); + + it("waits for degraded provider shutdown before fallback initialization", async () => { + const cfg = createCfg({ fallback: "fallback-provider" }); + const manager = await getPersistentManager(cfg); + await manager.sync({ reason: "test" }); + + let releaseProviderClose: () => void = () => {}; + providerCloseGate = new Promise((resolve) => { + releaseProviderClose = resolve; + }); + const fields = manager as unknown as { + provider: { + id: string; + model: string; + embedQuery: (text: string) => Promise; + embedBatch: (texts: string[]) => Promise; + close: () => Promise; + } | null; + markLocalEmbeddingProviderDegraded: (err: unknown) => void; + activateFallbackProvider: (reason: string) => Promise; + withTimeout: (promise: Promise, timeoutMs: number, message: string) => Promise; + }; + if (!fields.provider) { + throw new Error("Expected a test embedding provider"); + } + fields.provider.id = "local"; + fields.markLocalEmbeddingProviderDegraded(createLocalWorkerExitError()); + await vi.waitFor(() => expect(providerCloseCalls).toBe(1)); + + const callsBeforeFallback = providerCalls.length; + const fallbackPromise = fields.activateFallbackProvider("local worker exited"); + try { + await Promise.resolve(); + expect(providerCalls).toHaveLength(callsBeforeFallback); + } finally { + releaseProviderClose(); + providerCloseGate = null; + await fallbackPromise; + } + expect(providerCalls.slice(callsBeforeFallback).map((call) => call.provider)).toEqual([ + "fallback-provider", + ]); + }); + + it("retries failed provider retirement before fallback initialization", async () => { + const cfg = createCfg({ fallback: "fallback-provider" }); + const manager = await getPersistentManager(cfg); + await manager.sync({ reason: "test" }); + providerCloseFailuresRemaining = 1; + const fields = manager as unknown as { + activateFallbackProvider: (reason: string) => Promise; + }; + const callsBeforeFallback = providerCalls.length; + + await expect(fields.activateFallbackProvider("provider failed")).rejects.toThrow( + "provider close failed", + ); + expect(providerCalls).toHaveLength(callsBeforeFallback); + + await expect(fields.activateFallbackProvider("provider failed")).resolves.toBe(true); + expect(providerCloseCalls).toBe(2); + expect(providerCalls.slice(callsBeforeFallback).map((call) => call.provider)).toEqual([ + "fallback-provider", + ]); + }); + + it("waits for provider shutdown before retry initialization", async () => { + const cfg = createCfg({ provider: "openai" }); + const manager = await getPersistentManager(cfg); + await manager.sync({ reason: "test" }); + + let releaseProviderClose: () => void = () => {}; + providerCloseGate = new Promise((resolve) => { + releaseProviderClose = resolve; + }); + ( + manager as unknown as { + resetProviderInitializationForRetry: () => void; + } + ).resetProviderInitializationForRetry(); + await vi.waitFor(() => expect(providerCloseCalls).toBe(1)); + + const callsBeforeProbe = providerCalls.length; + const probePromise = manager.probeEmbeddingAvailability(); + try { + await Promise.resolve(); + expect(providerCalls).toHaveLength(callsBeforeProbe); + } finally { + releaseProviderClose(); + providerCloseGate = null; + await probePromise; + } + expect(providerCalls.slice(callsBeforeProbe).map((call) => call.provider)).toEqual(["openai"]); + }); + + it("waits for active provider shutdown before fallback initialization", async () => { + const cfg = createCfg({ + provider: "openai", + fallback: "fallback-provider", + hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, + }); + const manager = await getPersistentManager(cfg); + await manager.sync({ reason: "test" }); + + let releaseProviderClose: () => void = () => {}; + providerCloseGate = new Promise((resolve) => { + releaseProviderClose = resolve; + }); + const fields = manager as unknown as { + provider: { + embedQuery: (text: string) => Promise; + } | null; + }; + if (!fields.provider) { + throw new Error("Expected a test embedding provider"); + } + fields.provider.embedQuery = async () => { + throw new Error("embedding provider failed"); + }; + + const callsBeforeSearch = providerCalls.length; + const searchPromise = manager.search("alpha"); + let concurrentSearch: ReturnType = Promise.resolve([]); + try { + await vi.waitFor(() => expect(providerCloseCalls).toBe(1)); + concurrentSearch = manager.search("zebra"); + let concurrentSettled = false; + void concurrentSearch.then( + () => { + concurrentSettled = true; + }, + () => { + concurrentSettled = true; + }, + ); + await Promise.resolve(); + expect(concurrentSettled).toBe(false); + expect(providerCalls).toHaveLength(callsBeforeSearch); + } finally { + releaseProviderClose(); + providerCloseGate = null; + await Promise.allSettled([searchPromise, concurrentSearch]); + } + expect(providerCalls.slice(callsBeforeSearch).map((call) => call.provider)).toEqual([ + "fallback-provider", + ]); + await expect(concurrentSearch).resolves.toBeDefined(); + }); + + it("leases the indexing provider generation through chunk publication", async () => { + const manager = await getFreshManager( + createCfg({ + provider: "openai", + fallback: "fallback-provider", + cacheEnabled: true, + hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, + }), + "cli", + ); + managersForCleanup.add(manager); + const fields = manager as unknown as { + provider: { + id: string; + model: string; + embedBatch: (texts: string[]) => Promise; + } | null; + providerKey: string; + computeProviderKey: () => string; + ensureProviderInitialized: () => Promise; + markLocalEmbeddingProviderDegraded: (err: unknown) => void; + activateFallbackProvider: (reason: string) => Promise; + withTimeout: (promise: Promise, timeoutMs: number, message: string) => Promise; + indexFile: ( + entry: { + path: string; + absPath: string; + mtimeMs: number; + size: number; + hash: string; + content: string; + }, + options: { source: "memory"; content: string }, + ) => Promise; + ensureVectorReady: (dimensions?: number) => Promise; + db: { + prepare: (sql: string) => { + get: ( + ...params: unknown[] + ) => { model?: string; provider?: string; provider_key?: string } | undefined; + }; + }; + }; + await fields.ensureProviderInitialized(); + if (!fields.provider) { + throw new Error("Expected a test embedding provider"); + } + const indexedProvider = fields.provider; + indexedProvider.id = "local"; + fields.providerKey = fields.computeProviderKey(); + const indexedProviderKey = fields.providerKey; + const firstContent = "# Log\nFirst memory line indexed during provider fallback."; + const secondContent = "# Log\nSecond memory line indexed during provider fallback."; + + let releaseFirstEmbedding: () => void = () => {}; + let releaseSecondEmbedding: () => void = () => {}; + let markFirstEmbeddingStarted: () => void = () => {}; + let markSecondEmbeddingStarted: () => void = () => {}; + const firstEmbeddingGate = new Promise((resolve) => { + releaseFirstEmbedding = resolve; + }); + const secondEmbeddingGate = new Promise((resolve) => { + releaseSecondEmbedding = resolve; + }); + const firstEmbeddingStarted = new Promise((resolve) => { + markFirstEmbeddingStarted = resolve; + }); + const secondEmbeddingStarted = new Promise((resolve) => { + markSecondEmbeddingStarted = resolve; + }); + indexedProvider.embedBatch = async (texts) => { + if (texts.some((text) => text.includes("First"))) { + markFirstEmbeddingStarted(); + await firstEmbeddingGate; + } else { + markSecondEmbeddingStarted(); + await secondEmbeddingGate; + } + return texts.map(() => [1, 0, 0, 0]); + }; + let releasePublication: () => void = () => {}; + let markPublicationStarted: () => void = () => {}; + const publicationGate = new Promise((resolve) => { + releasePublication = resolve; + }); + const publicationStarted = new Promise((resolve) => { + markPublicationStarted = resolve; + }); + const ensureVectorReady = fields.ensureVectorReady.bind(manager); + let publicationCalls = 0; + fields.ensureVectorReady = async (dimensions) => { + publicationCalls += 1; + if (publicationCalls === 1) { + return await ensureVectorReady(dimensions); + } + markPublicationStarted(); + await publicationGate; + return await ensureVectorReady(dimensions); + }; + + const callsBeforeFallback = providerCalls.length; + const firstIndexPromise = fields.indexFile( + { + path: "memory/generation-race-first.md", + absPath: path.join(memoryDir, "generation-race-first.md"), + mtimeMs: Date.now(), + size: Buffer.byteLength(firstContent), + hash: hashText(firstContent), + content: firstContent, + }, + { source: "memory", content: firstContent }, + ); + const secondIndexPromise = fields.indexFile( + { + path: "memory/generation-race-second.md", + absPath: path.join(memoryDir, "generation-race-second.md"), + mtimeMs: Date.now(), + size: Buffer.byteLength(secondContent), + hash: hashText(secondContent), + content: secondContent, + }, + { source: "memory", content: secondContent }, + ); + let fallbackPromise: Promise | null = null; + try { + await fields.withTimeout( + Promise.all([firstEmbeddingStarted, secondEmbeddingStarted]), + 5_000, + "concurrent embeddings did not start", + ); + fields.markLocalEmbeddingProviderDegraded(createLocalWorkerExitError()); + await vi.waitFor(() => expect(fields.provider).toBeNull()); + fallbackPromise = fields.activateFallbackProvider("local worker exited"); + releaseFirstEmbedding(); + await firstIndexPromise; + expect(providerCloseCalls).toBe(0); + expect(providerCalls).toHaveLength(callsBeforeFallback); + + releaseSecondEmbedding(); + await fields.withTimeout(publicationStarted, 5_000, "publication did not start"); + expect(providerCloseCalls).toBe(0); + expect(providerCalls).toHaveLength(callsBeforeFallback); + + releasePublication(); + await secondIndexPromise; + await expect(fallbackPromise).resolves.toBe(true); + } finally { + releaseFirstEmbedding(); + releaseSecondEmbedding(); + releasePublication(); + await Promise.allSettled([ + firstIndexPromise, + secondIndexPromise, + ...(fallbackPromise ? [fallbackPromise] : []), + ]); + } + + expect(providerCalls.slice(callsBeforeFallback).map((call) => call.provider)).toEqual([ + "fallback-provider", + ]); + expect( + fields.db + .prepare("SELECT model FROM memory_index_chunks WHERE path = ?") + .get("memory/generation-race-second.md")?.model, + ).toBe(indexedProvider.model); + expect( + fields.db + .prepare("SELECT provider, model, provider_key FROM memory_embedding_cache LIMIT 1") + .get(), + ).toEqual({ + provider: indexedProvider.id, + model: indexedProvider.model, + provider_key: indexedProviderKey, + }); + }); +}); diff --git a/extensions/memory-core/src/memory/manager-provider-lifecycle.ts b/extensions/memory-core/src/memory/manager-provider-lifecycle.ts new file mode 100644 index 000000000000..98d799b21ab3 --- /dev/null +++ b/extensions/memory-core/src/memory/manager-provider-lifecycle.ts @@ -0,0 +1,607 @@ +// Memory Core plugin module owns embedding provider lifecycle. +import { resolveAgentConfig } from "openclaw/plugin-sdk/agent-runtime"; +import { + formatErrorMessage, + readErrorName, + toErrorObject, +} from "openclaw/plugin-sdk/error-runtime"; +import { listRegisteredMemoryEmbeddingProviderAdapters } from "openclaw/plugin-sdk/memory-core-host-embedding-registry"; +import { + createSubsystemLogger, + resolveAgentDir, + type OpenClawConfig, + type ResolvedMemorySearchConfig, +} from "openclaw/plugin-sdk/memory-core-host-engine-foundation"; +import type { + MemoryEmbeddingProbeResult, + MemorySearchRuntimeDebug, + MemorySyncParams, +} from "openclaw/plugin-sdk/memory-core-host-engine-storage"; +import { normalizeAgentId } from "openclaw/plugin-sdk/routing"; +import { redactSensitiveText } from "openclaw/plugin-sdk/security-runtime"; +import { + createEmbeddingProvider, + resolveEmbeddingProviderAdapterTransport, + type EmbeddingProvider, + type EmbeddingProviderRequest, + type EmbeddingProviderResult, +} from "./embeddings.js"; +import { MemoryManagerEmbeddingOps } from "./manager-embedding-ops.js"; +import { isLocalEmbeddingWorkerFailure } from "./manager-local-worker-errors.js"; +import { + createDegradedMemoryProviderLifecycle, + createPendingMemoryProviderLifecycle, + resolveMemoryPrimaryProviderRequest, + resolveMemoryProviderState, +} from "./manager-provider-state.js"; +import type { MemoryIndexIdentityState } from "./manager-reindex-state.js"; + +const EMBEDDING_PROBE_CACHE_TTL_MS = 30_000; +const log = createSubsystemLogger("memory"); + +export type MemoryEmbeddingProviderRequirement = { + mode: "fts-only" | "optional" | "required"; + provider: string; + configuredProvider?: string; +}; +export type MemoryEmbeddingBootstrapDebug = NonNullable< + MemorySearchRuntimeDebug["embeddingBootstrap"] +>; +type EmbeddingProbeCacheEntry = { + result: MemoryEmbeddingProbeResult; + checkedAtMs: number; + expireAtMs: number; +}; +const EMBEDDING_PROBE_CACHE = new Map(); + +export function clearMemoryEmbeddingProbeCache(): void { + EMBEDDING_PROBE_CACHE.clear(); +} + +export function resolveEffectiveMemorySearchSettings( + settings: ResolvedMemorySearchConfig, +): ResolvedMemorySearchConfig { + if (settings.provider !== "none" || !settings.store.vector.enabled) { + return settings; + } + return { + ...settings, + store: { + ...settings.store, + vector: { + ...settings.store.vector, + enabled: false, + }, + }, + }; +} + +function resolveConfiguredMemoryEmbeddingProvider(params: { + cfg: OpenClawConfig; + agentId: string; +}): string | undefined { + const agentEntry = resolveAgentConfig(params.cfg, normalizeAgentId(params.agentId)); + return agentEntry?.memory?.search?.provider ?? params.cfg.memory?.search?.provider; +} + +export function resolveMemoryEmbeddingProviderRequirement(params: { + cfg: OpenClawConfig; + agentId: string; + settings: ResolvedMemorySearchConfig; +}): MemoryEmbeddingProviderRequirement { + const configuredProvider = resolveConfiguredMemoryEmbeddingProvider(params)?.trim(); + if (params.settings.provider === "none" || configuredProvider === "none") { + return { mode: "fts-only", provider: params.settings.provider }; + } + const adapterTransport = resolveEmbeddingProviderAdapterTransport( + params.settings.provider, + params.cfg, + ); + if (!configuredProvider || configuredProvider === "auto" || adapterTransport === "local") { + return { mode: "optional", provider: params.settings.provider }; + } + return { + mode: "required", + provider: params.settings.provider, + configuredProvider, + }; +} + +export abstract class MemoryProviderLifecycle extends MemoryManagerEmbeddingOps { + protected abstract readonly cacheKey: string; + protected abstract readonly purpose: "default" | "status" | "cli"; + protected abstract readonly providerRequirement: MemoryEmbeddingProviderRequirement; + protected abstract readonly requestedProvider: EmbeddingProviderRequest; + protected abstract providerInitPromise: Promise | null; + protected abstract providerInitialized: boolean; + protected abstract embeddingBootstrapFailure?: MemoryEmbeddingBootstrapDebug; + protected abstract providerRetirementPromise: Promise; + protected abstract providersPendingRetirement: Set; + protected abstract closing: boolean; + protected abstract activeManagerOperations: number; + protected abstract managerIdleWaiters: Set<() => void>; + protected abstract indexIdentityDirty: boolean; + protected abstract indexIdentityState: MemoryIndexIdentityState; + protected abstract syncAdmitted( + params?: MemorySyncParams, + options?: { allowEmbeddingBootstrapFallback?: boolean; queuedSessionOwner?: boolean }, + ): Promise; + + protected applyProviderResult(providerResult: EmbeddingProviderResult): void { + const providerState = resolveMemoryProviderState(providerResult); + this.provider = providerState.provider; + this.fallbackFrom = providerState.fallbackFrom; + this.fallbackReason = providerState.fallbackReason; + this.providerUnavailableReason = providerState.providerUnavailableReason; + this.providerLifecycle = providerState.lifecycle; + this.providerRuntime = providerState.providerRuntime; + this.providerInitialized = true; + } + + protected markEmbeddingBootstrapFailure( + err: unknown, + options?: { retainProvider?: boolean; provider?: string }, + ): MemoryEmbeddingBootstrapDebug { + const rawErrorName = readErrorName(err).trim(); + const errorName = /^[A-Za-z][A-Za-z0-9_.-]{0,63}$/.test(rawErrorName) ? rawErrorName : ""; + const message = + redactSensitiveText(formatErrorMessage(err), { mode: "tools" }).trim() || + "embedding provider initialization failed"; + const reason = redactSensitiveText( + errorName && errorName !== "Error" ? `${errorName}: ${message}` : message, + { mode: "tools" }, + ); + // settings.provider is already resolved from "auto"; never trust an unknown + // error object's provider-shaped field for public diagnostics. + const provider = options?.provider ?? this.provider?.id ?? this.settings.provider; + const debug: MemoryEmbeddingBootstrapDebug = { + ok: false, + provider, + reason, + degradedTo: "keyword-only", + }; + if (!options?.retainProvider) { + this.provider = null; + this.providerRuntime = undefined; + } + this.providerInitialized = true; + this.providerUnavailableReason = reason; + this.providerLifecycle = createDegradedMemoryProviderLifecycle({ + providerId: provider, + reason, + }); + this.embeddingBootstrapFailure = debug; + this.providerKey = this.computeProviderKey(); + this.batch = this.resolveBatchConfig(); + this.vector.semanticAvailable = false; + this.cacheProbeResult({ ok: false, error: reason }); + return debug; + } + + protected async ensureEmbeddingProviderForSearch( + onDebug?: (debug: MemorySearchRuntimeDebug) => void, + ): Promise { + const failure = this.embeddingBootstrapFailure; + if (failure) { + const cached = this.getCachedEmbeddingAvailability(); + if (cached?.ok === false) { + onDebug?.({ backend: "builtin", embeddingBootstrap: failure }); + return true; + } + } + try { + await this.ensureProviderInitialized(); + } catch (err) { + if (this.providerRequirement.mode !== "optional") { + throw err; + } + const nextFailure = this.markEmbeddingBootstrapFailure(err); + onDebug?.({ backend: "builtin", embeddingBootstrap: nextFailure }); + return true; + } + if (!failure) { + return false; + } + if (!this.provider) { + const nextFailure: MemoryEmbeddingBootstrapDebug = { + ...failure, + reason: this.providerUnavailableReason ?? failure.reason, + }; + this.embeddingBootstrapFailure = nextFailure; + this.cacheProbeResult({ ok: false, error: nextFailure.reason }); + onDebug?.({ backend: "builtin", embeddingBootstrap: nextFailure }); + return true; + } + + const currentIdentity = this.refreshIndexIdentityDirty({ providerKeyKnown: true }); + let activeFailure = failure; + if (currentIdentity.status !== "valid") { + try { + await this.syncAdmitted({ reason: "search", force: true }); + } catch (err) { + const message = redactSensitiveText(formatErrorMessage(err), { mode: "tools" }); + log.warn(`memory sync failed (embedding-bootstrap-recovery): ${message}`); + activeFailure = this.markEmbeddingBootstrapFailure(err, { retainProvider: true }); + } + } + if ( + this.refreshIndexIdentityDirty({ providerKeyKnown: true }).status === "valid" && + (await this.confirmEmbeddingBootstrapRecovery()) + ) { + // A valid existing index skips recovery reindex, so explicitly restore the + // semantic readiness flag cleared when bootstrap degradation began. + this.vector.semanticAvailable = await this.probeVectorStoreAvailabilityAdmitted(); + this.clearEmbeddingBootstrapFailureAfterRecovery(); + return false; + } + activeFailure = this.embeddingBootstrapFailure ?? activeFailure; + onDebug?.({ backend: "builtin", embeddingBootstrap: activeFailure }); + return true; + } + + protected clearEmbeddingBootstrapFailureAfterRecovery(): void { + this.embeddingBootstrapFailure = undefined; + this.providerUnavailableReason = undefined; + if (this.provider) { + this.providerLifecycle = this.fallbackFrom + ? { + mode: "fallback-active", + providerId: this.provider.id, + fallbackFrom: this.fallbackFrom, + reason: this.fallbackReason ?? "fallback activated", + } + : { mode: "active", providerId: this.provider.id }; + } + EMBEDDING_PROBE_CACHE.delete(this.cacheKey); + } + + protected async confirmEmbeddingBootstrapRecovery(): Promise { + const cached = this.getCachedEmbeddingAvailability(); + if (cached) { + return cached.ok; + } + if (!this.provider) { + return false; + } + try { + await this.embedBatchWithRetry(["ping"]); + this.cacheProbeResult({ ok: true }); + return true; + } catch (err) { + this.markEmbeddingBootstrapFailure(err, { + retainProvider: true, + provider: this.provider.id, + }); + return false; + } + } + + protected async ensureProviderInitialized(): Promise { + if (this.providerInitialized) { + const bootstrapRetryDue = + this.embeddingBootstrapFailure !== undefined && + !this.provider && + this.getCachedEmbeddingAvailability() === null; + if (!bootstrapRetryDue) { + await this.getPendingFallbackProviderInitialization()?.catch(() => undefined); + return; + } + this.resetProviderInitializationForRetry(); + } + if (this.settings.provider === "none") { + this.applyProviderResult({ + provider: null, + requestedProvider: "none", + providerUnavailableReason: "No embedding provider available (FTS-only mode)", + }); + this.providerKey = this.computeProviderKey(); + this.batch = this.resolveBatchConfig(); + return; + } + if (!this.providerInitPromise) { + this.providerInitPromise = (async () => { + await this.getPendingFallbackProviderInitialization()?.catch(() => undefined); + await this.retireCurrentProvider(); + if (this.closed) { + return; + } + const providerResult = await createEmbeddingProvider({ + config: this.cfg, + agentDir: resolveAgentDir(this.cfg, this.agentId), + ...(this.acquireLocalService ? { acquireLocalService: this.acquireLocalService } : {}), + ...resolveMemoryPrimaryProviderRequest({ settings: this.settings }), + }); + this.applyProviderResult(providerResult); + this.providerKey = this.computeProviderKey(); + this.batch = this.resolveBatchConfig(); + })(); + } + try { + await this.providerInitPromise; + } catch (err) { + // Clear the cached rejected promise so subsequent calls can retry + // initialization instead of being permanently stuck with a stale failure. + this.providerInitPromise = null; + throw err; + } finally { + if (this.providerInitialized) { + this.providerInitPromise = null; + } + } + } + + protected resetProviderInitializationForRetry(): void { + void this.retireCurrentProvider(); + this.providerInitialized = false; + this.providerInitPromise = null; + this.providerUnavailableReason = undefined; + this.providerLifecycle = createPendingMemoryProviderLifecycle(this.requestedProvider); + } + + protected markLocalEmbeddingProviderDegraded(err: unknown): void { + if (this.provider?.id !== "local") { + return; + } + const workerFailure = isLocalEmbeddingWorkerFailure(err) + ? err + : err instanceof Error && isLocalEmbeddingWorkerFailure(err.cause) + ? err.cause + : null; + if (!workerFailure) { + return; + } + const message = formatErrorMessage(workerFailure); + const degradedProvider = this.provider; + void this.retireCurrentProvider(); + this.providerUnavailableReason = `Local embeddings degraded: ${message}`; + this.providerLifecycle = createDegradedMemoryProviderLifecycle({ + providerId: degradedProvider.id, + reason: message, + code: workerFailure.code, + }); + EMBEDDING_PROBE_CACHE.delete(this.cacheKey); + this.providerKey = this.computeProviderKey(); + this.batch = this.resolveBatchConfig(); + this.vector.semanticAvailable = false; + log.warn("memory embeddings: local provider degraded after worker failure", { + error: message, + }); + } + + protected override retireCurrentProvider(): Promise { + const provider = this.provider; + if (provider) { + this.provider = null; + this.providerRuntime = undefined; + this.providersPendingRetirement.add(provider); + } + if (this.providersPendingRetirement.size === 0) { + return this.providerRetirementPromise; + } + // Provider replacement must wait for the previous worker to exit; otherwise + // repeated retries can accumulate local workers on constrained hosts. + const retirement = this.providerRetirementPromise + .catch(() => {}) + .then(async () => { + let firstError: unknown; + let closeFailed = false; + for (const pendingProvider of this.providersPendingRetirement) { + try { + await this.awaitProviderIdle(pendingProvider); + await pendingProvider.close?.(); + this.providersPendingRetirement.delete(pendingProvider); + } catch (err) { + if (!closeFailed) { + firstError = err; + } + closeFailed = true; + } + } + if (closeFailed) { + throw toErrorObject(firstError, "Embedding provider retirement failed"); + } + }); + this.providerRetirementPromise = retirement; + void retirement.catch((err: unknown) => { + log.warn(`memory embeddings: failed to close previous provider: ${formatErrorMessage(err)}`); + }); + return retirement; + } + + protected async drainPendingProviderRetirements(): Promise { + const errors: unknown[] = []; + for ( + let attempt = 0; + attempt < 2 && (this.provider !== null || this.providersPendingRetirement.size > 0); + attempt += 1 + ) { + try { + await this.retireCurrentProvider(); + } catch (err) { + errors.push(err); + log.warn(`memory close: pending manager work failed: ${formatErrorMessage(err)}`); + } + } + return errors; + } + + protected isRequiredProviderUnavailable(): boolean { + return this.providerRequirement.mode === "required" && !this.provider; + } + + protected buildRequiredProviderUnavailableError(operation: "search" | "sync"): Error { + const registeredProviderIds = listRegisteredMemoryEmbeddingProviderAdapters() + .map((adapter) => adapter.id) + .toSorted(); + const registeredProviders = + registeredProviderIds.length > 0 ? registeredProviderIds.join(",") : "none"; + const reason = + this.providerUnavailableReason ?? + (this.providerLifecycle.mode === "fts-only" + ? this.providerLifecycle.reason + : "provider is unavailable"); + return new Error( + `Memory ${operation} unavailable: embedding provider "${this.settings.provider}" is configured but unavailable. ` + + `Reason: ${reason}. ` + + `agentId=${this.agentId} purpose=${this.purpose} lifecycle=${JSON.stringify(this.providerLifecycle)} ` + + `registeredMemoryEmbeddingProviders=${registeredProviders}`, + ); + } + + protected assertRequiredProviderAvailable(operation: "search" | "sync"): void { + if (this.isRequiredProviderUnavailable()) { + const error = this.buildRequiredProviderUnavailableError(operation); + this.resetProviderInitializationForRetry(); + throw error; + } + } + + protected refreshIndexIdentityDirty(params?: { providerKeyKnown?: boolean }) { + const provider = + this.settings.provider === "none" + ? null + : this.providerInitialized + ? this.provider + ? { id: this.provider.id, model: this.provider.model } + : null + : undefined; + const state = this.resolveCurrentIndexIdentityState({ + ...(provider !== undefined ? { provider } : {}), + providerKeyKnown: params?.providerKeyKnown, + }); + this.indexIdentityState = state; + this.indexIdentityDirty = + state.status === "mismatched" || + (state.status === "missing" && (this.sources.has("memory") || this.hasIndexedChunks())); + return state; + } + + protected refreshKeywordFallbackIndexIdentity() { + const meta = this.readMeta(); + const state = this.resolveCurrentIndexIdentityState({ + meta, + provider: meta && meta.provider !== "none" ? { id: meta.provider, model: meta.model } : null, + providerKeyKnown: false, + vectorReady: false, + }); + this.indexIdentityState = state; + this.indexIdentityDirty = + state.status === "mismatched" || + (state.status === "missing" && (this.sources.has("memory") || this.hasIndexedChunks())); + return state; + } + + protected async withManagerOperation(run: () => Promise): Promise { + if (this.closing || this.closed) { + throw new Error("Memory index manager is closed"); + } + this.activeManagerOperations += 1; + try { + return await run(); + } finally { + this.activeManagerOperations -= 1; + if (this.activeManagerOperations === 0) { + const waiters = Array.from(this.managerIdleWaiters); + this.managerIdleWaiters.clear(); + for (const resolve of waiters) { + resolve(); + } + } + } + } + + protected async awaitManagerIdle(): Promise { + if (this.activeManagerOperations === 0) { + return; + } + await new Promise((resolve) => { + this.managerIdleWaiters.add(resolve); + }); + } + + async probeVectorAvailability(): Promise { + return await this.withManagerOperation(async () => { + if (!this.vector.enabled) { + this.vector.semanticAvailable = false; + return false; + } + await this.ensureProviderInitialized(); + // FTS-only mode: vector search not available + if (!this.provider) { + this.vector.semanticAvailable = false; + return false; + } + const ready = await this.probeVectorStoreAvailabilityAdmitted(); + this.vector.semanticAvailable = ready; + return ready; + }); + } + + async probeVectorStoreAvailability(): Promise { + return await this.withManagerOperation( + async () => await this.probeVectorStoreAvailabilityAdmitted(), + ); + } + + private async probeVectorStoreAvailabilityAdmitted(): Promise { + if (!this.vector.enabled) { + this.vector.available = false; + return false; + } + return await this.ensureVectorReady(); + } + + protected cacheProbeResult(result: MemoryEmbeddingProbeResult): MemoryEmbeddingProbeResult { + const checkedAtMs = Date.now(); + EMBEDDING_PROBE_CACHE.set(this.cacheKey, { + result, + checkedAtMs, + expireAtMs: checkedAtMs + EMBEDDING_PROBE_CACHE_TTL_MS, + }); + return result; + } + + getCachedEmbeddingAvailability(): MemoryEmbeddingProbeResult | null { + const cached = EMBEDDING_PROBE_CACHE.get(this.cacheKey); + if (!cached) { + return null; + } + const nowMs = Date.now(); + if (nowMs >= cached.expireAtMs) { + EMBEDDING_PROBE_CACHE.delete(this.cacheKey); + return null; + } + return { + ...cached.result, + checked: true, + cached: true, + checkedAtMs: cached.checkedAtMs, + cacheExpiresAtMs: cached.expireAtMs, + }; + } + + async probeEmbeddingAvailability(): Promise { + return await this.withManagerOperation(async () => { + const cached = this.getCachedEmbeddingAvailability(); + if (cached) { + return cached; + } + await this.ensureProviderInitialized(); + // FTS-only mode: embeddings not available but search still works + if (!this.provider) { + return this.cacheProbeResult({ + ok: false, + error: + this.providerUnavailableReason ?? "No embedding provider available (FTS-only mode)", + }); + } + try { + await this.embedBatchWithRetry(["ping"]); + return this.cacheProbeResult({ ok: true }); + } catch (err) { + const message = formatErrorMessage(err); + return this.cacheProbeResult({ ok: false, error: message }); + } + }); + } +} diff --git a/extensions/memory-core/src/memory/manager.mistral-provider.test.ts b/extensions/memory-core/src/memory/manager-provider-state.test.ts similarity index 97% rename from extensions/memory-core/src/memory/manager.mistral-provider.test.ts rename to extensions/memory-core/src/memory/manager-provider-state.test.ts index 009d14023d5d..cded70b7dcef 100644 --- a/extensions/memory-core/src/memory/manager.mistral-provider.test.ts +++ b/extensions/memory-core/src/memory/manager-provider-state.test.ts @@ -172,10 +172,10 @@ describe("memory manager mistral provider wiring", () => { }; const remote = { baseUrl: "https://primary-openai.invalid/v1", - apiKey: "synthetic-primary-openai-api-key", + apiKey: "test-key", headers: { - Authorization: "Bearer synthetic-primary-openai-auth", - "X-OpenAI-Secret": "synthetic-primary-openai-header", + Authorization: "Bearer test-secret", + "X-OpenAI-Secret": "test-token", }, ...sharedRemote, }; diff --git a/extensions/memory-core/src/memory/manager-registry.test.ts b/extensions/memory-core/src/memory/manager-registry.test.ts new file mode 100644 index 000000000000..a1dfdd602c8a --- /dev/null +++ b/extensions/memory-core/src/memory/manager-registry.test.ts @@ -0,0 +1,809 @@ +// Memory Core tests cover manager registry behavior. +import { mkdirSync, rmSync } from "node:fs"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { clearMemoryEmbeddingProviders as clearRegistry } from "openclaw/plugin-sdk/memory-core-host-engine-embeddings"; +import { + closeOpenClawAgentDatabasesForTest, + closeOpenClawStateDatabaseForTest, +} from "openclaw/plugin-sdk/sqlite-runtime-testing"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { + configureMemoryCoreDreamingStateForTests, + resetMemoryCoreDreamingStateForTests, +} from "../test-helpers.js"; +import "./test-runtime-mocks.js"; +import { closeAllMemorySearchManagers, getMemorySearchManager } from "./index.js"; +import type { MemoryIndexManager } from "./manager.js"; +import { + closeAllMemoryIndexManagers, + closeMemoryIndexManagersForAgent, + MemoryIndexManager as RuntimeMemoryIndexManager, +} from "./manager.js"; +import { isolateMemoryManagerTestConfig } from "./test-config-helpers.js"; + +// This suite performs real sqlite/media indexing and can exceed the global +// timeout when it shares a packed CI extension shard. +vi.setConfig({ testTimeout: 240_000 }); + +afterAll(() => { + vi.resetConfig(); +}); + +let embedBatchCalls = 0; +let embeddedBatchTexts: string[] = []; +let embedBatchInputCalls = 0; +let providerRuntimeBatchCalls: string[][] = []; +let providerRuntimeBatchGate: Promise | null = null; +let providerRuntimeBatchErrors: unknown[] = []; +let providerRuntimeBatchFailuresRemaining = 0; +let providerRuntimeActiveBatchCalls = 0; +let providerRuntimeMaxActiveBatchCalls = 0; +let providerCloseCalls = 0; +let providerCloseFailuresRemaining = 0; +let providerCloseFailure: unknown = new Error("provider close failed"); +let providerCreationFailure: string | null = null; +let providerNullResult: string | null = null; +let providerCloseGate: Promise | null = null; +let providerInitGate: Promise | null = null; +let providerCalls: Array<{ provider?: string; model?: string; outputDimensionality?: number }> = []; +let forceNoProvider = false; + +const originalMemoryIndexStateDir = process.env.OPENCLAW_STATE_DIR; + +const identityAliasFixture = vi.hoisted(() => ({ + provider: "identity-alias-test", + canonicalModel: "hf:fixture/default-model.gguf", + cacheModel: "/fixture/cache/default-model.gguf", +})); + +function setMemoryIndexStateDir(stateDir: string): void { + Reflect.set(process.env, "OPENCLAW_STATE_DIR", stateDir); +} + +function restoreMemoryIndexStateDir(): void { + if (originalMemoryIndexStateDir === undefined) { + Reflect.deleteProperty(process.env, "OPENCLAW_STATE_DIR"); + } else { + Reflect.set(process.env, "OPENCLAW_STATE_DIR", originalMemoryIndexStateDir); + } +} + +vi.mock("./embeddings.js", async (importOriginal) => { + const actual = await importOriginal(); + const embedText = (text: string) => { + const lower = text.toLowerCase(); + const alpha = lower.split("alpha").length - 1; + const beta = lower.split("beta").length - 1; + const image = lower.split("image").length - 1; + const audio = lower.split("audio").length - 1; + return [alpha, beta, image, audio]; + }; + return { + ...actual, + resolveEmbeddingProviderFallbackModel: (providerId: string, fallbackSourceModel: string) => + providerId === "gemini" || providerId === "fallback-provider" + ? `${providerId}-embed` + : fallbackSourceModel, + resolveEmbeddingProviderAdapterId: ( + providerId: string, + config?: { + models?: { + providers?: Record; + }; + }, + ) => config?.models?.providers?.[providerId]?.api ?? providerId, + resolveEmbeddingProviderAdapterTransport: (providerId: string) => + providerId === "local" ? "local" : "remote", + resolveEmbeddingProviderIndexIdentity: (options: { provider?: string; model?: string }) => + options.provider === identityAliasFixture.provider + ? { + provider: { + id: identityAliasFixture.provider, + model: identityAliasFixture.canonicalModel, + }, + cacheKeyData: { + provider: identityAliasFixture.provider, + model: identityAliasFixture.canonicalModel, + }, + aliases: [ + { + model: identityAliasFixture.cacheModel, + cacheKeyData: { + provider: identityAliasFixture.provider, + model: identityAliasFixture.cacheModel, + }, + }, + ], + } + : undefined, + createEmbeddingProvider: async (options: { + provider?: string; + model?: string; + outputDimensionality?: number; + }) => { + providerCalls.push({ + provider: options.provider, + model: options.model, + outputDimensionality: options.outputDimensionality, + }); + await providerInitGate; + if (options.provider === providerCreationFailure) { + throw new Error(`provider creation failed: ${options.provider}`); + } + if (options.provider === providerNullResult) { + return { + provider: null, + requestedProvider: options.provider, + providerUnavailableReason: `provider unavailable: ${options.provider}`, + }; + } + if (forceNoProvider) { + return { + provider: null, + requestedProvider: options.provider ?? "auto", + providerUnavailableReason: "No API key found for provider", + }; + } + const providerId = + options.provider === "gemini" || + options.provider === "fallback-provider" || + options.provider === "batch-test" || + options.provider === "batch-wide-test" || + options.provider === identityAliasFixture.provider || + options.provider === "ollama" + ? options.provider + : "mock"; + const requestedModel = options.model ?? "mock-embed"; + const model = + providerId === identityAliasFixture.provider && + (requestedModel === identityAliasFixture.canonicalModel || + requestedModel === identityAliasFixture.cacheModel) + ? identityAliasFixture.canonicalModel + : requestedModel; + return { + requestedProvider: options.provider ?? "openai", + provider: { + id: providerId, + model, + close: async () => { + providerCloseCalls += 1; + await providerCloseGate; + if (providerCloseFailuresRemaining > 0) { + providerCloseFailuresRemaining -= 1; + throw providerCloseFailure; + } + }, + embedQuery: async (text: string) => embedText(text), + embedBatch: async (texts: string[]) => { + embedBatchCalls += 1; + embeddedBatchTexts.push(...texts); + return texts.map(embedText); + }, + ...(providerId === "gemini" || providerId === "fallback-provider" + ? { + embedBatchInputs: async ( + inputs: Array<{ + text: string; + parts?: Array< + | { type: "text"; text: string } + | { type: "inline-data"; mimeType: string; data: string } + >; + }>, + ) => { + embedBatchInputCalls += 1; + return inputs.map((input) => { + const inlineData = input.parts?.find((part) => part.type === "inline-data"); + if (inlineData?.type === "inline-data" && inlineData.data.length > 9000) { + throw new Error("payload too large"); + } + const mimeType = + inlineData?.type === "inline-data" ? inlineData.mimeType : undefined; + if (mimeType?.startsWith("image/")) { + return [0, 0, 1, 0]; + } + if (mimeType?.startsWith("audio/")) { + return [0, 0, 0, 1]; + } + return embedText(input.text); + }); + }, + } + : {}), + }, + ...(providerId === identityAliasFixture.provider + ? { + runtime: { + id: providerId, + cacheKeyData: { + provider: providerId, + model: identityAliasFixture.canonicalModel, + }, + indexIdentityAliases: [ + { + model: identityAliasFixture.cacheModel, + cacheKeyData: { + provider: providerId, + model: identityAliasFixture.cacheModel, + }, + }, + ], + }, + } + : providerId === "batch-test" || providerId === "batch-wide-test" + ? { + runtime: { + id: providerId, + ...(providerId === "batch-wide-test" ? { sourceWideBatchEmbed: true } : {}), + batchEmbed: async (batch: { chunks: Array<{ text: string }> }) => { + providerRuntimeActiveBatchCalls += 1; + providerRuntimeMaxActiveBatchCalls = Math.max( + providerRuntimeMaxActiveBatchCalls, + providerRuntimeActiveBatchCalls, + ); + try { + await providerRuntimeBatchGate; + providerRuntimeBatchCalls.push(batch.chunks.map((chunk) => chunk.text)); + if (providerRuntimeBatchErrors.length > 0) { + throw providerRuntimeBatchErrors.shift(); + } + if (providerRuntimeBatchFailuresRemaining > 0) { + providerRuntimeBatchFailuresRemaining -= 1; + throw new Error("provider runtime batch failed"); + } + return batch.chunks.map((chunk) => embedText(chunk.text)); + } finally { + providerRuntimeActiveBatchCalls -= 1; + } + }, + }, + } + : providerId === "gemini" || providerId === "fallback-provider" + ? { + runtime: { + id: providerId, + cacheKeyData: { + provider: providerId, + baseUrl: "https://generativelanguage.googleapis.com/v1beta", + model, + outputDimensionality: options.outputDimensionality, + headers: [], + }, + }, + } + : {}), + }; + }, + }; +}); + +describe("memory index", () => { + let fixtureRoot = ""; + let workspaceDir = ""; + let memoryDir = ""; + + const managersForCleanup = new Set(); + + beforeAll(async () => { + fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-mem-fixtures-")); + workspaceDir = path.join(fixtureRoot, "workspace"); + memoryDir = path.join(workspaceDir, "memory"); + }); + + afterAll(async () => { + await Promise.all(Array.from(managersForCleanup).map((manager) => manager.close())); + await fs.rm(fixtureRoot, { recursive: true, force: true }); + }); + + afterEach(async () => { + vi.useRealTimers(); + await Promise.all(Array.from(managersForCleanup).map((manager) => manager.close())); + await closeAllMemorySearchManagers(); + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + resetMemoryCoreDreamingStateForTests(); + clearRegistry(); + managersForCleanup.clear(); + restoreMemoryIndexStateDir(); + }); + + beforeEach(async () => { + vi.useRealTimers(); + clearRegistry(); + embedBatchCalls = 0; + embeddedBatchTexts = []; + embedBatchInputCalls = 0; + providerRuntimeBatchCalls = []; + providerRuntimeBatchGate = null; + providerRuntimeBatchErrors = []; + providerRuntimeBatchFailuresRemaining = 0; + providerRuntimeActiveBatchCalls = 0; + providerRuntimeMaxActiveBatchCalls = 0; + providerCloseCalls = 0; + providerCloseFailuresRemaining = 0; + providerCloseFailure = new Error("provider close failed"); + providerCreationFailure = null; + providerNullResult = null; + providerCloseGate = null; + providerInitGate = null; + providerCalls = []; + forceNoProvider = false; + + rmSync(workspaceDir, { recursive: true, force: true }); + mkdirSync(memoryDir, { recursive: true }); + setMemoryIndexStateDir(path.join(workspaceDir, ".state-memory-index")); + await configureMemoryCoreDreamingStateForTests(); + await fs.writeFile( + path.join(memoryDir, "2026-01-12.md"), + "# Log\nAlpha memory line.\nZebra memory line.", + ); + }); + + type TestCfg = Parameters[0]["cfg"]; + + function createCfg(params: { + extraPaths?: string[]; + sources?: Array<"memory" | "sessions">; + sessionMemory?: boolean; + rememberAcrossConversations?: boolean; + provider?: string; + fallback?: "none" | "gemini" | "fallback-provider"; + providerAliases?: NonNullable["providers"]>; + batchEnabled?: boolean; + model?: string; + outputDimensionality?: number; + multimodal?: { + enabled?: boolean; + modalities?: Array<"image" | "audio" | "all">; + maxFileBytes?: number; + }; + vectorEnabled?: boolean; + cacheEnabled?: boolean; + minScore?: number; + onSearch?: boolean; + hybrid?: { + enabled: boolean; + vectorWeight?: number; + textWeight?: number; + temporalDecay?: { enabled: boolean }; + }; + }): TestCfg { + return isolateMemoryManagerTestConfig({ + memory: { + search: { + ...(params.provider !== undefined ? { provider: params.provider } : {}), + model: params.model ?? "mock-embed", + fallback: params.fallback, + outputDimensionality: params.outputDimensionality, + store: { + vector: params.vectorEnabled !== undefined ? { enabled: params.vectorEnabled } : {}, + }, + remote: params.batchEnabled + ? { + batch: { enabled: true }, + } + : undefined, + query: { minScore: params.minScore ?? 0 }, + cache: params.cacheEnabled ? { enabled: true } : undefined, + extraPaths: params.extraPaths, + multimodal: params.multimodal, + sources: params.sources, + rememberAcrossConversations: + params.rememberAcrossConversations ?? params.sessionMemory ?? false, + }, + }, + + agents: { + defaults: { + workspace: workspaceDir, + }, + list: [{ id: "main", default: true }], + }, + models: params.providerAliases ? { providers: params.providerAliases } : undefined, + }); + } + + function requireManager( + result: Awaited>, + missingMessage = "manager missing", + ): MemoryIndexManager { + if (!result.manager) { + throw new Error(missingMessage); + } + return result.manager as unknown as MemoryIndexManager; + } + + async function getFreshManager( + cfg: TestCfg, + purpose?: "default" | "status" | "cli", + ): Promise { + const manager = requireManager(await getMemorySearchManager({ cfg, agentId: "main", purpose })); + managersForCleanup.add(manager); + return manager; + } + + it("waits for scoped manager close before initializing a replacement", async () => { + let releaseProviderClose: () => void = () => {}; + providerCloseGate = new Promise((resolve) => { + releaseProviderClose = resolve; + }); + const cfg = createCfg({ + hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, + }); + const first = requireManager(await getMemorySearchManager({ cfg, agentId: "main" })); + managersForCleanup.add(first); + await first.probeEmbeddingAvailability(); + const closePromise = closeMemoryIndexManagersForAgent({ agentId: "main" }); + const callsBeforeReplacement = providerCalls.length; + const secondPromise = getMemorySearchManager({ cfg, agentId: "main" }).then((result) => + requireManager(result), + ); + const concurrentSecondPromise = getMemorySearchManager({ cfg, agentId: "main" }).then( + (result) => requireManager(result), + ); + const secondProbe = secondPromise.then(async (manager) => { + await manager.probeEmbeddingAvailability(); + }); + let secondSettled = false; + void secondPromise.then( + () => { + secondSettled = true; + }, + () => { + secondSettled = true; + }, + ); + try { + await vi.waitFor(() => { + expect(providerCloseCalls).toBe(1); + }); + await Promise.resolve(); + expect(secondSettled).toBe(false); + expect(providerCalls).toHaveLength(callsBeforeReplacement); + } finally { + releaseProviderClose(); + providerCloseGate = null; + } + await closePromise; + const second = await secondPromise; + const concurrentSecond = await concurrentSecondPromise; + await secondProbe; + managersForCleanup.add(second); + expect(second === first).toBe(false); + expect(concurrentSecond).toBe(second); + + const third = requireManager(await getMemorySearchManager({ cfg, agentId: "main" })); + managersForCleanup.add(third); + expect(third).toBe(second); + }); + + it("does not reuse a cached manager after direct close starts", async () => { + let releaseProviderClose: () => void = () => {}; + providerCloseGate = new Promise((resolve) => { + releaseProviderClose = resolve; + }); + const cfg = createCfg({ + hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, + }); + const first = requireManager(await getMemorySearchManager({ cfg, agentId: "main" })); + managersForCleanup.add(first); + await first.probeEmbeddingAvailability(); + + const closePromise = first.close(); + const replacementPromise = getMemorySearchManager({ cfg, agentId: "main" }).then((result) => + requireManager(result), + ); + let replacementSettled = false; + void replacementPromise.then( + () => { + replacementSettled = true; + }, + () => { + replacementSettled = true; + }, + ); + try { + await vi.waitFor(() => expect(providerCloseCalls).toBe(1)); + await Promise.resolve(); + expect(replacementSettled).toBe(false); + } finally { + releaseProviderClose(); + providerCloseGate = null; + } + + await closePromise; + const replacement = await replacementPromise; + managersForCleanup.add(replacement); + expect(replacement === first).toBe(false); + }); + + it("serializes concurrent acquisitions with different cache identities", async () => { + const firstCfg = createCfg({ + model: "first-model", + hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, + }); + const first = requireManager(await getMemorySearchManager({ cfg: firstCfg, agentId: "main" })); + managersForCleanup.add(first); + await first.probeEmbeddingAvailability(); + let releaseProviderClose: () => void = () => {}; + providerCloseGate = new Promise((resolve) => { + releaseProviderClose = resolve; + }); + + const secondPromise = getMemorySearchManager({ + cfg: createCfg({ model: "second-model" }), + agentId: "main", + }).then((result) => requireManager(result)); + await vi.waitFor(() => expect(providerCloseCalls).toBe(1)); + const thirdPromise = getMemorySearchManager({ + cfg: createCfg({ model: "third-model" }), + agentId: "main", + }).then((result) => requireManager(result)); + try { + await Promise.resolve(); + expect(providerCalls).toHaveLength(1); + } finally { + releaseProviderClose(); + providerCloseGate = null; + } + + const [second, third] = await Promise.all([secondPromise, thirdPromise]); + managersForCleanup.add(second); + managersForCleanup.add(third); + expect(second === first).toBe(false); + expect(third === second).toBe(false); + expect((second as unknown as { closed: boolean }).closed).toBe(true); + expect((third as unknown as { closed: boolean }).closed).toBe(false); + }); + + it("canonicalizes agent ids before builtin manager acquisition", async () => { + const cfg = createCfg({ model: "canonical-model" }); + const first = await RuntimeMemoryIndexManager.get({ cfg, agentId: "Main-Agent" }); + const second = await RuntimeMemoryIndexManager.get({ cfg, agentId: "main-agent" }); + if (!first || !second) { + throw new Error("Expected canonical memory index managers"); + } + managersForCleanup.add(first); + managersForCleanup.add(second); + expect(second).toBe(first); + }); + + it("retires the prior builtin manager when an agent workspace changes", async () => { + const firstCfg = createCfg({ model: "workspace-model" }); + const secondCfg = createCfg({ model: "workspace-model" }); + if (!firstCfg.agents?.defaults || !secondCfg.agents?.defaults) { + throw new Error("Expected agent defaults"); + } + firstCfg.agents.defaults.workspace = path.join(fixtureRoot, "workspace-a"); + secondCfg.agents.defaults.workspace = path.join(fixtureRoot, "workspace-b"); + + const first = await RuntimeMemoryIndexManager.get({ cfg: firstCfg, agentId: "main" }); + const second = await RuntimeMemoryIndexManager.get({ cfg: secondCfg, agentId: "main" }); + if (!first || !second) { + throw new Error("Expected workspace memory index managers"); + } + managersForCleanup.add(first); + managersForCleanup.add(second); + expect(second === first).toBe(false); + expect((first as unknown as { closed: boolean }).closed).toBe(true); + }); + + it("does not block another agent while one scope retires its manager", async () => { + const firstCfg = createCfg({ + model: "first-model", + hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, + }); + const first = requireManager(await getMemorySearchManager({ cfg: firstCfg, agentId: "main" })); + managersForCleanup.add(first); + await first.probeEmbeddingAvailability(); + let releaseProviderClose: () => void = () => {}; + providerCloseGate = new Promise((resolve) => { + releaseProviderClose = resolve; + }); + + const replacementPromise = getMemorySearchManager({ + cfg: createCfg({ model: "second-model" }), + agentId: "main", + }); + await vi.waitFor(() => expect(providerCloseCalls).toBe(1)); + const otherAgentPromise = getMemorySearchManager({ + cfg: createCfg({ model: "other-model" }), + agentId: "other", + }); + let otherAgentSettled = false; + void otherAgentPromise.then( + () => { + otherAgentSettled = true; + }, + () => { + otherAgentSettled = true; + }, + ); + try { + await vi.waitFor(() => expect(otherAgentSettled).toBe(true)); + } finally { + releaseProviderClose(); + providerCloseGate = null; + } + + const otherAgent = requireManager(await otherAgentPromise); + const replacement = requireManager(await replacementPromise); + managersForCleanup.add(otherAgent); + managersForCleanup.add(replacement); + expect((otherAgent as unknown as { closed: boolean }).closed).toBe(false); + }); + + it("global teardown waits for an admitted builtin manager replacement", async () => { + const first = await RuntimeMemoryIndexManager.get({ + cfg: createCfg({ model: "first-model" }), + agentId: "main", + }); + if (!first) { + throw new Error("Expected first memory index manager"); + } + managersForCleanup.add(first); + await first.probeEmbeddingAvailability(); + let releaseProviderClose: () => void = () => {}; + providerCloseGate = new Promise((resolve) => { + releaseProviderClose = resolve; + }); + + const replacementPromise = RuntimeMemoryIndexManager.get({ + cfg: createCfg({ model: "second-model" }), + agentId: "main", + }); + await vi.waitFor(() => expect(providerCloseCalls).toBe(1)); + const globalClosePromise = closeAllMemoryIndexManagers(); + let globalCloseSettled = false; + void globalClosePromise.then( + () => { + globalCloseSettled = true; + }, + () => { + globalCloseSettled = true; + }, + ); + try { + await Promise.resolve(); + expect(globalCloseSettled).toBe(false); + } finally { + releaseProviderClose(); + providerCloseGate = null; + } + + const replacement = await replacementPromise; + await globalClosePromise; + if (!replacement) { + throw new Error("Expected replacement memory index manager"); + } + managersForCleanup.add(replacement); + expect((replacement as unknown as { closed: boolean }).closed).toBe(true); + }); + + it("retains a failed scoped close owner until provider retirement succeeds", async () => { + const cfg = createCfg({ + hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, + }); + const first = requireManager(await getMemorySearchManager({ cfg, agentId: "main" })); + managersForCleanup.add(first); + await first.probeEmbeddingAvailability(); + providerCloseFailuresRemaining = 2; + + await expect(closeMemoryIndexManagersForAgent({ agentId: "main" })).rejects.toThrow( + "provider close failed", + ); + expect(providerCloseCalls).toBe(2); + + let releaseProviderClose: () => void = () => {}; + providerCloseGate = new Promise((resolve) => { + releaseProviderClose = resolve; + }); + const callsBeforeReplacement = providerCalls.length; + const replacementPromise = getMemorySearchManager({ cfg, agentId: "main" }).then((result) => + requireManager(result), + ); + try { + await vi.waitFor(() => expect(providerCloseCalls).toBe(3)); + expect(providerCalls).toHaveLength(callsBeforeReplacement); + } finally { + releaseProviderClose(); + providerCloseGate = null; + } + + const replacement = await replacementPromise; + managersForCleanup.add(replacement); + expect(replacement === first).toBe(false); + }); + + it("retains a failed global close owner until provider retirement succeeds", async () => { + const cfg = createCfg({ + hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, + }); + const first = requireManager(await getMemorySearchManager({ cfg, agentId: "main" })); + managersForCleanup.add(first); + await first.probeEmbeddingAvailability(); + providerCloseFailuresRemaining = 2; + providerCloseFailure = undefined; + + let globalCloseRejected = false; + await closeAllMemorySearchManagers().then( + () => {}, + () => { + globalCloseRejected = true; + }, + ); + expect(globalCloseRejected).toBe(true); + expect(providerCloseCalls).toBe(2); + + let releaseProviderClose: () => void = () => {}; + providerCloseGate = new Promise((resolve) => { + releaseProviderClose = resolve; + }); + const callsBeforeReplacement = providerCalls.length; + const replacementPromise = getMemorySearchManager({ cfg, agentId: "main" }).then((result) => + requireManager(result), + ); + let concurrentGlobalClose: Promise = Promise.resolve(); + try { + await vi.waitFor(() => expect(providerCloseCalls).toBe(3)); + expect(providerCalls).toHaveLength(callsBeforeReplacement); + concurrentGlobalClose = closeAllMemorySearchManagers(); + } finally { + releaseProviderClose(); + providerCloseGate = null; + } + + const replacement = await replacementPromise; + await concurrentGlobalClose; + managersForCleanup.add(replacement); + expect(replacement === first).toBe(false); + expect((replacement as unknown as { closed: boolean }).closed).toBe(false); + }); + + it("does not reuse memory index managers across local-service hosts", async () => { + const cfg = createCfg({}); + const firstAcquire = vi.fn(async () => undefined); + const secondAcquire = vi.fn(async () => undefined); + const first = requireManager( + await getMemorySearchManager({ + cfg, + agentId: "main", + acquireLocalService: firstAcquire, + }), + ); + managersForCleanup.add(first); + + const second = requireManager( + await getMemorySearchManager({ + cfg, + agentId: "main", + acquireLocalService: secondAcquire, + }), + ); + managersForCleanup.add(second); + const secondAgain = requireManager( + await getMemorySearchManager({ + cfg, + agentId: "main", + acquireLocalService: secondAcquire, + }), + ); + + expect(Object.is(second, first)).toBe(false); + expect(Object.is(secondAgain, second)).toBe(true); + }); + + it("retries embedding provider close before releasing the manager", async () => { + providerCloseFailuresRemaining = 1; + const cfg = createCfg({ + hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, + }); + const manager = await getFreshManager(cfg); + + await manager.probeEmbeddingAvailability(); + await manager.close(); + + expect(providerCloseCalls).toBe(2); + }); +}); diff --git a/extensions/memory-core/src/memory/manager-registry.ts b/extensions/memory-core/src/memory/manager-registry.ts new file mode 100644 index 000000000000..eec08a9abf71 --- /dev/null +++ b/extensions/memory-core/src/memory/manager-registry.ts @@ -0,0 +1,256 @@ +// Memory Core plugin module owns manager cache and close serialization. +import { toErrorObject } from "openclaw/plugin-sdk/error-runtime"; +import { + createSubsystemLogger, + resolveGlobalSingleton, + type ResolvedMemorySearchConfig, +} from "openclaw/plugin-sdk/memory-core-host-engine-foundation"; +import { normalizeAgentId } from "openclaw/plugin-sdk/routing"; +import { + resolveMemoryCoreLocalServiceHostIdentity, + type MemoryCoreAcquireLocalService, +} from "./embedding-local-service.js"; +import { getOrCreateManagedCacheEntry, resolveSingletonManagedCache } from "./manager-cache.js"; + +const MEMORY_INDEX_MANAGER_CACHE_KEY = Symbol.for("openclaw.memoryIndexManagerCache"); +const MEMORY_INDEX_MANAGER_SCOPE_CLOSES_KEY = Symbol.for("openclaw.memoryIndexManagerScopeCloses"); +const MEMORY_INDEX_MANAGER_GLOBAL_LIFECYCLE_KEY = Symbol.for( + "openclaw.memoryIndexManagerGlobalLifecycle.v3", +); +const log = createSubsystemLogger("memory"); + +export type MemoryIndexManagerPurpose = "default" | "status" | "cli"; + +type ClosableMemoryManager = { + close(): Promise; +}; + +type PreparedMemoryManager = { + key: string; + transient: boolean; + create: () => Promise | T; + reuse: (manager: T) => boolean; +}; + +type MemoryManagerRegistryCallbacks = { + prepare: () => Promise | null> | PreparedMemoryManager | null; + close: (manager: T) => Promise; +}; + +type MemoryManagerRegistryGlobalLifecycle = { + closePromise: Promise | null; + closeFailed: boolean; +}; + +export function resolveMemoryIndexManagerCacheKey(params: { + agentId: string; + workspaceDir: string; + settings: ResolvedMemorySearchConfig; + providerRequirement: unknown; + purpose: MemoryIndexManagerPurpose; + acquireLocalService?: MemoryCoreAcquireLocalService; +}): string { + return [ + params.agentId, + params.workspaceDir, + JSON.stringify(params.settings), + JSON.stringify(params.providerRequirement), + resolveMemoryCoreLocalServiceHostIdentity(params.acquireLocalService), + params.purpose, + ].join(":"); +} + +export class MemoryManagerRegistry { + private readonly cache: Map; + private readonly pending: Map>; + private readonly scopeOperations: Map>; + private readonly globalLifecycle: MemoryManagerRegistryGlobalLifecycle; + + constructor() { + const managedCache = resolveSingletonManagedCache(MEMORY_INDEX_MANAGER_CACHE_KEY); + this.cache = managedCache.cache; + this.pending = managedCache.pending; + this.scopeOperations = resolveGlobalSingleton>>( + MEMORY_INDEX_MANAGER_SCOPE_CLOSES_KEY, + () => new Map(), + ); + this.globalLifecycle = resolveGlobalSingleton( + MEMORY_INDEX_MANAGER_GLOBAL_LIFECYCLE_KEY, + () => ({ closePromise: null, closeFailed: false }), + ); + } + + async acquire( + params: { agentId: string; purpose: MemoryIndexManagerPurpose }, + callbacks: MemoryManagerRegistryCallbacks, + ): Promise { + return await this.runScopeOperation(params, async () => { + if (this.globalLifecycle.closeFailed) { + await this.retryFailedGlobalClose(callbacks.close); + } + const prepared = await callbacks.prepare(); + if (!prepared) { + return null; + } + const getOrCreate = async () => + await getOrCreateManagedCacheEntry({ + cache: this.cache, + pending: this.pending, + key: prepared.key, + bypassCache: prepared.transient, + create: prepared.create, + }); + if (prepared.transient) { + return await getOrCreate(); + } + const cachedManager = this.cache.get(prepared.key); + await this.closeScopeUnlocked( + { + agentId: params.agentId, + purpose: params.purpose, + ...(cachedManager && prepared.reuse(cachedManager) ? { exceptKey: prepared.key } : {}), + }, + callbacks.close, + ); + return await getOrCreate(); + }); + } + + async closeAll(close: (manager: T) => Promise): Promise { + await this.runGlobalClose(async () => { + try { + await this.closeAllUnlocked(close); + this.globalLifecycle.closeFailed = false; + } catch (err) { + this.globalLifecycle.closeFailed = true; + throw err; + } + }); + } + + async closeForAgent(params: { + agentId: string; + purpose: MemoryIndexManagerPurpose; + close: (manager: T) => Promise; + }): Promise { + const scope = { agentId: normalizeAgentId(params.agentId), purpose: params.purpose }; + await this.runScopeOperation(scope, async () => { + await this.closeScopeUnlocked(scope, params.close); + }); + } + + deleteIfCurrent(key: string, manager: T): void { + if (this.cache.get(key) === manager) { + this.cache.delete(key); + } + } + + private async retryFailedGlobalClose(close: (manager: T) => Promise): Promise { + try { + await this.closeAllUnlocked(close); + this.globalLifecycle.closeFailed = false; + } catch (err) { + this.globalLifecycle.closeFailed = true; + throw err; + } + } + + private async runGlobalClose(operation: () => Promise): Promise { + const previous = this.globalLifecycle.closePromise ?? Promise.resolve(); + const closePromise = previous.then(operation, operation); + this.globalLifecycle.closePromise = closePromise; + await closePromise; + if (this.globalLifecycle.closePromise === closePromise) { + this.globalLifecycle.closePromise = null; + } + } + + private async runScopeOperation( + params: { agentId: string; purpose: MemoryIndexManagerPurpose }, + operation: () => Promise, + ): Promise { + while (this.globalLifecycle.closePromise) { + const globalClose = this.globalLifecycle.closePromise; + try { + await globalClose; + } catch { + if (this.globalLifecycle.closePromise === globalClose) { + await this.closeAll(async (manager) => await manager.close()); + } + } + } + const scopeKey = JSON.stringify([params.agentId, params.purpose]); + const previousOperation = this.scopeOperations.get(scopeKey) ?? Promise.resolve(); + const result = previousOperation.then(operation, operation); + const tail = result.then( + () => undefined, + () => undefined, + ); + this.scopeOperations.set(scopeKey, tail); + try { + return await result; + } finally { + if (this.scopeOperations.get(scopeKey) === tail) { + this.scopeOperations.delete(scopeKey); + } + } + } + + private async closeAllUnlocked(close: (manager: T) => Promise): Promise { + const scopedOperations = Array.from(this.scopeOperations.values()); + if (scopedOperations.length > 0) { + await Promise.allSettled(scopedOperations); + } + const pending = Array.from(this.pending.values()); + if (pending.length > 0) { + await Promise.allSettled(pending); + } + await this.closeEntries(Array.from(this.cache.entries()), close); + } + + private async closeScopeUnlocked( + params: { + agentId: string; + purpose: MemoryIndexManagerPurpose; + exceptKey?: string; + }, + close: (manager: T) => Promise, + ): Promise { + const isScopedKey = (key: string) => + key !== params.exceptKey && + key.startsWith(`${params.agentId}:`) && + key.endsWith(`:${params.purpose}`); + const pending = Array.from(this.pending.entries()) + .filter(([key]) => isScopedKey(key)) + .map(([, value]) => value); + if (pending.length > 0) { + await Promise.allSettled(pending); + } + await this.closeEntries( + Array.from(this.cache.entries()).filter(([key]) => isScopedKey(key)), + close, + params.agentId, + ); + } + + private async closeEntries( + entries: Array<[string, T]>, + close: (manager: T) => Promise, + agentId?: string, + ): Promise { + let firstError: unknown; + for (const [key, manager] of entries) { + try { + await close(manager); + this.deleteIfCurrent(key, manager); + } catch (err) { + firstError ??= err; + const scope = agentId ? ` for agent ${agentId}` : ""; + log.warn(`failed to close memory index manager${scope}: ${String(err)}`); + } + } + if (firstError !== undefined) { + throw toErrorObject(firstError, "Failed to close memory index manager"); + } + } +} diff --git a/extensions/memory-core/src/memory/manager-search-orchestration.test.ts b/extensions/memory-core/src/memory/manager-search-orchestration.test.ts new file mode 100644 index 000000000000..47087020c10c --- /dev/null +++ b/extensions/memory-core/src/memory/manager-search-orchestration.test.ts @@ -0,0 +1,848 @@ +// Memory Core tests cover manager search orchestration behavior. +import { mkdirSync, rmSync } from "node:fs"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { clearMemoryEmbeddingProviders as clearRegistry } from "openclaw/plugin-sdk/memory-core-host-engine-embeddings"; +import { resolveSessionTranscriptsDirForAgent } from "openclaw/plugin-sdk/memory-core-host-runtime-core"; +import { upsertSessionEntry } from "openclaw/plugin-sdk/session-store-runtime"; +import { appendSessionTranscriptMessageByIdentity } from "openclaw/plugin-sdk/session-transcript-runtime"; +import { + closeOpenClawAgentDatabasesForTest, + closeOpenClawStateDatabaseForTest, +} from "openclaw/plugin-sdk/sqlite-runtime-testing"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { + configureMemoryCoreDreamingStateForTests, + resetMemoryCoreDreamingStateForTests, +} from "../test-helpers.js"; +import "./test-runtime-mocks.js"; +import { closeAllMemorySearchManagers, getMemorySearchManager } from "./index.js"; +import type { MemoryIndexManager } from "./manager.js"; +import { isolateMemoryManagerTestConfig } from "./test-config-helpers.js"; + +// This suite performs real sqlite/media indexing and can exceed the global +// timeout when it shares a packed CI extension shard. +vi.setConfig({ testTimeout: 240_000 }); + +afterAll(() => { + vi.resetConfig(); +}); + +let embedBatchCalls = 0; +let embeddedBatchTexts: string[] = []; +let embedBatchInputCalls = 0; +let providerRuntimeBatchCalls: string[][] = []; +let providerRuntimeBatchGate: Promise | null = null; +let providerRuntimeBatchErrors: unknown[] = []; +let providerRuntimeBatchFailuresRemaining = 0; +let providerRuntimeActiveBatchCalls = 0; +let providerRuntimeMaxActiveBatchCalls = 0; +let providerCloseCalls = 0; +let providerCloseFailuresRemaining = 0; +let providerCloseFailure: unknown = new Error("provider close failed"); +let providerCreationFailure: string | null = null; +let providerNullResult: string | null = null; +let providerCloseGate: Promise | null = null; +let providerInitGate: Promise | null = null; +let providerCalls: Array<{ provider?: string; model?: string; outputDimensionality?: number }> = []; +let forceNoProvider = false; + +const originalMemoryIndexStateDir = process.env.OPENCLAW_STATE_DIR; + +const identityAliasFixture = vi.hoisted(() => ({ + provider: "identity-alias-test", + canonicalModel: "hf:fixture/default-model.gguf", + cacheModel: "/fixture/cache/default-model.gguf", +})); + +function setMemoryIndexStateDir(stateDir: string): void { + Reflect.set(process.env, "OPENCLAW_STATE_DIR", stateDir); +} + +function restoreMemoryIndexStateDir(): void { + if (originalMemoryIndexStateDir === undefined) { + Reflect.deleteProperty(process.env, "OPENCLAW_STATE_DIR"); + } else { + Reflect.set(process.env, "OPENCLAW_STATE_DIR", originalMemoryIndexStateDir); + } +} + +vi.mock("./embeddings.js", async (importOriginal) => { + const actual = await importOriginal(); + const embedText = (text: string) => { + const lower = text.toLowerCase(); + const alpha = lower.split("alpha").length - 1; + const beta = lower.split("beta").length - 1; + const image = lower.split("image").length - 1; + const audio = lower.split("audio").length - 1; + return [alpha, beta, image, audio]; + }; + return { + ...actual, + resolveEmbeddingProviderFallbackModel: (providerId: string, fallbackSourceModel: string) => + providerId === "gemini" || providerId === "fallback-provider" + ? `${providerId}-embed` + : fallbackSourceModel, + resolveEmbeddingProviderAdapterId: ( + providerId: string, + config?: { + models?: { + providers?: Record; + }; + }, + ) => config?.models?.providers?.[providerId]?.api ?? providerId, + resolveEmbeddingProviderAdapterTransport: (providerId: string) => + providerId === "local" ? "local" : "remote", + resolveEmbeddingProviderIndexIdentity: (options: { provider?: string; model?: string }) => + options.provider === identityAliasFixture.provider + ? { + provider: { + id: identityAliasFixture.provider, + model: identityAliasFixture.canonicalModel, + }, + cacheKeyData: { + provider: identityAliasFixture.provider, + model: identityAliasFixture.canonicalModel, + }, + aliases: [ + { + model: identityAliasFixture.cacheModel, + cacheKeyData: { + provider: identityAliasFixture.provider, + model: identityAliasFixture.cacheModel, + }, + }, + ], + } + : undefined, + createEmbeddingProvider: async (options: { + provider?: string; + model?: string; + outputDimensionality?: number; + }) => { + providerCalls.push({ + provider: options.provider, + model: options.model, + outputDimensionality: options.outputDimensionality, + }); + await providerInitGate; + if (options.provider === providerCreationFailure) { + throw new Error(`provider creation failed: ${options.provider}`); + } + if (options.provider === providerNullResult) { + return { + provider: null, + requestedProvider: options.provider, + providerUnavailableReason: `provider unavailable: ${options.provider}`, + }; + } + if (forceNoProvider) { + return { + provider: null, + requestedProvider: options.provider ?? "auto", + providerUnavailableReason: "No API key found for provider", + }; + } + const providerId = + options.provider === "gemini" || + options.provider === "fallback-provider" || + options.provider === "batch-test" || + options.provider === "batch-wide-test" || + options.provider === identityAliasFixture.provider || + options.provider === "ollama" + ? options.provider + : "mock"; + const requestedModel = options.model ?? "mock-embed"; + const model = + providerId === identityAliasFixture.provider && + (requestedModel === identityAliasFixture.canonicalModel || + requestedModel === identityAliasFixture.cacheModel) + ? identityAliasFixture.canonicalModel + : requestedModel; + return { + requestedProvider: options.provider ?? "openai", + provider: { + id: providerId, + model, + close: async () => { + providerCloseCalls += 1; + await providerCloseGate; + if (providerCloseFailuresRemaining > 0) { + providerCloseFailuresRemaining -= 1; + throw providerCloseFailure; + } + }, + embedQuery: async (text: string) => embedText(text), + embedBatch: async (texts: string[]) => { + embedBatchCalls += 1; + embeddedBatchTexts.push(...texts); + return texts.map(embedText); + }, + ...(providerId === "gemini" || providerId === "fallback-provider" + ? { + embedBatchInputs: async ( + inputs: Array<{ + text: string; + parts?: Array< + | { type: "text"; text: string } + | { type: "inline-data"; mimeType: string; data: string } + >; + }>, + ) => { + embedBatchInputCalls += 1; + return inputs.map((input) => { + const inlineData = input.parts?.find((part) => part.type === "inline-data"); + if (inlineData?.type === "inline-data" && inlineData.data.length > 9000) { + throw new Error("payload too large"); + } + const mimeType = + inlineData?.type === "inline-data" ? inlineData.mimeType : undefined; + if (mimeType?.startsWith("image/")) { + return [0, 0, 1, 0]; + } + if (mimeType?.startsWith("audio/")) { + return [0, 0, 0, 1]; + } + return embedText(input.text); + }); + }, + } + : {}), + }, + ...(providerId === identityAliasFixture.provider + ? { + runtime: { + id: providerId, + cacheKeyData: { + provider: providerId, + model: identityAliasFixture.canonicalModel, + }, + indexIdentityAliases: [ + { + model: identityAliasFixture.cacheModel, + cacheKeyData: { + provider: providerId, + model: identityAliasFixture.cacheModel, + }, + }, + ], + }, + } + : providerId === "batch-test" || providerId === "batch-wide-test" + ? { + runtime: { + id: providerId, + ...(providerId === "batch-wide-test" ? { sourceWideBatchEmbed: true } : {}), + batchEmbed: async (batch: { chunks: Array<{ text: string }> }) => { + providerRuntimeActiveBatchCalls += 1; + providerRuntimeMaxActiveBatchCalls = Math.max( + providerRuntimeMaxActiveBatchCalls, + providerRuntimeActiveBatchCalls, + ); + try { + await providerRuntimeBatchGate; + providerRuntimeBatchCalls.push(batch.chunks.map((chunk) => chunk.text)); + if (providerRuntimeBatchErrors.length > 0) { + throw providerRuntimeBatchErrors.shift(); + } + if (providerRuntimeBatchFailuresRemaining > 0) { + providerRuntimeBatchFailuresRemaining -= 1; + throw new Error("provider runtime batch failed"); + } + return batch.chunks.map((chunk) => embedText(chunk.text)); + } finally { + providerRuntimeActiveBatchCalls -= 1; + } + }, + }, + } + : providerId === "gemini" || providerId === "fallback-provider" + ? { + runtime: { + id: providerId, + cacheKeyData: { + provider: providerId, + baseUrl: "https://generativelanguage.googleapis.com/v1beta", + model, + outputDimensionality: options.outputDimensionality, + headers: [], + }, + }, + } + : {}), + }; + }, + }; +}); + +describe("memory index", () => { + let fixtureRoot = ""; + let workspaceDir = ""; + let memoryDir = ""; + + const managersForCleanup = new Set(); + + beforeAll(async () => { + fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-mem-fixtures-")); + workspaceDir = path.join(fixtureRoot, "workspace"); + memoryDir = path.join(workspaceDir, "memory"); + }); + + afterAll(async () => { + await Promise.all(Array.from(managersForCleanup).map((manager) => manager.close())); + await fs.rm(fixtureRoot, { recursive: true, force: true }); + }); + + afterEach(async () => { + vi.useRealTimers(); + await Promise.all(Array.from(managersForCleanup).map((manager) => manager.close())); + await closeAllMemorySearchManagers(); + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + resetMemoryCoreDreamingStateForTests(); + clearRegistry(); + managersForCleanup.clear(); + restoreMemoryIndexStateDir(); + }); + + beforeEach(async () => { + vi.useRealTimers(); + clearRegistry(); + embedBatchCalls = 0; + embeddedBatchTexts = []; + embedBatchInputCalls = 0; + providerRuntimeBatchCalls = []; + providerRuntimeBatchGate = null; + providerRuntimeBatchErrors = []; + providerRuntimeBatchFailuresRemaining = 0; + providerRuntimeActiveBatchCalls = 0; + providerRuntimeMaxActiveBatchCalls = 0; + providerCloseCalls = 0; + providerCloseFailuresRemaining = 0; + providerCloseFailure = new Error("provider close failed"); + providerCreationFailure = null; + providerNullResult = null; + providerCloseGate = null; + providerInitGate = null; + providerCalls = []; + forceNoProvider = false; + + rmSync(workspaceDir, { recursive: true, force: true }); + mkdirSync(memoryDir, { recursive: true }); + setMemoryIndexStateDir(path.join(workspaceDir, ".state-memory-index")); + await configureMemoryCoreDreamingStateForTests(); + await fs.writeFile( + path.join(memoryDir, "2026-01-12.md"), + "# Log\nAlpha memory line.\nZebra memory line.", + ); + }); + + function resetManagerForTest(manager: MemoryIndexManager) { + // These tests reuse managers for performance. Clear the index + embedding + // cache to keep each test fully isolated. + const db = ( + manager as unknown as { + db: { + exec: (sql: string) => void; + prepare: (sql: string) => { get: (name: string) => { name?: string } | undefined }; + }; + } + ).db; + for (const table of [ + "memory_index_sources", + "memory_index_chunks", + "memory_embedding_cache", + "memory_index_chunks_fts", + "memory_index_chunks_vec", + ]) { + const existingTable = db + .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?") + .get(table); + if (existingTable?.name === table) { + db.exec(`DELETE FROM ${table}`); + } + } + (manager as unknown as { dirty: boolean }).dirty = true; + (manager as unknown as { sessionsDirty: boolean }).sessionsDirty = false; + (manager as unknown as { sessionsDirtyFiles: Set }).sessionsDirtyFiles.clear(); + } + + type TestCfg = Parameters[0]["cfg"]; + + function createCfg(params: { + extraPaths?: string[]; + sources?: Array<"memory" | "sessions">; + sessionMemory?: boolean; + rememberAcrossConversations?: boolean; + provider?: string; + fallback?: "none" | "gemini" | "fallback-provider"; + providerAliases?: NonNullable["providers"]>; + batchEnabled?: boolean; + model?: string; + outputDimensionality?: number; + multimodal?: { + enabled?: boolean; + modalities?: Array<"image" | "audio" | "all">; + maxFileBytes?: number; + }; + vectorEnabled?: boolean; + cacheEnabled?: boolean; + minScore?: number; + onSearch?: boolean; + hybrid?: { + enabled: boolean; + vectorWeight?: number; + textWeight?: number; + temporalDecay?: { enabled: boolean }; + }; + }): TestCfg { + return isolateMemoryManagerTestConfig({ + memory: { + search: { + ...(params.provider !== undefined ? { provider: params.provider } : {}), + model: params.model ?? "mock-embed", + fallback: params.fallback, + outputDimensionality: params.outputDimensionality, + store: { + vector: params.vectorEnabled !== undefined ? { enabled: params.vectorEnabled } : {}, + }, + remote: params.batchEnabled + ? { + batch: { enabled: true }, + } + : undefined, + query: { minScore: params.minScore ?? 0 }, + cache: params.cacheEnabled ? { enabled: true } : undefined, + extraPaths: params.extraPaths, + multimodal: params.multimodal, + sources: params.sources, + rememberAcrossConversations: + params.rememberAcrossConversations ?? params.sessionMemory ?? false, + }, + }, + + agents: { + defaults: { + workspace: workspaceDir, + }, + list: [{ id: "main", default: true }], + }, + models: params.providerAliases ? { providers: params.providerAliases } : undefined, + }); + } + + async function seedMemoryIndexSessionTranscript(params: { + messages: Array<{ + content: string; + role: "assistant" | "user"; + senderIsOwner?: boolean; + timestamp: number | string; + }>; + sessionId: string; + sessionKey?: string; + }): Promise { + const sessionsDir = resolveSessionTranscriptsDirForAgent("main"); + const storePath = path.join(sessionsDir, "sessions.json"); + const sessionKey = params.sessionKey ?? `agent:main:memory:${params.sessionId}`; + // Message timestamps are behavioral inputs; entry freshness only keeps the + // fixture out of real session-retention maintenance as wall time advances. + const updatedAt = Date.now(); + await fs.mkdir(sessionsDir, { recursive: true }); + await upsertSessionEntry({ + agentId: "main", + sessionKey, + storePath, + entry: { + sessionId: params.sessionId, + updatedAt, + }, + }); + for (const message of params.messages) { + await appendSessionTranscriptMessageByIdentity({ + agentId: "main", + sessionId: params.sessionId, + sessionKey, + storePath, + message: { + role: message.role, + timestamp: message.timestamp, + content: [{ type: "text", text: message.content }], + ...(message.senderIsOwner ? { __openclaw: { senderIsOwner: true } } : {}), + }, + }); + } + } + + function requireManager( + result: Awaited>, + missingMessage = "manager missing", + ): MemoryIndexManager { + if (!result.manager) { + throw new Error(missingMessage); + } + return result.manager as unknown as MemoryIndexManager; + } + + async function getPersistentManager(cfg: TestCfg): Promise { + const result = await getMemorySearchManager({ cfg, agentId: "main" }); + const manager = requireManager(result); + managersForCleanup.add(manager); + resetManagerForTest(manager); + return manager; + } + + async function getFreshManager( + cfg: TestCfg, + purpose?: "default" | "status" | "cli", + ): Promise { + const manager = requireManager(await getMemorySearchManager({ cfg, agentId: "main", purpose })); + managersForCleanup.add(manager); + return manager; + } + + async function expectHybridKeywordSearchFindsMemory(cfg: TestCfg) { + const manager = await getFreshManager(cfg); + try { + const status = manager.status(); + if (!status.fts?.available) { + return; + } + + await manager.sync({ reason: "test" }); + const results = await manager.search("zebra"); + expect(results.length).toBeGreaterThan(0); + expect(results[0]?.path).toContain("memory/2026-01-12.md"); + } finally { + await manager.close?.(); + } + } + + async function getFtsSessionManager(params: { + stateDirName: string; + }): Promise { + forceNoProvider = true; + setMemoryIndexStateDir(path.join(workspaceDir, params.stateDirName)); + const cfg = createCfg({ + provider: "none", + sources: ["memory", "sessions"], + sessionMemory: true, + minScore: 0, + hybrid: { enabled: true, vectorWeight: 0.7, textWeight: 0.3 }, + }); + const result = await getMemorySearchManager({ cfg, agentId: "main" }); + const manager = requireManager(result); + managersForCleanup.add(manager); + resetManagerForTest(manager); + return manager.status().fts?.available ? manager : null; + } + + it("finds keyword matches via hybrid search when query embedding is zero", async () => { + await expectHybridKeywordSearchFindsMemory( + createCfg({ + hybrid: { enabled: true, vectorWeight: 0, textWeight: 1 }, + }), + ); + }); + + it("retries transient query embedding transport failures during search", async () => { + const cfg = createCfg({ + hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, + }); + const manager = await getPersistentManager(cfg); + await manager.sync({ reason: "test" }); + + let queryCalls = 0; + ( + manager as unknown as { + provider: { + id: string; + model: string; + embedQuery: (text: string) => Promise; + embedBatch: (texts: string[]) => Promise; + close: () => Promise; + }; + waitForEmbeddingRetry: (delayMs: number, action: string) => Promise; + } + ).provider = { + id: "mock", + model: "mock-embed", + embedQuery: async () => { + queryCalls += 1; + if (queryCalls === 1) { + throw new Error("TypeError: fetch failed | other side closed"); + } + return [1, 0, 0, 0]; + }, + embedBatch: async (texts: string[]) => texts.map(() => [1, 0, 0, 0]), + close: async () => {}, + }; + ( + manager as unknown as { + waitForEmbeddingRetry: (delayMs: number, action: string) => Promise; + } + ).waitForEmbeddingRetry = async () => {}; + + const results = await manager.search("alpha"); + + expect(queryCalls).toBe(2); + expect(results.some((result) => result.path.endsWith("memory/2026-01-12.md"))).toBe(true); + }); + + it("fails search after bounded query embedding retries are exhausted", async () => { + const cfg = createCfg({ + hybrid: { enabled: true, vectorWeight: 0.5, textWeight: 0.5 }, + }); + const manager = await getPersistentManager(cfg); + await manager.sync({ reason: "test" }); + + let queryCalls = 0; + ( + manager as unknown as { + provider: { + id: string; + model: string; + embedQuery: (text: string) => Promise; + embedBatch: (texts: string[]) => Promise; + close: () => Promise; + }; + } + ).provider = { + id: "mock", + model: "mock-embed", + embedQuery: async () => { + queryCalls += 1; + throw new Error("TypeError: fetch failed | other side closed"); + }, + embedBatch: async (texts: string[]) => texts.map(() => [1, 0, 0, 0]), + close: async () => {}, + }; + ( + manager as unknown as { + waitForEmbeddingRetry: (delayMs: number, action: string) => Promise; + } + ).waitForEmbeddingRetry = async () => {}; + + await expect(manager.search("alpha")).rejects.toThrow("fetch failed"); + expect(queryCalls).toBe(3); + }); + + it("preserves keyword-only hybrid hits when minScore exceeds text weight", async () => { + await expectHybridKeywordSearchFindsMemory( + createCfg({ + minScore: 0.35, + hybrid: { enabled: true, vectorWeight: 0.7, textWeight: 0.3 }, + }), + ); + }); + + it("supplements thin strict FTS results for conversational queries", async () => { + const cases = [ + { + query: "that thing we discussed about the API", + strictFile: "strict-english.md", + strictText: "That thing we discussed about the API belongs in the first draft.", + recallFile: "recall-english.md", + recallText: "API authentication uses short-lived OAuth tokens.", + }, + { + query: "ayer hablamos sobre estrategia de despliegue", + strictFile: "strict-spanish.md", + strictText: "Ayer hablamos sobre estrategia de despliegue para la primera region.", + recallFile: "recall-spanish.md", + recallText: "La estrategia de despliegue requiere una ventana de mantenimiento.", + }, + ] as const; + for (const entry of cases) { + await fs.writeFile(path.join(memoryDir, entry.strictFile), entry.strictText); + await fs.writeFile(path.join(memoryDir, entry.recallFile), entry.recallText); + } + + const manager = await getPersistentManager( + createCfg({ + minScore: 0, + hybrid: { enabled: true, vectorWeight: 0.7, textWeight: 0.3 }, + }), + ); + await manager.sync({ reason: "test" }); + const provider = Reflect.get(manager, "provider") as { + embedQuery: (text: string) => Promise; + }; + const embedQuerySpy = vi.spyOn(provider, "embedQuery"); + + for (const entry of cases) { + const results = await manager.search(entry.query, { maxResults: 6 }); + expect(results.some((result) => result.path.endsWith(`memory/${entry.recallFile}`))).toBe( + true, + ); + } + expect(embedQuerySpy).toHaveBeenCalledTimes(cases.length); + }); + + it("bounds per-keyword FTS fallback in provider-backed hybrid search", async () => { + const cfg = createCfg({ + minScore: 0.35, + hybrid: { enabled: true, vectorWeight: 0.7, textWeight: 0.3 }, + }); + const manager = await getPersistentManager(cfg); + await manager.sync({ reason: "test" }); + + const db = ( + manager as unknown as { + db: { + prepare: (sql: string) => unknown; + }; + } + ).db; + const originalPrepare = db.prepare.bind(db); + let ftsSelects = 0; + const prepareSpy = vi.spyOn(db, "prepare").mockImplementation((sql: string) => { + if ( + sql.includes("FROM memory_index_chunks_fts") && + sql.includes("WHERE memory_index_chunks_fts MATCH ?") + ) { + ftsSelects += 1; + } + return originalPrepare(sql); + }); + + try { + const results = await manager.search( + "zebra project router gateway session transcript approval command owner workspace token budget retry queue", + { maxResults: 5 }, + ); + + expect(results.length).toBeGreaterThan(0); + expect(results[0]?.path).toContain("memory/2026-01-12.md"); + expect(ftsSelects).toBeGreaterThan(1); + expect(ftsSelects).toBeLessThanOrEqual(7); + } finally { + prepareSpy.mockRestore(); + } + }); + + it("preserves fallback body boosts through hybrid weighting", async () => { + const manager = await getPersistentManager( + createCfg({ + minScore: 0, + hybrid: { enabled: true, vectorWeight: 0, textWeight: 1 }, + }), + ); + await fs.writeFile( + path.join(memoryDir, "body.md"), + "Alpha gamma alpha gamma strongest fallback body match.", + ); + await fs.writeFile(path.join(memoryDir, "alpha.md"), "Unrelated path-only candidate."); + await manager.sync({ reason: "test" }); + + const results = await manager.search("alpha gamma", { maxResults: 2, minScore: 0 }); + + expect(results.map((entry) => entry.path)).toEqual(["memory/body.md", "memory/alpha.md"]); + expect(results[0]?.score).toBeGreaterThan(results[1]?.score ?? 0); + }); + + it("bootstraps an empty index on first search so session transcript hits are available", async () => { + try { + const manager = await getFtsSessionManager({ + stateDirName: ".state-session-bootstrap", + }); + if (!manager) { + return; + } + + await seedMemoryIndexSessionTranscript({ + sessionId: "session-bootstrap", + messages: [ + { + role: "assistant", + timestamp: "2026-04-07T15:25:04.113Z", + content: "The current Project Nebula codename is ORBIT-10.", + }, + ], + }); + + const results = await manager.search("current Project Nebula codename ORBIT-10", { + minScore: 0, + maxResults: 3, + }); + + expect(results[0]?.source).toBe("sessions"); + expect(results[0]?.snippet).toContain("ORBIT-10"); + } finally { + restoreMemoryIndexStateDir(); + } + }); + + it("keeps remember-only session transcripts out of ordinary manager searches", async () => { + forceNoProvider = true; + setMemoryIndexStateDir(path.join(workspaceDir, ".state-remember-search-sources")); + try { + const cfg = createCfg({ + provider: "none", + rememberAcrossConversations: true, + minScore: 0, + hybrid: { enabled: true, vectorWeight: 0.7, textWeight: 0.3 }, + }); + const manager = await getFreshManager(cfg); + managersForCleanup.add(manager); + if (!manager.status().fts?.available) { + return; + } + + await seedMemoryIndexSessionTranscript({ + sessionId: "remember-only", + messages: [ + { + role: "assistant", + timestamp: "2026-04-07T15:25:04.113Z", + content: "Recall-only canary is NEBULA-47.", + }, + ], + }); + + await manager.sync({ reason: "test", force: true }); + + await expect( + manager.search("Recall-only canary NEBULA-47", { minScore: 0 }), + ).resolves.toEqual([]); + const trustedResults = await manager.search("Recall-only canary NEBULA-47", { + minScore: 0, + sources: ["sessions"], + }); + expect(trustedResults[0]?.source).toBe("sessions"); + } finally { + restoreMemoryIndexStateDir(); + } + }); + + it("returns before provider or index bootstrap for a blank query", async () => { + const manager = await getPersistentManager( + createCfg({ provider: "required-provider", hybrid: { enabled: true } }), + ); + providerCalls = []; + + await expect(manager.search(" \n\t ")).resolves.toStrictEqual([]); + + expect(providerCalls).toHaveLength(0); + }); + + it("waits for dirty sync before querying", async () => { + forceNoProvider = true; + const manager = await getPersistentManager( + createCfg({ provider: "none", minScore: 0, onSearch: true, hybrid: { enabled: true } }), + ); + await manager.sync({ reason: "test" }); + await fs.writeFile( + path.join(memoryDir, "search-sync.md"), + "Current memory appears only after the dirty search sync.", + ); + await vi.waitFor(() => expect(manager.status().dirty).toBe(true)); + + const results = await manager.search("current dirty search sync", { + maxResults: 5, + minScore: 0, + }); + + expect(results.some((entry) => entry.path === "memory/search-sync.md")).toBe(true); + }); +}); diff --git a/extensions/memory-core/src/memory/manager-search-orchestration.ts b/extensions/memory-core/src/memory/manager-search-orchestration.ts new file mode 100644 index 000000000000..8a74712978d2 --- /dev/null +++ b/extensions/memory-core/src/memory/manager-search-orchestration.ts @@ -0,0 +1,495 @@ +// Memory Core plugin module owns public search orchestration. +import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; +import { classifyMemoryMultimodalPath } from "openclaw/plugin-sdk/memory-core-host-engine-embeddings"; +import { createSubsystemLogger } from "openclaw/plugin-sdk/memory-core-host-engine-foundation"; +import { + MEMORY_INDEX_FTS_TABLE, + MEMORY_INDEX_VECTOR_TABLE, + type MemorySearchManager, + type MemorySearchResult, + type MemorySource, +} from "openclaw/plugin-sdk/memory-core-host-engine-storage"; +import { redactSensitiveText } from "openclaw/plugin-sdk/security-runtime"; +import { uniqueValues } from "openclaw/plugin-sdk/string-coerce-runtime"; +import { + mergeHybridResults, + selectHybridSearchResults, + type HybridSearchResult, +} from "./hybrid.js"; +import { applyImportanceMultiplier } from "./importance.js"; +import { startAsyncSearchSync } from "./manager-async-state.js"; +import { MemoryKeywordRetrieval, type KeywordSearchHit } from "./manager-keyword-retrieval.js"; +import { resolveMemorySearchPreflight } from "./manager-search-preflight.js"; +import { resolveExactPathSpecificity, searchVector } from "./manager-search.js"; +import { applyProjectRanking } from "./project-ranking.js"; +import { applyTemporalDecayToHybridResults } from "./temporal-decay.js"; + +const SNIPPET_MAX_CHARS = 700; +const VECTOR_TABLE = MEMORY_INDEX_VECTOR_TABLE; +const FTS_TABLE = MEMORY_INDEX_FTS_TABLE; +const log = createSubsystemLogger("memory"); +type MemoryIndexSearchOptions = NonNullable[1]>; + +export abstract class MemorySearchOrchestration extends MemoryKeywordRetrieval { + protected abstract sessionWarm: Set; + + protected async warmSession(sessionKey?: string): Promise { + if (!this.settings.sync.onSessionStart) { + return; + } + const key = sessionKey?.trim() || ""; + if (key && this.sessionWarm.has(key)) { + return; + } + void this.sync({ reason: "session-start" }).catch((err: unknown) => { + log.warn(`memory sync failed (session-start): ${String(err)}`); + }); + if (key) { + this.sessionWarm.add(key); + } + } + + async search(query: string, opts?: MemoryIndexSearchOptions): Promise { + const normalizedQuery = query.trim(); + if (!normalizedQuery) { + return []; + } + const maxResults = opts?.maxResults ?? this.settings.query.maxResults; + const minScore = opts?.minScore ?? this.settings.query.minScore; + const hasActiveProject = (opts?.activeProjectKeys?.length ?? 0) > 0; + const candidateMaxResults = hasActiveProject + ? Math.min(200, Math.max(maxResults, maxResults * 4)) + : maxResults; + const candidateMinScore = hasActiveProject ? minScore / 1.15 : minScore; + const results = await this.searchCandidates(normalizedQuery, { + ...opts, + maxResults: candidateMaxResults, + minScore: candidateMinScore, + }); + return hasActiveProject + ? results.filter((entry) => entry.score >= minScore).slice(0, maxResults) + : results; + } + + private async searchCandidates( + normalizedQuery: string, + opts?: MemoryIndexSearchOptions, + ): Promise { + return await this.withManagerOperation(async () => { + opts?.onDebug?.({ backend: "builtin" }); + if (this.providerRequirement.mode === "required") { + await this.ensureProviderInitialized(); + this.assertRequiredProviderAvailable("search"); + } + let hasIndexedContent = this.hasIndexedContent(); + if (!hasIndexedContent) { + try { + // A fresh process can receive its first search before background watch/session + // syncs have built the index. Force one synchronous bootstrap so the first + // lookup after restart does not fail closed with empty results. + await this.syncAdmitted( + { reason: "search", force: true }, + { allowEmbeddingBootstrapFallback: true }, + ); + } catch (err) { + if (this.providerRequirement.mode === "optional" && this.shouldFallbackOnError(err)) { + const failedProvider = this.provider?.id ?? this.settings.provider; + await this.retireCurrentProvider().catch((retireErr: unknown) => { + const message = redactSensitiveText(formatErrorMessage(retireErr), { + mode: "tools", + }); + log.warn(`memory search-bootstrap: failed to retire embedding provider: ${message}`); + }); + this.markEmbeddingBootstrapFailure(err, { provider: failedProvider }); + await this.syncAdmitted({ reason: "search", force: true }).catch( + (fallbackErr: unknown) => { + const message = redactSensitiveText(formatErrorMessage(fallbackErr), { + mode: "tools", + }); + log.warn(`memory sync failed (search-bootstrap-fallback): ${message}`); + }, + ); + } else { + log.warn(`memory sync failed (search-bootstrap): ${String(err)}`); + } + } + hasIndexedContent = this.hasIndexedContent(); + } + const preflight = resolveMemorySearchPreflight({ + query: normalizedQuery, + hasIndexedContent, + }); + if (!preflight.shouldSearch) { + if (this.embeddingBootstrapFailure) { + opts?.onDebug?.({ + backend: "builtin", + embeddingBootstrap: this.embeddingBootstrapFailure, + }); + } + return []; + } + const cleaned = preflight.normalizedQuery; + const embeddingBootstrapKeywordOnly = await this.ensureEmbeddingProviderForSearch( + opts?.onDebug, + ); + void this.warmSession(opts?.sessionKey); + await startAsyncSearchSync({ + enabled: this.settings.sync.onSearch, + dirty: this.dirty, + sessionsDirty: this.sessionsDirty, + sync: async (params) => await this.syncAdmitted(params), + onError: (err) => { + log.warn(`memory sync failed (search): ${String(err)}`); + }, + }); + if ( + !embeddingBootstrapKeywordOnly && + preflight.shouldInitializeProvider && + !this.provider && + (this.providerLifecycle.mode === "pending" || + (this.providerLifecycle.mode === "degraded" && + this.providerLifecycle.providerId !== this.settings.provider)) + ) { + // A failed fallback must yield ownership back to the configured primary. + // Reinitialize it before identity validation; leaving the lifecycle pending + // makes a valid existing index look mismatched and drops keyword results. + this.resetProviderInitializationForRetry(); + await this.ensureProviderInitialized(); + } + this.assertRequiredProviderAvailable("search"); + if ( + !embeddingBootstrapKeywordOnly && + !this.provider && + this.providerLifecycle.mode === "degraded" + ) { + const activatedFallback = await this.activateFallbackProvider( + this.providerLifecycle.reason, + ).catch((fallbackErr: unknown) => { + log.warn( + `memory search: failed to activate fallback provider: ${formatErrorMessage(fallbackErr)}`, + ); + return false; + }); + if (activatedFallback) { + this.refreshIndexIdentityDirty({ + providerKeyKnown: this.providerInitialized, + }); + } + } + const indexIdentity = embeddingBootstrapKeywordOnly + ? this.refreshKeywordFallbackIndexIdentity() + : this.refreshIndexIdentityDirty({ + providerKeyKnown: this.providerInitialized, + }); + if (indexIdentity.status !== "valid") { + return []; + } + const minScore = opts?.minScore ?? this.settings.query.minScore; + const maxResults = opts?.maxResults ?? this.settings.query.maxResults; + const searchSources = + opts?.sources && opts.sources.length > 0 + ? uniqueValues(opts.sources).filter((s) => this.sources.has(s)) + : undefined; + if ( + opts?.sources && + opts.sources.length > 0 && + (!searchSources || searchSources.length === 0) + ) { + return []; + } + // The manager may index recall-only transcripts without making them part of + // ordinary searches. Trusted recall passes an explicit source override; + // every other caller defaults to the configured search corpus. + const sourceFilterList = searchSources ?? this.settings.searchSources; + const hybrid = this.settings.query.hybrid; + const candidates = Math.min( + 200, + Math.max(1, Math.floor(maxResults * hybrid.candidateMultiplier)), + ); + + // FTS-only mode: no embedding provider available + if (embeddingBootstrapKeywordOnly || !this.provider) { + this.assertRequiredProviderAvailable("search"); + if (!this.fts.enabled || !this.fts.available) { + log.warn("memory search: no provider and FTS unavailable"); + return []; + } + + const keywordResults = await this.searchKeywordWithFallback( + cleaned, + candidates, + { + boostFallbackRanking: true, + }, + sourceFilterList, + ).catch((err: unknown) => { + log.warn(`memory search: FTS keyword query failed: ${formatErrorMessage(err)}`); + return []; + }); + + return await this.finalizeKeywordOnlyResults({ + results: keywordResults, + temporalDecay: hybrid.temporalDecay, + maxResults, + minScore, + activeProjectKeys: opts?.activeProjectKeys, + }); + } + let semanticProvider = this.provider; + let semanticProviderRuntime = this.providerRuntime; + let vectorProviderIdentity = { + model: semanticProvider.model, + aliases: this.resolveProviderIndexIdentities() + .slice(1) + .map((identity) => identity.model), + }; + + // If FTS isn't available, hybrid mode cannot use keyword search; degrade to vector-only. + const loadKeywordResults = async () => + hybrid.enabled && this.fts.enabled && this.fts.available + ? await this.searchKeywordWithFallback( + cleaned, + candidates, + { boostFallbackRanking: true }, + sourceFilterList, + ).catch((err: unknown) => { + log.warn( + `memory search: FTS hybrid keyword query failed: ${formatErrorMessage(err)}`, + ); + return []; + }) + : []; + let keywordResults: Awaited> = []; + let queryVec: number[]; + const releaseSemanticProvider = this.acquireProviderUse(semanticProvider); + try { + keywordResults = await loadKeywordResults(); + // lexicalOnly is a reply-path contract: no query embedding, no vector + // search, no network. Callers accept keyword-only recall quality. + if (opts?.lexicalOnly) { + return await this.finalizeKeywordOnlyResults({ + results: keywordResults, + temporalDecay: hybrid.temporalDecay, + maxResults, + minScore, + activeProjectKeys: opts?.activeProjectKeys, + }); + } + try { + queryVec = await this.embedQueryWithRetry( + cleaned, + opts?.signal, + semanticProvider, + false, + semanticProviderRuntime, + ); + } catch (err) { + releaseSemanticProvider(); + this.markLocalEmbeddingProviderDegraded(err); + // An aborted caller already stopped waiting; skip fallback-provider + // activation so the abandoned search stops instead of re-embedding. + if (opts?.signal?.aborted) { + throw err; + } + const message = formatErrorMessage(err); + const activatedFallback = this.shouldFallbackOnError(err) + ? await this.activateFallbackProvider(message).catch((fallbackErr: unknown) => { + log.warn( + `memory search: failed to activate fallback provider: ${formatErrorMessage(fallbackErr)}`, + ); + return false; + }) + : false; + if (activatedFallback) { + if ( + this.refreshIndexIdentityDirty({ + providerKeyKnown: this.providerInitialized, + }).status !== "valid" + ) { + return []; + } + if (!this.provider) { + return []; + } + semanticProvider = this.provider; + semanticProviderRuntime = this.providerRuntime; + vectorProviderIdentity = { + model: semanticProvider.model, + aliases: this.resolveProviderIndexIdentities() + .slice(1) + .map((identity) => identity.model), + }; + const releaseFallbackProvider = this.acquireProviderUse(semanticProvider); + try { + keywordResults = await loadKeywordResults(); + queryVec = await this.embedQueryWithRetry( + cleaned, + opts?.signal, + semanticProvider, + false, + semanticProviderRuntime, + ); + } catch (fallbackErr) { + releaseFallbackProvider(); + this.markLocalEmbeddingProviderDegraded(fallbackErr); + throw fallbackErr; + } finally { + releaseFallbackProvider(); + } + } else if (!this.provider && this.fts.enabled && this.fts.available) { + this.assertRequiredProviderAvailable("search"); + log.warn( + `memory search: embeddings unavailable; using keyword-only results: ${message}`, + ); + return await this.finalizeKeywordOnlyResults({ + results: keywordResults, + temporalDecay: hybrid.temporalDecay, + maxResults, + minScore, + activeProjectKeys: opts?.activeProjectKeys, + }); + } else { + throw err; + } + } + } finally { + releaseSemanticProvider(); + } + const hasVector = queryVec.some((v) => v !== 0); + const vectorResults = hasVector + ? await this.searchVector( + queryVec, + candidates, + sourceFilterList, + vectorProviderIdentity, + ).catch((err: unknown) => { + log.warn(`memory search: vector query failed: ${formatErrorMessage(err)}`); + return []; + }) + : []; + + if (!hybrid.enabled || !this.fts.enabled || !this.fts.available) { + const decayed = await applyTemporalDecayToHybridResults({ + results: vectorResults, + temporalDecay: hybrid.temporalDecay, + workspaceDir: this.workspaceDir, + }); + return applyProjectRanking(applyImportanceMultiplier(decayed), opts?.activeProjectKeys) + .filter((entry) => entry.score >= minScore) + .slice(0, maxResults); + } + + const merged = await this.mergeHybridResults({ + query: cleaned, + vector: vectorResults, + keyword: keywordResults, + vectorWeight: hybrid.vectorWeight, + textWeight: hybrid.textWeight, + mmr: hybrid.mmr, + temporalDecay: hybrid.temporalDecay, + activeProjectKeys: opts?.activeProjectKeys, + }); + return selectHybridSearchResults({ + merged, + keyword: keywordResults, + maxResults, + minScore, + }); + }); + } + + private hasIndexedContent(): boolean { + const chunkRow = this.db.prepare(`SELECT 1 as found FROM memory_index_chunks LIMIT 1`).get() as + | { + found?: number; + } + | undefined; + if (chunkRow?.found === 1) { + return true; + } + if (!this.fts.enabled || !this.fts.available) { + return false; + } + const ftsRow = this.db.prepare(`SELECT 1 as found FROM ${FTS_TABLE} LIMIT 1`).get() as + | { + found?: number; + } + | undefined; + return ftsRow?.found === 1; + } + + private async searchVector( + queryVec: number[], + limit: number, + sourceFilterList: MemorySource[], + providerIdentity: { model: string; aliases: string[] }, + ): Promise> { + const results = await searchVector({ + db: this.db, + vectorTable: VECTOR_TABLE, + providerModel: providerIdentity.model, + providerModelAliases: providerIdentity.aliases, + queryVec, + limit, + snippetMaxChars: SNIPPET_MAX_CHARS, + ensureVectorReady: async (dimensions) => await this.ensureVectorReady(dimensions), + sourceFilterVec: this.buildSourceFilter("c", sourceFilterList), + sourceFilterChunks: this.buildSourceFilter(undefined, sourceFilterList), + }); + return this.attachRecallMetadata( + results.map((entry) => entry as MemorySearchResult & { id: string }), + ); + } + + private mergeHybridResults(params: { + query: string; + vector: Array; + keyword: KeywordSearchHit[]; + vectorWeight: number; + textWeight: number; + mmr?: { enabled: boolean; lambda: number }; + temporalDecay?: { enabled: boolean; halfLifeDays: number }; + activeProjectKeys?: readonly string[]; + }): Promise[]> { + return mergeHybridResults({ + vector: params.vector.map((r) => ({ + id: r.id, + path: r.path, + startLine: r.startLine, + endLine: r.endLine, + source: r.source, + snippet: r.snippet, + vectorScore: r.score, + importance: r.importance, + triggers: r.triggers, + projectKey: r.projectKey, + exactPathSpecificity: resolveExactPathSpecificity(params.query, r.path), + ...(r.provenance ? { provenance: r.provenance } : {}), + })), + keyword: params.keyword.map((r) => ({ + id: r.id, + path: r.path, + startLine: r.startLine, + endLine: r.endLine, + source: r.source, + snippet: r.snippet, + textScore: r.textScore, + importance: r.importance, + triggers: r.triggers, + projectKey: r.projectKey, + rankingScore: r.score, + pathScore: r.pathScore, + exactPathSpecificity: r.exactPathSpecificity, + ...(r.provenance ? { provenance: r.provenance } : {}), + })), + vectorWeight: params.vectorWeight, + textWeight: params.textWeight, + isNonTextMediaPath: (path) => + classifyMemoryMultimodalPath(path, this.settings.multimodal) !== null, + mmr: params.mmr, + temporalDecay: params.temporalDecay, + activeProjectKeys: params.activeProjectKeys, + workspaceDir: this.workspaceDir, + }); + } +} diff --git a/extensions/memory-core/src/memory/manager.session-reindex.test.ts b/extensions/memory-core/src/memory/manager-session-reindex.test.ts similarity index 100% rename from extensions/memory-core/src/memory/manager.session-reindex.test.ts rename to extensions/memory-core/src/memory/manager-session-reindex.test.ts diff --git a/extensions/memory-core/src/memory/manager.vector-dedupe.test.ts b/extensions/memory-core/src/memory/manager-vector-write.test.ts similarity index 100% rename from extensions/memory-core/src/memory/manager.vector-dedupe.test.ts rename to extensions/memory-core/src/memory/manager-vector-write.test.ts diff --git a/extensions/memory-core/src/memory/manager.async-search.test.ts b/extensions/memory-core/src/memory/manager.async-search.test.ts deleted file mode 100644 index 08e635d0b070..000000000000 --- a/extensions/memory-core/src/memory/manager.async-search.test.ts +++ /dev/null @@ -1,143 +0,0 @@ -// Memory Core tests cover manager.async search plugin behavior. -import { describe, expect, it, vi } from "vitest"; -import { awaitPendingManagerWork, startAsyncSearchSync } from "./manager-async-state.js"; -import { MemoryIndexManager } from "./manager.js"; - -describe("memory search async sync", () => { - it("returns before provider or index bootstrap for a blank query", async () => { - const manager = Object.create(MemoryIndexManager.prototype) as MemoryIndexManager; - const ensureProviderInitialized = vi.fn(async () => {}); - const assertRequiredProviderAvailable = vi.fn(); - const hasIndexedContent = vi.fn(() => false); - const sync = vi.fn(async () => {}); - Object.assign(manager as unknown as Record, { - providerRequirement: { mode: "required" }, - ensureProviderInitialized, - assertRequiredProviderAvailable, - hasIndexedContent, - sync, - }); - - await expect(manager.search(" \n\t ")).resolves.toStrictEqual([]); - expect(ensureProviderInitialized).not.toHaveBeenCalled(); - expect(assertRequiredProviderAvailable).not.toHaveBeenCalled(); - expect(hasIndexedContent).not.toHaveBeenCalled(); - expect(sync).not.toHaveBeenCalled(); - }); - - it("waits for dirty sync before querying", async () => { - let releaseSync = () => {}; - const pendingSync = new Promise((resolve) => { - releaseSync = () => resolve(); - }); - const syncMock = vi.fn(async () => { - return pendingSync; - }); - const queryMock = vi.fn(async () => []); - const manager = Object.create(MemoryIndexManager.prototype) as MemoryIndexManager; - Object.assign(manager as unknown as Record, { - providerRequirement: { mode: "fts-only", provider: "none" }, - hasIndexedContent: () => true, - settings: { - sync: { onSearch: true }, - query: { - minScore: 0, - maxResults: 5, - hybrid: { - enabled: true, - candidateMultiplier: 2, - temporalDecay: { enabled: false, halfLifeDays: 30 }, - }, - }, - }, - warmSession: vi.fn(), - ensureProviderInitialized: vi.fn(async () => {}), - assertRequiredProviderAvailable: vi.fn(), - dirty: true, - sessionsDirty: false, - syncAdmitted: syncMock, - provider: null, - providerLifecycle: { mode: "fts-only", reason: "test" }, - refreshIndexIdentityDirty: () => ({ status: "valid" }), - sources: new Set(["memory"]), - fts: { enabled: true, available: true }, - searchKeywordWithFallback: queryMock, - workspaceDir: "", - }); - - const searchPromise = manager.search("current memory"); - await vi.waitFor(() => expect(syncMock).toHaveBeenCalledWith({ reason: "search" })); - expect(queryMock).not.toHaveBeenCalled(); - - expect(syncMock).toHaveBeenCalledTimes(1); - releaseSync(); - await searchPromise; - expect(queryMock).toHaveBeenCalledTimes(1); - }); - - it("waits for in-flight search sync during close", async () => { - let releaseSync = () => {}; - const pendingSync = new Promise((resolve) => { - releaseSync = () => resolve(); - }); - - let closed = false; - const closePromise = awaitPendingManagerWork({ pendingSync }).then(() => { - closed = true; - }); - - await Promise.resolve(); - expect(closed).toBe(false); - - releaseSync(); - await closePromise; - }); - - it("reports pending sync failures during close", async () => { - const onError = vi.fn(); - const syncError = new Error("sync failed"); - - await awaitPendingManagerWork({ - pendingSync: Promise.reject(syncError), - onError, - }); - - expect(onError).toHaveBeenCalledWith(syncError); - }); - - it("reports pending provider initialization failures during close", async () => { - const onError = vi.fn(); - const providerError = new Error("provider init failed"); - - await awaitPendingManagerWork({ - pendingProviderInit: Promise.reject(providerError), - onError, - }); - - expect(onError).toHaveBeenCalledWith(providerError); - }); - - it("does not report errors for completed pending close work", async () => { - const onError = vi.fn(); - - await awaitPendingManagerWork({ - pendingSync: Promise.resolve(), - pendingProviderInit: Promise.resolve(), - onError, - }); - - expect(onError).not.toHaveBeenCalled(); - }); - - it("skips background search sync when search-triggered sync is disabled", async () => { - const syncMock = vi.fn(async () => {}); - await startAsyncSearchSync({ - enabled: false, - dirty: true, - sessionsDirty: false, - sync: syncMock, - onError: vi.fn(), - }); - expect(syncMock).not.toHaveBeenCalled(); - }); -}); diff --git a/extensions/memory-core/src/memory/manager.reindex-recovery.test.ts b/extensions/memory-core/src/memory/manager.reindex-recovery.test.ts index 300bebde50b2..99472a5eda27 100644 --- a/extensions/memory-core/src/memory/manager.reindex-recovery.test.ts +++ b/extensions/memory-core/src/memory/manager.reindex-recovery.test.ts @@ -7,9 +7,9 @@ import type { OpenClawConfig } from "openclaw/plugin-sdk/memory-core-host-engine import { resolveOpenClawAgentSqlitePath } from "openclaw/plugin-sdk/sqlite-runtime"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { resetEmbeddingMocks } from "./embedding.test-mocks.js"; -import type { MemoryIndexManager } from "./index.js"; import { acquireMemoryReindexLock } from "./manager-reindex-lock.js"; import type { MemoryIndexMeta } from "./manager-reindex-state.js"; +import type { MemoryIndexManager } from "./manager.js"; type SyncArchiveParams = { needsFullReindex: boolean; targetArchiveFiles?: string[] }; diff --git a/extensions/memory-core/src/memory/manager.ts b/extensions/memory-core/src/memory/manager.ts index 73a5bcfc382f..59f6863e713c 100644 --- a/extensions/memory-core/src/memory/manager.ts +++ b/extensions/memory-core/src/memory/manager.ts @@ -1,89 +1,47 @@ -// Memory Core plugin module implements manager behavior. +// Memory Core plugin module implements the concrete memory index manager. import type { DatabaseSync } from "node:sqlite"; -import type { FSWatcher } from "chokidar"; -import { resolveAgentConfig } from "openclaw/plugin-sdk/agent-runtime"; -import { - formatErrorMessage, - readErrorName, - toErrorObject, -} from "openclaw/plugin-sdk/error-runtime"; -import { listRegisteredMemoryEmbeddingProviderAdapters } from "openclaw/plugin-sdk/memory-core-host-embedding-registry"; -import { classifyMemoryMultimodalPath } from "openclaw/plugin-sdk/memory-core-host-engine-embeddings"; +import { formatErrorMessage, toErrorObject } from "openclaw/plugin-sdk/error-runtime"; import { createSubsystemLogger, - resolveGlobalSingleton, - resolveAgentDir, resolveAgentWorkspaceDir, resolveMemorySearchConfig, type OpenClawConfig, type ResolvedMemorySearchConfig, } from "openclaw/plugin-sdk/memory-core-host-engine-foundation"; -import { extractKeywords } from "openclaw/plugin-sdk/memory-core-host-engine-sessions"; import { - readCuratedProjectMemoryCandidates, readMemoryFile, - readCuratedMemoryTriggerCandidates, - readMemoryRecallMetadata, MEMORY_EMBEDDING_CACHE_TABLE, - MEMORY_INDEX_FTS_TABLE, - MEMORY_INDEX_PATHS_FTS_TABLE, MEMORY_INDEX_VECTOR_TABLE, - type MemoryEmbeddingProbeResult, type MemoryProviderStatus, type MemorySearchManager, - type MemorySearchRuntimeDebug, - type MemorySearchResult, type MemorySessionSyncTarget, type MemorySource, type MemorySyncParams, } from "openclaw/plugin-sdk/memory-core-host-engine-storage"; import { normalizeAgentId } from "openclaw/plugin-sdk/routing"; -import { redactSensitiveText } from "openclaw/plugin-sdk/security-runtime"; -import { uniqueValues } from "openclaw/plugin-sdk/string-coerce-runtime"; -import { - resolveMemoryCoreLocalServiceHostIdentity, - type MemoryCoreAcquireLocalService, -} from "./embedding-local-service.js"; -import { - createEmbeddingProvider, - resolveEmbeddingProviderAdapterTransport, - type EmbeddingProvider, - type EmbeddingProviderId, - type EmbeddingProviderRequest, - type EmbeddingProviderResult, - type EmbeddingProviderRuntime, -} from "./embeddings.js"; -import { - bm25RankToScore, - buildFtsQuery, - mergeHybridResults, - scoreExactPathTieForTemporalDecay, - selectHybridSearchResults, - type HybridSearchResult, -} from "./hybrid.js"; -import { applyImportanceMultiplier } from "./importance.js"; -import { awaitPendingManagerWork, startAsyncSearchSync } from "./manager-async-state.js"; +import type { MemoryCoreAcquireLocalService } from "./embedding-local-service.js"; +import type { EmbeddingProvider, EmbeddingProviderRequest } from "./embeddings.js"; +import { awaitPendingManagerWork } from "./manager-async-state.js"; import { MEMORY_BATCH_FAILURE_LIMIT } from "./manager-batch-state.js"; -import { getOrCreateManagedCacheEntry, resolveSingletonManagedCache } from "./manager-cache.js"; import { closeMemoryDatabase } from "./manager-db.js"; -import { MemoryManagerEmbeddingOps } from "./manager-embedding-ops.js"; -import { isLocalEmbeddingWorkerFailure } from "./manager-local-worker-errors.js"; import { - createDegradedMemoryProviderLifecycle, + clearMemoryEmbeddingProbeCache, + resolveEffectiveMemorySearchSettings, + resolveMemoryEmbeddingProviderRequirement, + type MemoryEmbeddingBootstrapDebug, + type MemoryEmbeddingProviderRequirement, +} from "./manager-provider-lifecycle.js"; +import { createPendingMemoryProviderLifecycle, - resolveMemoryPrimaryProviderRequest, - resolveMemoryProviderState, type MemoryProviderLifecycleState, } from "./manager-provider-state.js"; -import type { MemoryIndexIdentityState } from "./manager-reindex-state.js"; -import { resolveMemorySearchPreflight } from "./manager-search-preflight.js"; import { - resolveExactPathSpecificity, - searchKeyword, - searchPathKeyword, - searchVector, - type ExactPathSpecificity, -} from "./manager-search.js"; + MemoryManagerRegistry, + resolveMemoryIndexManagerCacheKey, + type MemoryIndexManagerPurpose, +} from "./manager-registry.js"; +import type { MemoryIndexIdentityState } from "./manager-reindex-state.js"; +import { MemorySearchOrchestration } from "./manager-search-orchestration.js"; import { collectMemoryStatusAggregate, resolveInitialMemoryDirty, @@ -91,8 +49,6 @@ import { } from "./manager-status-state.js"; import { enqueueMemoryTargetedSessionSync } from "./manager-sync-control.js"; import { resolvePersistedMemoryVectorIndexState } from "./manager-vector-rebuild-state.js"; -import { applyProjectRanking } from "./project-ranking.js"; -import { applyTemporalDecayToHybridResults } from "./temporal-decay.js"; const LOCAL_EMBEDDING_RUNTIME_FACTS = Symbol.for("openclaw.localEmbeddingRuntimeFacts"); @@ -104,346 +60,44 @@ function getLocalEmbeddingRuntimeFacts(provider: EmbeddingProvider | null): unkn return typeof getRuntimeFacts === "function" ? getRuntimeFacts() : undefined; } -const SNIPPET_MAX_CHARS = 700; -const VECTOR_TABLE = MEMORY_INDEX_VECTOR_TABLE; -const FTS_TABLE = MEMORY_INDEX_FTS_TABLE; -const PATH_FTS_TABLE = MEMORY_INDEX_PATHS_FTS_TABLE; -const EMBEDDING_CACHE_TABLE = MEMORY_EMBEDDING_CACHE_TABLE; -const MEMORY_INDEX_MANAGER_CACHE_KEY = Symbol.for("openclaw.memoryIndexManagerCache"); -const MEMORY_INDEX_MANAGER_SCOPE_CLOSES_KEY = Symbol.for("openclaw.memoryIndexManagerScopeCloses"); -const MEMORY_INDEX_MANAGER_GLOBAL_LIFECYCLE_KEY = Symbol.for( - "openclaw.memoryIndexManagerGlobalLifecycle.v3", -); -const EMBEDDING_PROBE_CACHE_TTL_MS = 30_000; -const KEYWORD_FALLBACK_SEARCH_TERM_LIMIT = 6; -const EXACT_PATH_CANDIDATE_LIMIT = 200; const log = createSubsystemLogger("memory"); -type MemoryIndexManagerPurpose = "default" | "status" | "cli"; -type MemoryEmbeddingProviderRequirement = { - mode: "fts-only" | "optional" | "required"; - provider: string; - configuredProvider?: string; -}; -type MemoryEmbeddingBootstrapDebug = NonNullable; - -const { cache: INDEX_CACHE, pending: INDEX_CACHE_PENDING } = - resolveSingletonManagedCache(MEMORY_INDEX_MANAGER_CACHE_KEY); -const INDEX_SCOPE_CLOSES = resolveGlobalSingleton>>( - MEMORY_INDEX_MANAGER_SCOPE_CLOSES_KEY, - () => new Map(), -); -const INDEX_GLOBAL_LIFECYCLE = resolveGlobalSingleton<{ - closePromise: Promise | null; - closeFailed: boolean; -}>(MEMORY_INDEX_MANAGER_GLOBAL_LIFECYCLE_KEY, () => ({ - closePromise: null, - closeFailed: false, -})); - -async function runMemoryIndexManagerGlobalClose(operation: () => Promise): Promise { - const previous = INDEX_GLOBAL_LIFECYCLE.closePromise ?? Promise.resolve(); - const closePromise = previous.then(operation, operation); - INDEX_GLOBAL_LIFECYCLE.closePromise = closePromise; - await closePromise; - if (INDEX_GLOBAL_LIFECYCLE.closePromise === closePromise) { - INDEX_GLOBAL_LIFECYCLE.closePromise = null; - } -} - -async function closeAllMemoryIndexManagersUnlocked(): Promise { - const scopedCloses = Array.from(INDEX_SCOPE_CLOSES.values()); - if (scopedCloses.length > 0) { - await Promise.allSettled(scopedCloses); - } - const pending = Array.from(INDEX_CACHE_PENDING.values()); - if (pending.length > 0) { - await Promise.allSettled(pending); - } - const entries = Array.from(INDEX_CACHE.entries()); - let firstError: unknown; - let closeFailed = false; - for (const [key, manager] of entries) { - try { - await manager.close(); - if (INDEX_CACHE.get(key) === manager) { - INDEX_CACHE.delete(key); - } - } catch (err) { - if (!closeFailed) { - firstError = err; - } - closeFailed = true; - log.warn(`failed to close memory index manager: ${String(err)}`); - } - } - if (closeFailed) { - throw firstError; - } -} - -type EmbeddingProbeCacheEntry = { - result: MemoryEmbeddingProbeResult; - checkedAtMs: number; - expireAtMs: number; -}; - -type KeywordSearchHit = MemorySearchResult & { - id: string; - textScore: number; - pathScore: number; - exactPathSpecificity: ExactPathSpecificity; -}; - -function compareKeywordSearchHits( - a: KeywordSearchHit, - b: KeywordSearchHit, - preferExactBody = true, -): number { - const specificityDelta = b.exactPathSpecificity - a.exactPathSpecificity; - if (specificityDelta !== 0) { - return specificityDelta; - } - if (preferExactBody && a.exactPathSpecificity > 0) { - const bodyPresenceDelta = Number(b.textScore > 0) - Number(a.textScore > 0); - if (bodyPresenceDelta !== 0) { - return bodyPresenceDelta; - } - } - // Score carries body relevance plus any configured decay. Exact tiers ignore - // path BM25 because specificity already owns path precedence. - const relevanceDelta = b.score - a.score; - if (relevanceDelta !== 0) { - return relevanceDelta; - } - const textDelta = b.textScore - a.textScore; - if (textDelta !== 0) { - return textDelta; - } - if (a.exactPathSpecificity === 0) { - const pathDelta = b.pathScore - a.pathScore; - if (pathDelta !== 0) { - return pathDelta; - } - } - return a.path.localeCompare(b.path) || a.startLine - b.startLine || a.id.localeCompare(b.id); -} - -const EMBEDDING_PROBE_CACHE = new Map(); +const INDEX_MANAGER_REGISTRY = new MemoryManagerRegistry(); export async function closeAllMemoryIndexManagers(): Promise { - EMBEDDING_PROBE_CACHE.clear(); - await runMemoryIndexManagerGlobalClose(async () => { - try { - await closeAllMemoryIndexManagersUnlocked(); - INDEX_GLOBAL_LIFECYCLE.closeFailed = false; - } catch (err) { - INDEX_GLOBAL_LIFECYCLE.closeFailed = true; - throw err; - } - }); + clearMemoryEmbeddingProbeCache(); + await INDEX_MANAGER_REGISTRY.closeAll(async (manager) => await manager.close()); } -export async function closeMemoryIndexManagersForAgent(params: { - cfg: OpenClawConfig; - agentId: string; -}): Promise { - await closeMemoryIndexManagersForScope({ - agentId: normalizeAgentId(params.agentId), +export async function closeMemoryIndexManagersForAgent(params: { agentId: string }): Promise { + await INDEX_MANAGER_REGISTRY.closeForAgent({ + agentId: params.agentId, purpose: "default", + close: async (manager) => await manager.close(), }); } -function resolveEffectiveMemorySearchSettings( - settings: ResolvedMemorySearchConfig, -): ResolvedMemorySearchConfig { - if (settings.provider !== "none" || !settings.store.vector.enabled) { - return settings; - } - return { - ...settings, - store: { - ...settings.store, - vector: { - ...settings.store.vector, - enabled: false, - }, - }, - }; -} - -function resolveConfiguredMemoryEmbeddingProvider(params: { - cfg: OpenClawConfig; - agentId: string; -}): string | undefined { - const agentEntry = resolveAgentConfig(params.cfg, normalizeAgentId(params.agentId)); - return agentEntry?.memory?.search?.provider ?? params.cfg.memory?.search?.provider; -} - -function resolveMemoryEmbeddingProviderRequirement(params: { - cfg: OpenClawConfig; - agentId: string; - settings: ResolvedMemorySearchConfig; -}): MemoryEmbeddingProviderRequirement { - const configuredProvider = resolveConfiguredMemoryEmbeddingProvider(params)?.trim(); - if (params.settings.provider === "none" || configuredProvider === "none") { - return { mode: "fts-only", provider: params.settings.provider }; - } - const adapterTransport = resolveEmbeddingProviderAdapterTransport( - params.settings.provider, - params.cfg, - ); - if (!configuredProvider || configuredProvider === "auto" || adapterTransport === "local") { - return { mode: "optional", provider: params.settings.provider }; - } - return { - mode: "required", - provider: params.settings.provider, - configuredProvider, - }; -} - -function resolveMemoryIndexManagerCacheKey(params: { - agentId: string; - workspaceDir: string; - settings: ResolvedMemorySearchConfig; - providerRequirement: MemoryEmbeddingProviderRequirement; - purpose: MemoryIndexManagerPurpose; - acquireLocalService?: MemoryCoreAcquireLocalService; -}): string { - return [ - params.agentId, - params.workspaceDir, - JSON.stringify(params.settings), - JSON.stringify(params.providerRequirement), - resolveMemoryCoreLocalServiceHostIdentity(params.acquireLocalService), - params.purpose, - ].join(":"); -} - -function isMemoryIndexManagerCacheKeyInScope( - key: string, - params: { - agentId: string; - purpose: MemoryIndexManagerPurpose; - }, -): boolean { - return key.startsWith(`${params.agentId}:`) && key.endsWith(`:${params.purpose}`); -} - -function resolveMemoryIndexManagerScopeKey(params: { - agentId: string; - purpose: MemoryIndexManagerPurpose; -}): string { - return JSON.stringify([params.agentId, params.purpose]); -} - -async function runMemoryIndexManagerScopeOperation( - params: { - agentId: string; - purpose: MemoryIndexManagerPurpose; - }, - operation: () => Promise, -): Promise { - while (INDEX_GLOBAL_LIFECYCLE.closePromise) { - const globalClose = INDEX_GLOBAL_LIFECYCLE.closePromise; - try { - await globalClose; - } catch { - if (INDEX_GLOBAL_LIFECYCLE.closePromise === globalClose) { - await closeAllMemoryIndexManagers(); - } - } - } - const scopeKey = resolveMemoryIndexManagerScopeKey(params); - const previousOperation = INDEX_SCOPE_CLOSES.get(scopeKey) ?? Promise.resolve(); - const result = previousOperation.then(operation, operation); - const tail = result.then( - () => undefined, - () => undefined, - ); - INDEX_SCOPE_CLOSES.set(scopeKey, tail); - try { - return await result; - } finally { - if (INDEX_SCOPE_CLOSES.get(scopeKey) === tail) { - INDEX_SCOPE_CLOSES.delete(scopeKey); - } - } -} - -async function closeMemoryIndexManagersForScopeUnlocked(params: { - agentId: string; - purpose: MemoryIndexManagerPurpose; - exceptKey?: string; -}): Promise { - const isScopedKey = (key: string) => - key !== params.exceptKey && isMemoryIndexManagerCacheKeyInScope(key, params); - const pending = Array.from(INDEX_CACHE_PENDING.entries()) - .filter(([key]) => isScopedKey(key)) - .map(([, value]) => value); - if (pending.length > 0) { - await Promise.allSettled(pending); - } - const entries = Array.from(INDEX_CACHE.entries()).filter(([key]) => isScopedKey(key)); - let firstError: unknown; - let closeFailed = false; - for (const [key, manager] of entries) { - try { - await manager.close(); - if (INDEX_CACHE.get(key) === manager) { - INDEX_CACHE.delete(key); - } - } catch (err) { - if (!closeFailed) { - firstError = err; - } - closeFailed = true; - log.warn(`failed to close memory index manager for agent ${params.agentId}: ${String(err)}`); - } - } - if (closeFailed) { - throw firstError; - } -} - -async function closeMemoryIndexManagersForScope(params: { - agentId: string; - purpose: MemoryIndexManagerPurpose; - exceptKey?: string; -}): Promise { - await runMemoryIndexManagerScopeOperation(params, async () => { - await closeMemoryIndexManagersForScopeUnlocked(params); - }); -} - -type MemoryIndexSearchOptions = NonNullable[1]>; - -export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements MemorySearchManager { - private readonly cacheKey: string; - private readonly purpose: MemoryIndexManagerPurpose; +export class MemoryIndexManager extends MemorySearchOrchestration implements MemorySearchManager { + protected readonly cacheKey: string; + protected readonly purpose: MemoryIndexManagerPurpose; protected override readonly acquireLocalService?: MemoryCoreAcquireLocalService; protected readonly cfg: OpenClawConfig; protected readonly agentId: string; protected readonly workspaceDir: string; protected readonly settings: ResolvedMemorySearchConfig; - private readonly providerRequirement: MemoryEmbeddingProviderRequirement; - protected override provider: EmbeddingProvider | null; - private readonly requestedProvider: EmbeddingProviderRequest; - private providerInitPromise: Promise | null = null; - private providerInitialized = false; - private embeddingBootstrapFailure?: MemoryEmbeddingBootstrapDebug; - private providerRetirementPromise: Promise = Promise.resolve(); - private providersPendingRetirement = new Set(); + protected readonly providerRequirement: MemoryEmbeddingProviderRequirement; + protected readonly requestedProvider: EmbeddingProviderRequest; + protected providerInitPromise: Promise | null = null; + protected providerInitialized = false; + protected embeddingBootstrapFailure?: MemoryEmbeddingBootstrapDebug; + protected providerRetirementPromise: Promise = Promise.resolve(); + protected providersPendingRetirement = new Set(); private closePromise: Promise | null = null; private closeTeardownComplete = false; - private closing = false; - private activeManagerOperations = 0; - private managerIdleWaiters = new Set<() => void>(); - protected override fallbackFrom?: EmbeddingProviderId; - protected override fallbackReason?: string; + protected closing = false; + protected activeManagerOperations = 0; + protected managerIdleWaiters = new Set<() => void>(); protected providerUnavailableReason?: string; protected override providerLifecycle: MemoryProviderLifecycleState; - protected override providerRuntime?: EmbeddingProviderRuntime; protected batch: { enabled: boolean; wait: boolean; @@ -456,8 +110,6 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem protected batchFailureLastProvider?: string; protected batchFailureLock: Promise = Promise.resolve(); protected db: DatabaseSync; - protected override readonly sources: Set; - protected override providerKey: string; protected readonly cache: { enabled: boolean; maxEntries?: number }; protected readonly vector: { enabled: boolean; @@ -467,51 +119,19 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem loadError?: string; dims?: number; }; - protected override readonly fts: { - enabled: boolean; - available: boolean; - loadError?: string; - }; - protected override vectorReady: Promise | null = null; - protected override watcher: FSWatcher | null = null; - protected override watchTimer: NodeJS.Timeout | null = null; - protected override sessionWatchTimer: NodeJS.Timeout | null = null; - protected override sessionUnsubscribe: (() => void) | null = null; - protected override intervalTimer: NodeJS.Timeout | null = null; - protected override memoryWatchPressureStartupTimer: NodeJS.Timeout | null = null; - protected override closed = false; - protected override dirty = false; - protected override sessionsDirty = false; - protected override sessionsDirtyFiles = new Set(); - protected override sessionPendingFiles = new Set(); - protected override sessionPendingTargets = new Map(); - private indexIdentityDirty = false; - private sessionWarm = new Set(); + protected indexIdentityDirty = false; + protected sessionWarm = new Set(); private syncing: Promise | null = null; private queuedArchiveFiles = new Set(); private queuedSessions = new Map(); private queuedForce = false; private queuedProgressCallbacks = new Set>(); private queuedSessionSync: Promise | null = null; - private indexIdentityState: MemoryIndexIdentityState = { + protected indexIdentityState: MemoryIndexIdentityState = { status: "missing", reason: "index metadata is missing", }; - private static async loadProviderResult(params: { - cfg: OpenClawConfig; - agentId: string; - settings: ResolvedMemorySearchConfig; - acquireLocalService?: MemoryCoreAcquireLocalService; - }): Promise { - return await createEmbeddingProvider({ - config: params.cfg, - agentDir: resolveAgentDir(params.cfg, params.agentId), - ...(params.acquireLocalService ? { acquireLocalService: params.acquireLocalService } : {}), - ...resolveMemoryPrimaryProviderRequest({ settings: params.settings }), - }); - } - static async get(params: { cfg: OpenClawConfig; agentId: string; @@ -521,89 +141,57 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem const agentId = normalizeAgentId(params.agentId); const purpose = params.purpose === "status" || params.purpose === "cli" ? params.purpose : "default"; - return await runMemoryIndexManagerScopeOperation({ agentId, purpose }, async () => { - if (INDEX_GLOBAL_LIFECYCLE.closeFailed) { - try { - await closeAllMemoryIndexManagersUnlocked(); - INDEX_GLOBAL_LIFECYCLE.closeFailed = false; - } catch (err) { - INDEX_GLOBAL_LIFECYCLE.closeFailed = true; - throw err; - } - } - return await MemoryIndexManager.getWithinGlobalLifecycle({ ...params, agentId }); - }); - } - - private static async getWithinGlobalLifecycle(params: { - cfg: OpenClawConfig; - agentId: string; - purpose?: MemoryIndexManagerPurpose; - acquireLocalService?: MemoryCoreAcquireLocalService; - }): Promise { - const { cfg, agentId } = params; - const settings = resolveMemorySearchConfig(cfg, agentId); - if (!settings) { - return null; - } - const workspaceDir = resolveAgentWorkspaceDir(cfg, agentId); - const purpose = - params.purpose === "status" || params.purpose === "cli" ? params.purpose : "default"; - const providerRequirement = resolveMemoryEmbeddingProviderRequirement({ - cfg, - agentId, - settings, - }); - const key = resolveMemoryIndexManagerCacheKey({ - agentId, - workspaceDir, - settings, - providerRequirement, - purpose, - acquireLocalService: params.acquireLocalService, - }); - const transient = purpose === "status" || purpose === "cli"; - const getOrCreate = async () => - await getOrCreateManagedCacheEntry({ - cache: INDEX_CACHE, - pending: INDEX_CACHE_PENDING, - key, - bypassCache: transient, - create: async () => { - const manager = new MemoryIndexManager({ - cacheKey: key, - cfg, + return await INDEX_MANAGER_REGISTRY.acquire( + { agentId, purpose }, + { + prepare: () => { + const settings = resolveMemorySearchConfig(params.cfg, agentId); + if (!settings) { + return null; + } + const workspaceDir = resolveAgentWorkspaceDir(params.cfg, agentId); + const providerRequirement = resolveMemoryEmbeddingProviderRequirement({ + cfg: params.cfg, + agentId, + settings, + }); + const key = resolveMemoryIndexManagerCacheKey({ agentId, workspaceDir, settings, providerRequirement, - purpose: params.purpose, + purpose, acquireLocalService: params.acquireLocalService, }); - // Lightweight dirty-file detection for status mode: check for unindexed - // session files on disk without triggering a full sync. This runs before - // any caller reads manager.status(), so the dirty flag is accurate when - // status() reads sessionsDirty. - if (purpose === "status" && manager.sources.has("sessions")) { - try { - await manager.markSessionStartupCatchupDirtyFiles(); - } catch (err) { - log.warn("memory status session dirty detection failed: " + String(err)); - } - } - return manager; + return { + key, + transient: purpose === "status" || purpose === "cli", + create: async () => { + const manager = new MemoryIndexManager({ + cacheKey: key, + cfg: params.cfg, + agentId, + workspaceDir, + settings, + providerRequirement, + purpose: params.purpose, + acquireLocalService: params.acquireLocalService, + }); + if (purpose === "status" && manager.sources.has("sessions")) { + try { + await manager.markSessionStartupCatchupDirtyFiles(); + } catch (err) { + log.warn("memory status session dirty detection failed: " + String(err)); + } + } + return manager; + }, + reuse: (manager) => !manager.closing && !manager.closed, + }; }, - }); - if (transient) { - return await getOrCreate(); - } - const cachedManager = INDEX_CACHE.get(key); - await closeMemoryIndexManagersForScopeUnlocked({ - agentId, - purpose, - ...(cachedManager?.closing || cachedManager?.closed ? {} : { exceptKey: key }), - }); - return await getOrCreate(); + close: async (manager) => await manager.close(), + }, + ); } private constructor(params: { @@ -613,7 +201,6 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem workspaceDir: string; settings: ResolvedMemorySearchConfig; providerRequirement: MemoryEmbeddingProviderRequirement; - providerResult?: EmbeddingProviderResult; purpose?: MemoryIndexManagerPurpose; acquireLocalService?: MemoryCoreAcquireLocalService; }) { @@ -628,13 +215,11 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem this.workspaceDir = params.workspaceDir; this.settings = effectiveSettings; this.providerRequirement = params.providerRequirement; - this.provider = null; this.requestedProvider = effectiveSettings.provider; this.providerLifecycle = createPendingMemoryProviderLifecycle(this.requestedProvider); - if (params.providerResult) { - this.applyProviderResult(params.providerResult); + for (const source of effectiveSettings.sources) { + this.sources.add(source); } - this.sources = new Set(effectiveSettings.sources); this.db = this.openDatabase(); try { this.providerKey = this.computeProviderKey(); @@ -642,7 +227,7 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem enabled: effectiveSettings.cache.enabled, maxEntries: effectiveSettings.cache.maxEntries, }; - this.fts = { enabled: effectiveSettings.query.hybrid.enabled, available: false }; + this.fts.enabled = effectiveSettings.query.hybrid.enabled; this.ensureSchema(); this.vector = { enabled: effectiveSettings.store.vector.enabled, @@ -655,7 +240,7 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem } const initialIndexIdentity = this.resolveCurrentIndexIdentityState({ meta, - providerKeyKnown: Boolean(params.providerResult), + providerKeyKnown: false, }); this.indexIdentityState = initialIndexIdentity; this.indexIdentityDirty = @@ -700,1193 +285,6 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem } } - private applyProviderResult(providerResult: EmbeddingProviderResult): void { - const providerState = resolveMemoryProviderState(providerResult); - this.provider = providerState.provider; - this.fallbackFrom = providerState.fallbackFrom; - this.fallbackReason = providerState.fallbackReason; - this.providerUnavailableReason = providerState.providerUnavailableReason; - this.providerLifecycle = providerState.lifecycle; - this.providerRuntime = providerState.providerRuntime; - this.providerInitialized = true; - } - - private markEmbeddingBootstrapFailure( - err: unknown, - options?: { retainProvider?: boolean; provider?: string }, - ): MemoryEmbeddingBootstrapDebug { - const rawErrorName = readErrorName(err).trim(); - const errorName = /^[A-Za-z][A-Za-z0-9_.-]{0,63}$/.test(rawErrorName) ? rawErrorName : ""; - const message = - redactSensitiveText(formatErrorMessage(err), { mode: "tools" }).trim() || - "embedding provider initialization failed"; - const reason = redactSensitiveText( - errorName && errorName !== "Error" ? `${errorName}: ${message}` : message, - { mode: "tools" }, - ); - // settings.provider is already resolved from "auto"; never trust an unknown - // error object's provider-shaped field for public diagnostics. - const provider = options?.provider ?? this.provider?.id ?? this.settings.provider; - const debug: MemoryEmbeddingBootstrapDebug = { - ok: false, - provider, - reason, - degradedTo: "keyword-only", - }; - if (!options?.retainProvider) { - this.provider = null; - this.providerRuntime = undefined; - } - this.providerInitialized = true; - this.providerUnavailableReason = reason; - this.providerLifecycle = createDegradedMemoryProviderLifecycle({ - providerId: provider, - reason, - }); - this.embeddingBootstrapFailure = debug; - this.providerKey = this.computeProviderKey(); - this.batch = this.resolveBatchConfig(); - this.vector.semanticAvailable = false; - this.cacheProbeResult({ ok: false, error: reason }); - return debug; - } - - private async ensureEmbeddingProviderForSearch( - onDebug?: (debug: MemorySearchRuntimeDebug) => void, - ): Promise { - const failure = this.embeddingBootstrapFailure; - if (failure) { - const cached = this.getCachedEmbeddingAvailability(); - if (cached?.ok === false) { - onDebug?.({ backend: "builtin", embeddingBootstrap: failure }); - return true; - } - } - try { - await this.ensureProviderInitialized(); - } catch (err) { - if (this.providerRequirement.mode !== "optional") { - throw err; - } - const nextFailure = this.markEmbeddingBootstrapFailure(err); - onDebug?.({ backend: "builtin", embeddingBootstrap: nextFailure }); - return true; - } - if (!failure) { - return false; - } - if (!this.provider) { - const nextFailure: MemoryEmbeddingBootstrapDebug = { - ...failure, - reason: this.providerUnavailableReason ?? failure.reason, - }; - this.embeddingBootstrapFailure = nextFailure; - this.cacheProbeResult({ ok: false, error: nextFailure.reason }); - onDebug?.({ backend: "builtin", embeddingBootstrap: nextFailure }); - return true; - } - - const currentIdentity = this.refreshIndexIdentityDirty({ providerKeyKnown: true }); - let activeFailure = failure; - if (currentIdentity.status !== "valid") { - try { - await this.syncAdmitted({ reason: "search", force: true }); - } catch (err) { - const message = redactSensitiveText(formatErrorMessage(err), { mode: "tools" }); - log.warn(`memory sync failed (embedding-bootstrap-recovery): ${message}`); - activeFailure = this.markEmbeddingBootstrapFailure(err, { retainProvider: true }); - } - } - if ( - this.refreshIndexIdentityDirty({ providerKeyKnown: true }).status === "valid" && - (await this.confirmEmbeddingBootstrapRecovery()) - ) { - // A valid existing index skips recovery reindex, so explicitly restore the - // semantic readiness flag cleared when bootstrap degradation began. - this.vector.semanticAvailable = await this.probeVectorStoreAvailabilityAdmitted(); - this.clearEmbeddingBootstrapFailureAfterRecovery(); - return false; - } - activeFailure = this.embeddingBootstrapFailure ?? activeFailure; - onDebug?.({ backend: "builtin", embeddingBootstrap: activeFailure }); - return true; - } - - private clearEmbeddingBootstrapFailureAfterRecovery(): void { - this.embeddingBootstrapFailure = undefined; - this.providerUnavailableReason = undefined; - if (this.provider) { - this.providerLifecycle = this.fallbackFrom - ? { - mode: "fallback-active", - providerId: this.provider.id, - fallbackFrom: this.fallbackFrom, - reason: this.fallbackReason ?? "fallback activated", - } - : { mode: "active", providerId: this.provider.id }; - } - EMBEDDING_PROBE_CACHE.delete(this.cacheKey); - } - - private async confirmEmbeddingBootstrapRecovery(): Promise { - const cached = this.getCachedEmbeddingAvailability(); - if (cached) { - return cached.ok; - } - if (!this.provider) { - return false; - } - try { - await this.embedBatchWithRetry(["ping"]); - this.cacheProbeResult({ ok: true }); - return true; - } catch (err) { - this.markEmbeddingBootstrapFailure(err, { - retainProvider: true, - provider: this.provider.id, - }); - return false; - } - } - - private async ensureProviderInitialized(): Promise { - if (this.providerInitialized) { - const bootstrapRetryDue = - this.embeddingBootstrapFailure !== undefined && - !this.provider && - this.getCachedEmbeddingAvailability() === null; - if (!bootstrapRetryDue) { - await this.getPendingFallbackProviderInitialization()?.catch(() => undefined); - return; - } - this.resetProviderInitializationForRetry(); - } - if (this.settings.provider === "none") { - this.applyProviderResult({ - provider: null, - requestedProvider: "none", - providerUnavailableReason: "No embedding provider available (FTS-only mode)", - }); - this.providerKey = this.computeProviderKey(); - this.batch = this.resolveBatchConfig(); - return; - } - if (!this.providerInitPromise) { - this.providerInitPromise = (async () => { - await this.getPendingFallbackProviderInitialization()?.catch(() => undefined); - await this.retireCurrentProvider(); - if (this.closed) { - return; - } - const providerResult = await MemoryIndexManager.loadProviderResult({ - cfg: this.cfg, - agentId: this.agentId, - settings: this.settings, - acquireLocalService: this.acquireLocalService, - }); - this.applyProviderResult(providerResult); - this.providerKey = this.computeProviderKey(); - this.batch = this.resolveBatchConfig(); - })(); - } - try { - await this.providerInitPromise; - } catch (err) { - // Clear the cached rejected promise so subsequent calls can retry - // initialization instead of being permanently stuck with a stale failure. - this.providerInitPromise = null; - throw err; - } finally { - if (this.providerInitialized) { - this.providerInitPromise = null; - } - } - } - - protected resetProviderInitializationForRetry(): void { - void this.retireCurrentProvider(); - this.providerInitialized = false; - this.providerInitPromise = null; - this.providerUnavailableReason = undefined; - this.providerLifecycle = createPendingMemoryProviderLifecycle(this.requestedProvider); - } - - protected markLocalEmbeddingProviderDegraded(err: unknown): void { - if (this.provider?.id !== "local") { - return; - } - const workerFailure = isLocalEmbeddingWorkerFailure(err) - ? err - : err instanceof Error && isLocalEmbeddingWorkerFailure(err.cause) - ? err.cause - : null; - if (!workerFailure) { - return; - } - const message = formatErrorMessage(workerFailure); - const degradedProvider = this.provider; - void this.retireCurrentProvider(); - this.providerUnavailableReason = `Local embeddings degraded: ${message}`; - this.providerLifecycle = createDegradedMemoryProviderLifecycle({ - providerId: degradedProvider.id, - reason: message, - code: workerFailure.code, - }); - EMBEDDING_PROBE_CACHE.delete(this.cacheKey); - this.providerKey = this.computeProviderKey(); - this.batch = this.resolveBatchConfig(); - this.vector.semanticAvailable = false; - log.warn("memory embeddings: local provider degraded after worker failure", { - error: message, - }); - } - - protected override retireCurrentProvider(): Promise { - const provider = this.provider; - if (provider) { - this.provider = null; - this.providerRuntime = undefined; - this.providersPendingRetirement.add(provider); - } - if (this.providersPendingRetirement.size === 0) { - return this.providerRetirementPromise; - } - // Provider replacement must wait for the previous worker to exit; otherwise - // repeated retries can accumulate local workers on constrained hosts. - const retirement = this.providerRetirementPromise - .catch(() => {}) - .then(async () => { - let firstError: unknown; - let closeFailed = false; - for (const pendingProvider of this.providersPendingRetirement) { - try { - await this.awaitProviderIdle(pendingProvider); - await pendingProvider.close?.(); - this.providersPendingRetirement.delete(pendingProvider); - } catch (err) { - if (!closeFailed) { - firstError = err; - } - closeFailed = true; - } - } - if (closeFailed) { - throw toErrorObject(firstError, "Embedding provider retirement failed"); - } - }); - this.providerRetirementPromise = retirement; - void retirement.catch((err: unknown) => { - log.warn(`memory embeddings: failed to close previous provider: ${formatErrorMessage(err)}`); - }); - return retirement; - } - - private async drainPendingProviderRetirements(): Promise { - const errors: unknown[] = []; - for ( - let attempt = 0; - attempt < 2 && (this.provider !== null || this.providersPendingRetirement.size > 0); - attempt += 1 - ) { - try { - await this.retireCurrentProvider(); - } catch (err) { - errors.push(err); - log.warn(`memory close: pending manager work failed: ${formatErrorMessage(err)}`); - } - } - return errors; - } - - protected isRequiredProviderUnavailable(): boolean { - return this.providerRequirement.mode === "required" && !this.provider; - } - - protected buildRequiredProviderUnavailableError(operation: "search" | "sync"): Error { - const registeredProviderIds = listRegisteredMemoryEmbeddingProviderAdapters() - .map((adapter) => adapter.id) - .toSorted(); - const registeredProviders = - registeredProviderIds.length > 0 ? registeredProviderIds.join(",") : "none"; - const reason = - this.providerUnavailableReason ?? - (this.providerLifecycle.mode === "fts-only" - ? this.providerLifecycle.reason - : "provider is unavailable"); - return new Error( - `Memory ${operation} unavailable: embedding provider "${this.settings.provider}" is configured but unavailable. ` + - `Reason: ${reason}. ` + - `agentId=${this.agentId} purpose=${this.purpose} lifecycle=${JSON.stringify(this.providerLifecycle)} ` + - `registeredMemoryEmbeddingProviders=${registeredProviders}`, - ); - } - - protected assertRequiredProviderAvailable(operation: "search" | "sync"): void { - if (this.isRequiredProviderUnavailable()) { - const error = this.buildRequiredProviderUnavailableError(operation); - this.resetProviderInitializationForRetry(); - throw error; - } - } - - async warmSession(sessionKey?: string): Promise { - if (!this.settings.sync.onSessionStart) { - return; - } - const key = sessionKey?.trim() || ""; - if (key && this.sessionWarm.has(key)) { - return; - } - void this.sync({ reason: "session-start" }).catch((err: unknown) => { - log.warn(`memory sync failed (session-start): ${String(err)}`); - }); - if (key) { - this.sessionWarm.add(key); - } - } - - private refreshIndexIdentityDirty(params?: { providerKeyKnown?: boolean }) { - const provider = - this.settings.provider === "none" - ? null - : this.providerInitialized - ? this.provider - ? { id: this.provider.id, model: this.provider.model } - : null - : undefined; - const state = this.resolveCurrentIndexIdentityState({ - ...(provider !== undefined ? { provider } : {}), - providerKeyKnown: params?.providerKeyKnown, - }); - this.indexIdentityState = state; - this.indexIdentityDirty = - state.status === "mismatched" || - (state.status === "missing" && (this.sources.has("memory") || this.hasIndexedChunks())); - return state; - } - - private refreshKeywordFallbackIndexIdentity() { - const meta = this.readMeta(); - const state = this.resolveCurrentIndexIdentityState({ - meta, - provider: meta && meta.provider !== "none" ? { id: meta.provider, model: meta.model } : null, - providerKeyKnown: false, - vectorReady: false, - }); - this.indexIdentityState = state; - this.indexIdentityDirty = - state.status === "mismatched" || - (state.status === "missing" && (this.sources.has("memory") || this.hasIndexedChunks())); - return state; - } - - private async withManagerOperation(run: () => Promise): Promise { - if (this.closing || this.closed) { - throw new Error("Memory index manager is closed"); - } - this.activeManagerOperations += 1; - try { - return await run(); - } finally { - this.activeManagerOperations -= 1; - if (this.activeManagerOperations === 0) { - const waiters = Array.from(this.managerIdleWaiters); - this.managerIdleWaiters.clear(); - for (const resolve of waiters) { - resolve(); - } - } - } - } - - private async awaitManagerIdle(): Promise { - if (this.activeManagerOperations === 0) { - return; - } - await new Promise((resolve) => { - this.managerIdleWaiters.add(resolve); - }); - } - - async search(query: string, opts?: MemoryIndexSearchOptions): Promise { - const normalizedQuery = query.trim(); - if (!normalizedQuery) { - return []; - } - const maxResults = opts?.maxResults ?? this.settings.query.maxResults; - const minScore = opts?.minScore ?? this.settings.query.minScore; - const hasActiveProject = (opts?.activeProjectKeys?.length ?? 0) > 0; - const candidateMaxResults = hasActiveProject - ? Math.min(200, Math.max(maxResults, maxResults * 4)) - : maxResults; - const candidateMinScore = hasActiveProject ? minScore / 1.15 : minScore; - const results = await this.searchCandidates(normalizedQuery, { - ...opts, - maxResults: candidateMaxResults, - minScore: candidateMinScore, - }); - return hasActiveProject - ? results.filter((entry) => entry.score >= minScore).slice(0, maxResults) - : results; - } - - private async searchCandidates( - normalizedQuery: string, - opts?: MemoryIndexSearchOptions, - ): Promise { - return await this.withManagerOperation(async () => { - opts?.onDebug?.({ backend: "builtin" }); - if (this.providerRequirement.mode === "required") { - await this.ensureProviderInitialized(); - this.assertRequiredProviderAvailable("search"); - } - let hasIndexedContent = this.hasIndexedContent(); - if (!hasIndexedContent) { - try { - // A fresh process can receive its first search before background watch/session - // syncs have built the index. Force one synchronous bootstrap so the first - // lookup after restart does not fail closed with empty results. - await this.syncAdmitted( - { reason: "search", force: true }, - { allowEmbeddingBootstrapFallback: true }, - ); - } catch (err) { - if (this.providerRequirement.mode === "optional" && this.shouldFallbackOnError(err)) { - const failedProvider = this.provider?.id ?? this.settings.provider; - await this.retireCurrentProvider().catch((retireErr: unknown) => { - const message = redactSensitiveText(formatErrorMessage(retireErr), { - mode: "tools", - }); - log.warn(`memory search-bootstrap: failed to retire embedding provider: ${message}`); - }); - this.markEmbeddingBootstrapFailure(err, { provider: failedProvider }); - await this.syncAdmitted({ reason: "search", force: true }).catch( - (fallbackErr: unknown) => { - const message = redactSensitiveText(formatErrorMessage(fallbackErr), { - mode: "tools", - }); - log.warn(`memory sync failed (search-bootstrap-fallback): ${message}`); - }, - ); - } else { - log.warn(`memory sync failed (search-bootstrap): ${String(err)}`); - } - } - hasIndexedContent = this.hasIndexedContent(); - } - const preflight = resolveMemorySearchPreflight({ - query: normalizedQuery, - hasIndexedContent, - }); - if (!preflight.shouldSearch) { - if (this.embeddingBootstrapFailure) { - opts?.onDebug?.({ - backend: "builtin", - embeddingBootstrap: this.embeddingBootstrapFailure, - }); - } - return []; - } - const cleaned = preflight.normalizedQuery; - const embeddingBootstrapKeywordOnly = await this.ensureEmbeddingProviderForSearch( - opts?.onDebug, - ); - void this.warmSession(opts?.sessionKey); - await startAsyncSearchSync({ - enabled: this.settings.sync.onSearch, - dirty: this.dirty, - sessionsDirty: this.sessionsDirty, - sync: async (params) => await this.syncAdmitted(params), - onError: (err) => { - log.warn(`memory sync failed (search): ${String(err)}`); - }, - }); - if ( - !embeddingBootstrapKeywordOnly && - preflight.shouldInitializeProvider && - !this.provider && - (this.providerLifecycle.mode === "pending" || - (this.providerLifecycle.mode === "degraded" && - this.providerLifecycle.providerId !== this.settings.provider)) - ) { - // A failed fallback must yield ownership back to the configured primary. - // Reinitialize it before identity validation; leaving the lifecycle pending - // makes a valid existing index look mismatched and drops keyword results. - this.resetProviderInitializationForRetry(); - await this.ensureProviderInitialized(); - } - this.assertRequiredProviderAvailable("search"); - if ( - !embeddingBootstrapKeywordOnly && - !this.provider && - this.providerLifecycle.mode === "degraded" - ) { - const activatedFallback = await this.activateFallbackProvider( - this.providerLifecycle.reason, - ).catch((fallbackErr: unknown) => { - log.warn( - `memory search: failed to activate fallback provider: ${formatErrorMessage(fallbackErr)}`, - ); - return false; - }); - if (activatedFallback) { - this.refreshIndexIdentityDirty({ - providerKeyKnown: this.providerInitialized, - }); - } - } - const indexIdentity = embeddingBootstrapKeywordOnly - ? this.refreshKeywordFallbackIndexIdentity() - : this.refreshIndexIdentityDirty({ - providerKeyKnown: this.providerInitialized, - }); - if (indexIdentity.status !== "valid") { - return []; - } - const minScore = opts?.minScore ?? this.settings.query.minScore; - const maxResults = opts?.maxResults ?? this.settings.query.maxResults; - const searchSources = - opts?.sources && opts.sources.length > 0 - ? uniqueValues(opts.sources).filter((s) => this.sources.has(s)) - : undefined; - if ( - opts?.sources && - opts.sources.length > 0 && - (!searchSources || searchSources.length === 0) - ) { - return []; - } - // The manager may index recall-only transcripts without making them part of - // ordinary searches. Trusted recall passes an explicit source override; - // every other caller defaults to the configured search corpus. - const sourceFilterList = searchSources ?? this.settings.searchSources; - const hybrid = this.settings.query.hybrid; - const candidates = Math.min( - 200, - Math.max(1, Math.floor(maxResults * hybrid.candidateMultiplier)), - ); - - // FTS-only mode: no embedding provider available - if (embeddingBootstrapKeywordOnly || !this.provider) { - this.assertRequiredProviderAvailable("search"); - if (!this.fts.enabled || !this.fts.available) { - log.warn("memory search: no provider and FTS unavailable"); - return []; - } - - const keywordResults = await this.searchKeywordWithFallback( - cleaned, - candidates, - { - boostFallbackRanking: true, - }, - sourceFilterList, - ).catch((err: unknown) => { - log.warn(`memory search: FTS keyword query failed: ${formatErrorMessage(err)}`); - return []; - }); - - return await this.finalizeKeywordOnlyResults({ - results: keywordResults, - temporalDecay: hybrid.temporalDecay, - maxResults, - minScore, - activeProjectKeys: opts?.activeProjectKeys, - }); - } - let semanticProvider = this.provider; - let semanticProviderRuntime = this.providerRuntime; - let vectorProviderIdentity = { - model: semanticProvider.model, - aliases: this.resolveProviderIndexIdentities() - .slice(1) - .map((identity) => identity.model), - }; - - // If FTS isn't available, hybrid mode cannot use keyword search; degrade to vector-only. - const loadKeywordResults = async () => - hybrid.enabled && this.fts.enabled && this.fts.available - ? await this.searchKeywordWithFallback( - cleaned, - candidates, - { boostFallbackRanking: true }, - sourceFilterList, - ).catch((err: unknown) => { - log.warn( - `memory search: FTS hybrid keyword query failed: ${formatErrorMessage(err)}`, - ); - return []; - }) - : []; - let keywordResults: Awaited> = []; - let queryVec: number[]; - const releaseSemanticProvider = this.acquireProviderUse(semanticProvider); - try { - keywordResults = await loadKeywordResults(); - // lexicalOnly is a reply-path contract: no query embedding, no vector - // search, no network. Callers accept keyword-only recall quality. - if (opts?.lexicalOnly) { - return await this.finalizeKeywordOnlyResults({ - results: keywordResults, - temporalDecay: hybrid.temporalDecay, - maxResults, - minScore, - activeProjectKeys: opts?.activeProjectKeys, - }); - } - try { - queryVec = await this.embedQueryWithRetry( - cleaned, - opts?.signal, - semanticProvider, - false, - semanticProviderRuntime, - ); - } catch (err) { - releaseSemanticProvider(); - this.markLocalEmbeddingProviderDegraded(err); - // An aborted caller already stopped waiting; skip fallback-provider - // activation so the abandoned search stops instead of re-embedding. - if (opts?.signal?.aborted) { - throw err; - } - const message = formatErrorMessage(err); - const activatedFallback = this.shouldFallbackOnError(err) - ? await this.activateFallbackProvider(message).catch((fallbackErr: unknown) => { - log.warn( - `memory search: failed to activate fallback provider: ${formatErrorMessage(fallbackErr)}`, - ); - return false; - }) - : false; - if (activatedFallback) { - if ( - this.refreshIndexIdentityDirty({ - providerKeyKnown: this.providerInitialized, - }).status !== "valid" - ) { - return []; - } - if (!this.provider) { - return []; - } - semanticProvider = this.provider; - semanticProviderRuntime = this.providerRuntime; - vectorProviderIdentity = { - model: semanticProvider.model, - aliases: this.resolveProviderIndexIdentities() - .slice(1) - .map((identity) => identity.model), - }; - const releaseFallbackProvider = this.acquireProviderUse(semanticProvider); - try { - keywordResults = await loadKeywordResults(); - queryVec = await this.embedQueryWithRetry( - cleaned, - opts?.signal, - semanticProvider, - false, - semanticProviderRuntime, - ); - } catch (fallbackErr) { - releaseFallbackProvider(); - this.markLocalEmbeddingProviderDegraded(fallbackErr); - throw fallbackErr; - } finally { - releaseFallbackProvider(); - } - } else if (!this.provider && this.fts.enabled && this.fts.available) { - this.assertRequiredProviderAvailable("search"); - log.warn( - `memory search: embeddings unavailable; using keyword-only results: ${message}`, - ); - return await this.finalizeKeywordOnlyResults({ - results: keywordResults, - temporalDecay: hybrid.temporalDecay, - maxResults, - minScore, - activeProjectKeys: opts?.activeProjectKeys, - }); - } else { - throw err; - } - } - } finally { - releaseSemanticProvider(); - } - const hasVector = queryVec.some((v) => v !== 0); - const vectorResults = hasVector - ? await this.searchVector( - queryVec, - candidates, - sourceFilterList, - vectorProviderIdentity, - ).catch((err: unknown) => { - log.warn(`memory search: vector query failed: ${formatErrorMessage(err)}`); - return []; - }) - : []; - - if (!hybrid.enabled || !this.fts.enabled || !this.fts.available) { - const decayed = await applyTemporalDecayToHybridResults({ - results: vectorResults, - temporalDecay: hybrid.temporalDecay, - workspaceDir: this.workspaceDir, - }); - return applyProjectRanking(applyImportanceMultiplier(decayed), opts?.activeProjectKeys) - .filter((entry) => entry.score >= minScore) - .slice(0, maxResults); - } - - const merged = await this.mergeHybridResults({ - query: cleaned, - vector: vectorResults, - keyword: keywordResults, - vectorWeight: hybrid.vectorWeight, - textWeight: hybrid.textWeight, - mmr: hybrid.mmr, - temporalDecay: hybrid.temporalDecay, - activeProjectKeys: opts?.activeProjectKeys, - }); - return selectHybridSearchResults({ - merged, - keyword: keywordResults, - maxResults, - minScore, - }); - }); - } - - private selectScoredResults( - results: T[], - maxResults: number, - minScore: number, - relaxedMinScore = minScore, - ): T[] { - const strict = results.filter((entry) => entry.score >= minScore); - if (strict.length > 0) { - return strict.slice(0, maxResults); - } - return results.filter((entry) => entry.score >= relaxedMinScore).slice(0, maxResults); - } - - async listTriggerCandidates(opts?: { - limit?: number; - activeProjectKeys?: string[]; - }): Promise { - const limit = Math.max(1, Math.min(512, Math.floor(opts?.limit ?? 512))); - return this.toCuratedMemorySearchResults( - readCuratedMemoryTriggerCandidates(this.db, limit, opts?.activeProjectKeys), - ); - } - - async listCuratedProjectCandidates(opts: { - activeProjectKeys: string[]; - limit?: number; - }): Promise { - const limit = Math.max(1, Math.min(512, Math.floor(opts.limit ?? 48))); - return this.toCuratedMemorySearchResults( - readCuratedProjectMemoryCandidates(this.db, limit, opts.activeProjectKeys), - ); - } - - private toCuratedMemorySearchResults( - rows: ReturnType, - ): MemorySearchResult[] { - return rows.map((row) => { - const result: MemorySearchResult = { - path: row.path, - startLine: row.start_line, - endLine: row.end_line, - score: 0, - snippet: row.text, - source: "memory", - }; - if (typeof row.importance === "number") { - result.importance = row.importance; - } - if (typeof row.triggers === "string" && row.triggers.trim()) { - result.triggers = row.triggers.trim(); - } - if (typeof row.project_key === "string" && row.project_key.trim()) { - result.projectKey = row.project_key.trim(); - } - return result; - }); - } - - private rankKeywordOnlyResults( - results: KeywordSearchHit[], - preferExactBody = true, - ): KeywordSearchHit[] { - return results - .toSorted((left, right) => compareKeywordSearchHits(left, right, preferExactBody)) - .map((entry) => - entry.exactPathSpecificity > 0 ? Object.assign(entry, { score: 1 }) : entry, - ); - } - - private async finalizeKeywordOnlyResults(params: { - results: KeywordSearchHit[]; - temporalDecay?: { enabled: boolean; halfLifeDays: number }; - maxResults: number; - minScore: number; - activeProjectKeys?: readonly string[]; - }): Promise { - const appliesTemporalDecay = params.temporalDecay?.enabled === true; - const decayInputs = appliesTemporalDecay - ? params.results.map((entry) => { - if (entry.exactPathSpecificity === 0) { - return entry; - } - const contentScore = entry.textScore > 0 ? entry.score : 0; - return { ...entry, score: scoreExactPathTieForTemporalDecay(contentScore) }; - }) - : params.results; - const decayed = await applyTemporalDecayToHybridResults({ - results: decayInputs, - temporalDecay: params.temporalDecay, - workspaceDir: this.workspaceDir, - }); - const ranked = applyProjectRanking( - this.rankKeywordOnlyResults(applyImportanceMultiplier(decayed), !appliesTemporalDecay), - params.activeProjectKeys, - ); - return this.toMemorySearchResults( - this.selectScoredResults(ranked, params.maxResults, params.minScore, 0), - ); - } - - private hasIndexedContent(): boolean { - const chunkRow = this.db.prepare(`SELECT 1 as found FROM memory_index_chunks LIMIT 1`).get() as - | { - found?: number; - } - | undefined; - if (chunkRow?.found === 1) { - return true; - } - if (!this.fts.enabled || !this.fts.available) { - return false; - } - const ftsRow = this.db.prepare(`SELECT 1 as found FROM ${FTS_TABLE} LIMIT 1`).get() as - | { - found?: number; - } - | undefined; - return ftsRow?.found === 1; - } - - private async searchVector( - queryVec: number[], - limit: number, - sourceFilterList: MemorySource[], - providerIdentity: { model: string; aliases: string[] }, - ): Promise> { - const results = await searchVector({ - db: this.db, - vectorTable: VECTOR_TABLE, - providerModel: providerIdentity.model, - providerModelAliases: providerIdentity.aliases, - queryVec, - limit, - snippetMaxChars: SNIPPET_MAX_CHARS, - ensureVectorReady: async (dimensions) => await this.ensureVectorReady(dimensions), - sourceFilterVec: this.buildSourceFilter("c", sourceFilterList), - sourceFilterChunks: this.buildSourceFilter(undefined, sourceFilterList), - }); - return this.attachRecallMetadata( - results.map((entry) => entry as MemorySearchResult & { id: string }), - ); - } - - private attachRecallMetadata(results: T[]): T[] { - if (results.length === 0) { - return results; - } - const metadataById = readMemoryRecallMetadata( - this.db, - results.map((entry) => entry.id), - ); - return results.map((entry) => { - const row = metadataById.get(entry.id); - return { - ...entry, - ...(typeof row?.importance === "number" ? { importance: row.importance } : {}), - ...(typeof row?.triggers === "string" && row.triggers.trim() - ? { triggers: row.triggers.trim() } - : {}), - ...(typeof row?.project_key === "string" && row.project_key.trim() - ? { projectKey: row.project_key.trim() } - : {}), - }; - }); - } - - private buildFtsQuery(raw: string): string | null { - return buildFtsQuery(raw); - } - - private async searchKeyword( - query: string, - limit: number, - options?: { - boostFallbackRanking?: boolean; - exactPathQuery?: string; - rankingQuery?: string; - }, - sourceFilterList?: MemorySource[], - ): Promise { - if (!this.fts.enabled || !this.fts.available) { - return []; - } - const bodySearch = searchKeyword({ - db: this.db, - ftsTable: FTS_TABLE, - query, - ftsTokenizer: this.settings.store.fts.tokenizer, - limit, - snippetMaxChars: SNIPPET_MAX_CHARS, - sourceFilter: this.buildSourceFilter(undefined, sourceFilterList), - buildFtsQuery: (raw) => this.buildFtsQuery(raw), - bm25RankToScore, - boostFallbackRanking: options?.boostFallbackRanking, - rankingQuery: options?.rankingQuery, - }).catch((err: unknown) => { - log.warn(`memory search: body keyword query failed: ${formatErrorMessage(err)}`); - return []; - }); - const exactPathQuery = options?.exactPathQuery ?? query; - const pathSearch = searchPathKeyword({ - db: this.db, - pathFtsTable: PATH_FTS_TABLE, - query, - exactPathQuery, - exactPathLimit: EXACT_PATH_CANDIDATE_LIMIT, - ftsTokenizer: this.settings.store.fts.tokenizer, - limit, - snippetMaxChars: SNIPPET_MAX_CHARS, - sourceFilter: this.buildSourceFilter(PATH_FTS_TABLE, sourceFilterList), - buildFtsQuery: (raw) => this.buildFtsQuery(raw), - bm25RankToScore, - }).catch((err: unknown) => { - log.warn(`memory search: path keyword query failed: ${formatErrorMessage(err)}`); - return []; - }); - const [bodyResults, pathResults] = await Promise.all([bodySearch, pathSearch]); - const merged = this.mergeKeywordSearchHits( - [ - bodyResults.map((entry) => - Object.assign(entry, { - exactPathSpecificity: resolveExactPathSpecificity(exactPathQuery, entry.path), - pathScore: 0, - }), - ), - pathResults, - ], - exactPathQuery, - ); - return this.attachRecallMetadata(this.limitKeywordSearchHits(merged, limit)); - } - - private async searchKeywordWithFallback( - query: string, - limit: number, - options: { boostFallbackRanking?: boolean } | undefined, - sourceFilterList: MemorySource[], - ): Promise { - const fullQueryResults = await this.searchKeyword( - query, - limit, - options, - sourceFilterList, - ).catch(() => []); - const nonExactResults = fullQueryResults.filter((result) => result.exactPathSpecificity === 0); - if (nonExactResults.length >= limit) { - return fullQueryResults; - } - - // Supplement thin candidate pools for conversational queries, but cap the - // extra FTS probes so long prompts cannot fan out into unbounded sqlite work. - const fallbackTerms = this.resolveKeywordFallbackTerms(query); - if (fallbackTerms.length === 0) { - return fullQueryResults; - } - const strictFtsQuery = this.buildFtsQuery(query)?.toLowerCase(); - const keywordFtsQuery = this.buildFtsQuery(fallbackTerms.join(" "))?.toLowerCase(); - if (fullQueryResults.length > 0 && strictFtsQuery === keywordFtsQuery) { - // Expansion did not normalize this already-matching keyword query; OR - // probes can only weaken its strict relevance before importance ranking. - return fullQueryResults; - } - - const resultSets = await Promise.all( - fallbackTerms.map((term) => - this.searchKeyword( - term, - limit, - { ...options, exactPathQuery: query, rankingQuery: query }, - sourceFilterList, - ).catch(() => []), - ), - ); - return this.limitKeywordSearchHits( - this.mergeKeywordSearchHits([fullQueryResults, ...resultSets], query), - limit, - ); - } - - private resolveKeywordFallbackTerms(query: string): string[] { - const normalizedQuery = query.trim().toLowerCase(); - const keywords = extractKeywords(query, { - ftsTokenizer: this.settings.store.fts.tokenizer, - }).filter((term) => term !== normalizedQuery); - return keywords.slice(0, KEYWORD_FALLBACK_SEARCH_TERM_LIMIT); - } - - private mergeKeywordSearchHits( - resultSets: KeywordSearchHit[][], - exactPathQuery?: string, - ): KeywordSearchHit[] { - const seenIds = new Map(); - for (const results of resultSets) { - for (const result of results) { - const existing = seenIds.get(result.id); - if (!existing) { - seenIds.set(result.id, result); - continue; - } - const existingHasBody = existing.textScore > 0; - const resultHasBody = result.textScore > 0; - const existingBodyScore = existingHasBody ? existing.score : 0; - const resultBodyScore = resultHasBody ? result.score : 0; - existing.textScore = Math.max(existing.textScore, result.textScore); - existing.pathScore = Math.max(existing.pathScore, result.pathScore); - existing.exactPathSpecificity = Math.max( - existing.exactPathSpecificity, - result.exactPathSpecificity, - ) as ExactPathSpecificity; - const bodyScore = Math.max(existingBodyScore, resultBodyScore); - existing.score = bodyScore > 0 ? bodyScore : existing.pathScore; - // Path hits project the first chunk; keep a real body-match snippet - // authoritative when both retrieval surfaces find the same document. - if ( - (resultHasBody && !existingHasBody) || - (resultHasBody === existingHasBody && result.snippet.length > existing.snippet.length) - ) { - existing.snippet = result.snippet; - } - } - } - const merged = [...seenIds.values()]; - if (exactPathQuery !== undefined) { - // Fallback terms broaden lexical recall, but only the original user query - // can claim exact path, basename, or stem precedence. - for (const result of merged) { - result.exactPathSpecificity = resolveExactPathSpecificity(exactPathQuery, result.path); - } - } - for (const result of merged) { - if (result.textScore === 0) { - // A uniform exact-only baseline lets temporal decay order otherwise - // equivalent filename hits without reusing incomparable path BM25. - result.score = result.exactPathSpecificity > 0 ? 1 : result.pathScore; - } - } - return merged.toSorted(compareKeywordSearchHits); - } - - private limitKeywordSearchHits( - results: KeywordSearchHit[], - nonExactLimit: number, - ): KeywordSearchHit[] { - const ranked = results.toSorted(compareKeywordSearchHits); - const exactBody = ranked - .filter((entry) => entry.exactPathSpecificity > 0 && entry.textScore > 0) - .slice(0, nonExactLimit); - const exactPathOnly = ranked.filter( - (entry) => entry.exactPathSpecificity > 0 && entry.textScore === 0, - ); - const boundedExact = exactBody.concat(exactPathOnly).toSorted(compareKeywordSearchHits); - const selectedPathKeys = new Set(); - for (const entry of boundedExact) { - selectedPathKeys.add(`${entry.source}:${entry.path}`); - if (selectedPathKeys.size === EXACT_PATH_CANDIDATE_LIMIT) { - break; - } - } - const exact = boundedExact.filter((entry) => - selectedPathKeys.has(`${entry.source}:${entry.path}`), - ); - const nonExact = ranked - .filter((entry) => entry.exactPathSpecificity === 0) - .slice(0, nonExactLimit); - return exact.concat(nonExact); - } - - private toMemorySearchResults(results: KeywordSearchHit[]): MemorySearchResult[] { - return results.map( - ({ - id: _id, - pathScore: _pathScore, - exactPathSpecificity: _exactPathSpecificity, - ...result - }) => result, - ); - } - - private mergeHybridResults(params: { - query: string; - vector: Array; - keyword: KeywordSearchHit[]; - vectorWeight: number; - textWeight: number; - mmr?: { enabled: boolean; lambda: number }; - temporalDecay?: { enabled: boolean; halfLifeDays: number }; - activeProjectKeys?: readonly string[]; - }): Promise[]> { - return mergeHybridResults({ - vector: params.vector.map((r) => ({ - id: r.id, - path: r.path, - startLine: r.startLine, - endLine: r.endLine, - source: r.source, - snippet: r.snippet, - vectorScore: r.score, - importance: r.importance, - triggers: r.triggers, - projectKey: r.projectKey, - exactPathSpecificity: resolveExactPathSpecificity(params.query, r.path), - ...(r.provenance ? { provenance: r.provenance } : {}), - })), - keyword: params.keyword.map((r) => ({ - id: r.id, - path: r.path, - startLine: r.startLine, - endLine: r.endLine, - source: r.source, - snippet: r.snippet, - textScore: r.textScore, - importance: r.importance, - triggers: r.triggers, - projectKey: r.projectKey, - rankingScore: r.score, - pathScore: r.pathScore, - exactPathSpecificity: r.exactPathSpecificity, - ...(r.provenance ? { provenance: r.provenance } : {}), - })), - vectorWeight: params.vectorWeight, - textWeight: params.textWeight, - isNonTextMediaPath: (path) => - classifyMemoryMultimodalPath(path, this.settings.multimodal) !== null, - mmr: params.mmr, - temporalDecay: params.temporalDecay, - activeProjectKeys: params.activeProjectKeys, - workspaceDir: this.workspaceDir, - }); - } - async sync(params?: MemorySyncParams): Promise { if (this.closing || this.closed) { return; @@ -1904,7 +302,7 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem return await this.syncAdmitted(params); } - private async syncAdmitted( + protected async syncAdmitted( params?: MemorySyncParams, options?: { allowEmbeddingBootstrapFallback?: boolean; @@ -2104,9 +502,9 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem enabled: true, entries: ( - this.db.prepare(`SELECT COUNT(*) as c FROM ${EMBEDDING_CACHE_TABLE}`).get() as - | { c: number } - | undefined + this.db + .prepare(`SELECT COUNT(*) as c FROM ${MEMORY_EMBEDDING_CACHE_TABLE}`) + .get() as { c: number } | undefined )?.c ?? 0, maxEntries: this.cache.maxEntries, } @@ -2123,7 +521,7 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem enabled: this.vector.enabled, index: resolvePersistedMemoryVectorIndexState({ db: this.db, - vectorTable: VECTOR_TABLE, + vectorTable: MEMORY_INDEX_VECTOR_TABLE, metaVectorDims: this.vector.dims, hasSemanticChunks: this.hasSemanticChunks(), }), @@ -2155,92 +553,6 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem }; } - async probeVectorAvailability(): Promise { - return await this.withManagerOperation(async () => { - if (!this.vector.enabled) { - this.vector.semanticAvailable = false; - return false; - } - await this.ensureProviderInitialized(); - // FTS-only mode: vector search not available - if (!this.provider) { - this.vector.semanticAvailable = false; - return false; - } - const ready = await this.probeVectorStoreAvailabilityAdmitted(); - this.vector.semanticAvailable = ready; - return ready; - }); - } - - async probeVectorStoreAvailability(): Promise { - return await this.withManagerOperation( - async () => await this.probeVectorStoreAvailabilityAdmitted(), - ); - } - - private async probeVectorStoreAvailabilityAdmitted(): Promise { - if (!this.vector.enabled) { - this.vector.available = false; - return false; - } - return await this.ensureVectorReady(); - } - - private cacheProbeResult(result: MemoryEmbeddingProbeResult): MemoryEmbeddingProbeResult { - const checkedAtMs = Date.now(); - EMBEDDING_PROBE_CACHE.set(this.cacheKey, { - result, - checkedAtMs, - expireAtMs: checkedAtMs + EMBEDDING_PROBE_CACHE_TTL_MS, - }); - return result; - } - - getCachedEmbeddingAvailability(): MemoryEmbeddingProbeResult | null { - const cached = EMBEDDING_PROBE_CACHE.get(this.cacheKey); - if (!cached) { - return null; - } - const nowMs = Date.now(); - if (nowMs >= cached.expireAtMs) { - EMBEDDING_PROBE_CACHE.delete(this.cacheKey); - return null; - } - return { - ...cached.result, - checked: true, - cached: true, - checkedAtMs: cached.checkedAtMs, - cacheExpiresAtMs: cached.expireAtMs, - }; - } - - async probeEmbeddingAvailability(): Promise { - return await this.withManagerOperation(async () => { - const cached = this.getCachedEmbeddingAvailability(); - if (cached) { - return cached; - } - await this.ensureProviderInitialized(); - // FTS-only mode: embeddings not available but search still works - if (!this.provider) { - return this.cacheProbeResult({ - ok: false, - error: - this.providerUnavailableReason ?? "No embedding provider available (FTS-only mode)", - }); - } - try { - await this.embedBatchWithRetry(["ping"]); - return this.cacheProbeResult({ ok: true }); - } catch (err) { - const message = formatErrorMessage(err); - return this.cacheProbeResult({ ok: false, error: message }); - } - }); - } - async close(): Promise { const existingClose = this.closePromise; if (existingClose) { @@ -2264,9 +576,7 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem if (this.providersPendingRetirement.size > 0) { throw toErrorObject(retirementErrors.at(-1), "Embedding provider retirement failed"); } - if (INDEX_CACHE.get(this.cacheKey) === this) { - INDEX_CACHE.delete(this.cacheKey); - } + INDEX_MANAGER_REGISTRY.deleteIfCurrent(this.cacheKey, this); } private async closeOnce(): Promise { @@ -2379,9 +689,7 @@ export class MemoryIndexManager extends MemoryManagerEmbeddingOps implements Mem if (closeError) { throw toErrorObject(closeError, "Non-Error thrown"); } - if (INDEX_CACHE.get(this.cacheKey) === this) { - INDEX_CACHE.delete(this.cacheKey); - } + INDEX_MANAGER_REGISTRY.deleteIfCurrent(this.cacheKey, this); } } @@ -2391,5 +699,3 @@ function hasTargetedSessionSyncParams(params: MemorySyncParams | undefined): boo params?.archiveFiles?.some((sessionFile) => sessionFile.trim().length > 0), ); } - -/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/extensions/memory-core/src/memory/manager.watcher-config.test.ts b/extensions/memory-core/src/memory/manager.watcher-config.test.ts index e12374bf8274..94678a102e62 100644 --- a/extensions/memory-core/src/memory/manager.watcher-config.test.ts +++ b/extensions/memory-core/src/memory/manager.watcher-config.test.ts @@ -170,11 +170,8 @@ vi.mock("./embeddings.js", () => ({ })); import { clearMemoryEmbeddingProviders as clearRegistry } from "openclaw/plugin-sdk/memory-core-host-engine-embeddings"; -import { - closeAllMemorySearchManagers, - getMemorySearchManager, - type MemoryIndexManager, -} from "./index.js"; +import { closeAllMemorySearchManagers, getMemorySearchManager } from "./index.js"; +import type { MemoryIndexManager } from "./manager.js"; import { isolateMemoryManagerTestConfig } from "./test-config-helpers.js"; describe("memory watcher config", () => { diff --git a/extensions/memory-core/src/memory/search-manager.test.ts b/extensions/memory-core/src/memory/search-manager.test.ts index 16b5e900678c..9c0146d6cef8 100644 --- a/extensions/memory-core/src/memory/search-manager.test.ts +++ b/extensions/memory-core/src/memory/search-manager.test.ts @@ -67,6 +67,6 @@ describe("builtin memory search manager", () => { await closeMemorySearchManager({ cfg, agentId: " Main " }); - expect(closeMemoryIndexManagersForAgent).toHaveBeenCalledWith({ cfg, agentId: "main" }); + expect(closeMemoryIndexManagersForAgent).toHaveBeenCalledWith({ agentId: "main" }); }); }); diff --git a/extensions/memory-core/src/memory/search-manager.ts b/extensions/memory-core/src/memory/search-manager.ts index 207ee3318c41..bc927ed5116f 100644 --- a/extensions/memory-core/src/memory/search-manager.ts +++ b/extensions/memory-core/src/memory/search-manager.ts @@ -70,7 +70,6 @@ export async function closeMemorySearchManager(params: { } const { closeMemoryIndexManagersForAgent } = await loadManagerRuntime(); await closeMemoryIndexManagersForAgent({ - cfg: params.cfg, agentId: normalizeAgentId(params.agentId), }); } diff --git a/extensions/memory-core/src/memory/test-manager-helpers.ts b/extensions/memory-core/src/memory/test-manager-helpers.ts index ca753d7871ab..469d6f769493 100644 --- a/extensions/memory-core/src/memory/test-manager-helpers.ts +++ b/extensions/memory-core/src/memory/test-manager-helpers.ts @@ -1,7 +1,7 @@ import { createLazyRuntimeModule } from "openclaw/plugin-sdk/lazy-runtime"; // Memory Core helper module supports test manager helpers behavior. import type { OpenClawConfig } from "openclaw/plugin-sdk/memory-core-host-engine-foundation"; -import type { MemoryIndexManager } from "./index.js"; +import type { MemoryIndexManager } from "./manager.js"; const ensureEmbeddingMocksLoaded = createLazyRuntimeModule(() => import("./embedding.test-mocks.js").then(() => undefined), diff --git a/extensions/memory-core/src/memory/manager.read-file.test.ts b/packages/memory-host-sdk/src/host/read-file-manager-compat.test.ts similarity index 100% rename from extensions/memory-core/src/memory/manager.read-file.test.ts rename to packages/memory-host-sdk/src/host/read-file-manager-compat.test.ts diff --git a/src/scripts/test-projects.test.ts b/src/scripts/test-projects.test.ts index bd1919476422..3d5d779945e1 100644 --- a/src/scripts/test-projects.test.ts +++ b/src/scripts/test-projects.test.ts @@ -562,6 +562,12 @@ describe("test-projects args", () => { forwardedArgs: [], includePatterns: [ "extensions/memory-core/src/memory/index.test.ts", + "extensions/memory-core/src/memory/manager-keyword-retrieval.test.ts", + "extensions/memory-core/src/memory/manager-provider-lifecycle-fallback.test.ts", + "extensions/memory-core/src/memory/manager-provider-lifecycle-leases.test.ts", + "extensions/memory-core/src/memory/manager-provider-lifecycle.test.ts", + "extensions/memory-core/src/memory/manager-registry.test.ts", + "extensions/memory-core/src/memory/manager-search-orchestration.test.ts", "extensions/memory-core/src/memory/manager.fts-only-reindex.test.ts", "extensions/memory-core/src/memory/manager.legacy-migration-cleanup.test.ts", "extensions/memory-core/src/memory/manager.reindex-recovery.test.ts", diff --git a/test/scripts/check-runtime-sidecar-loaders.test.ts b/test/scripts/check-runtime-sidecar-loaders.test.ts index d6edd59fd92e..562dc89c62b7 100644 --- a/test/scripts/check-runtime-sidecar-loaders.test.ts +++ b/test/scripts/check-runtime-sidecar-loaders.test.ts @@ -1,11 +1,78 @@ // Check Runtime Sidecar Loaders tests cover check runtime sidecar loaders script behavior. +import { existsSync, readFileSync } from "node:fs"; +import { dirname, relative, resolve } from "node:path"; +import ts from "typescript"; import { describe, expect, it } from "vitest"; import { collectTsdownEntrySources, findRuntimeSidecarLoaderViolations, } from "../../scripts/check-runtime-sidecar-loaders.mts"; +function listRuntimeStaticSpecifiers(sourcePath: string): string[] { + const source = readFileSync(sourcePath, "utf8"); + const sourceFile = ts.createSourceFile(sourcePath, source, ts.ScriptTarget.Latest, true); + return sourceFile.statements.flatMap((statement) => { + if ( + ts.isImportDeclaration(statement) && + ts.isStringLiteral(statement.moduleSpecifier) && + !statement.importClause?.isTypeOnly + ) { + return [statement.moduleSpecifier.text]; + } + if ( + ts.isExportDeclaration(statement) && + statement.moduleSpecifier && + ts.isStringLiteral(statement.moduleSpecifier) && + !statement.isTypeOnly && + !( + statement.exportClause && + ts.isNamedExports(statement.exportClause) && + statement.exportClause.elements.every((element) => element.isTypeOnly) + ) + ) { + return [statement.moduleSpecifier.text]; + } + return []; + }); +} + +function resolveLocalSource(importerPath: string, specifier: string): string | null { + if (!specifier.startsWith(".")) { + return null; + } + const resolved = resolve(dirname(importerPath), specifier); + const candidates = [resolved, resolved.replace(/\.js$/, ".ts"), resolve(resolved, "index.ts")]; + return candidates.find((candidate) => existsSync(candidate)) ?? null; +} + +function collectRuntimeStaticGraph(entryPath: string): Set { + const pending = [entryPath]; + const visited = new Set(); + for (const sourcePath of pending) { + if (visited.has(sourcePath)) { + continue; + } + visited.add(sourcePath); + for (const specifier of listRuntimeStaticSpecifiers(sourcePath)) { + const resolved = resolveLocalSource(sourcePath, specifier); + if (resolved && !visited.has(resolved)) { + pending.push(resolved); + } + } + } + return visited; +} + describe("check-runtime-sidecar-loaders", () => { + it("keeps the memory runtime facade out of the manager sidecar graph", () => { + const sourcePath = new URL("../../extensions/memory-core/runtime-api.ts", import.meta.url); + const runtimeGraph = [...collectRuntimeStaticGraph(sourcePath.pathname)].map((filePath) => + relative(resolve(dirname(sourcePath.pathname), "../.."), filePath), + ); + + expect(runtimeGraph.filter((filePath) => /(^|\/)manager(?:-|\.)/.test(filePath))).toEqual([]); + }); + it("flags hidden createRequire runtime sidecars that are not build entries", () => { const source = ` import { createRequire } from "node:module";