diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256 index 81ee859b5a7d..3c2299c90f82 100644 --- a/docs/.generated/plugin-sdk-api-baseline.sha256 +++ b/docs/.generated/plugin-sdk-api-baseline.sha256 @@ -1,4 +1,4 @@ -1ab1c0c13b0ee6dc1ef34bb8f7688107a3f3a36dbbf5c0de8aed1c4c002ce773 module/index +4f222069c2ba0c355b72e48a27da2294484dcce0a9b4b3fd3a7ea90a134130c7 module/index f8baddc86ac0e924c7806957f90a6ebdd5916a362ad24b665824503ee77fc6a9 module/access-groups 7a8753532854ff6f6d2a0c8e6386653adce285f9f6a9b5e56e88c9ea49e79cc2 module/account-core e7519fd1b7ff7003a6c8b9b640ffedd443a3754a10efc82877d48cd98cc6ba28 module/account-helpers @@ -13,7 +13,7 @@ e5e67ddf3cab38fcbf9220bc3160715897e2709d9a9ff6ff36f1ecc9453c2367 module/agent-c 30452bae2a689fb75dcb6dba9ef11e5b95f9cbe0c62eb190a0fcb82d0a19ae52 module/agent-core 74daa746deb548379d3f0d6eac3c4d082df1034c4360cc03bf51fee0f10a2e4d module/agent-harness e09226afd443cee02ed648f032f53bfeb60585617bb523a649995764d3c38da9 module/agent-harness-exec-review-runtime -1a89aff5b4fe0775a776e934a99df462589cf346542acd52fe7ea4a1279ce35d module/agent-harness-runtime +6131781d5c77140711114835b64e087cc81eb87fde6644dd6b2b1f3dafb28d32 module/agent-harness-runtime ec22d7a039fb58d0b8343ad149322960d3d8ca58b3f4c70f2fa8a099f8186d0c module/agent-harness-task-runtime 5f63bf587bf3547d59d0dc5d0dc2fee54745aa6edaab4aa3ae700dba03443edb module/agent-harness-tool-runtime 5168648cd946abad8a92822889f13ceacc87ed502314a66190d0b1eb8ebe76ea module/agent-media-payload @@ -35,13 +35,13 @@ a5e9215460bc99fb6e6cdd4bc67eb6466a5e99f28279e13d1769599c18dfe0b3 module/approva 6c81b9122ab0a5c3190700c0a234047274a8d48edddcc8f26d3859b886696068 module/async-lock-runtime ad60ccc4fe9084d47f0477e02d9296bacad32f26d7456e2a84be8d25a53a25c2 module/boolean-param 333a906d4ad9d3e89102ab7eb9059a7f9c7277328041223ee3b0713442f56a29 module/browser-config -1fcdba7e1d79400bf585f3e0278633c52d4b2d5d1ef8f9ecab5745bfef63c486 module/bundled-channel-config-schema +95772920b307d37a852d9f1b879b5f1fbdc35520da39c803d290c2e23306b00b module/bundled-channel-config-schema b6e53fb9c69840d71785325cd9a244809ffd6d3f0dd08ed0cccf816dd993f210 module/channel-actions fcfd76dbfe818e8e8574a425f340fee935acf53d4484e961f54c3ec760fb15fe module/channel-activity-runtime 24c53c9cefacd8c1bec2aa91ba2b01e606f8b8477bc5cb2f99567225a4dcf67c module/channel-config-helpers a82cc04b1cda7a647850a13c3de552ab638550c7042f887f32bb0ced7227d143 module/channel-config-primitives 929b3f55b60e3885e33644ed3e1757b82349e1beeeede02eda80f7bf7eea4631 module/channel-config-schema -d73888857348fc8ccc21d6e7113a6c2594a2c1b70dc80763987199a718d17a9e module/channel-config-schema-legacy +19cdd42fdae93a84200c96237bf71de49fef9db5fecc9879288de82ce1107f75 module/channel-config-schema-legacy 8740f7cc786a380043e41aaf710ac47931b3587c122bb152d5f0231f98fd5351 module/channel-config-writes 2dd98659d9600e755f09ef00dd91c36692b8562ed3700461937e94f6cd1e640a module/channel-contract 2a5b6694afc08c18248cacaa2af6bf4d448a646799684c7f7be825c4159fc7cb module/channel-core @@ -89,17 +89,17 @@ a224ce0381a32ed7a9d2261669c444f7f46701c15bf2f6121e9b057d9910da78 module/command eb4c757fe0086c1dbfa4c3f3caf3dcff0d3cab3924c608237f08f740a6ee5f59 module/command-status 10f073c8c7c7384843cb89548e01cf66241e23ebabea01ccc1f9ac7e774f563e module/command-status-runtime d340686cf814326b5a554a32cc5ca324a9e1254a933c4d9d8d1ada5ac7109a10 module/command-surface -965428a4ea5cb85a6cdebbf2a976123c1f8056298b75134589d701dfd3adb29d module/compat +251197439b471bc18d7470023b75a4b342884d7096217de9c7e8831ad037818f module/compat 2cf2f7732af5491a14466fc984f833ddce2b0f6a20b1c82796c3ee856e2966ce module/concurrency-runtime a821f9cc4e6f9339399d99e73f58c3b00baf139aa9d85c22d73c89d8be5702d2 module/config-contracts 20f3f8042de53e4eee61b64de9102c8c202b9299e6a29235647a4729f70145f2 module/config-mutation 316949815affe623ac63951a5db580527f02663576dffa084f02768f612c0c1c module/config-runtime -0468278d29417b56a187fdc8af8c7e29cd5363f90d0fc98352747bc2dccd68b1 module/config-schema +76000159a81c16a708d605473d9babb6f902299c8c15066fb708a16c65482d98 module/config-schema bfe6eec12f45bc2fda6df28699681b22dfd819a2261aabe8e0c789a20a017058 module/config-types 42d15153981cfe3adc1d5f91621434c56f07a9bd48c15ce34742f72dd040c142 module/context-visibility-runtime 03636897fb99cb73e4d8620c8a0e0d72b4d52fc32bf94f525af2aa88c489c6c2 module/conversation-binding-runtime c1ea9510dfda047609a99d5d2cd1f1560f5d469a36e6b695766213d695c25b0f module/conversation-runtime -5dd9d423f43dc332073f5b37ef087425da046d639a518af7e0354ebaff561dd0 module/core +d3363f5751848982fa35722402c27a5563d9efd22e0735b83e75629f9361f41b module/core 93b61bc7893ed37047c97c10360588c0acf5df72cf49c54678ce68ed8d07f65c module/cron-store-runtime e6fc74e18435cd4639801d9bd1f224154d1cd9405cdb3ba91a24b67c0792e9d2 module/dangerous-name-runtime 4af19d59c2f18674e7d7f7dc1b358b644dc707e6bd601dc47168bd9e4a669940 module/dedupe-runtime @@ -111,7 +111,7 @@ feb743a5f25297abc7bb47d22ef060e0ed54b0cff69ac50885681dfade970f8e module/deliver 371ee1fd78810526745c93c24c4b85562c981676adcc33aaa0c627f5d2d45810 module/direct-dm-guard-policy 91278c800e0f87d7111f226e1cfcb6f29552936fec7215b3b69be4da7e2ee8fb module/directory-config-runtime ea81ef06956c1bc0853fa00afbbc2b5a4019116aaf8a436e1b27d06f7a2c9e88 module/directory-runtime -e324635b9484d58d1fc0ddf210a817ef0738b31bac58ad4762a8cd9cf1440981 module/discord +04f2efcb8ad7b36eca498d0bbf13090ea7a642ac346e06e91b629a7704a52629 module/discord f41f9b34ab771c894293453bcdf072860c7cd509dd4e0172156634a816b8d727 module/document-extractor 3ac20ebba52de5a2f18807c0c8ade6e61f59e260a35fc1d077c9dedb9960dabd module/embedding-providers 46c05a90b66032d1d7ad08445840a4bf81aa2bd325348f87710ad4538daf38f6 module/error-runtime @@ -159,7 +159,7 @@ a5f59c9acbcaa3f82247bf806eb5ba08032373fb853719f0ec9457690f16fc70 module/media-m c5e3eb1a584f4b8126d9d6c177a840ec9103671e8d1242634ee67db9b5b9e573 module/media-understanding c0ffaed532578cf33493992e1ff806b2268b8e3774a92edbaede5cf5bda162a6 module/media-understanding-runtime 927014fec9d2e9384c696db06138a2e0f6a9cb1e33b9ec4bb1a8dba84adf3520 module/meeting-runtime -f6e428d474a1c7c9354068d4fc8391b4231a9e99d15ee5ea5fdf4180a7b28159 module/memory-core +bdcdb7db74523c557b6bcaa98a47210ad90f16bc97f9143227d03108c7fd8c23 module/memory-core 6ae3c62fd51b36439cd3f22286f2e52c6bb27bc85de648650b805c3b7603a487 module/memory-core-engine-runtime 2bc363010e81c6990c30fd6743e3ffa37320ec39b91abfc222302dc812d8bd33 module/memory-core-host-embedding-registry ae67c256150b14f1a58738dce0a703071035b86ec26fc344ab03e942253aa72c module/memory-core-host-engine-embeddings @@ -174,7 +174,7 @@ deb9953dbae4323348197f7aa90de855446d89db6abaac8434104464b16341f5 module/memory- f8eb835f0769eba8324033141ecddb9f45bdb3d3241d2a5959fe8426ba6a2280 module/memory-core-host-runtime-files ebbf937b2258487f773b8a6fbd6c648feea96399cae5fbea18ec7d29e027523d module/memory-core-host-secret 16bdd82751898ffa98157fd27c86fa0eb9ea415fb8ed703eb4e9b170f0a9501b module/memory-core-host-status -0f911f63448af365031f92f0c90c3e852ad047e60a85ac42a41d3b90bc320cd4 module/memory-host-core +646773d8282a2ac6a89101685c406200935ddb3459a2830e820da132fa433b3c module/memory-host-core 7d68cbdfa0d5fad1ab32ba3792787daf386bc4ff7975fd5e25117e92563a2897 module/memory-host-events 059856bc3671ccba7a2191f47c0295e0a91a374f7048cc2bc22784b389d15182 module/memory-host-files ba07f497aa5469915ebb5b6787148ff1f0f267d7e48c48f0c95d32b7dda8d91a module/memory-host-markdown @@ -314,7 +314,7 @@ c3c90ba2b012732413c5275d65a1c0ff07a937cd67d04d1acbd6207243544d43 module/tool-re 9d6ab352913a573b226e054e1dc8c6d088493aea9954950c65585923b5b6895a module/tool-send 5d02a59fe9a44a4aaaf027427238874d5ff883688b22b213e05c9f171edfbff5 module/transcripts 14ec8a05b813027a5afb9b92eab79995e0d61768545783081ff1515a84383b2c module/transport-ready-runtime -7626acd98e981a1fd00f1df3dd4ea6b1e21f4b18107d06efac7ee697c3538c99 module/tts-runtime +eb92e9bf85a08cf263e61b5b3497010b1de73f19bdcc5e826b0b9416ad053b90 module/tts-runtime bf0ffa969b46486f52201900956ed266107f062b22e9be73e5b81356a32185f1 module/types 20946e039f15f52023735836f5d48c72a51c4ace80c019edcb0301571cc79101 module/video-generation 37c7c8e392583c905c4ad6546dfe4f76e52771ef0272b41a57fd4418d70f638e module/video-generation-core diff --git a/docs/cli/wiki.md b/docs/cli/wiki.md index 5a2f281ddfe7..33c2cd0f9004 100644 --- a/docs/cli/wiki.md +++ b/docs/cli/wiki.md @@ -120,10 +120,7 @@ openclaw wiki get ### `wiki compile` -Rebuild indexes, related blocks, dashboards, and compiled digests. Writes stable machine-facing artifacts under: - -- `.openclaw-wiki/cache/agent-digest.json` -- `.openclaw-wiki/cache/claims.jsonl` +Rebuild indexes, related blocks, dashboards, and the compiled query/prompt snapshot. The snapshot is persisted in OpenClaw's shared SQLite plugin state and kept in memory for synchronous prompt projection; it does not create cache files in the vault. If `render.createDashboards` is enabled, compile also refreshes report pages. diff --git a/docs/concepts/context-engine.md b/docs/concepts/context-engine.md index 23dd8cbe8350..5c9895380582 100644 --- a/docs/concepts/context-engine.md +++ b/docs/concepts/context-engine.md @@ -168,7 +168,10 @@ export default function register(api) { The factory `ctx` includes optional `config`, `agentDir`, and `workspaceDir` values so plugins can initialize per-agent or per-workspace state before the -first lifecycle hook runs. +first lifecycle call. Before a non-legacy `assemble()` call, the host completes +registered async memory prompt preparation. The synchronous +`buildMemorySystemPromptAddition(...)` helper reads that immutable run snapshot; +pass the supplied tool, citation, agent, and session context through unchanged. Then enable it in config: diff --git a/docs/plugins/architecture-internals.md b/docs/plugins/architecture-internals.md index 4e9bd0ebed94..02e4df32c585 100644 --- a/docs/plugins/architecture-internals.md +++ b/docs/plugins/architecture-internals.md @@ -1100,6 +1100,12 @@ export default function (api) { The factory `ctx` exposes optional `config`, `agentDir`, and `workspaceDir` values for construction-time initialization. +The host completes registered async memory prompt preparation before calling a +non-legacy engine's `assemble()`. `buildMemorySystemPromptAddition(...)` stays +synchronous and reads that immutable run snapshot while `assemble()` is active. +Pass the supplied tool and citation context through unchanged so the snapshot +cannot cross run boundaries. + `assemble()` may return `contextProjection` when the active harness has a persistent backend thread. Omit it for legacy per-turn projection. Return `{ mode: "thread_bootstrap", epoch }` when the assembled context should be diff --git a/docs/plugins/memory-wiki.md b/docs/plugins/memory-wiki.md index 0ec88944ade9..7e5f77da1977 100644 --- a/docs/plugins/memory-wiki.md +++ b/docs/plugins/memory-wiki.md @@ -196,17 +196,26 @@ claims: ## Compile pipeline -Compile reads wiki pages, normalizes summaries, and emits stable -machine-facing artifacts under: - -- `.openclaw-wiki/cache/agent-digest.json` -- `.openclaw-wiki/cache/claims.jsonl` - -Agents and runtime code read these digests instead of scraping Markdown. +Compile reads wiki pages, normalizes summaries, and persists a machine-facing +snapshot in OpenClaw's shared SQLite plugin state. Runtime code uses the +lifecycle-owned owner snapshot to load SQLite during async prompt preparation; +synchronous prompt assembly never scrapes Markdown or reads cache files. Compiled output also powers first-pass wiki indexing for search/get, claim-id lookup back to owning pages, compact prompt supplements, and report generation. +Source edits and vault restores become machine-facing only after the next +compile. Restarting or refreshing the plugin lifecycle compares the vault's +causally chained compile publication with SQLite and rejects a snapshot from a +newer, rolled-back state. A compiler that started before the rollback cannot +publish against the restored predecessor. Prompt preparation does not poll the +vault or install file watchers. +After rollback quarantine, a compile in the running process clears the owner +immediately; a separate compiler process requires plugin lifecycle refresh so +the daemon can confirm the new durable publication. +Compiled caches are rebuildable: cache rows from before publication epochs are +treated as misses and replaced by the next compile; they are not migrated. + ## Dashboards and health reports When `render.createDashboards` is enabled, compile maintains dashboards under @@ -270,7 +279,7 @@ plugin supports corpus selection. ## Prompt and context behavior When `context.includeCompiledDigestPrompt` is enabled, memory prompt sections -append a compact compiled snapshot from `agent-digest.json`: top pages only, +append a compact compiled snapshot from plugin state: top pages only, top claims only, contradiction count, question count, confidence/freshness qualifiers. This is opt-in because it changes prompt shape; it mainly matters for context engines or prompt assembly that explicitly consume memory diff --git a/docs/plugins/sdk-overview.md b/docs/plugins/sdk-overview.md index 04bff5d3a2c6..948d6f4431c1 100644 --- a/docs/plugins/sdk-overview.md +++ b/docs/plugins/sdk-overview.md @@ -197,6 +197,7 @@ advertised node command. | `api.registerInteractiveHandler(registration)` | Interactive handler | | `api.registerAgentToolResultMiddleware(...)` | Runtime tool-result middleware | | `api.registerMemoryPromptSupplement(builder)` | Additive memory-adjacent prompt section | +| `api.registerMemoryPromptPreparation(prepare)` | Async preparation for a memory-adjacent prompt section | | `api.registerMemoryCorpusSupplement(adapter)` | Additive memory search/read corpus | | `api.registerHostedMediaResolver(resolver)` | Resolver for browser-style hosted media URLs | | `api.registerMcpServerConnectionResolver(...)` | Per-requester MCP transport (`url`/`headers`) for a static server name | @@ -308,6 +309,15 @@ capturing one global path during registration. If an agent id is required but missing in a multi-agent operation, fail closed rather than choosing an arbitrary agent. +Use `registerMemoryPromptPreparation(...)` when prompt text depends on async +plugin state. The callback runs once before each full agent prompt and receives +the same tool, agent, session, and sandbox context as synchronous memory prompt +builders. Validate the current storage-owner instance before loading persisted +state, then return only lines for that run. OpenClaw freezes those lines and +hands the immutable result to synchronous prompt assembly. Keep persistence, +atomic replacement, and owner-removal deletion inside the owning plugin; do not +poll or read files from a prompt builder. + Telegram interactive handlers can return `{ submitText }` to route text through Telegram's normal inbound agent path after the handler succeeds. OpenClaw keeps the callback button when inbound policy skips the text or processing fails, so diff --git a/docs/refactor/database-first.md b/docs/refactor/database-first.md index 41bd27105bcd..bc8a3d8dc447 100644 --- a/docs/refactor/database-first.md +++ b/docs/refactor/database-first.md @@ -1342,10 +1342,10 @@ sessionId})`; create, branch, continue, list, and fork flows live in their per vault/run id instead of writing `.openclaw-wiki/import-runs/*.json`. Rollback snapshots remain explicit vault files until import-run snapshot archival is moved into blob storage. -- Memory Wiki compiled digests now store SQLite plugin blob rows instead of - writing `.openclaw-wiki/cache/agent-digest.json` and - `.openclaw-wiki/cache/claims.jsonl`. The migration provider imports old cache - files and removes the cache directory when it becomes empty. +- Memory Wiki compiled digests now store compressed SQLite plugin-blob rows + instead of writing `.openclaw-wiki/cache/agent-digest.json` and + `.openclaw-wiki/cache/claims.jsonl`. The cache is rebuildable, so doctor + deletes old cache files without importing them. - ClawHub skill install tracking now stores one SQLite plugin-state row per workspace/skill instead of writing or reading `.clawhub/lock.json` and `.clawhub/origin.json` sidecars at runtime. Runtime code uses tracked-install diff --git a/extensions/codex/src/app-server/attempt-context.test.ts b/extensions/codex/src/app-server/attempt-context.test.ts index fcbda2ffb70a..de9610da946c 100644 --- a/extensions/codex/src/app-server/attempt-context.test.ts +++ b/extensions/codex/src/app-server/attempt-context.test.ts @@ -180,12 +180,14 @@ describe("Codex app-server attempt context", () => { sessionKey: "agent:marketing-agent:session-1", sessionAgentId: "marketing-agent", memoryToolNames: ["memory_search", "memory_get"], + sandboxed: true, }); expect(context.memoryToolRouted).toBe(true); expect(observedContext).toMatchObject({ agentId: "marketing-agent", agentSessionKey: "agent:marketing-agent:session-1", + sandboxed: true, }); expect(context.memoryCollaborationInstructions).toContain( "agent=marketing-agent session=agent:marketing-agent:session-1", diff --git a/extensions/codex/src/app-server/attempt-context.ts b/extensions/codex/src/app-server/attempt-context.ts index ee95d3ac5d4e..0b46879e616d 100644 --- a/extensions/codex/src/app-server/attempt-context.ts +++ b/extensions/codex/src/app-server/attempt-context.ts @@ -15,7 +15,10 @@ import { type EmbeddedRunAttemptResult, } from "openclaw/plugin-sdk/agent-harness-runtime"; import { resolveAgentWorkspaceDir } from "openclaw/plugin-sdk/agent-runtime"; -import { buildMemorySystemPromptAddition } from "openclaw/plugin-sdk/core"; +import { + buildMemorySystemPromptAddition, + prepareMemorySystemPromptAddition, +} from "openclaw/plugin-sdk/core"; import { MESSAGE_TOOL_DELIVERY_HINTS } from "openclaw/plugin-sdk/message-tool-delivery-hints"; import type { CodexDynamicToolFunctionSpec, CodexDynamicToolSpec, JsonValue } from "./protocol.js"; import { flattenCodexDynamicToolFunctions } from "./protocol.js"; @@ -170,6 +173,7 @@ export async function buildCodexWorkspaceBootstrapContext(params: { sessionKey: string; sessionAgentId: string; memoryToolNames: readonly string[]; + sandboxed?: boolean; }): Promise { try { const memoryToolsAvailable = @@ -257,13 +261,14 @@ export async function buildCodexWorkspaceBootstrapContext(params: { turnScopedDeveloperInstructionFiles, ), memoryCollaborationInstructions: shouldInjectCodexOpenClawPromptContext(params.params) - ? renderCodexWorkspaceMemoryCollaborationInstructions({ + ? await renderCodexWorkspaceMemoryCollaborationInstructions({ files: memoryReferenceFiles, toolNames: params.memoryToolNames, memoryToolRouted: memoryToolsAvailable, citationsMode: params.params.config?.memory?.citations, agentId: params.params.agentId ?? params.sessionAgentId, agentSessionKey: params.sessionKey, + sandboxed: params.sandboxed, }) : undefined, heartbeatCollaborationInstructions: @@ -857,20 +862,22 @@ function renderCodexWorkspaceMemoryReference(params: { return lines.join("\n").trim(); } -function renderCodexWorkspaceMemoryCollaborationInstructions(params: { +async function renderCodexWorkspaceMemoryCollaborationInstructions(params: { files: EmbeddedContextFile[]; toolNames: readonly string[]; memoryToolRouted: boolean; citationsMode?: Parameters[0]["citationsMode"]; agentId?: string; agentSessionKey?: string; -}): string | undefined { + sandboxed?: boolean; +}): Promise { const memoryRecallInstructions = params.memoryToolRouted - ? renderCodexMemoryRecallInstructions({ + ? await renderCodexMemoryRecallInstructions({ toolNames: params.toolNames, citationsMode: params.citationsMode, agentId: params.agentId, agentSessionKey: params.agentSessionKey, + sandboxed: params.sandboxed, }) : undefined; const memoryReferenceInstructions = renderCodexWorkspaceMemoryReference({ @@ -881,18 +888,20 @@ function renderCodexWorkspaceMemoryCollaborationInstructions(params: { return sections.length > 0 ? sections.join("\n\n") : undefined; } -function renderCodexMemoryRecallInstructions(params: { +async function renderCodexMemoryRecallInstructions(params: { toolNames: readonly string[]; citationsMode?: Parameters[0]["citationsMode"]; agentId?: string; agentSessionKey?: string; -}): string | undefined { + sandboxed?: boolean; +}): Promise { const availableTools = new Set(params.toolNames); - const memoryPrompt = buildMemorySystemPromptAddition({ + const memoryPrompt = await prepareMemorySystemPromptAddition({ availableTools, citationsMode: params.citationsMode, agentId: params.agentId, agentSessionKey: params.agentSessionKey, + sandboxed: params.sandboxed, }); if (!memoryPrompt) { // Memory recall policy belongs to the active memory plugin. diff --git a/extensions/codex/src/app-server/run-attempt-context.ts b/extensions/codex/src/app-server/run-attempt-context.ts index ecb95fb9e5af..6d014847f90a 100644 --- a/extensions/codex/src/app-server/run-attempt-context.ts +++ b/extensions/codex/src/app-server/run-attempt-context.ts @@ -56,6 +56,7 @@ export async function prepareCodexAttemptContext( usesSupervisionConnection, resolvedWorkspace, initialInactiveThreadBootstrapBindingForcedFreshStart, + sandbox, } = connection; const { toolBridge } = attemptTools; const activeTranscriptTarget = { @@ -139,6 +140,7 @@ export async function prepareCodexAttemptContext( sessionKey: contextSessionKey, sessionAgentId, memoryToolNames, + sandboxed: sandbox?.enabled === true, }); const baseDeveloperInstructions = joinPresentSections( buildDeveloperInstructions(runtimeParams, { dynamicTools: toolBridge.availableSpecs }), diff --git a/extensions/codex/src/app-server/run-attempt-prompt.ts b/extensions/codex/src/app-server/run-attempt-prompt.ts index 948ff1f07222..ae3b6784a27f 100644 --- a/extensions/codex/src/app-server/run-attempt-prompt.ts +++ b/extensions/codex/src/app-server/run-attempt-prompt.ts @@ -65,6 +65,7 @@ export async function prepareCodexAttemptPrompt(context: CodexAttemptContext) { appServer, contextSessionKey, effectiveWorkspace, + sandbox, } = connection; const { toolBridge } = attemptTools; const applyFreshThreadContinuityProjection = () => { @@ -96,6 +97,7 @@ export async function prepareCodexAttemptPrompt(context: CodexAttemptContext) { .filter(isNonEmptyString), ), citationsMode: params.config?.memory?.citations, + sandboxed: sandbox?.enabled === true, modelId: effectiveRuntimeModelId, contextEngineHostSupport: CODEX_APP_SERVER_CONTEXT_ENGINE_HOST, providerId: effectiveRuntimeProviderId, diff --git a/extensions/memory-wiki/README.md b/extensions/memory-wiki/README.md index 8dacb84be386..906ffe7485e3 100644 --- a/extensions/memory-wiki/README.md +++ b/extensions/memory-wiki/README.md @@ -143,7 +143,7 @@ The plugin initializes a vault like this: Generated content stays inside managed blocks. Human note blocks are preserved. -Key beliefs can live in structured `claims` frontmatter with per-claim evidence, confidence, and status. Compile also emits machine-readable digests under `.openclaw-wiki/cache/` so agent/runtime consumers do not have to scrape markdown pages. +Key beliefs can live in structured `claims` frontmatter with per-claim evidence, confidence, and status. Compile also persists a machine-readable snapshot in OpenClaw plugin state so agent/runtime consumers do not have to scrape markdown pages. When `render.createBacklinks` is enabled, compile adds deterministic `## Related` blocks to pages. Those blocks list source pages, pages that reference the current page, and nearby pages that share the same source ids. @@ -198,7 +198,7 @@ The plugin also registers a non-exclusive memory corpus supplement, so shared `m `wiki_apply` accepts structured `claims` payloads for synthesis and metadata updates, so the wiki can store claim-level evidence instead of only page-level prose. -When `context.includeCompiledDigestPrompt` is enabled, the memory prompt supplement also appends a compact snapshot from `.openclaw-wiki/cache/agent-digest.json`. Legacy prompt assembly sees that automatically, and non-legacy context engines can pick it up when they explicitly consume memory prompt supplements via `buildActiveMemoryPromptSection(...)`. +When `context.includeCompiledDigestPrompt` is enabled, the memory prompt supplement also appends a compact snapshot from the lifecycle-owned in-memory cache. Legacy prompt assembly sees that automatically, and non-legacy context engines can pick it up when they explicitly consume memory prompt supplements via `buildActiveMemoryPromptSection(...)`. ## Gateway RPC @@ -233,5 +233,8 @@ unknown ids fail in multi-agent setups. - Bridge mode reads the active memory plugin through public seams only. - Agent scope is incompatible with `unsafe-local` and official Obsidian CLI actions. - Wiki pages are compiled artifacts, not the ultimate source of truth. Keep provenance attached to raw sources, memory artifacts, and daily notes. -- The compiled agent digests in `.openclaw-wiki/cache/agent-digest.json` and `.openclaw-wiki/cache/claims.jsonl` are the stable machine-facing view of the wiki. +- The compiled snapshot in shared SQLite plugin state is the stable machine-facing view of the wiki. +- After editing or restoring vault files, compile again before expecting tools or prompts to use that source state. Lifecycle refresh rejects SQLite snapshots newer than a restored vault, and causal publication chaining rejects compilers started before the restore, without polling or watching files. +- Rollback quarantine clears immediately for an in-process compile. After a separate compiler process publishes, refresh the plugin lifecycle so the daemon can validate that durable publication. +- Pre-publication-epoch cache rows are rebuildable misses, not migrated state; the next compile replaces them. - Obsidian CLI support requires the official `obsidian` CLI to be installed and available on `PATH`. diff --git a/extensions/memory-wiki/doctor-contract-api.test.ts b/extensions/memory-wiki/doctor-contract-api.test.ts index 2eaad2362cf9..399d3d51220c 100644 --- a/extensions/memory-wiki/doctor-contract-api.test.ts +++ b/extensions/memory-wiki/doctor-contract-api.test.ts @@ -20,8 +20,11 @@ import { resolveMemoryWikiSourceSyncStatePath, } from "./src/source-sync-state.js"; -function requireStateMigration(index: number) { - return expectDefined(stateMigrations[index], `Memory Wiki state migration ${index}`); +function requireStateMigration(id: string) { + return expectDefined( + stateMigrations.find((migration) => migration.id === id), + `Memory Wiki state migration ${id}`, + ); } const tempDirs: string[] = []; @@ -76,6 +79,70 @@ describe("memory-wiki doctor source sync migration", () => { ); }); + it("deletes rebuildable compiled cache files without importing them", async () => { + const stateDir = await makeTempDir(); + const vaultRoot = path.join(stateDir, "vault"); + const cacheDir = path.join(vaultRoot, ".openclaw-wiki", "cache"); + const legacyPaths = [ + path.join(cacheDir, "agent-digest.json"), + path.join(cacheDir, "claims.jsonl"), + ]; + await fs.mkdir(cacheDir, { recursive: true }); + await Promise.all(legacyPaths.map((filePath) => fs.writeFile(filePath, "stale\n", "utf8"))); + const params = migrationParams({ stateDir, vaultRoot }); + const migration = requireStateMigration("memory-wiki-compiled-cache-file-cleanup"); + + await expect(migration.detectLegacyState(params)).resolves.toEqual({ + preview: legacyPaths.map((filePath) => + expect.stringContaining(`Remove rebuildable Memory Wiki compiled cache: ${filePath}`), + ), + }); + await expect(migration.migrateLegacyState(params)).resolves.toEqual({ + changes: legacyPaths.map( + (filePath) => `Removed rebuildable Memory Wiki compiled cache: ${filePath}`, + ), + warnings: [], + }); + await Promise.all( + legacyPaths.map((filePath) => + expect(fs.stat(filePath)).rejects.toMatchObject({ code: "ENOENT" }), + ), + ); + }); + + it("skips configured vaults that have not been initialized", async () => { + const stateDir = await makeTempDir(); + const vaultRoot = path.join(stateDir, "missing-vault"); + const params = migrationParams({ stateDir, vaultRoot }); + const migration = requireStateMigration("memory-wiki-compiled-cache-file-cleanup"); + + await expect(migration.detectLegacyState(params)).resolves.toBeNull(); + await expect(migration.migrateLegacyState(params)).resolves.toEqual({ + changes: [], + warnings: [], + }); + }); + + it("does not follow a symlinked legacy cache directory", async () => { + const stateDir = await makeTempDir(); + const vaultRoot = path.join(stateDir, "vault"); + const externalCacheDir = path.join(stateDir, "external-cache"); + const externalCachePath = path.join(externalCacheDir, "agent-digest.json"); + await fs.mkdir(path.join(vaultRoot, ".openclaw-wiki"), { recursive: true }); + await fs.mkdir(externalCacheDir, { recursive: true }); + await fs.writeFile(externalCachePath, "private\n", "utf8"); + await fs.symlink(externalCacheDir, path.join(vaultRoot, ".openclaw-wiki", "cache")); + const params = migrationParams({ stateDir, vaultRoot }); + const migration = requireStateMigration("memory-wiki-compiled-cache-file-cleanup"); + + await expect(migration.detectLegacyState(params)).resolves.toBeNull(); + await expect(migration.migrateLegacyState(params)).resolves.toEqual({ + changes: [], + warnings: [], + }); + await expect(fs.readFile(externalCachePath, "utf8")).resolves.toBe("private\n"); + }); + it("detects and migrates legacy source-sync.json into plugin state", async () => { const stateDir = await makeTempDir(); const vaultRoot = path.join(stateDir, "vault"); @@ -98,7 +165,7 @@ describe("memory-wiki doctor source sync migration", () => { })}\n`, ); const params = migrationParams({ stateDir, vaultRoot }); - const migration = requireStateMigration(0); + const migration = requireStateMigration("memory-wiki-source-sync-json-to-plugin-state"); await expect(migration.detectLegacyState(params)).resolves.toEqual({ preview: [expect.stringContaining("Memory Wiki source sync:")], @@ -246,7 +313,11 @@ describe("memory-wiki doctor source sync migration", () => { }, }); - await expect(requireStateMigration(0).migrateLegacyState(params)).resolves.toEqual({ + await expect( + requireStateMigration("memory-wiki-source-sync-json-to-plugin-state").migrateLegacyState( + params, + ), + ).resolves.toEqual({ changes: [ "Migrated Memory Wiki source sync -> plugin state (1 imported, 1 existing)", expect.stringContaining("Archived Memory Wiki source-sync legacy source ->"), @@ -303,13 +374,14 @@ describe("memory-wiki doctor source sync migration", () => { } const params = migrationParams({ stateDir, vaultRoot, agentIds }); - await expect(requireStateMigration(0).detectLegacyState(params)).resolves.toEqual({ + const migration = requireStateMigration("memory-wiki-source-sync-json-to-plugin-state"); + await expect(migration.detectLegacyState(params)).resolves.toEqual({ preview: [ expect.stringContaining(path.join(vaultRoot, "support")), expect.stringContaining(path.join(vaultRoot, "marketing")), ], }); - await expect(requireStateMigration(0).migrateLegacyState(params)).resolves.toMatchObject({ + await expect(migration.migrateLegacyState(params)).resolves.toMatchObject({ warnings: [], }); diff --git a/extensions/memory-wiki/doctor-contract-api.ts b/extensions/memory-wiki/doctor-contract-api.ts index 058eedeca20d..8b024d785c22 100644 --- a/extensions/memory-wiki/doctor-contract-api.ts +++ b/extensions/memory-wiki/doctor-contract-api.ts @@ -1,4 +1,4 @@ -// Memory Wiki doctor contract migrates shipped source-sync state. +// Memory Wiki doctor contract owns legacy state cleanup and migrations. import fs from "node:fs/promises"; import path from "node:path"; import type { OpenClawConfig } from "openclaw/plugin-sdk/plugin-entry"; @@ -7,6 +7,8 @@ import { legacyStateFileExists, type PluginDoctorStateMigration, } from "openclaw/plugin-sdk/runtime-doctor"; +import { FsSafeError, root as fsRoot } from "openclaw/plugin-sdk/security-runtime"; +import { LEGACY_MEMORY_WIKI_COMPILED_CACHE_PATHS } from "./src/compiled-cache.js"; import { resolveMemoryWikiAgentConfig, resolveMemoryWikiConfig, @@ -38,6 +40,39 @@ function resolveHomeDir(env: NodeJS.ProcessEnv): string | undefined { return env.HOME?.trim() || env.USERPROFILE?.trim() || undefined; } +function isMissingPathError(error: unknown): boolean { + return ( + (error instanceof FsSafeError && error.code === "not-found") || + (isRecord(error) && error.code === "ENOENT") + ); +} + +async function safeLegacyCacheFileExists( + vaultRoot: Awaited>, + relativePath: string, +): Promise { + try { + const stat = await vaultRoot.stat(relativePath); + return stat.isFile; + } catch (error) { + if (isMissingPathError(error) || error instanceof FsSafeError) { + return false; + } + throw error; + } +} + +async function openExistingVaultRoot(vaultRoot: string) { + try { + return await fsRoot(vaultRoot); + } catch (error) { + if (isMissingPathError(error)) { + return null; + } + throw error; + } +} + function readConfiguredPluginConfig(config: OpenClawConfig): MemoryWikiPluginConfig | undefined { const entries = config.plugins?.entries; const pluginEntry = isRecord(entries) ? entries["memory-wiki"] : undefined; @@ -105,6 +140,64 @@ function countImportRunStateRows( } export const stateMigrations: PluginDoctorStateMigration[] = [ + { + id: "memory-wiki-compiled-cache-file-cleanup", + label: "Memory Wiki compiled cache files", + async detectLegacyState(params) { + const previews: string[] = []; + for (const vaultRoot of resolveConfiguredVaultRoots({ + config: params.config, + env: params.env, + })) { + const root = await openExistingVaultRoot(vaultRoot); + if (!root) { + continue; + } + const stalePaths = ( + await Promise.all( + LEGACY_MEMORY_WIKI_COMPILED_CACHE_PATHS.map(async (relativePath) => { + const filePath = path.join(vaultRoot, relativePath); + return (await safeLegacyCacheFileExists(root, relativePath)) ? filePath : null; + }), + ) + ).filter((filePath): filePath is string => Boolean(filePath)); + for (const filePath of stalePaths) { + previews.push(`- Remove rebuildable Memory Wiki compiled cache: ${filePath}`); + } + } + return previews.length > 0 ? { preview: previews } : null; + }, + async migrateLegacyState(params) { + const changes: string[] = []; + const warnings: string[] = []; + for (const vaultRoot of resolveConfiguredVaultRoots({ + config: params.config, + env: params.env, + })) { + const root = await openExistingVaultRoot(vaultRoot); + if (!root) { + continue; + } + for (const relativePath of LEGACY_MEMORY_WIKI_COMPILED_CACHE_PATHS) { + const filePath = path.join(vaultRoot, relativePath); + if (!(await safeLegacyCacheFileExists(root, relativePath))) { + continue; + } + try { + await root.remove(relativePath); + changes.push(`Removed rebuildable Memory Wiki compiled cache: ${filePath}`); + } catch (error) { + if (!isMissingPathError(error)) { + warnings.push( + `Failed removing rebuildable Memory Wiki compiled cache ${filePath}: ${String(error)}`, + ); + } + } + } + } + return { changes, warnings }; + }, + }, { id: "memory-wiki-source-sync-json-to-plugin-state", label: "Memory Wiki source sync state", diff --git a/extensions/memory-wiki/index.test.ts b/extensions/memory-wiki/index.test.ts index 5cf274732b9d..791399f30218 100644 --- a/extensions/memory-wiki/index.test.ts +++ b/extensions/memory-wiki/index.test.ts @@ -1,8 +1,23 @@ // Memory Wiki tests cover index plugin behavior. +import fs from "node:fs/promises"; import path from "node:path"; import { describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "./api.js"; import plugin from "./index.js"; +import { + createMemoryWikiCompiledCachePublicationId, + loadMemoryWikiCompiledCache, + resolveMemoryWikiCompiledCacheGeneration, + writeMemoryWikiCompiledCache, + type MemoryWikiCompiledCacheSnapshot, +} from "./src/compiled-cache.js"; +import { resolveMemoryWikiConfig } from "./src/config.js"; +import { + appendMemoryWikiLog, + loadMemoryWikiValidatedVaultIdentity, + loadMemoryWikiVaultIdentity, + resolveMemoryWikiVaultSourceGeneration, +} from "./src/log.js"; import { createMemoryWikiTestHarness } from "./src/test-helpers.js"; const toolMocks = vi.hoisted(() => { @@ -32,14 +47,20 @@ describe("memory-wiki plugin", () => { registerCli, registerGatewayMethod, registerMemoryCorpusSupplement, + registerMemoryPromptPreparation, registerMemoryPromptSupplement, + registerService, registerTool, } = createPluginApi(); plugin.register(api); expect(registerMemoryCorpusSupplement).toHaveBeenCalledTimes(1); + expect(registerMemoryPromptPreparation).toHaveBeenCalledTimes(1); expect(registerMemoryPromptSupplement).toHaveBeenCalledTimes(1); + expect(registerService).toHaveBeenCalledWith( + expect.objectContaining({ id: "memory-wiki-compiled-cache-owner-cleanup" }), + ); expect(registerGatewayMethod.mock.calls.map((call) => call[0])).toEqual([ "wiki.status", "wiki.importRuns", @@ -132,4 +153,72 @@ describe("memory-wiki plugin", () => { expect(() => factory({ agentId: "finance" })).toThrow("Unknown memory-wiki agentId: finance"); } }); + + it("activates an initialized legacy vault before an external compile", async () => { + const rootDir = await createTempDir("memory-wiki-index-legacy-vault-"); + await fs.mkdir(path.join(rootDir, ".openclaw-wiki"), { recursive: true }); + await fs.writeFile(path.join(rootDir, ".openclaw-wiki", "log.jsonl"), "", "utf8"); + const { api, registerService } = createPluginApi(); + api.pluginConfig = { vault: { path: rootDir } }; + + plugin.register(api); + const service = registerService.mock.calls[0]?.[0]; + await service?.start?.(); + + await expect(loadMemoryWikiVaultIdentity(rootDir)).resolves.toMatchObject({ + vaultGeneration: expect.any(String), + }); + }); + + it("clears active owners before a fallible lifecycle identity refresh", async () => { + const rootDir = await createTempDir("memory-wiki-index-refresh-failure-"); + const { api, registerService } = createPluginApi(); + api.pluginConfig = { vault: { path: rootDir } }; + plugin.register(api); + const config = resolveMemoryWikiConfig(api.pluginConfig); + await fs.mkdir(path.join(rootDir, ".openclaw-wiki"), { recursive: true }); + await fs.writeFile(path.join(rootDir, ".openclaw-wiki", "log.jsonl"), "", "utf8"); + const service = registerService.mock.calls[0]?.[0]; + await service?.start?.(); + + const snapshot: MemoryWikiCompiledCacheSnapshot = { + digest: { claimCount: 0, contradictionCount: 0, pages: [] }, + claims: [], + }; + const publicationId = createMemoryWikiCompiledCachePublicationId(); + const reservationId = createMemoryWikiCompiledCachePublicationId(); + const parentPublicationId = (await loadMemoryWikiVaultIdentity(rootDir)) + .compiledCachePublicationId; + await appendMemoryWikiLog(rootDir, { + type: "compile", + timestamp: "2026-07-17T00:00:00.000Z", + details: { compiledCacheReservationId: reservationId }, + }); + const sourceGeneration = await resolveMemoryWikiVaultSourceGeneration(rootDir); + await appendMemoryWikiLog(rootDir, { + type: "compile", + timestamp: "2026-07-17T00:00:00.000Z", + details: { + compiledCachePublicationId: publicationId, + compiledCacheParentPublicationId: parentPublicationId, + compiledCacheReservationId: reservationId, + compiledCacheSourceGeneration: sourceGeneration, + }, + }); + await writeMemoryWikiCompiledCache( + config, + snapshot, + resolveMemoryWikiCompiledCacheGeneration(snapshot), + publicationId, + parentPublicationId, + async () => {}, + async () => {}, + () => loadMemoryWikiValidatedVaultIdentity(rootDir), + ); + await expect(loadMemoryWikiCompiledCache(config)).resolves.toEqual(snapshot); + + vi.spyOn(fs, "readFile").mockRejectedValueOnce(new Error("transient restore read failure")); + await expect(service?.start?.()).rejects.toThrow("transient restore read failure"); + await expect(loadMemoryWikiCompiledCache(config)).resolves.toBeNull(); + }); }); diff --git a/extensions/memory-wiki/index.ts b/extensions/memory-wiki/index.ts index df021546ff25..894e711ac905 100644 --- a/extensions/memory-wiki/index.ts +++ b/extensions/memory-wiki/index.ts @@ -1,6 +1,16 @@ // Memory Wiki plugin entrypoint registers its OpenClaw integration. +import fs from "node:fs/promises"; +import path from "node:path"; import { definePluginEntry, type OpenClawConfig } from "./api.js"; import { registerWikiCli } from "./src/cli.js"; +import { + activateMemoryWikiCompiledCacheOwner, + configureMemoryWikiCompiledCacheStore, + createMemoryWikiCompiledCacheStore, + deactivateMemoryWikiCompiledCacheOwnersExcept, + reconcileMemoryWikiCompiledCacheOwner, + resolveMemoryWikiCompiledCacheOwnerId, +} from "./src/compiled-cache.js"; import { memoryWikiConfigSchema, resolveMemoryWikiAgentConfig, @@ -14,7 +24,14 @@ import { configureMemoryWikiImportRunStateStore, createMemoryWikiImportRunStateStore, } from "./src/import-runs-state.js"; -import { createWikiPromptSectionBuilder } from "./src/prompt-section.js"; +import { + ensureMemoryWikiVaultGeneration, + loadMemoryWikiValidatedVaultIdentity, +} from "./src/log.js"; +import { + createWikiPromptSectionBuilder, + createWikiPromptSectionPreparer, +} from "./src/prompt-section.js"; import { configureMemoryWikiSourceSyncStateStore, createMemoryWikiSourceSyncStateStore, @@ -27,6 +44,36 @@ import { createWikiStatusTool, } from "./src/tool.js"; +async function loadConfiguredVaultIdentity(vaultRoot: string): Promise<{ + vaultGeneration: string; + compiledCachePublicationId: string | null; +} | null> { + const identity = await loadMemoryWikiValidatedVaultIdentity(vaultRoot); + if (identity.vaultGeneration) { + return { + vaultGeneration: identity.vaultGeneration, + compiledCachePublicationId: identity.compiledCachePublicationId, + }; + } + try { + const stat = await fs.stat(path.join(vaultRoot, ".openclaw-wiki", "log.jsonl")); + if (!stat.isFile()) { + return null; + } + } catch (error) { + if (error instanceof Error && "code" in error && error.code === "ENOENT") { + return null; + } + throw error; + } + // Cache data is rebuildable, but an initialized pre-generation vault still + // needs a stable owner identity before an external compiler can publish it. + return { + vaultGeneration: await ensureMemoryWikiVaultGeneration(vaultRoot), + compiledCachePublicationId: null, + }; +} + export default definePluginEntry({ id: "memory-wiki", name: "Memory Wiki", @@ -56,8 +103,62 @@ export default definePluginEntry({ configureMemoryWikiImportRunStateStore( createMemoryWikiImportRunStateStore(api.runtime.state.openKeyedStore), ); + const compiledCacheStore = createMemoryWikiCompiledCacheStore(api.runtime.state.openBlobStore, { + onReadError(error) { + api.logger.warn(`memory-wiki: compiled cache unavailable: ${String(error)}`); + }, + }); + configureMemoryWikiCompiledCacheStore(compiledCacheStore); + api.registerService({ + id: "memory-wiki-compiled-cache-owner-cleanup", + async start() { + const appConfig = getAppConfig(); + const activeConfigs = + config.vault.scope === "global" + ? [resolveConfig(undefined, appConfig)] + : resolveMemoryWikiConfiguredAgentIds(appConfig).map((agentId) => + resolveConfig(agentId, appConfig), + ); + // Clear every previously trusted owner before fallible vault reads. A failed + // lifecycle refresh must leave prompt preparation closed, not stale-but-active. + deactivateMemoryWikiCompiledCacheOwnersExcept(new Set()); + const preparedOwners: Array<{ + config: ReturnType; + identity: { + vaultGeneration: string; + compiledCachePublicationId: string | null; + }; + }> = []; + for (const activeConfig of activeConfigs) { + const identity = await loadConfiguredVaultIdentity(activeConfig.vault.path); + if (identity) { + preparedOwners.push({ config: activeConfig, identity }); + } + } + const activeOwnerIds = new Set(); + try { + for (const { config: activeConfig, identity } of preparedOwners) { + activateMemoryWikiCompiledCacheOwner( + activeConfig, + identity.vaultGeneration, + identity.compiledCachePublicationId, + ); + await reconcileMemoryWikiCompiledCacheOwner(activeConfig, () => + loadMemoryWikiValidatedVaultIdentity(activeConfig.vault.path), + ); + activeOwnerIds.add(resolveMemoryWikiCompiledCacheOwnerId(activeConfig)); + } + } catch (error) { + deactivateMemoryWikiCompiledCacheOwnersExcept(new Set()); + throw error; + } + deactivateMemoryWikiCompiledCacheOwnersExcept(activeOwnerIds); + await compiledCacheStore.deleteOwnersExcept(activeOwnerIds); + }, + }); - api.registerMemoryPromptSupplement(createWikiPromptSectionBuilder({ config, resolveConfig })); + api.registerMemoryPromptSupplement(createWikiPromptSectionBuilder()); + api.registerMemoryPromptPreparation(createWikiPromptSectionPreparer({ config, resolveConfig })); api.registerMemoryCorpusSupplement(createWikiCorpusSupplement({ resolveConfig, getAppConfig })); registerMemoryWikiGatewayMethods({ api, diff --git a/extensions/memory-wiki/src/bridge.test.ts b/extensions/memory-wiki/src/bridge.test.ts index e893ee56baa7..abc1993076f9 100644 --- a/extensions/memory-wiki/src/bridge.test.ts +++ b/extensions/memory-wiki/src/bridge.test.ts @@ -147,7 +147,7 @@ describe("syncMemoryWikiBridgeSources", () => { const logLines = (await fs.readFile(path.join(vaultDir, ".openclaw-wiki", "log.jsonl"), "utf8")) .trim() .split("\n"); - expect(logLines).toHaveLength(2); + expect(logLines).toHaveLength(3); }); it("skips generated artifacts from its own vault", async () => { diff --git a/extensions/memory-wiki/src/compile.test.ts b/extensions/memory-wiki/src/compile.test.ts index 69fc0a4407c3..78ea6f873136 100644 --- a/extensions/memory-wiki/src/compile.test.ts +++ b/extensions/memory-wiki/src/compile.test.ts @@ -4,6 +4,7 @@ import os from "node:os"; import path from "node:path"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { compileMemoryWikiVault } from "./compile.js"; +import { loadMemoryWikiCompiledCache } from "./compiled-cache.js"; import { renderWikiMarkdown, WIKI_RAW_SOURCE_MARKER } from "./markdown.js"; import { writeMemoryWikiSourceSyncState } from "./source-sync-state.js"; import { createMemoryWikiTestHarness } from "./test-helpers.js"; @@ -57,12 +58,12 @@ describe("compileMemoryWikiVault", () => { return page; } - function expectDigestCluster(clusters: T[], key: string): T { - const cluster = clusters.find((candidate) => candidate.key === key); - if (!cluster) { - throw new Error(`Expected digest contradiction cluster ${key}`); + async function expectCompiledCache(config: Parameters[0]) { + const snapshot = await loadMemoryWikiCompiledCache(config); + if (!snapshot) { + throw new Error(`Expected compiled cache for ${config.vault.path}`); } - return cluster; + return snapshot; } it("writes root and directory indexes for native markdown", async () => { @@ -104,21 +105,14 @@ describe("compileMemoryWikiVault", () => { await expect(fs.readFile(path.join(rootDir, "sources", "index.md"), "utf8")).resolves.toContain( "[Alpha](alpha.md)", ); - const agentDigest = JSON.parse( - await fs.readFile(path.join(rootDir, ".openclaw-wiki", "cache", "agent-digest.json"), "utf8"), - ) as { - claimCount: number; - pages: Array<{ path: string; claimCount: number; topClaims: Array<{ text: string }> }>; - }; + const { digest: agentDigest, claims } = await expectCompiledCache(config); expect(agentDigest.claimCount).toBe(1); const alphaPage = expectDigestPage(agentDigest.pages, "sources/alpha.md"); expect(alphaPage.claimCount).toBe(1); expect(alphaPage.topClaims.map((claim) => claim.text)).toEqual([ "Alpha is the canonical source page.", ]); - await expect( - fs.readFile(path.join(rootDir, ".openclaw-wiki", "cache", "claims.jsonl"), "utf8"), - ).resolves.toContain('"text":"Alpha is the canonical source page."'); + expect(claims.map((claim) => claim.text)).toContain("Alpha is the canonical source page."); }); it("excludes malformed pages from indexes, digests, counts, and page writes (#96125)", async () => { @@ -167,9 +161,9 @@ describe("compileMemoryWikiVault", () => { await expect(fs.readFile(path.join(rootDir, "index.md"), "utf8")).resolves.not.toContain( "Broken", ); - await expect( - fs.readFile(path.join(rootDir, ".openclaw-wiki", "cache", "agent-digest.json"), "utf8"), - ).resolves.not.toContain("syntheses/broken.md"); + expect((await expectCompiledCache(config)).digest.pages.map((page) => page.path)).not.toContain( + "syntheses/broken.md", + ); }); it.each([ @@ -695,17 +689,7 @@ describe("compileMemoryWikiVault", () => { await expect( fs.readFile(path.join(rootDir, "reports", "stale-pages.md"), "utf8"), ).resolves.toContain("Tracked Raw Alpha Source"); - const agentDigest = JSON.parse( - await fs.readFile(path.join(rootDir, ".openclaw-wiki", "cache", "agent-digest.json"), "utf8"), - ) as { - claimHealth: { missingEvidence: number; freshness: { unknown: number } }; - contradictionClusters: Array<{ key: string }>; - }; - expect(agentDigest.claimHealth.missingEvidence).toBeGreaterThanOrEqual(1); - expect(agentDigest.claimHealth.freshness.unknown).toBeGreaterThanOrEqual(1); - expect(expectDigestCluster(agentDigest.contradictionClusters, "claim.alpha.db").key).toBe( - "claim.alpha.db", - ); + expect((await expectCompiledCache(config)).digest.contradictionCount).toBeGreaterThanOrEqual(1); }); it("excludes concept and synthesis pages from stale-pages report", async () => { @@ -886,25 +870,13 @@ describe("compileMemoryWikiVault", () => { fs.readFile(path.join(rootDir, "reports", "privacy-review.md"), "utf8"), ).resolves.toContain("[Brad Groux](../entities/brad.md)"); - const agentDigest = JSON.parse( - await fs.readFile(path.join(rootDir, ".openclaw-wiki", "cache", "agent-digest.json"), "utf8"), - ) as { - pages: Array<{ - path: string; - canonicalId?: string; - aliases?: string[]; - personCard?: { lane?: string }; - relationshipCount?: number; - }>; - }; + const { digest: agentDigest, claims } = await expectCompiledCache(config); const bradPage = expectDigestPage(agentDigest.pages, "entities/brad.md"); expect(bradPage.canonicalId).toBe("maintainer.brad-groux"); expect(bradPage.aliases).toEqual(["brad"]); expect(bradPage.personCard?.lane).toBe("Microsoft Teams"); expect(bradPage.relationshipCount).toBe(1); - await expect( - fs.readFile(path.join(rootDir, ".openclaw-wiki", "cache", "claims.jsonl"), "utf8"), - ).resolves.toContain('"evidenceKinds":["maintainer-whois"]'); + expect(claims.flatMap((claim) => claim.evidenceKinds ?? [])).toContain("maintainer-whois"); }); it("ignores generated related links when computing backlinks on repeated compile", async () => { diff --git a/extensions/memory-wiki/src/compile.ts b/extensions/memory-wiki/src/compile.ts index 60f1b5ed5795..285b45801a2e 100644 --- a/extensions/memory-wiki/src/compile.ts +++ b/extensions/memory-wiki/src/compile.ts @@ -27,8 +27,19 @@ import { type WikiFreshnessLevel, type WikiPageContradictionCluster, } from "./claim-health.js"; +import { + createMemoryWikiCompiledCachePublicationId, + resolveMemoryWikiCompiledCacheGeneration, + writeMemoryWikiCompiledCache, + type MemoryWikiCompiledCacheSnapshot, +} from "./compiled-cache.js"; import type { ResolvedMemoryWikiConfig } from "./config.js"; -import { appendMemoryWikiLog } from "./log.js"; +import { + appendMemoryWikiLog, + loadMemoryWikiValidatedVaultIdentity, + loadMemoryWikiVaultIdentity, + resolveMemoryWikiVaultSourceGeneration, +} from "./log.js"; import { formatWikiLink, isUnmanagedRawSourceSummary, @@ -55,8 +66,6 @@ const COMPILE_PAGE_GROUPS: Array<{ kind: WikiPageKind; dir: string; heading: str { kind: "synthesis", dir: "syntheses", heading: "Syntheses" }, { kind: "report", dir: "reports", heading: "Reports" }, ]; -const AGENT_DIGEST_PATH = ".openclaw-wiki/cache/agent-digest.json"; -const CLAIMS_DIGEST_PATH = ".openclaw-wiki/cache/claims.jsonl"; const READ_PAGE_SUMMARIES_CONCURRENCY = 16; const MAX_RELATED_PAGES_PER_SECTION = 12; const MAX_SHARED_SOURCE_FANOUT = 24; @@ -1090,76 +1099,6 @@ function buildDirectoryIndexBody(params: { }); } -type AgentDigestClaim = { - id?: string; - text: string; - status: string; - confidence?: number; - evidenceCount: number; - missingEvidence: boolean; - evidence: WikiClaim["evidence"]; - freshnessLevel: WikiFreshnessLevel; - lastTouchedAt?: string; -}; - -type AgentDigestPage = { - id?: string; - title: string; - kind: WikiPageKind; - path: string; - pageType?: string; - entityType?: string; - canonicalId?: string; - aliases: string[]; - sourceIds: string[]; - questions: string[]; - contradictions: string[]; - confidence?: number; - privacyTier?: string; - personCard?: WikiPageSummary["personCard"]; - bestUsedFor: string[]; - notEnoughFor: string[]; - relationshipCount: number; - topRelationships: WikiRelationship[]; - freshnessLevel: WikiFreshnessLevel; - lastTouchedAt?: string; - lastRefreshedAt?: string; - claimCount: number; - topClaims: AgentDigestClaim[]; -}; - -type AgentDigestClaimHealthSummary = { - freshness: Record; - contested: number; - lowConfidence: number; - missingEvidence: number; -}; - -type AgentDigestContradictionCluster = { - key: string; - label: string; - kind: "claim-id" | "page-note"; - entryCount: number; - paths: string[]; -}; - -type AgentDigest = { - pageCounts: Record; - claimCount: number; - claimHealth: AgentDigestClaimHealthSummary; - contradictionClusters: AgentDigestContradictionCluster[]; - pages: AgentDigestPage[]; -}; - -function createFreshnessSummary(): Record { - return { - fresh: 0, - aging: 0, - stale: 0, - unknown: 0, - }; -} - function rankFreshnessLevel(level: WikiFreshnessLevel): number { switch (level) { case "fresh": @@ -1190,65 +1129,12 @@ function sortClaims(page: WikiPageSummary): WikiClaim[] { }); } -function buildAgentDigestClaimHealthSummary( - pages: WikiPageSummary[], -): AgentDigestClaimHealthSummary { - const freshness = createFreshnessSummary(); - let contested = 0; - let lowConfidence = 0; - let missingEvidence = 0; - - for (const claim of collectWikiClaimHealth(pages)) { - freshness[claim.freshness.level] += 1; - if (isClaimHealthContested(claim)) { - contested += 1; - } - if (typeof claim.confidence === "number" && claim.confidence < 0.5) { - lowConfidence += 1; - } - if (claim.missingEvidence) { - missingEvidence += 1; - } - } - - return { - freshness, - contested, - lowConfidence, - missingEvidence, - }; -} - -function buildAgentDigestContradictionClusters( - pages: WikiPageSummary[], -): AgentDigestContradictionCluster[] { - const pageClusters = buildPageContradictionClusters(pages).map((cluster) => ({ - key: cluster.key, - label: cluster.label, - kind: "page-note" as const, - entryCount: cluster.entries.length, - paths: uniqueStrings(cluster.entries.map((entry) => entry.pagePath)).toSorted(), - })); - const claimClusters = buildClaimContradictionClusters({ pages }).map((cluster) => ({ - key: cluster.key, - label: cluster.label, - kind: "claim-id" as const, - entryCount: cluster.entries.length, - paths: uniqueStrings(cluster.entries.map((entry) => entry.pagePath)).toSorted(), - })); - return [...pageClusters, ...claimClusters].toSorted((left, right) => - left.label.localeCompare(right.label), - ); -} - -function buildAgentDigest(params: { - pages: WikiPageSummary[]; - pageCounts: Record; -}): AgentDigest { - const pages = [...params.pages] +function buildCompiledCacheSnapshot( + pagesInput: WikiPageSummary[], +): MemoryWikiCompiledCacheSnapshot { + const pages = [...pagesInput] .toSorted((left, right) => left.relativePath.localeCompare(right.relativePath)) .map((page) => { - const pageFreshness = assessPageFreshness(page); return Object.assign( {}, page.id ? { id: page.id } : {}, @@ -1268,12 +1154,8 @@ function buildAgentDigest(params: { page.pageType ? { pageType: page.pageType } : {}, page.entityType ? { entityType: page.entityType } : {}, page.canonicalId ? { canonicalId: page.canonicalId } : {}, - typeof page.confidence === "number" ? { confidence: page.confidence } : {}, page.privacyTier ? { privacyTier: page.privacyTier } : {}, page.personCard ? { personCard: page.personCard } : {}, - { freshnessLevel: pageFreshness.level }, - pageFreshness.lastTouchedAt ? { lastTouchedAt: pageFreshness.lastTouchedAt } : {}, - page.lastRefreshedAt ? { lastRefreshedAt: page.lastRefreshedAt } : {}, { claimCount: page.claims.length, topClaims: sortClaims(page) @@ -1289,33 +1171,18 @@ function buildAgentDigest(params: { }, typeof claim.confidence === "number" ? { confidence: claim.confidence } : {}, { - evidenceCount: claim.evidence.length, - missingEvidence: claim.evidence.length === 0, - evidence: [...claim.evidence], freshnessLevel: freshness.level, }, - freshness.lastTouchedAt ? { lastTouchedAt: freshness.lastTouchedAt } : {}, ); }), }, ); }); - return { - pageCounts: params.pageCounts, - claimCount: params.pages.reduce((total, page) => total + page.claims.length, 0), - claimHealth: buildAgentDigestClaimHealthSummary(params.pages), - contradictionClusters: buildAgentDigestContradictionClusters(params.pages), - pages, - }; -} - -function buildClaimsDigestLines(params: { pages: WikiPageSummary[] }): string[] { - return params.pages + const claims = pagesInput .flatMap((page) => sortClaims(page).map((claim) => { const freshness = assessClaimFreshness({ page, claim }); - return JSON.stringify({ - ...(claim.id ? { id: claim.id } : {}), + return Object.assign({}, claim.id ? { id: claim.id } : {}, { pageId: page.id, pageTitle: page.title, pageKind: page.kind, @@ -1338,51 +1205,25 @@ function buildClaimsDigestLines(params: { pages: WikiPageSummary[] }): string[] ].flatMap((entry) => entry ?? []), ), ], - evidenceCount: claim.evidence.length, - missingEvidence: claim.evidence.length === 0, - evidence: claim.evidence, freshnessLevel: freshness.level, lastTouchedAt: freshness.lastTouchedAt, }); }), ) - .toSorted((left, right) => left.localeCompare(right)); -} - -async function writeAgentDigestArtifacts(params: { - rootDir: string; - pages: WikiPageSummary[]; - pageCounts: Record; -}): Promise { - const updatedFiles: string[] = []; - const agentDigestPath = path.join(params.rootDir, AGENT_DIGEST_PATH); - const claimsDigestPath = path.join(params.rootDir, CLAIMS_DIGEST_PATH); - const agentDigest = `${JSON.stringify( - buildAgentDigest({ - pages: params.pages, - pageCounts: params.pageCounts, - }), - null, - 2, - )}\n`; - const claimsDigest = withTrailingNewline( - buildClaimsDigestLines({ pages: params.pages }).join("\n"), - ); - - for (const [filePath, content] of [ - [agentDigestPath, agentDigest], - [claimsDigestPath, claimsDigest], - ] as const) { - const relativePath = path.relative(params.rootDir, filePath); - const root = await fsRoot(params.rootDir); - const existing = await root.readText(relativePath).catch(() => ""); - if (existing === content) { - continue; - } - await root.write(relativePath, content); - updatedFiles.push(filePath); - } - return updatedFiles; + .toSorted( + (left, right) => + left.pagePath.localeCompare(right.pagePath) || left.text.localeCompare(right.text), + ); + return { + digest: { + claimCount: claims.length, + contradictionCount: + buildPageContradictionClusters(pagesInput).length + + buildClaimContradictionClusters({ pages: pagesInput }).length, + pages, + }, + claims, + }; } async function compileMemoryWikiVaultUnlocked( @@ -1390,6 +1231,27 @@ async function compileMemoryWikiVaultUnlocked( ): Promise { await initializeMemoryWikiVault(config); const rootDir = config.vault.path; + const compiledInputIdentity = await loadMemoryWikiVaultIdentity(rootDir); + if (!compiledInputIdentity.vaultGeneration) { + throw new Error(`Memory Wiki vault generation is missing: ${rootDir}`); + } + const compiledCacheReservationId = createMemoryWikiCompiledCachePublicationId(); + await appendMemoryWikiLog(rootDir, { + type: "compile", + timestamp: new Date().toISOString(), + details: { + compiledCacheReservationId, + compiledCacheParentPublicationId: compiledInputIdentity.compiledCachePublicationId, + }, + }); + const reservedIdentity = await loadMemoryWikiVaultIdentity(rootDir); + if ( + reservedIdentity.vaultGeneration !== compiledInputIdentity.vaultGeneration || + reservedIdentity.compiledCacheReservationId !== compiledCacheReservationId || + reservedIdentity.compiledCachePublicationId !== compiledInputIdentity.compiledCachePublicationId + ) { + throw new Error("Memory Wiki vault changed before its compiled cache scan began."); + } const sourceSyncState = await readMemoryWikiSourceSyncState(rootDir); const managedImportedSourcePagePaths = new Set( Object.values(sourceSyncState.entries).map((entry) => entry.pagePath.split(path.sep).join("/")), @@ -1413,12 +1275,10 @@ async function compileMemoryWikiVaultUnlocked( pages = scan.pages; } const counts = buildPageCounts(pages); - const digestUpdatedFiles = await writeAgentDigestArtifacts({ - rootDir, - pages, - pageCounts: counts, - }); - updatedFiles.push(...digestUpdatedFiles); + const compiledSnapshot = buildCompiledCacheSnapshot(pages); + const compiledCacheGeneration = resolveMemoryWikiCompiledCacheGeneration(compiledSnapshot); + const compiledCachePublicationId = createMemoryWikiCompiledCachePublicationId(); + let compiledCacheSourceGeneration: string | undefined; const rootIndexPath = path.join(rootDir, "index.md"); if ( @@ -1451,16 +1311,72 @@ async function compileMemoryWikiVaultUnlocked( } } - if (updatedFiles.length > 0) { - await appendMemoryWikiLog(rootDir, { - type: "compile", - timestamp: new Date().toISOString(), - details: { - pageCounts: counts, - updatedFiles: updatedFiles.map((filePath) => path.relative(rootDir, filePath)), - }, - }); - } + // Persist an immutable candidate, then commit its causal publication. A stale + // compiler cannot overwrite the accepted row or activate before validation. + await writeMemoryWikiCompiledCache( + config, + compiledSnapshot, + compiledCacheGeneration, + compiledCachePublicationId, + compiledInputIdentity.compiledCachePublicationId, + async () => { + const currentIdentity = await loadMemoryWikiVaultIdentity(rootDir); + if ( + currentIdentity.vaultGeneration !== compiledInputIdentity.vaultGeneration || + currentIdentity.compiledCacheReservationId !== compiledCacheReservationId || + currentIdentity.compiledCachePublicationId !== + compiledInputIdentity.compiledCachePublicationId + ) { + throw new Error("Memory Wiki vault changed while its compiled cache was being built."); + } + const sourceGenerationBeforeScan = await resolveMemoryWikiVaultSourceGeneration(rootDir); + const verifiedScan = await readPageSummaries(rootDir); + const verifiedGeneration = resolveMemoryWikiCompiledCacheGeneration( + buildCompiledCacheSnapshot(verifiedScan.pages), + ); + const sourceGenerationAfterScan = await resolveMemoryWikiVaultSourceGeneration(rootDir); + if ( + verifiedGeneration !== compiledCacheGeneration || + sourceGenerationAfterScan !== sourceGenerationBeforeScan + ) { + throw new Error("Memory Wiki vault changed while its compiled cache was being published."); + } + compiledCacheSourceGeneration = sourceGenerationAfterScan; + const verifiedIdentity = await loadMemoryWikiVaultIdentity(rootDir); + if ( + verifiedIdentity.vaultGeneration !== compiledInputIdentity.vaultGeneration || + verifiedIdentity.compiledCacheReservationId !== compiledCacheReservationId || + verifiedIdentity.compiledCachePublicationId !== + compiledInputIdentity.compiledCachePublicationId + ) { + throw new Error("Memory Wiki vault changed while its compiled cache was being verified."); + } + }, + async () => { + if (!compiledCacheSourceGeneration) { + throw new Error("Memory Wiki compiled cache source generation is missing."); + } + await appendMemoryWikiLog(rootDir, { + type: "compile", + timestamp: new Date().toISOString(), + details: { + compiledCachePublicationId, + compiledCacheParentPublicationId: compiledInputIdentity.compiledCachePublicationId, + compiledCacheReservationId, + compiledCacheSourceGeneration, + }, + }); + }, + () => loadMemoryWikiValidatedVaultIdentity(rootDir), + ); + await appendMemoryWikiLog(rootDir, { + type: "compile", + timestamp: new Date().toISOString(), + details: { + pageCounts: counts, + updatedFiles: updatedFiles.map((filePath) => path.relative(rootDir, filePath)), + }, + }); return { vaultRoot: rootDir, diff --git a/extensions/memory-wiki/src/compiled-cache.integration.test.ts b/extensions/memory-wiki/src/compiled-cache.integration.test.ts new file mode 100644 index 000000000000..8f56ad5b3822 --- /dev/null +++ b/extensions/memory-wiki/src/compiled-cache.integration.test.ts @@ -0,0 +1,685 @@ +// Memory Wiki compiled cache tests cover compile, prepare, query, restart, and owner cleanup. +import { createHash } from "node:crypto"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { gzipSync } from "node:zlib"; +import type { OpenBlobStoreOptions } from "openclaw/plugin-sdk/plugin-state-runtime"; +import { + createPluginBlobStoreForTests, + resetPluginBlobStoreForTests, +} from "openclaw/plugin-sdk/plugin-state-test-runtime"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { compileMemoryWikiVault } from "./compile.js"; +import { + activateMemoryWikiCompiledCacheOwner, + configureMemoryWikiCompiledCacheStore, + createMemoryWikiCompiledCachePublicationId, + createMemoryWikiCompiledCacheStore, + deactivateMemoryWikiCompiledCacheOwnersExcept, + loadMemoryWikiCompiledCache, + reconcileMemoryWikiCompiledCacheOwner, + resolveMemoryWikiCompiledCacheGeneration, + resolveMemoryWikiCompiledCacheOwnerId, + writeMemoryWikiCompiledCache, + type MemoryWikiCompiledCacheSnapshot, +} from "./compiled-cache.js"; +import { resolveMemoryWikiAgentConfig, resolveMemoryWikiConfig } from "./config.js"; +import { + appendMemoryWikiLog, + loadMemoryWikiValidatedVaultIdentity, + loadMemoryWikiVaultIdentity, + resolveMemoryWikiVaultSourceGeneration, +} from "./log.js"; +import { renderWikiMarkdown } from "./markdown.js"; +import { createWikiPromptSectionPreparer } from "./prompt-section.js"; +import { getMemoryWikiPage } from "./query.js"; +import { createMemoryWikiTestHarness } from "./test-helpers.js"; +import { initializeMemoryWikiVault } from "./vault.js"; + +const { createTempDir, createVault } = createMemoryWikiTestHarness(); +let blobStateDir = ""; +let blobStoreEnv: NodeJS.ProcessEnv = {}; + +function createCacheStore() { + return createMemoryWikiCompiledCacheStore((options: OpenBlobStoreOptions) => + createPluginBlobStoreForTests("memory-wiki", options, blobStoreEnv), + ); +} + +async function createPersistentVault( + options?: Parameters[0], +): Promise>> { + const vault = await createVault(options); + // The shared unit harness installs its in-memory cache store. These lifecycle + // tests deliberately switch back to the SQLite-backed plugin Blob test store. + configureMemoryWikiCompiledCacheStore(createCacheStore()); + return vault; +} + +async function activateVault(config: ReturnType): Promise { + const identity = await loadMemoryWikiValidatedVaultIdentity(config.vault.path); + if (!identity.vaultGeneration) { + throw new Error(`Expected vault generation for ${config.vault.path}`); + } + activateMemoryWikiCompiledCacheOwner( + config, + identity.vaultGeneration, + identity.compiledCachePublicationId, + ); + await reconcileMemoryWikiCompiledCacheOwner(config, () => + loadMemoryWikiValidatedVaultIdentity(config.vault.path), + ); +} + +function snapshot(text: string): MemoryWikiCompiledCacheSnapshot { + return { + digest: { + claimCount: 1, + contradictionCount: 0, + pages: [ + { + title: "Snapshot", + kind: "entity", + path: "entities/snapshot.md", + aliases: [], + sourceIds: [], + questions: [], + contradictions: [], + bestUsedFor: [], + notEnoughFor: [], + relationshipCount: 0, + topRelationships: [], + claimCount: 1, + topClaims: [{ text, status: "supported", freshnessLevel: "fresh" }], + }, + ], + }, + claims: [ + { + pageTitle: "Snapshot", + pageKind: "entity", + pagePath: "entities/snapshot.md", + text, + }, + ], + }; +} + +function createDeferred(): { promise: Promise; resolve: () => void } { + let resolve!: () => void; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} + +async function publishSnapshot( + config: ReturnType, + value: MemoryWikiCompiledCacheSnapshot, +): Promise { + const generation = resolveMemoryWikiCompiledCacheGeneration(value); + const publicationId = createMemoryWikiCompiledCachePublicationId(); + const reservationId = createMemoryWikiCompiledCachePublicationId(); + const parentPublicationId = (await loadMemoryWikiVaultIdentity(config.vault.path)) + .compiledCachePublicationId; + await appendMemoryWikiLog(config.vault.path, { + type: "compile", + timestamp: "2026-07-17T00:00:00.000Z", + details: { compiledCacheReservationId: reservationId }, + }); + const sourceGeneration = await resolveMemoryWikiVaultSourceGeneration(config.vault.path); + await appendMemoryWikiLog(config.vault.path, { + type: "compile", + timestamp: "2026-07-17T00:00:00.000Z", + details: { + compiledCachePublicationId: publicationId, + compiledCacheParentPublicationId: parentPublicationId, + compiledCacheReservationId: reservationId, + compiledCacheSourceGeneration: sourceGeneration, + }, + }); + await writeMemoryWikiCompiledCache( + config, + value, + generation, + publicationId, + parentPublicationId, + async () => {}, + async () => {}, + () => loadMemoryWikiValidatedVaultIdentity(config.vault.path), + ); + return publicationId; +} + +async function preparePrompt(config: ReturnType): Promise { + return ( + await createWikiPromptSectionPreparer({ config, resolveConfig: () => config })({ + availableTools: new Set(), + }) + ).join("\n"); +} + +describe("Memory Wiki compiled cache lifecycle", () => { + beforeEach(async () => { + resetPluginBlobStoreForTests(); + configureMemoryWikiCompiledCacheStore(undefined); + blobStateDir = await createTempDir("memory-wiki-compiled-cache-state-"); + blobStoreEnv = { ...process.env, OPENCLAW_STATE_DIR: blobStateDir }; + configureMemoryWikiCompiledCacheStore(createCacheStore()); + }); + + afterEach(async () => { + configureMemoryWikiCompiledCacheStore(undefined); + resetPluginBlobStoreForTests(); + blobStateDir = ""; + blobStoreEnv = {}; + }); + + it("round-trips compile through async preparation and claim query after restart", async () => { + const { rootDir, config } = await createPersistentVault({ + initialize: true, + config: { context: { includeCompiledDigestPrompt: true } }, + }); + await fs.writeFile( + path.join(rootDir, "entities", "alpha.md"), + renderWikiMarkdown({ + frontmatter: { + pageType: "entity", + id: "entity.alpha", + title: "Alpha", + claims: [ + { + id: "claim.alpha.db", + text: "Alpha uses PostgreSQL for production writes.", + status: "supported", + confidence: 0.91, + evidence: [{ sourceId: "source.alpha", lines: "1-2" }], + }, + ], + }, + body: "# Alpha\n\nDatabase notes.\n", + }), + "utf8", + ); + + await compileMemoryWikiVault(config); + + await expect(preparePrompt(config)).resolves.toContain( + "Alpha uses PostgreSQL for production writes.", + ); + await expect(getMemoryWikiPage({ config, lookup: "claim.alpha.db" })).resolves.toMatchObject({ + path: "entities/alpha.md", + title: "Alpha", + }); + + configureMemoryWikiCompiledCacheStore(undefined); + configureMemoryWikiCompiledCacheStore(createCacheStore()); + await activateVault(config); + + await expect(preparePrompt(config)).resolves.toContain( + "Alpha uses PostgreSQL for production writes.", + ); + }); + + it("ignores legacy files and rebuilds only on compile", async () => { + const { rootDir, config } = await createPersistentVault({ + initialize: true, + config: { context: { includeCompiledDigestPrompt: true } }, + }); + const legacyPath = path.join(rootDir, ".openclaw-wiki", "cache", "agent-digest.json"); + await fs.mkdir(path.dirname(legacyPath), { recursive: true }); + await fs.writeFile(legacyPath, JSON.stringify({ claimCount: 1, pages: [] }), "utf8"); + await fs.writeFile( + path.join(rootDir, "entities", "fresh.md"), + renderWikiMarkdown({ + frontmatter: { + pageType: "entity", + id: "entity.fresh", + title: "Fresh", + claims: [{ text: "Fresh cache content.", status: "supported" }], + }, + body: "# Fresh\n", + }), + "utf8", + ); + + await expect(preparePrompt(config)).resolves.not.toContain("Fresh cache content."); + await compileMemoryWikiVault(config); + + await expect(preparePrompt(config)).resolves.toContain("Fresh cache content."); + await expect(fs.readFile(legacyPath, "utf8")).resolves.toContain("claimCount"); + }); + + it("persists snapshots beyond the keyed-state value limit", async () => { + const { config } = await createPersistentVault({ initialize: true }); + const text = Array.from({ length: 4096 }, (_, index) => + createHash("sha256").update(String(index)).digest("hex"), + ).join(""); + expect(gzipSync(text).byteLength).toBeGreaterThan(65_536); + await publishSnapshot(config, snapshot(text)); + + configureMemoryWikiCompiledCacheStore(undefined); + configureMemoryWikiCompiledCacheStore(createCacheStore()); + await activateVault(config); + + expect((await loadMemoryWikiCompiledCache(config))?.claims[0]?.text).toBe(text); + }); + + it("loads an externally compiled generation after lifecycle refresh without polling", async () => { + const { config } = await createPersistentVault({ + initialize: true, + config: { context: { includeCompiledDigestPrompt: true } }, + }); + await publishSnapshot(config, snapshot("before")); + await expect(preparePrompt(config)).resolves.toContain("before"); + + const nextSnapshot = snapshot("after"); + const nextGeneration = resolveMemoryWikiCompiledCacheGeneration(nextSnapshot); + const nextPublicationId = createMemoryWikiCompiledCachePublicationId(); + const nextReservationId = createMemoryWikiCompiledCachePublicationId(); + const parentPublicationId = (await loadMemoryWikiVaultIdentity(config.vault.path)) + .compiledCachePublicationId; + await appendMemoryWikiLog(config.vault.path, { + type: "compile", + timestamp: "2026-07-17T00:01:00.000Z", + details: { compiledCacheReservationId: nextReservationId }, + }); + const sourceGeneration = await resolveMemoryWikiVaultSourceGeneration(config.vault.path); + await appendMemoryWikiLog(config.vault.path, { + type: "compile", + timestamp: "2026-07-17T00:01:00.000Z", + details: { + compiledCachePublicationId: nextPublicationId, + compiledCacheParentPublicationId: parentPublicationId, + compiledCacheReservationId: nextReservationId, + compiledCacheSourceGeneration: sourceGeneration, + }, + }); + await createCacheStore().write(config, nextSnapshot, nextGeneration, nextPublicationId); + + await expect(preparePrompt(config)).resolves.not.toContain("after"); + await activateVault(config); + await expect(preparePrompt(config)).resolves.toContain("after"); + }); + + it("defers a publication that completes during lifecycle reconciliation", async () => { + const { config } = await createPersistentVault({ initialize: true }); + await publishSnapshot(config, snapshot("before")); + const nextSnapshot = snapshot("during reconciliation"); + const nextGeneration = resolveMemoryWikiCompiledCacheGeneration(nextSnapshot); + const nextPublicationId = createMemoryWikiCompiledCachePublicationId(); + const nextReservationId = createMemoryWikiCompiledCachePublicationId(); + const parentPublicationId = (await loadMemoryWikiVaultIdentity(config.vault.path)) + .compiledCachePublicationId; + await appendMemoryWikiLog(config.vault.path, { + type: "compile", + timestamp: "2026-07-17T00:01:00.000Z", + details: { compiledCacheReservationId: nextReservationId }, + }); + const sourceGeneration = await resolveMemoryWikiVaultSourceGeneration(config.vault.path); + const externalStore = createCacheStore(); + let publishDuringLookup = true; + const reconcilingStore = createMemoryWikiCompiledCacheStore( + (options: OpenBlobStoreOptions) => { + const blobStore = createPluginBlobStoreForTests("memory-wiki", options, blobStoreEnv); + return { + ...blobStore, + async lookup(key) { + const entry = await blobStore.lookup(key); + if (publishDuringLookup) { + publishDuringLookup = false; + await appendMemoryWikiLog(config.vault.path, { + type: "compile", + timestamp: "2026-07-17T00:01:00.000Z", + details: { + compiledCachePublicationId: nextPublicationId, + compiledCacheParentPublicationId: parentPublicationId, + compiledCacheReservationId: nextReservationId, + compiledCacheSourceGeneration: sourceGeneration, + }, + }); + await externalStore.write(config, nextSnapshot, nextGeneration, nextPublicationId); + } + return entry; + }, + }; + }, + ); + configureMemoryWikiCompiledCacheStore(reconcilingStore); + + await activateVault(config); + + await expect(loadMemoryWikiCompiledCache(config)).resolves.toBeNull(); + await activateVault(config); + expect((await loadMemoryWikiCompiledCache(config))?.claims[0]?.text).toBe( + "during reconciliation", + ); + }); + + it("reads the stable owner row directly without enumerating stale metadata", async () => { + const { config } = await createPersistentVault({ initialize: true }); + const reader = createMemoryWikiCompiledCacheStore((options: OpenBlobStoreOptions) => { + const store = createPluginBlobStoreForTests("memory-wiki", options, blobStoreEnv); + return { + ...store, + async entries() { + throw new Error("read must not enumerate owner rows"); + }, + }; + }); + configureMemoryWikiCompiledCacheStore(reader); + await publishSnapshot(config, snapshot("authoritative")); + + expect((await loadMemoryWikiCompiledCache(config))?.claims[0]?.text).toBe("authoritative"); + }); + + it("preserves vault identity across atomic edits to user-managed scaffold files", async () => { + const { rootDir, config } = await createPersistentVault({ initialize: true }); + await publishSnapshot(config, snapshot("still current")); + const replacement = path.join(rootDir, "WIKI.md.replacement"); + await fs.writeFile(replacement, "# Edited wiki\n", "utf8"); + await fs.rename(replacement, path.join(rootDir, "WIKI.md")); + + expect((await loadMemoryWikiCompiledCache(config))?.claims[0]?.text).toBe("still current"); + }); + + it("rejects claims newer than a restored vault after lifecycle refresh", async () => { + const { rootDir, config } = await createPersistentVault({ initialize: true }); + await publishSnapshot(config, snapshot("backup")); + const logPath = path.join(rootDir, ".openclaw-wiki", "log.jsonl"); + const backupLog = await fs.readFile(logPath, "utf8"); + const newerSnapshot = snapshot("Private post-backup claim."); + const preRestorePublicationId = await publishSnapshot(config, newerSnapshot); + await fs.writeFile(logPath, backupLog, "utf8"); + + configureMemoryWikiCompiledCacheStore(undefined); + configureMemoryWikiCompiledCacheStore(createCacheStore()); + await activateVault(config); + + await expect(loadMemoryWikiCompiledCache(config)).resolves.toBeNull(); + + const delayedSnapshot = snapshot("Private delayed pre-restore claim."); + await createCacheStore().write( + config, + delayedSnapshot, + resolveMemoryWikiCompiledCacheGeneration(delayedSnapshot), + createMemoryWikiCompiledCachePublicationId(), + ); + await expect(loadMemoryWikiCompiledCache(config)).resolves.toBeNull(); + + const republishedId = createMemoryWikiCompiledCachePublicationId(); + const staleReservationId = createMemoryWikiCompiledCachePublicationId(); + const newerGeneration = resolveMemoryWikiCompiledCacheGeneration(newerSnapshot); + const sourceGeneration = await resolveMemoryWikiVaultSourceGeneration(config.vault.path); + await appendMemoryWikiLog(config.vault.path, { + type: "compile", + timestamp: "2026-07-17T00:02:00.000Z", + details: { + compiledCachePublicationId: republishedId, + compiledCacheParentPublicationId: preRestorePublicationId, + compiledCacheReservationId: staleReservationId, + compiledCacheSourceGeneration: sourceGeneration, + }, + }); + await expect( + writeMemoryWikiCompiledCache( + config, + newerSnapshot, + newerGeneration, + republishedId, + preRestorePublicationId, + async () => {}, + async () => {}, + () => loadMemoryWikiValidatedVaultIdentity(config.vault.path), + ), + ).rejects.toThrow("vault changed"); + + await expect(loadMemoryWikiCompiledCache(config)).resolves.toBeNull(); + await activateVault(config); + await expect(loadMemoryWikiCompiledCache(config)).resolves.toBeNull(); + }); + + it("rejects a reserved publication when its identical parent was restored", async () => { + const { rootDir, config } = await createPersistentVault({ initialize: true }); + await publishSnapshot(config, snapshot("backup")); + const logPath = path.join(rootDir, ".openclaw-wiki", "log.jsonl"); + const backupLog = await fs.readFile(logPath, "utf8"); + const parentPublicationId = (await loadMemoryWikiVaultIdentity(rootDir)) + .compiledCachePublicationId; + const reservedPublicationId = createMemoryWikiCompiledCachePublicationId(); + const reservationId = createMemoryWikiCompiledCachePublicationId(); + await appendMemoryWikiLog(rootDir, { + type: "compile", + timestamp: "2026-07-17T00:03:00.000Z", + details: { + compiledCacheReservationId: reservationId, + compiledCacheParentPublicationId: parentPublicationId, + }, + }); + const compiledAfterBackup = snapshot("Private content scanned after backup."); + await fs.writeFile(logPath, backupLog, "utf8"); + const sourceGeneration = await resolveMemoryWikiVaultSourceGeneration(rootDir); + + await expect( + writeMemoryWikiCompiledCache( + config, + compiledAfterBackup, + resolveMemoryWikiCompiledCacheGeneration(compiledAfterBackup), + reservedPublicationId, + parentPublicationId, + async () => {}, + async () => { + await appendMemoryWikiLog(rootDir, { + type: "compile", + timestamp: "2026-07-17T00:03:01.000Z", + details: { + compiledCachePublicationId: reservedPublicationId, + compiledCacheParentPublicationId: parentPublicationId, + compiledCacheReservationId: reservationId, + compiledCacheSourceGeneration: sourceGeneration, + }, + }); + }, + () => loadMemoryWikiValidatedVaultIdentity(rootDir), + ), + ).rejects.toThrow("vault changed"); + await expect(loadMemoryWikiCompiledCache(config)).resolves.toBeNull(); + await activateVault(config); + expect((await loadMemoryWikiCompiledCache(config))?.claims[0]?.text).toBe("backup"); + }); + + it("keeps a committed publication when an older writer is rejected", async () => { + const { config } = await createPersistentVault({ initialize: true }); + const parentPublicationId = await publishSnapshot(config, snapshot("parent")); + const stalePublicationId = createMemoryWikiCompiledCachePublicationId(); + const staleReservationId = createMemoryWikiCompiledCachePublicationId(); + await appendMemoryWikiLog(config.vault.path, { + type: "compile", + timestamp: "2026-07-17T00:03:30.000Z", + details: { compiledCacheReservationId: staleReservationId }, + }); + const staleSnapshot = snapshot("stale candidate"); + const sourceGeneration = await resolveMemoryWikiVaultSourceGeneration(config.vault.path); + const callbackEntered = createDeferred(); + const releaseCallback = createDeferred(); + const staleWrite = writeMemoryWikiCompiledCache( + config, + staleSnapshot, + resolveMemoryWikiCompiledCacheGeneration(staleSnapshot), + stalePublicationId, + parentPublicationId, + async () => {}, + async () => { + callbackEntered.resolve(); + await releaseCallback.promise; + await appendMemoryWikiLog(config.vault.path, { + type: "compile", + timestamp: "2026-07-17T00:04:00.000Z", + details: { + compiledCachePublicationId: stalePublicationId, + compiledCacheParentPublicationId: parentPublicationId, + compiledCacheReservationId: staleReservationId, + compiledCacheSourceGeneration: sourceGeneration, + }, + }); + }, + () => loadMemoryWikiValidatedVaultIdentity(config.vault.path), + ); + await callbackEntered.promise; + await publishSnapshot(config, snapshot("accepted successor")); + releaseCallback.resolve(); + + await expect(staleWrite).rejects.toThrow("vault changed"); + expect((await loadMemoryWikiCompiledCache(config))?.claims[0]?.text).toBe("accepted successor"); + }); + + it("loads a prepared snapshot without prompt-path file I/O", async () => { + const { config } = await createPersistentVault({ + initialize: true, + config: { context: { includeCompiledDigestPrompt: true } }, + }); + await publishSnapshot(config, snapshot("prepared")); + const stat = vi.spyOn(fs, "stat"); + const readFile = vi.spyOn(fs, "readFile"); + + await expect(preparePrompt(config)).resolves.toContain("prepared"); + expect(stat).not.toHaveBeenCalled(); + expect(readFile).not.toHaveBeenCalled(); + }); + + it("treats transient SQLite read failures as a recoverable cache miss", async () => { + const { config } = await createPersistentVault({ initialize: true }); + const errors: unknown[] = []; + let failNextRead = false; + const store = createMemoryWikiCompiledCacheStore( + (options: OpenBlobStoreOptions) => { + const blobStore = createPluginBlobStoreForTests("memory-wiki", options, blobStoreEnv); + return { + ...blobStore, + async lookup(key) { + if (failNextRead) { + failNextRead = false; + throw new Error("transient SQLite failure"); + } + return await blobStore.lookup(key); + }, + }; + }, + { onReadError: (error) => errors.push(error) }, + ); + configureMemoryWikiCompiledCacheStore(store); + await publishSnapshot(config, snapshot("recoverable")); + failNextRead = true; + + await expect(loadMemoryWikiCompiledCache(config)).resolves.toBeNull(); + expect(errors).toHaveLength(1); + expect((await loadMemoryWikiCompiledCache(config))?.claims[0]?.text).toBe("recoverable"); + }); + + it("keeps a restored owner closed when lifecycle reconciliation fails", async () => { + const { rootDir, config } = await createPersistentVault({ initialize: true }); + await publishSnapshot(config, snapshot("backup")); + const logPath = path.join(rootDir, ".openclaw-wiki", "log.jsonl"); + const backupLog = await fs.readFile(logPath, "utf8"); + await publishSnapshot(config, snapshot("Private newer claim.")); + await fs.writeFile(logPath, backupLog, "utf8"); + + const errors: unknown[] = []; + let failNextRead = true; + const store = createMemoryWikiCompiledCacheStore( + (options: OpenBlobStoreOptions) => { + const blobStore = createPluginBlobStoreForTests("memory-wiki", options, blobStoreEnv); + return { + ...blobStore, + async lookup(key) { + if (failNextRead) { + failNextRead = false; + throw new Error("transient reconciliation failure"); + } + return await blobStore.lookup(key); + }, + }; + }, + { onReadError: (error) => errors.push(error) }, + ); + configureMemoryWikiCompiledCacheStore(undefined); + configureMemoryWikiCompiledCacheStore(store); + await expect(activateVault(config)).rejects.toThrow("transient reconciliation failure"); + + await expect(loadMemoryWikiCompiledCache(config)).resolves.toBeNull(); + await expect(loadMemoryWikiCompiledCache(config)).resolves.toBeNull(); + expect(errors).toHaveLength(1); + }); + + it("rejects a predecessor snapshot when a vault path is reused", async () => { + const { rootDir, config } = await createPersistentVault({ + initialize: true, + config: { context: { includeCompiledDigestPrompt: true } }, + }); + await publishSnapshot(config, snapshot("Private predecessor content.")); + await fs.rm(rootDir, { recursive: true, force: true }); + await fs.mkdir(path.join(rootDir, ".openclaw-wiki"), { recursive: true }); + await Promise.all([ + fs.writeFile(path.join(rootDir, "WIKI.md"), "# Replacement\n", "utf8"), + fs.writeFile( + path.join(rootDir, ".openclaw-wiki", "log.jsonl"), + `${JSON.stringify({ + type: "vault-generation", + timestamp: "2026-07-17T00:00:00.000Z", + details: { vaultGeneration: "replacement-generation" }, + })}\n`, + "utf8", + ), + ]); + await initializeMemoryWikiVault(config); + + await expect(preparePrompt(config)).resolves.not.toContain("Private predecessor content."); + await expect(loadMemoryWikiCompiledCache(config)).resolves.toBeNull(); + }); + + it("atomically replaces one stable owner row when the configured vault moves", async () => { + const { config: firstConfig } = await createPersistentVault({ initialize: true }); + const { config: secondConfig } = await createPersistentVault({ initialize: true }); + const store = createCacheStore(); + configureMemoryWikiCompiledCacheStore(store); + + await activateVault(firstConfig); + await publishSnapshot(firstConfig, snapshot("first")); + await activateVault(secondConfig); + await publishSnapshot(secondConfig, snapshot("second")); + + await expect(loadMemoryWikiCompiledCache(firstConfig)).resolves.toBeNull(); + expect((await loadMemoryWikiCompiledCache(secondConfig))?.claims[0]?.text).toBe("second"); + }); + + it("deletes cache rows when their agent owner is removed", async () => { + const rootDir = path.join((await createPersistentVault()).rootDir, "agents"); + const appConfig = { + agents: { list: [{ id: "support", default: true }, { id: "marketing" }] }, + }; + const baseConfig = resolveMemoryWikiConfig({ vault: { scope: "agent", path: rootDir } }); + const support = resolveMemoryWikiAgentConfig({ + config: baseConfig, + appConfig, + agentId: "support", + }); + const marketing = resolveMemoryWikiAgentConfig({ + config: baseConfig, + appConfig, + agentId: "marketing", + }); + for (const config of [support, marketing]) { + await initializeMemoryWikiVault(config); + await publishSnapshot(config, snapshot(config.agentId ?? "unknown")); + } + const store = createCacheStore(); + configureMemoryWikiCompiledCacheStore(store); + + const activeOwners = new Set([resolveMemoryWikiCompiledCacheOwnerId(support)]); + deactivateMemoryWikiCompiledCacheOwnersExcept(activeOwners); + await store.deleteOwnersExcept(activeOwners); + + await expect(loadMemoryWikiCompiledCache(marketing)).resolves.toBeNull(); + await expect(loadMemoryWikiCompiledCache(support)).resolves.not.toBeNull(); + }); +}); diff --git a/extensions/memory-wiki/src/compiled-cache.ts b/extensions/memory-wiki/src/compiled-cache.ts new file mode 100644 index 000000000000..ea7eeabefeb4 --- /dev/null +++ b/extensions/memory-wiki/src/compiled-cache.ts @@ -0,0 +1,465 @@ +// Memory Wiki compiled cache ownership and persistence. +import { createHash, randomUUID } from "node:crypto"; +import path from "node:path"; +import { gunzipSync, gzipSync } from "node:zlib"; +import type { PluginBlobStore } from "openclaw/plugin-sdk/plugin-state-runtime"; +import type { WikiFreshnessLevel } from "./claim-health.js"; +import type { ResolvedMemoryWikiConfig } from "./config.js"; +import type { WikiPageKind, WikiPageSummary, WikiRelationship } from "./markdown.js"; + +export const LEGACY_MEMORY_WIKI_COMPILED_CACHE_PATHS = [ + ".openclaw-wiki/cache/agent-digest.json", + ".openclaw-wiki/cache/claims.jsonl", +] as const; + +const COMPILED_CACHE_NAMESPACE = "compiled-cache"; +const COMPILED_CACHE_MAX_ENTRIES = 256; +const COMPILED_CACHE_MAX_BYTES_PER_ENTRY = 100 * 1024 * 1024; +const COMPILED_CACHE_MAX_BYTES = 512 * 1024 * 1024; +const COMPILED_CACHE_VERSION = 2; + +export type MemoryWikiCompiledDigestClaim = { + id?: string; + text: string; + status: string; + confidence?: number; + freshnessLevel: WikiFreshnessLevel; +}; + +export type MemoryWikiCompiledDigestPage = { + id?: string; + title: string; + kind: WikiPageKind; + path: string; + pageType?: string; + entityType?: string; + canonicalId?: string; + aliases: string[]; + sourceIds: string[]; + questions: string[]; + contradictions: string[]; + privacyTier?: string; + personCard?: WikiPageSummary["personCard"]; + bestUsedFor: string[]; + notEnoughFor: string[]; + relationshipCount: number; + topRelationships: WikiRelationship[]; + claimCount: number; + topClaims: MemoryWikiCompiledDigestClaim[]; +}; + +export type MemoryWikiCompiledClaim = { + id?: string; + pageId?: string; + pageTitle: string; + pageKind: WikiPageKind; + pagePath: string; + pageType?: string; + entityType?: string; + canonicalId?: string; + aliases?: string[]; + text: string; + status?: string; + confidence?: number; + sourceIds?: string[]; + evidenceKinds?: string[]; + privacyTiers?: string[]; + freshnessLevel?: string; + lastTouchedAt?: string; +}; + +export type MemoryWikiCompiledCacheSnapshot = { + digest: { + claimCount: number; + contradictionCount: number; + pages: MemoryWikiCompiledDigestPage[]; + }; + claims: MemoryWikiCompiledClaim[]; +}; + +type CompiledCacheMetadata = { + version: typeof COMPILED_CACHE_VERSION; + ownerId: string; + vaultPath: string; + vaultGeneration: string; + publicationId: string; + generation: string; + encoding: "gzip-json"; +}; + +type ActiveVault = { + path: string; + vaultGeneration: string; + compiledCachePublicationId?: string; + reconciled: boolean; +}; + +type MemoryWikiCompiledCacheStore = { + read(config: ResolvedMemoryWikiConfig): Promise; + write( + config: ResolvedMemoryWikiConfig, + snapshot: MemoryWikiCompiledCacheSnapshot, + generation: string, + publicationId: string, + ): Promise; + reconcile( + config: ResolvedMemoryWikiConfig, + loadDurableIdentity: () => Promise<{ + vaultGeneration: string | null; + compiledCachePublicationId: string | null; + }>, + ): Promise; + delete(config: ResolvedMemoryWikiConfig): Promise; + deletePublication(config: ResolvedMemoryWikiConfig, publicationId: string): Promise; + deleteOwnersExcept(ownerIds: ReadonlySet): Promise; +}; + +let configuredStore: MemoryWikiCompiledCacheStore | undefined; +const activeVaults = new Map(); + +export function resolveMemoryWikiCompiledCacheOwnerId(config: ResolvedMemoryWikiConfig): string { + if (config.vault.scope === "global") { + return "global"; + } + const agentId = config.agentId?.trim(); + if (!agentId) { + throw new Error("Memory Wiki agent-scoped compiled cache requires an agent owner."); + } + return `agent:${agentId}`; +} + +function ownerKeyPrefix(ownerId: string): string { + return `owner:${createHash("sha256").update(ownerId).digest("hex")}:publication:`; +} + +function publicationKey(ownerId: string, publicationId: string): string { + return `${ownerKeyPrefix(ownerId)}${createHash("sha256").update(publicationId).digest("hex")}`; +} + +function isMetadata(value: CompiledCacheMetadata | undefined): value is CompiledCacheMetadata { + return ( + value?.version === COMPILED_CACHE_VERSION && + typeof value.ownerId === "string" && + typeof value.vaultPath === "string" && + typeof value.vaultGeneration === "string" && + typeof value.publicationId === "string" && + typeof value.generation === "string" && + value.encoding === "gzip-json" + ); +} + +export function activateMemoryWikiCompiledCacheOwner( + config: ResolvedMemoryWikiConfig, + vaultGeneration: string, + compiledCachePublicationId?: string | null, +): void { + const normalizedVaultGeneration = vaultGeneration.trim(); + if (!normalizedVaultGeneration) { + throw new Error("Memory Wiki vault generation must not be empty."); + } + activeVaults.set(resolveMemoryWikiCompiledCacheOwnerId(config), { + path: path.resolve(config.vault.path), + vaultGeneration: normalizedVaultGeneration, + compiledCachePublicationId: compiledCachePublicationId?.trim() || undefined, + reconciled: false, + }); +} + +export function deactivateMemoryWikiCompiledCacheOwnersExcept(ownerIds: ReadonlySet): void { + for (const ownerId of activeVaults.keys()) { + if (!ownerIds.has(ownerId)) { + activeVaults.delete(ownerId); + } + } +} + +function resolveActiveVault(config: ResolvedMemoryWikiConfig): ActiveVault | null { + const active = activeVaults.get(resolveMemoryWikiCompiledCacheOwnerId(config)); + if (!active || active.path !== path.resolve(config.vault.path)) { + return null; + } + return active; +} + +function parseSnapshot( + bytes: Uint8Array, + generation: string, +): MemoryWikiCompiledCacheSnapshot | null { + try { + const serialized = gunzipSync(bytes).toString("utf8"); + if (createHash("sha256").update(serialized).digest("hex") !== generation) { + return null; + } + const parsed = JSON.parse(serialized) as MemoryWikiCompiledCacheSnapshot; + if ( + !parsed || + typeof parsed !== "object" || + !parsed.digest || + typeof parsed.digest !== "object" || + !Array.isArray(parsed.digest.pages) || + !Array.isArray(parsed.claims) + ) { + return null; + } + return parsed; + } catch { + return null; + } +} + +export function resolveMemoryWikiCompiledCacheGeneration( + snapshot: MemoryWikiCompiledCacheSnapshot, +): string { + return createHash("sha256").update(JSON.stringify(snapshot)).digest("hex"); +} + +export function createMemoryWikiCompiledCachePublicationId(): string { + return randomUUID(); +} + +export function createMemoryWikiCompiledCacheStore( + openBlobStore: (options: { + namespace: string; + maxEntries: number; + maxBytesPerEntry: number; + maxBytesPerNamespace: number; + overflowPolicy: "evict-oldest"; + }) => PluginBlobStore, + options: { onReadError?: (error: unknown) => void } = {}, +): MemoryWikiCompiledCacheStore { + const store = openBlobStore({ + namespace: COMPILED_CACHE_NAMESPACE, + maxEntries: COMPILED_CACHE_MAX_ENTRIES, + maxBytesPerEntry: COMPILED_CACHE_MAX_BYTES_PER_ENTRY, + maxBytesPerNamespace: COMPILED_CACHE_MAX_BYTES, + overflowPolicy: "evict-oldest", + }); + async function deleteKey(key: string): Promise { + await store.delete(key); + } + + return { + async read(config) { + const ownerId = resolveMemoryWikiCompiledCacheOwnerId(config); + const activeVault = resolveActiveVault(config); + if (!activeVault?.reconciled || !activeVault.compiledCachePublicationId) { + return null; + } + const key = publicationKey(ownerId, activeVault.compiledCachePublicationId); + const entry = await store.lookup(key).catch((error: unknown) => { + options.onReadError?.(error); + return undefined; + }); + if (!entry) { + return null; + } + const metadata = entry.metadata; + const vaultPath = path.resolve(config.vault.path); + if (!isMetadata(metadata) || metadata.ownerId !== ownerId) { + return null; + } + // Compile or lifecycle refresh owns source changes; prompt preparation never polls files. + // Every run still binds SQLite to that owner snapshot before exposing immutable lines. + if ( + metadata.vaultPath !== vaultPath || + metadata.vaultGeneration !== activeVault.vaultGeneration + ) { + return null; + } + if (metadata.publicationId !== activeVault.compiledCachePublicationId) { + return null; + } + const snapshot = parseSnapshot(entry.bytes, metadata.generation); + if (!snapshot) { + return null; + } + if (resolveActiveVault(config) !== activeVault) { + return null; + } + return snapshot; + }, + + async write(config, snapshot, generation, publicationId) { + const ownerId = resolveMemoryWikiCompiledCacheOwnerId(config); + const vaultPath = path.resolve(config.vault.path); + const activeVault = resolveActiveVault(config); + if (!activeVault) { + throw new Error(`Memory Wiki vault is not active: ${vaultPath}`); + } + const serialized = JSON.stringify(snapshot); + if (createHash("sha256").update(serialized).digest("hex") !== generation) { + throw new Error("Memory Wiki compiled cache generation does not match its snapshot."); + } + const metadata: CompiledCacheMetadata = { + version: COMPILED_CACHE_VERSION, + ownerId, + vaultPath, + vaultGeneration: activeVault.vaultGeneration, + publicationId, + generation, + encoding: "gzip-json", + }; + await store.register(publicationKey(ownerId, publicationId), gzipSync(serialized), metadata); + return activeVault; + }, + + async reconcile(config, loadDurableIdentity) { + const ownerId = resolveMemoryWikiCompiledCacheOwnerId(config); + const activeVault = resolveActiveVault(config); + if (!activeVault) { + return; + } + const durableIdentity = await loadDurableIdentity(); + if (durableIdentity.compiledCachePublicationId) { + try { + await store.lookup(publicationKey(ownerId, durableIdentity.compiledCachePublicationId)); + } catch (error) { + options.onReadError?.(error); + throw error; + } + } + const confirmedIdentity = await loadDurableIdentity(); + if (resolveActiveVault(config) !== activeVault) { + return; + } + if ( + !confirmedIdentity.vaultGeneration || + confirmedIdentity.vaultGeneration !== durableIdentity.vaultGeneration || + confirmedIdentity.compiledCachePublicationId !== durableIdentity.compiledCachePublicationId + ) { + activeVaults.delete(ownerId); + return; + } + // SQLite is observed before the durable identity reread. A cross-process write that + // races this boundary stays unreadable until the next lifecycle refresh. + activeVaults.set(ownerId, { + path: activeVault.path, + vaultGeneration: confirmedIdentity.vaultGeneration, + compiledCachePublicationId: confirmedIdentity.compiledCachePublicationId ?? undefined, + reconciled: true, + }); + }, + + async delete(config) { + const ownerId = resolveMemoryWikiCompiledCacheOwnerId(config); + for (const entry of await store.entries()) { + if (isMetadata(entry.metadata) && entry.metadata.ownerId === ownerId) { + await deleteKey(entry.key); + } + } + }, + + async deletePublication(config, publicationId) { + await deleteKey(publicationKey(resolveMemoryWikiCompiledCacheOwnerId(config), publicationId)); + }, + + async deleteOwnersExcept(ownerIds) { + let deleted = 0; + for (const entry of await store.entries()) { + const metadata = entry.metadata; + if (isMetadata(metadata) && ownerIds.has(metadata.ownerId)) { + continue; + } + await deleteKey(entry.key); + deleted += 1; + } + return deleted; + }, + }; +} + +export function configureMemoryWikiCompiledCacheStore( + store: MemoryWikiCompiledCacheStore | undefined, +): void { + configuredStore = store; + if (!store) { + activeVaults.clear(); + } +} + +function requireConfiguredStore(): MemoryWikiCompiledCacheStore { + if (!configuredStore) { + throw new Error("Memory Wiki compiled cache store is not configured."); + } + return configuredStore; +} + +export async function loadMemoryWikiCompiledCache( + config: ResolvedMemoryWikiConfig, +): Promise { + return await requireConfiguredStore().read(config); +} + +export async function invalidateMemoryWikiCompiledCache( + config: ResolvedMemoryWikiConfig, +): Promise { + await requireConfiguredStore().delete(config); +} + +export async function reconcileMemoryWikiCompiledCacheOwner( + config: ResolvedMemoryWikiConfig, + loadDurableIdentity: () => Promise<{ + vaultGeneration: string | null; + compiledCachePublicationId: string | null; + }>, +): Promise { + await requireConfiguredStore().reconcile(config, loadDurableIdentity); +} + +export async function writeMemoryWikiCompiledCache( + config: ResolvedMemoryWikiConfig, + snapshot: MemoryWikiCompiledCacheSnapshot, + generation: string, + publicationId: string, + parentPublicationId: string | null, + validatePublication: () => Promise, + commitPublication: () => Promise, + loadDurableIdentity: () => Promise<{ + vaultGeneration: string | null; + compiledCachePublicationId: string | null; + }>, +): Promise { + const store = requireConfiguredStore(); + const activeVault = await store.write(config, snapshot, generation, publicationId); + try { + await validatePublication(); + } catch (error) { + await store.deletePublication(config, publicationId); + throw error; + } + try { + await commitPublication(); + } catch (error) { + const identity = await loadDurableIdentity().catch(() => undefined); + if (identity?.compiledCachePublicationId !== publicationId) { + await store.deletePublication(config, publicationId); + } + throw error; + } + // The publication committed. If validation fails, retain its immutable row + // so a later lifecycle refresh can reconcile it. + const durableIdentity = await loadDurableIdentity(); + if ( + durableIdentity.vaultGeneration !== activeVault.vaultGeneration || + durableIdentity.compiledCachePublicationId !== publicationId + ) { + await store.deletePublication(config, publicationId); + if (resolveActiveVault(config) === activeVault) { + activeVaults.delete(resolveMemoryWikiCompiledCacheOwnerId(config)); + } + throw new Error("Memory Wiki vault changed while its compiled cache was being published."); + } + if (parentPublicationId) { + await store.deletePublication(config, parentPublicationId); + } + // The publication is durable. A concurrent lifecycle refresh owns in-memory + // activation; retaining this row lets its next refresh reconcile safely. + if (resolveActiveVault(config) !== activeVault) { + return; + } + activeVaults.set(resolveMemoryWikiCompiledCacheOwnerId(config), { + ...activeVault, + compiledCachePublicationId: publicationId, + reconciled: true, + }); +} diff --git a/extensions/memory-wiki/src/log.ts b/extensions/memory-wiki/src/log.ts index f500bb4fc6c6..acea3c77851c 100644 --- a/extensions/memory-wiki/src/log.ts +++ b/extensions/memory-wiki/src/log.ts @@ -1,14 +1,36 @@ // Memory Wiki plugin module implements log behavior. +import { createHash, randomUUID } from "node:crypto"; +import type { Dirent } from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; import { appendRegularFile } from "openclaw/plugin-sdk/security-runtime"; type MemoryWikiLogEntry = { - type: "init" | "ingest" | "okf-import" | "compile" | "lint"; + type: "init" | "vault-generation" | "ingest" | "okf-import" | "compile" | "lint"; timestamp: string; details?: Record; }; +const VAULT_GENERATION_FIELD = "vaultGeneration"; +const COMPILED_CACHE_RESERVATION_ID_FIELD = "compiledCacheReservationId"; +const COMPILED_CACHE_PUBLICATION_ID_FIELD = "compiledCachePublicationId"; +const COMPILED_CACHE_PARENT_PUBLICATION_ID_FIELD = "compiledCacheParentPublicationId"; +const COMPILED_CACHE_SOURCE_GENERATION_FIELD = "compiledCacheSourceGeneration"; +const COMPILED_SOURCE_DIRECTORIES = [ + "sources", + "entities", + "concepts", + "syntheses", + "reports", +] as const; + +type MemoryWikiVaultIdentity = { + vaultGeneration: string | null; + compiledCacheReservationId: string | null; + compiledCachePublicationId: string | null; + compiledCacheSourceGeneration: string | null; +}; + export async function appendMemoryWikiLog( vaultRoot: string, entry: MemoryWikiLogEntry, @@ -21,3 +43,165 @@ export async function appendMemoryWikiLog( rejectSymlinkParents: true, }); } + +export async function loadMemoryWikiVaultIdentity( + vaultRoot: string, +): Promise { + let raw: string; + try { + raw = await fs.readFile(path.join(vaultRoot, ".openclaw-wiki", "log.jsonl"), "utf8"); + } catch (error) { + if (error instanceof Error && "code" in error && error.code === "ENOENT") { + return { + vaultGeneration: null, + compiledCacheReservationId: null, + compiledCachePublicationId: null, + compiledCacheSourceGeneration: null, + }; + } + throw error; + } + let vaultGeneration: string | null = null; + let compiledCacheReservationId: string | null = null; + let compiledCachePublicationId: string | null = null; + let compiledCacheSourceGeneration: string | null = null; + for (const line of raw.split(/\r?\n/)) { + try { + const parsed = JSON.parse(line) as MemoryWikiLogEntry; + const candidateVaultGeneration = parsed.details?.[VAULT_GENERATION_FIELD]; + if ( + !vaultGeneration && + typeof candidateVaultGeneration === "string" && + candidateVaultGeneration.trim() + ) { + vaultGeneration = candidateVaultGeneration.trim(); + } + const candidateReservationId = parsed.details?.[COMPILED_CACHE_RESERVATION_ID_FIELD]; + const normalizedReservationId = + typeof candidateReservationId === "string" && candidateReservationId.trim() + ? candidateReservationId.trim() + : undefined; + const candidateCompiledCachePublicationId = + parsed.details?.[COMPILED_CACHE_PUBLICATION_ID_FIELD]; + if ( + typeof candidateCompiledCachePublicationId === "string" && + candidateCompiledCachePublicationId.trim() + ) { + const candidateParent = parsed.details?.[COMPILED_CACHE_PARENT_PUBLICATION_ID_FIELD]; + const normalizedParent = + candidateParent === null + ? null + : typeof candidateParent === "string" && candidateParent.trim() + ? candidateParent.trim() + : undefined; + const candidateSourceGeneration = parsed.details?.[COMPILED_CACHE_SOURCE_GENERATION_FIELD]; + const normalizedSourceGeneration = + typeof candidateSourceGeneration === "string" && candidateSourceGeneration.trim() + ? candidateSourceGeneration.trim() + : undefined; + // A commit must reference both the prior publication and a reservation + // already present in the log; it cannot recreate either after rollback. + if ( + normalizedParent === compiledCachePublicationId && + normalizedReservationId === compiledCacheReservationId && + normalizedSourceGeneration + ) { + compiledCachePublicationId = candidateCompiledCachePublicationId.trim(); + compiledCacheSourceGeneration = normalizedSourceGeneration; + } + } else if (normalizedReservationId) { + compiledCacheReservationId = normalizedReservationId; + } + } catch { + // Audit logs may contain a partial final line after an interrupted append. + } + } + return { + vaultGeneration, + compiledCacheReservationId, + compiledCachePublicationId, + compiledCacheSourceGeneration, + }; +} + +export async function resolveMemoryWikiVaultSourceGeneration(vaultRoot: string): Promise { + const files = ( + await Promise.all( + COMPILED_SOURCE_DIRECTORIES.map(async (relativeDir) => { + const dirPath = path.join(vaultRoot, relativeDir); + let entries: Dirent[]; + try { + entries = await fs.readdir(dirPath, { withFileTypes: true, recursive: true }); + } catch (error) { + if (error instanceof Error && "code" in error && error.code === "ENOENT") { + return []; + } + throw error; + } + return entries + .filter((entry) => entry.isFile() && entry.name.endsWith(".md")) + .map((entry) => { + const absolutePath = path.join(entry.parentPath ?? dirPath, entry.name); + return { + absolutePath, + relativePath: path.relative(vaultRoot, absolutePath).split(path.sep).join("/"), + }; + }) + .filter((entry) => path.basename(entry.relativePath) !== "index.md"); + }), + ) + ) + .flat() + .toSorted((left, right) => left.relativePath.localeCompare(right.relativePath)); + const hash = createHash("sha256"); + for (const file of files) { + const relativePath = Buffer.from(file.relativePath); + const pathLength = Buffer.allocUnsafe(4); + pathLength.writeUInt32BE(relativePath.byteLength); + const contentDigest = createHash("sha256") + .update(await fs.readFile(file.absolutePath)) + .digest(); + hash.update(pathLength).update(relativePath).update(contentDigest); + } + return hash.digest("hex"); +} + +export async function loadMemoryWikiValidatedVaultIdentity( + vaultRoot: string, +): Promise { + const identity = await loadMemoryWikiVaultIdentity(vaultRoot); + if (!identity.compiledCachePublicationId || !identity.compiledCacheSourceGeneration) { + return identity; + } + if ( + (await resolveMemoryWikiVaultSourceGeneration(vaultRoot)) === + identity.compiledCacheSourceGeneration + ) { + return identity; + } + return { + ...identity, + compiledCachePublicationId: null, + compiledCacheSourceGeneration: null, + }; +} + +async function loadMemoryWikiVaultGeneration(vaultRoot: string): Promise { + return (await loadMemoryWikiVaultIdentity(vaultRoot)).vaultGeneration; +} + +export async function ensureMemoryWikiVaultGeneration(vaultRoot: string): Promise { + const existing = await loadMemoryWikiVaultGeneration(vaultRoot); + if (existing) { + return existing; + } + const candidate = randomUUID(); + await appendMemoryWikiLog(vaultRoot, { + type: "vault-generation", + timestamp: new Date().toISOString(), + details: { [VAULT_GENERATION_FIELD]: candidate }, + }); + // Concurrent initialization can append two candidates. The first durable + // audit entry owns the vault generation, so every caller converges on it. + return (await loadMemoryWikiVaultGeneration(vaultRoot)) ?? candidate; +} diff --git a/extensions/memory-wiki/src/prompt-section.test.ts b/extensions/memory-wiki/src/prompt-section.test.ts index 1d43dfdf0d33..cb266833bb57 100644 --- a/extensions/memory-wiki/src/prompt-section.test.ts +++ b/extensions/memory-wiki/src/prompt-section.test.ts @@ -1,43 +1,141 @@ -// Memory Wiki tests cover prompt section plugin behavior. +// Memory Wiki tests cover synchronous guidance and async compiled prompt preparation. +import fsSync from "node:fs"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { expectDefined } from "@openclaw/normalization-core"; -import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../api.js"; +import { + activateMemoryWikiCompiledCacheOwner, + configureMemoryWikiCompiledCacheStore, + createMemoryWikiCompiledCachePublicationId, + resolveMemoryWikiCompiledCacheGeneration, + writeMemoryWikiCompiledCache, + type MemoryWikiCompiledCacheSnapshot, + type MemoryWikiCompiledDigestClaim, + type MemoryWikiCompiledDigestPage, +} from "./compiled-cache.js"; import { resolveMemoryWikiAgentConfig, resolveMemoryWikiConfig, type ResolvedMemoryWikiConfig, } from "./config.js"; -import { createWikiPromptSectionBuilder } from "./prompt-section.js"; +import { + appendMemoryWikiLog, + ensureMemoryWikiVaultGeneration, + loadMemoryWikiValidatedVaultIdentity, + loadMemoryWikiVaultIdentity, + resolveMemoryWikiVaultSourceGeneration, +} from "./log.js"; +import { + createWikiPromptSectionBuilder, + createWikiPromptSectionPreparer, +} from "./prompt-section.js"; +import { createMemoryWikiTestHarness } from "./test-helpers.js"; +const { configureCompiledCacheStore } = createMemoryWikiTestHarness(); let suiteRoot = ""; beforeAll(async () => { suiteRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-wiki-prompt-suite-")); }); +afterEach(() => { + vi.restoreAllMocks(); + configureMemoryWikiCompiledCacheStore(undefined); +}); + afterAll(async () => { if (suiteRoot) { await fs.rm(suiteRoot, { recursive: true, force: true }); } }); -function createStaticWikiPromptSectionBuilder(config: ResolvedMemoryWikiConfig) { - return createWikiPromptSectionBuilder({ config, resolveConfig: () => config }); +type PromptPageFixture = Pick & + Omit, "topClaims"> & { + topClaims?: Array & { text: string }>; + }; + +async function seedCompiledDigest(params: { + config: ResolvedMemoryWikiConfig; + claimCount: number; + contradictionCount?: number; + pages: PromptPageFixture[]; +}): Promise { + configureCompiledCacheStore(); + await fs.mkdir(path.join(params.config.vault.path, ".openclaw-wiki"), { recursive: true }); + await Promise.all([ + fs.writeFile(path.join(params.config.vault.path, "WIKI.md"), "# Memory Wiki\n", "utf8"), + fs.writeFile(path.join(params.config.vault.path, ".openclaw-wiki", "log.jsonl"), "", "utf8"), + ]); + activateMemoryWikiCompiledCacheOwner( + params.config, + await ensureMemoryWikiVaultGeneration(params.config.vault.path), + ); + const snapshot: MemoryWikiCompiledCacheSnapshot = { + digest: { + claimCount: params.claimCount, + contradictionCount: params.contradictionCount ?? 0, + pages: params.pages.map((page, index) => ({ + ...page, + path: page.path ?? `entities/page-${index}.md`, + aliases: page.aliases ?? [], + sourceIds: page.sourceIds ?? [], + questions: page.questions ?? [], + contradictions: page.contradictions ?? [], + bestUsedFor: page.bestUsedFor ?? [], + notEnoughFor: page.notEnoughFor ?? [], + relationshipCount: page.relationshipCount ?? 0, + topRelationships: page.topRelationships ?? [], + topClaims: (page.topClaims ?? []).map((claim) => + Object.assign({ status: "supported", freshnessLevel: "unknown" }, claim), + ), + })), + }, + claims: [], + }; + const publicationId = createMemoryWikiCompiledCachePublicationId(); + const reservationId = createMemoryWikiCompiledCachePublicationId(); + const parentPublicationId = (await loadMemoryWikiVaultIdentity(params.config.vault.path)) + .compiledCachePublicationId; + await appendMemoryWikiLog(params.config.vault.path, { + type: "compile", + timestamp: "2026-07-17T00:00:00.000Z", + details: { compiledCacheReservationId: reservationId }, + }); + const sourceGeneration = await resolveMemoryWikiVaultSourceGeneration(params.config.vault.path); + await appendMemoryWikiLog(params.config.vault.path, { + type: "compile", + timestamp: "2026-07-17T00:00:00.000Z", + details: { + compiledCachePublicationId: publicationId, + compiledCacheParentPublicationId: parentPublicationId, + compiledCacheReservationId: reservationId, + compiledCacheSourceGeneration: sourceGeneration, + }, + }); + await writeMemoryWikiCompiledCache( + params.config, + snapshot, + resolveMemoryWikiCompiledCacheGeneration(snapshot), + publicationId, + parentPublicationId, + async () => {}, + async () => {}, + () => loadMemoryWikiValidatedVaultIdentity(params.config.vault.path), + ); } -const buildDefaultWikiPromptSection = createStaticWikiPromptSectionBuilder( - resolveMemoryWikiConfig({ - vault: { path: "" }, - context: { includeCompiledDigestPrompt: false }, - }), -); +function createStaticPreparer(config: ResolvedMemoryWikiConfig) { + return createWikiPromptSectionPreparer({ config, resolveConfig: () => config }); +} + +describe("Memory Wiki prompt section", () => { + const buildGuidance = createWikiPromptSectionBuilder(); -describe("default wiki prompt section", () => { it("prefers shared memory corpus guidance when memory tools are available", () => { - const lines = buildDefaultWikiPromptSection({ + const lines = buildGuidance({ availableTools: new Set(["memory_search", "memory_get", "wiki_search", "wiki_get"]), }); @@ -47,121 +145,26 @@ describe("default wiki prompt section", () => { }); it("stays empty when no wiki or memory-adjacent tools are registered", () => { - expect( - buildDefaultWikiPromptSection({ availableTools: new Set(["web_search"]) }), - ).toStrictEqual([]); + expect(buildGuidance({ availableTools: new Set(["web_search"]) })).toStrictEqual([]); }); - it("can append a compact compiled digest snapshot when enabled", async () => { - const rootDir = path.join(suiteRoot, "digest-enabled"); - await fs.mkdir(path.join(rootDir, ".openclaw-wiki", "cache"), { recursive: true }); - await fs.writeFile( - path.join(rootDir, ".openclaw-wiki", "cache", "agent-digest.json"), - JSON.stringify( - { - claimCount: 8, - contradictionClusters: [{ key: "claim.alpha.db" }], - pages: [ - { - title: "Alpha", - kind: "entity", - claimCount: 3, - questions: ["Still active?"], - contradictions: ["Conflicts with source.beta"], - topClaims: [ - { - text: "Alpha uses PostgreSQL for production writes.", - status: "supported", - confidence: 0.91, - freshnessLevel: "fresh", - }, - ], - }, - ], - }, - null, - 2, - ), - "utf8", - ); - const builder = createStaticWikiPromptSectionBuilder( - resolveMemoryWikiConfig({ - vault: { path: rootDir }, - context: { includeCompiledDigestPrompt: true }, - }), - ); - - const lines = builder({ availableTools: new Set(["web_search"]) }); - - expect(lines.join("\n")).toContain("## Compiled Wiki Snapshot"); - expect(lines.join("\n")).toContain( - "Alpha: entity, 3 claims, 1 open questions, 1 contradiction notes", - ); - expect(lines.join("\n")).toContain("Alpha uses PostgreSQL for production writes."); - }); - - it("keeps the digest snapshot disabled by default", async () => { - const rootDir = path.join(suiteRoot, "digest-disabled"); - await fs.mkdir(path.join(rootDir, ".openclaw-wiki", "cache"), { recursive: true }); - await fs.writeFile( - path.join(rootDir, ".openclaw-wiki", "cache", "agent-digest.json"), - JSON.stringify({ - claimCount: 1, - pages: [{ title: "Alpha", kind: "entity", claimCount: 1, topClaims: [] }], - }), - "utf8", - ); - const builder = createStaticWikiPromptSectionBuilder( - resolveMemoryWikiConfig({ - vault: { path: rootDir }, - }), - ); - - expect(builder({ availableTools: new Set(["web_search"]) })).toStrictEqual([]); - }); - - it("stabilizes digest prompt ordering for prompt-cache-friendly output", async () => { - const rootDir = path.join(suiteRoot, "digest-stable"); - const digestPath = path.join(rootDir, ".openclaw-wiki", "cache", "agent-digest.json"); - await fs.mkdir(path.dirname(digestPath), { recursive: true }); - - const builder = createStaticWikiPromptSectionBuilder( - resolveMemoryWikiConfig({ - vault: { path: rootDir }, - context: { includeCompiledDigestPrompt: true }, - }), - ); - - const firstDigest = { - claimCount: 6, - contradictionClusters: [{ key: "claim.alpha.db" }], + it("prepares a compact compiled digest from SQLite", async () => { + const config = resolveMemoryWikiConfig({ + vault: { path: path.join(suiteRoot, "digest-enabled") }, + context: { includeCompiledDigestPrompt: true }, + }); + await seedCompiledDigest({ + config, + claimCount: 8, + contradictionCount: 1, pages: [ - { - title: "Zulu", - kind: "concept", - claimCount: 2, - questions: [], - contradictions: [], - topClaims: [ - { - text: "Zulu fallback note.", - confidence: 0.3, - freshnessLevel: "stale", - }, - ], - }, { title: "Alpha", kind: "entity", - claimCount: 4, + claimCount: 3, questions: ["Still active?"], contradictions: ["Conflicts with source.beta"], topClaims: [ - { - text: "Alpha was renamed in 2026.", - confidence: 0.42, - freshnessLevel: "aging", - }, { text: "Alpha uses PostgreSQL for production writes.", confidence: 0.91, @@ -170,36 +173,84 @@ describe("default wiki prompt section", () => { ], }, ], - }; - const firstPage = expectDefined(firstDigest.pages[0], "first Memory Wiki digest page"); - const secondPage = expectDefined(firstDigest.pages[1], "second Memory Wiki digest page"); - const secondDigest = { - ...firstDigest, - pages: [ - { - ...secondPage, - topClaims: secondPage.topClaims.toReversed(), - }, - firstPage, - ], - }; + }); - await fs.writeFile(digestPath, JSON.stringify(firstDigest, null, 2), "utf8"); - const firstLines = builder({ availableTools: new Set(["web_search"]) }); + const lines = await createStaticPreparer(config)({ availableTools: new Set() }); - await fs.writeFile(digestPath, JSON.stringify(secondDigest, null, 2), "utf8"); - const secondLines = builder({ availableTools: new Set(["web_search"]) }); + expect(lines.join("\n")).toContain("## Compiled Wiki Snapshot"); + expect(lines.join("\n")).toContain( + "Alpha: entity, 3 claims, 1 open questions, 1 contradiction notes", + ); + expect(lines.join("\n")).toContain("Alpha uses PostgreSQL for production writes."); + }); + + it("keeps the digest disabled by default", async () => { + const config = resolveMemoryWikiConfig({ + vault: { path: path.join(suiteRoot, "digest-disabled") }, + }); + await seedCompiledDigest({ + config, + claimCount: 1, + pages: [{ title: "Alpha", kind: "entity", claimCount: 1 }], + }); + + await expect(createStaticPreparer(config)({ availableTools: new Set() })).resolves.toEqual([]); + }); + + it("stabilizes digest ordering for prompt-cache-friendly output", async () => { + const config = resolveMemoryWikiConfig({ + vault: { path: path.join(suiteRoot, "digest-stable") }, + context: { includeCompiledDigestPrompt: true }, + }); + const pages: PromptPageFixture[] = [ + { + title: "Zulu", + kind: "concept", + claimCount: 2, + topClaims: [{ text: "Zulu fallback note.", confidence: 0.3, freshnessLevel: "stale" }], + }, + { + title: "Alpha", + kind: "entity", + claimCount: 4, + questions: ["Still active?"], + contradictions: ["Conflicts with source.beta"], + topClaims: [ + { text: "Alpha was renamed in 2026.", confidence: 0.42, freshnessLevel: "aging" }, + { + text: "Alpha uses PostgreSQL for production writes.", + confidence: 0.91, + freshnessLevel: "fresh", + }, + ], + }, + ]; + await seedCompiledDigest({ config, claimCount: 6, contradictionCount: 1, pages }); + const firstLines = await createStaticPreparer(config)({ availableTools: new Set() }); + const firstPage = expectDefined(pages[0], "first Memory Wiki digest page"); + const secondPage = expectDefined(pages[1], "second Memory Wiki digest page"); + await seedCompiledDigest({ + config, + claimCount: 6, + contradictionCount: 1, + pages: [{ ...secondPage, topClaims: secondPage.topClaims?.toReversed() }, firstPage], + }); + const secondLines = await createStaticPreparer(config)({ availableTools: new Set() }); expect(firstLines).toEqual(secondLines); expect(firstLines.join("\n")).toContain( - "Alpha uses PostgreSQL for production writes. (confidence 0.91, freshness fresh)", - ); - expect(firstLines.join("\n")).toContain( - "Alpha was renamed in 2026. (confidence 0.42, freshness aging)", + "Alpha uses PostgreSQL for production writes. (status supported, confidence 0.91, freshness fresh)", ); }); - it("reads only the invoking agent's compiled digest", async () => { + it("does no filesystem work during synchronous prompt guidance", () => { + const readFileSync = vi.spyOn(fsSync, "readFileSync"); + + expect(buildGuidance({ availableTools: new Set(["web_search"]) })).toEqual([]); + expect(readFileSync).not.toHaveBeenCalled(); + }); + + it("prepares only the invoking agent's compiled digest", async () => { const rootDir = path.join(suiteRoot, "agent-digests"); const appConfig = { agents: { list: [{ id: "support", default: true }, { id: "marketing" }] }, @@ -212,45 +263,32 @@ describe("default wiki prompt section", () => { ["support", "SUPPORT_SENTINEL"], ["marketing", "MARKETING_SENTINEL"], ] as const) { - const digestPath = path.join( - rootDir, - agentId, - ".openclaw-wiki", - "cache", - "agent-digest.json", - ); - await fs.mkdir(path.dirname(digestPath), { recursive: true }); - await fs.writeFile( - digestPath, - JSON.stringify({ - claimCount: 1, - pages: [ - { - title: agentId, - kind: "entity", - claimCount: 1, - topClaims: [{ text: marker }], - }, - ], - }), - "utf8", - ); + const agentConfig = resolveMemoryWikiAgentConfig({ config, appConfig, agentId }); + await seedCompiledDigest({ + config: agentConfig, + claimCount: 1, + pages: [ + { + title: agentId, + kind: "entity", + claimCount: 1, + topClaims: [{ text: marker }], + }, + ], + }); } - const builder = createWikiPromptSectionBuilder({ + const prepare = createWikiPromptSectionPreparer({ config, resolveConfig: (agentId) => resolveMemoryWikiAgentConfig({ config, appConfig, agentId }), }); - const support = builder({ availableTools: new Set(["web_search"]), agentId: "support" }); - const marketing = builder({ - availableTools: new Set(["web_search"]), - agentId: "marketing", - }); + const support = await prepare({ availableTools: new Set(), agentId: "support" }); + const marketing = await prepare({ availableTools: new Set(), agentId: "marketing" }); expect(support.join("\n")).toContain("SUPPORT_SENTINEL"); expect(support.join("\n")).not.toContain("MARKETING_SENTINEL"); expect(marketing.join("\n")).toContain("MARKETING_SENTINEL"); expect(marketing.join("\n")).not.toContain("SUPPORT_SENTINEL"); - expect(builder({ availableTools: new Set(["web_search"]) })).toStrictEqual([]); + await expect(prepare({ availableTools: new Set() })).resolves.toEqual([]); }); }); diff --git a/extensions/memory-wiki/src/prompt-section.ts b/extensions/memory-wiki/src/prompt-section.ts index 824eaf1e250b..e30c04ea0950 100644 --- a/extensions/memory-wiki/src/prompt-section.ts +++ b/extensions/memory-wiki/src/prompt-section.ts @@ -1,51 +1,17 @@ // Memory Wiki plugin module implements prompt section behavior. -import fs from "node:fs"; -import path from "node:path"; import type { MemoryPromptSectionBuilder } from "openclaw/plugin-sdk/memory-host-core"; +import { + loadMemoryWikiCompiledCache, + type MemoryWikiCompiledCacheSnapshot, + type MemoryWikiCompiledDigestClaim, + type MemoryWikiCompiledDigestPage, +} from "./compiled-cache.js"; import type { MemoryWikiConfigResolver, ResolvedMemoryWikiConfig } from "./config.js"; -const AGENT_DIGEST_PATH = ".openclaw-wiki/cache/agent-digest.json"; const DIGEST_MAX_PAGES = 4; const DIGEST_MAX_CLAIMS_PER_PAGE = 2; -type PromptDigestClaim = { - text: string; - status?: string; - confidence?: number; - freshnessLevel?: string; -}; - -type PromptDigestPage = { - title: string; - kind: string; - claimCount: number; - questions?: string[]; - contradictions?: string[]; - topClaims?: PromptDigestClaim[]; -}; - -type PromptDigest = { - pageCounts?: Record; - claimCount?: number; - contradictionClusters?: Array; - pages?: PromptDigestPage[]; -}; - -function tryReadPromptDigest(config: ResolvedMemoryWikiConfig): PromptDigest | null { - const digestPath = path.join(config.vault.path, AGENT_DIGEST_PATH); - try { - const raw = fs.readFileSync(digestPath, "utf8"); - const parsed = JSON.parse(raw) as PromptDigest; - if (!parsed || typeof parsed !== "object") { - return null; - } - return parsed; - } catch { - return null; - } -} - -function rankPromptDigestPage(page: PromptDigestPage): number { +function rankPromptDigestPage(page: MemoryWikiCompiledDigestPage): number { return ( (page.contradictions?.length ?? 0) * 6 + (page.questions?.length ?? 0) * 4 + @@ -67,7 +33,9 @@ function rankPromptClaimFreshness(level?: string): number { } } -function sortPromptClaims(claims: PromptDigestClaim[]): PromptDigestClaim[] { +function sortPromptClaims( + claims: MemoryWikiCompiledDigestClaim[], +): MemoryWikiCompiledDigestClaim[] { return [...claims].toSorted((left, right) => { const leftConfidence = typeof left.confidence === "number" ? left.confidence : -1; const rightConfidence = typeof right.confidence === "number" ? right.confidence : -1; @@ -83,7 +51,7 @@ function sortPromptClaims(claims: PromptDigestClaim[]): PromptDigestClaim[] { }); } -function formatPromptClaim(claim: PromptDigestClaim): string { +function formatPromptClaim(claim: MemoryWikiCompiledDigestClaim): string { const qualifiers = [ claim.status?.trim() ? `status ${claim.status.trim()}` : null, typeof claim.confidence === "number" ? `confidence ${claim.confidence.toFixed(2)}` : null, @@ -95,11 +63,9 @@ function formatPromptClaim(claim: PromptDigestClaim): string { return `${claim.text} (${qualifiers.join(", ")})`; } -function buildDigestPromptSection(config: ResolvedMemoryWikiConfig): string[] { - if (!config.context.includeCompiledDigestPrompt) { - return []; - } - const digest = tryReadPromptDigest(config); +function buildDigestPromptSection( + digest: MemoryWikiCompiledCacheSnapshot["digest"] | undefined, +): string[] { if (!digest?.pages?.length) { return []; } @@ -129,9 +95,7 @@ function buildDigestPromptSection(config: ResolvedMemoryWikiConfig): string[] { "## Compiled Wiki Snapshot", `Compiled wiki currently tracks ${digest.claimCount ?? 0} claims across ${selectedPages.length} high-signal pages.`, ]; - if (Array.isArray(digest.contradictionClusters)) { - lines.push(`Contradiction clusters: ${digest.contradictionClusters.length}.`); - } + lines.push(`Contradiction clusters: ${digest.contradictionCount}.`); for (const page of selectedPages) { const details = [ page.kind, @@ -214,20 +178,24 @@ function buildWikiToolGuidance(availableTools: Set): string[] { return lines; } -export function createWikiPromptSectionBuilder(params: { +export function createWikiPromptSectionBuilder(): MemoryPromptSectionBuilder { + return ({ availableTools }) => buildWikiToolGuidance(availableTools); +} + +export function createWikiPromptSectionPreparer(params: { config: ResolvedMemoryWikiConfig; resolveConfig: MemoryWikiConfigResolver; -}): MemoryPromptSectionBuilder { - return ({ availableTools, agentId }) => { - // Prompt contexts without an agent must not fall back to another agent's digest. - const digestLines = - params.config.vault.scope === "agent" && !agentId - ? [] - : buildDigestPromptSection(params.resolveConfig(agentId)); - const toolGuidance = buildWikiToolGuidance(availableTools); - if (digestLines.length === 0 && toolGuidance.length === 0) { +}) { + return async ({ agentId }: Parameters[0]) => { + // Context-free preparation must not choose or disclose another agent's vault. + if (params.config.vault.scope === "agent" && !agentId) { return []; } - return [...toolGuidance, ...digestLines]; + const config = params.resolveConfig(agentId); + if (!config.context.includeCompiledDigestPrompt) { + return []; + } + const snapshot = await loadMemoryWikiCompiledCache(config); + return buildDigestPromptSection(snapshot?.digest); }; } diff --git a/extensions/memory-wiki/src/query.ts b/extensions/memory-wiki/src/query.ts index 2b0c764d2faa..7cfddc4acecc 100644 --- a/extensions/memory-wiki/src/query.ts +++ b/extensions/memory-wiki/src/query.ts @@ -21,19 +21,21 @@ import { import pMap, { pMapSkip } from "p-map"; import type { OpenClawConfig } from "../api.js"; import { assessClaimFreshness, isClaimContestedStatus } from "./claim-health.js"; +import { + loadMemoryWikiCompiledCache, + type MemoryWikiCompiledClaim, + type MemoryWikiCompiledDigestPage, +} from "./compiled-cache.js"; import type { ResolvedMemoryWikiConfig, WikiSearchBackend, WikiSearchCorpus } from "./config.js"; import { parseWikiMarkdown, toWikiPageSummary, type WikiClaim, type WikiPageSummary, - type WikiRelationship, } from "./markdown.js"; import { initializeMemoryWikiVault } from "./vault.js"; const QUERY_DIRS = ["entities", "concepts", "sources", "syntheses", "reports"] as const; -const AGENT_DIGEST_PATH = ".openclaw-wiki/cache/agent-digest.json"; -const CLAIMS_DIGEST_PATH = ".openclaw-wiki/cache/claims.jsonl"; const QUERY_PAGE_READ_CONCURRENCY = 16; const RELATED_BLOCK_PATTERN = /[\s\S]*?/g; @@ -104,45 +106,8 @@ export const WIKI_SEARCH_MODES = [ export type WikiSearchMode = (typeof WIKI_SEARCH_MODES)[number]; -type QueryDigestPage = { - id?: string; - title: string; - kind: WikiPageSummary["kind"]; - path: string; - pageType?: string; - entityType?: string; - canonicalId?: string; - aliases?: string[]; - sourceIds: string[]; - questions: string[]; - contradictions: string[]; - privacyTier?: string; - personCard?: WikiPageSummary["personCard"]; - bestUsedFor?: string[]; - notEnoughFor?: string[]; - relationshipCount?: number; - topRelationships?: WikiRelationship[]; -}; - -type QueryDigestClaim = { - id?: string; - pageId?: string; - pageTitle: string; - pageKind: WikiPageSummary["kind"]; - pagePath: string; - pageType?: string; - entityType?: string; - canonicalId?: string; - aliases?: string[]; - text: string; - status?: string; - confidence?: number; - sourceIds?: string[]; - evidenceKinds?: string[]; - privacyTiers?: string[]; - freshnessLevel?: string; - lastTouchedAt?: string; -}; +type QueryDigestPage = MemoryWikiCompiledDigestPage; +type QueryDigestClaim = MemoryWikiCompiledClaim; type QueryDigestBundle = { pages: QueryDigestPage[]; @@ -285,51 +250,11 @@ async function readQueryableWikiPagesByPaths( ); } -function parseClaimsDigest(raw: string): QueryDigestClaim[] { - return raw.split(/\r?\n/).flatMap((line) => { - const trimmed = line.trim(); - if (!trimmed) { - return []; - } - try { - const parsed = JSON.parse(trimmed) as QueryDigestClaim; - if (!parsed || typeof parsed !== "object" || typeof parsed.pagePath !== "string") { - return []; - } - return [parsed]; - } catch { - return []; - } - }); -} - -async function readQueryDigestBundle(rootDir: string): Promise { - const [agentDigestRaw, claimsDigestRaw] = await Promise.all([ - fs.readFile(path.join(rootDir, AGENT_DIGEST_PATH), "utf8").catch(() => null), - fs.readFile(path.join(rootDir, CLAIMS_DIGEST_PATH), "utf8").catch(() => null), - ]); - if (!agentDigestRaw && !claimsDigestRaw) { - return null; - } - - const pages = (() => { - if (!agentDigestRaw) { - return []; - } - try { - const parsed = JSON.parse(agentDigestRaw) as { pages?: QueryDigestPage[] }; - return Array.isArray(parsed.pages) ? parsed.pages : []; - } catch { - return []; - } - })(); - const claims = claimsDigestRaw ? parseClaimsDigest(claimsDigestRaw) : []; - - if (pages.length === 0 && claims.length === 0) { - return null; - } - - return { pages, claims }; +async function readQueryDigestBundle( + config: ResolvedMemoryWikiConfig, +): Promise { + const snapshot = await loadMemoryWikiCompiledCache(config); + return snapshot ? { pages: snapshot.digest.pages, claims: snapshot.claims } : null; } function buildSnippet(raw: string, query: string): string { @@ -1407,12 +1332,13 @@ function canReadSessionMemoryPath(params: { } async function searchWikiCorpus(params: { - rootDir: string; + config: ResolvedMemoryWikiConfig; query: string; maxResults: number; mode: WikiSearchMode; }): Promise { - const digest = await readQueryDigestBundle(params.rootDir); + const digest = await readQueryDigestBundle(params.config); + const rootDir = params.config.vault.path; const candidatePaths = digest ? buildDigestCandidatePaths({ digest, @@ -1424,8 +1350,8 @@ async function searchWikiCorpus(params: { const seenPaths = new Set(); const candidatePages = candidatePaths.length > 0 - ? await readQueryableWikiPagesByPaths(params.rootDir, candidatePaths) - : await readQueryableWikiPages(params.rootDir); + ? await readQueryableWikiPagesByPaths(rootDir, candidatePaths) + : await readQueryableWikiPages(rootDir); for (const page of candidatePages) { seenPaths.add(page.relativePath); } @@ -1437,10 +1363,10 @@ async function searchWikiCorpus(params: { return results; } - const remainingPaths = (await listWikiMarkdownFiles(params.rootDir)).filter( + const remainingPaths = (await listWikiMarkdownFiles(rootDir)).filter( (relativePath) => !seenPaths.has(relativePath), ); - const remainingPages = await readQueryableWikiPagesByPaths(params.rootDir, remainingPaths); + const remainingPages = await readQueryableWikiPagesByPaths(rootDir, remainingPaths); return [ ...results, ...remainingPages @@ -1499,7 +1425,7 @@ export async function searchMemoryWiki(params: { const wikiResults = shouldSearchWiki(effectiveConfig) ? await searchWikiCorpus({ - rootDir: effectiveConfig.vault.path, + config: effectiveConfig, query: params.query, maxResults, mode, @@ -1568,7 +1494,7 @@ export async function getMemoryWikiPage(params: { const lineCount = normalizePositiveInteger(params.lineCount, 200); if (shouldSearchWiki(effectiveConfig)) { - const digest = await readQueryDigestBundle(effectiveConfig.vault.path); + const digest = await readQueryDigestBundle(effectiveConfig); const digestClaimPagePath = digest ? resolveDigestClaimLookup(digest, params.lookup) : null; const digestLookupPage = digestClaimPagePath ? (( diff --git a/extensions/memory-wiki/src/test-helpers.ts b/extensions/memory-wiki/src/test-helpers.ts index e37cb0c97398..7b5b671613df 100644 --- a/extensions/memory-wiki/src/test-helpers.ts +++ b/extensions/memory-wiki/src/test-helpers.ts @@ -1,11 +1,20 @@ // Memory Wiki helper module supports test helpers behavior. import fs from "node:fs/promises"; import path from "node:path"; -import type { PluginStateEntry } from "openclaw/plugin-sdk/plugin-state-runtime"; +import type { + PluginBlobEntry, + PluginBlobEntryInfo, + PluginBlobStore, + PluginStateEntry, +} from "openclaw/plugin-sdk/plugin-state-runtime"; import { createTestPluginApi } from "openclaw/plugin-sdk/plugin-test-api"; import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path"; import { afterEach, vi } from "vitest"; import type { OpenClawPluginApi } from "../api.js"; +import { + configureMemoryWikiCompiledCacheStore, + createMemoryWikiCompiledCacheStore, +} from "./compiled-cache.js"; import { resolveMemoryWikiConfig, type MemoryWikiPluginConfig, @@ -25,7 +34,9 @@ type MemoryWikiPluginApiHarness = { registerCli: ReturnType; registerGatewayMethod: ReturnType; registerMemoryCorpusSupplement: ReturnType; + registerMemoryPromptPreparation: ReturnType; registerMemoryPromptSupplement: ReturnType; + registerService: ReturnType; registerTool: ReturnType; }; @@ -69,16 +80,91 @@ function createMemoryKeyedStore() { }; } +function createMemoryBlobStore() { + const values = new Map>(); + const register = async ( + key: string, + bytes: Uint8Array, + metadata: T, + opts?: { ttlMs?: number }, + ) => { + values.set(key, { + key, + bytes, + metadata, + sizeBytes: bytes.byteLength, + createdAt: Date.now(), + ...(opts?.ttlMs ? { expiresAt: Date.now() + opts.ttlMs } : {}), + }); + }; + return { + register, + async registerIfAbsent(key: string, bytes: Uint8Array, metadata: T, opts?: { ttlMs?: number }) { + if (values.has(key)) { + return false; + } + await register(key, bytes, metadata, opts); + return true; + }, + async lookup(key: string) { + return values.get(key); + }, + async entries() { + return [...values.values()].map(({ bytes: _bytes, ...entry }) => entry); + }, + async delete(key: string) { + return values.delete(key); + }, + async deleteExpiredKey(key: string) { + const entry = values.get(key); + if (!entry?.expiresAt || entry.expiresAt > Date.now()) { + return undefined; + } + values.delete(key); + const { bytes: _bytes, ...info } = entry; + return info; + }, + async deleteExpired() { + const expired: PluginBlobEntryInfo[] = []; + for (const key of values.keys()) { + const entry = values.get(key); + if (!entry?.expiresAt || entry.expiresAt > Date.now()) { + continue; + } + values.delete(key); + const { bytes: _bytes, ...info } = entry; + expired.push(info); + } + return expired; + }, + async clear() { + values.clear(); + }, + }; +} + export function createMemoryWikiTestHarness() { const tempDirs: string[] = []; + let compiledBlobStore = createMemoryBlobStore(); + + function configureCompiledCacheStore(): void { + configureMemoryWikiCompiledCacheStore( + createMemoryWikiCompiledCacheStore( + () => compiledBlobStore as unknown as PluginBlobStore, + ), + ); + } afterEach(async () => { + configureMemoryWikiCompiledCacheStore(undefined); + compiledBlobStore = createMemoryBlobStore(); await Promise.all( tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true })), ); }); async function createTempDir(prefix: string): Promise { + configureCompiledCacheStore(); const tempDir = await fs.mkdtemp(path.join(resolvePreferredOpenClawTmpDir(), prefix)); tempDirs.push(tempDir); return tempDir; @@ -90,6 +176,7 @@ export function createMemoryWikiTestHarness() { config?: MemoryWikiPluginConfig; initialize?: boolean; }): Promise { + configureCompiledCacheStore(); const rootDir = options?.rootDir ?? (await createTempDir(options?.prefix ?? "memory-wiki-test-")); const config = resolveMemoryWikiConfig( @@ -114,7 +201,9 @@ export function createMemoryWikiTestHarness() { const registerCli = vi.fn(); const registerGatewayMethod = vi.fn(); const registerMemoryCorpusSupplement = vi.fn(); + const registerMemoryPromptPreparation = vi.fn(); const registerMemoryPromptSupplement = vi.fn(); + const registerService = vi.fn(); const registerTool = vi.fn(); const api = createTestPluginApi({ id: "memory-wiki", @@ -124,12 +213,15 @@ export function createMemoryWikiTestHarness() { runtime: { state: { openKeyedStore: vi.fn(() => createMemoryKeyedStore()), + openBlobStore: vi.fn(() => createMemoryBlobStore()), }, } as unknown as OpenClawPluginApi["runtime"], registerCli, registerGatewayMethod, registerMemoryCorpusSupplement, + registerMemoryPromptPreparation, registerMemoryPromptSupplement, + registerService, registerTool, }); @@ -138,12 +230,15 @@ export function createMemoryWikiTestHarness() { registerCli, registerGatewayMethod, registerMemoryCorpusSupplement, + registerMemoryPromptPreparation, registerMemoryPromptSupplement, + registerService, registerTool, }; } return { + configureCompiledCacheStore, createPluginApi, createTempDir, createVault, diff --git a/extensions/memory-wiki/src/vault.test.ts b/extensions/memory-wiki/src/vault.test.ts index 78f0b627e9ff..c1d082b76e74 100644 --- a/extensions/memory-wiki/src/vault.test.ts +++ b/extensions/memory-wiki/src/vault.test.ts @@ -35,6 +35,12 @@ describe("initializeMemoryWikiVault", () => { await expect(fs.readFile(path.join(rootDir, "WIKI.md"), "utf8")).resolves.toContain( "Render mode: `obsidian`", ); + await expect(fs.readFile(path.join(rootDir, "WIKI.md"), "utf8")).resolves.toContain( + "snapshots live in OpenClaw plugin state", + ); + await expect(fs.access(path.join(rootDir, ".openclaw-wiki", "cache"))).rejects.toThrow( + /ENOENT/, + ); await expect(fs.access(path.join(rootDir, ".openclaw-wiki", "state.json"))).rejects.toThrow( /ENOENT/, ); diff --git a/extensions/memory-wiki/src/vault.ts b/extensions/memory-wiki/src/vault.ts index 4fc055f07144..cf027edbe5aa 100644 --- a/extensions/memory-wiki/src/vault.ts +++ b/extensions/memory-wiki/src/vault.ts @@ -6,8 +6,17 @@ import { withTrailingNewline, } from "openclaw/plugin-sdk/memory-host-markdown"; import { FsSafeError, pathExists, root as fsRoot } from "openclaw/plugin-sdk/security-runtime"; +import { + activateMemoryWikiCompiledCacheOwner, + invalidateMemoryWikiCompiledCache, + reconcileMemoryWikiCompiledCacheOwner, +} from "./compiled-cache.js"; import type { ResolvedMemoryWikiConfig } from "./config.js"; -import { appendMemoryWikiLog } from "./log.js"; +import { + appendMemoryWikiLog, + ensureMemoryWikiVaultGeneration, + loadMemoryWikiValidatedVaultIdentity, +} from "./log.js"; import { WIKI_RAW_SOURCE_MARKER } from "./markdown.js"; import { resolveMemoryWikiTimestamp } from "./time.js"; @@ -20,9 +29,10 @@ const WIKI_VAULT_DIRECTORIES = [ "_attachments", "_views", ".openclaw-wiki", - ".openclaw-wiki/cache", ] as const; +const WIKI_VAULT_SCAFFOLD = ["AGENTS.md", "WIKI.md", "index.md", ".openclaw-wiki/log.jsonl"]; + type InitializeMemoryWikiVaultResult = { rootDir: string; created: boolean; @@ -50,7 +60,7 @@ function buildAgentsMarkdown(): string { - Preserve human notes outside managed markers. - Prefer source-backed claims over wiki-to-wiki citation loops. - Prefer structured \`claims\` with evidence over burying key beliefs only in prose. -- Use \`.openclaw-wiki/cache/agent-digest.json\` and \`claims.jsonl\` for machine reads; markdown pages are the human view. +- Use the wiki tools for machine reads; Markdown pages are the human view. `); } @@ -68,7 +78,7 @@ This vault is maintained by the OpenClaw memory-wiki plugin. - Raw sources remain the evidence layer. - To keep unmanaged raw Markdown in \`sources/\`, add \`${WIKI_RAW_SOURCE_MARKER}\` near the top of the page. - Wiki pages are the human-readable synthesis layer. -- \`.openclaw-wiki/cache/agent-digest.json\` is the agent-facing compiled digest. +- Compiled query and prompt snapshots live in OpenClaw plugin state, not vault files. ## Notes @@ -101,11 +111,21 @@ export async function initializeMemoryWikiVault( const rootDir = config.vault.path; const createdDirectories: string[] = []; const createdFiles: string[] = []; + const rootCreated = !(await pathExists(rootDir)); - if (!(await pathExists(rootDir))) { + if (rootCreated) { createdDirectories.push(rootDir); } await fs.mkdir(rootDir, { recursive: true }); + const hadVaultScaffold = ( + await Promise.all( + WIKI_VAULT_SCAFFOLD.map((relativePath) => pathExists(path.join(rootDir, relativePath))), + ) + ).every(Boolean); + if (!hadVaultScaffold) { + // Missing scaffold means a new/recreated vault, even when its parent directory survived. + await invalidateMemoryWikiCompiledCache(config); + } for (const relativeDir of WIKI_VAULT_DIRECTORIES) { const fullPath = path.join(rootDir, relativeDir); @@ -136,6 +156,16 @@ export async function initializeMemoryWikiVault( }, }); } + const vaultGeneration = await ensureMemoryWikiVaultGeneration(rootDir); + const identity = await loadMemoryWikiValidatedVaultIdentity(rootDir); + activateMemoryWikiCompiledCacheOwner( + config, + vaultGeneration, + identity.compiledCachePublicationId, + ); + await reconcileMemoryWikiCompiledCacheOwner(config, () => + loadMemoryWikiValidatedVaultIdentity(rootDir), + ); return { rootDir, diff --git a/scripts/check-database-first-legacy-stores.mjs b/scripts/check-database-first-legacy-stores.mjs index 865e97bcf25d..b485fcf39a04 100644 --- a/scripts/check-database-first-legacy-stores.mjs +++ b/scripts/check-database-first-legacy-stores.mjs @@ -161,9 +161,7 @@ const allowedRuntimeMigrationPaths = [ const allowedFixturePaths = new Set(["extensions/qa-lab/src/providers/shared/auth-store.ts"]); -const allowedCurrentLegacyWriteViolations = [ - "extensions/memory-wiki/src/compile.ts:legacy store filesystem write:root.write(relativePath, content)", -]; +const allowedCurrentLegacyWriteViolations = []; const sourceFileExtensions = new Set([".cjs", ".cts", ".js", ".mjs", ".mts", ".ts", ".tsx"]); diff --git a/scripts/plugin-sdk-surface-report.mjs b/scripts/plugin-sdk-surface-report.mjs index e1e26c6ee369..104bac5b652c 100644 --- a/scripts/plugin-sdk-surface-report.mjs +++ b/scripts/plugin-sdk-surface-report.mjs @@ -264,7 +264,8 @@ export function readPluginSdkSurfaceBudgets(env = process.env) { // +23: core channel, envelope, direct-DM, feedback, legacy-payload, and memory contracts. // +81: meeting-runtime barrel: browser meeting-bot core behind MeetingPlatformAdapter. // +3: question-gateway-runtime resolver plus request/result types. - 8152, + // +1: async memory prompt preparation registration. + 8153, env, ), publicFunctionExports: readPluginSdkSurfaceBudgetEnv( @@ -299,7 +300,8 @@ export function readPluginSdkSurfaceBudgets(env = process.env) { // +13: core channel, envelope, direct-DM, feedback, legacy-payload, and memory operations. // +32: meeting-runtime barrel: browser meeting-bot core behind MeetingPlatformAdapter. // +1: question-gateway-runtime resolver. - 4534, + // +1: async memory prompt preparation registration. + 4535, env, ), publicDeprecatedExports: readPluginSdkSurfaceBudgetEnv( diff --git a/src/agents/embedded-agent-runner/compact.ts b/src/agents/embedded-agent-runner/compact.ts index 6d63dd246928..5f7cd61bb0b2 100644 --- a/src/agents/embedded-agent-runner/compact.ts +++ b/src/agents/embedded-agent-runner/compact.ts @@ -102,6 +102,7 @@ import { selectAgentHarnessForPreparedModelProviders, } from "../harness/selection.js"; import { resolveHeartbeatPromptForSystemPrompt } from "../heartbeat-system-prompt.js"; +import { prepareAgentMemoryPrompt } from "../memory-prompt-prepare.js"; import { applyAuthHeaderOverride, applyLocalNoAuthHeaderOverride, @@ -1320,6 +1321,14 @@ async function compactEmbeddedAgentSessionDirectOnce( }; const promptContribution = runtimePlan.prompt.resolveSystemPromptContribution(promptContributionContext); + const preparedMemoryPrompt = await prepareAgentMemoryPrompt({ + enabled: promptMode === "full", + toolNames: effectiveTools.map((tool) => tool.name), + citationsMode: params.config?.memory?.citations, + agentId: runtimeInfo.agentId, + agentSessionKey: runtimeInfo.sessionKey, + sandboxed: sandboxInfo?.enabled === true, + }); const buildSystemPromptText = (defaultThinkLevel: ThinkLevel) => { const builtSystemPrompt = buildEmbeddedSystemPrompt({ config: params.config, @@ -1354,6 +1363,7 @@ async function compactEmbeddedAgentSessionDirectOnce( userTime, userTimeFormat, contextFiles, + preparedMemoryPrompt, promptContribution, nativeCommandGuidanceLines, }); diff --git a/src/agents/embedded-agent-runner/run/attempt-execution-phase.ts b/src/agents/embedded-agent-runner/run/attempt-execution-phase.ts index 39cd3f6ae603..8a56bf33bc24 100644 --- a/src/agents/embedded-agent-runner/run/attempt-execution-phase.ts +++ b/src/agents/embedded-agent-runner/run/attempt-execution-phase.ts @@ -77,6 +77,7 @@ export async function runEmbeddedAttemptExecutionPhase( isRawModelRun: input.isRawModelRun, ...(orphanRepair ? { orphanRepair } : {}), replayAllowedToolNames, + sandboxed: input.setup.sandbox?.enabled === true, sessionAgentId: input.setup.sessionAgentId, settingsManager, systemPromptText: sessionRuntimeState.systemPromptText, diff --git a/src/agents/embedded-agent-runner/run/attempt-history-prepare.ts b/src/agents/embedded-agent-runner/run/attempt-history-prepare.ts index 1060a9ddf317..e8411daa6478 100644 --- a/src/agents/embedded-agent-runner/run/attempt-history-prepare.ts +++ b/src/agents/embedded-agent-runner/run/attempt-history-prepare.ts @@ -59,6 +59,7 @@ export async function prepareEmbeddedAttemptHistory(input: { isRawModelRun: boolean; orphanRepair?: OrphanRepairPlan; replayAllowedToolNames: Set; + sandboxed: boolean; sessionAgentId: string; settingsManager: SettingsManager; systemPromptText: string; @@ -225,6 +226,7 @@ export async function prepareEmbeddedAttemptHistory(input: { tokenBudget: messageBudget, availableTools: new Set(input.capabilityToolNames), citationsMode: attempt.config?.memory?.citations, + sandboxed: input.sandboxed, modelId: attempt.modelId, maxOutputTokens: reserveTokens, contextEngineHostSupport: OPENCLAW_EMBEDDED_CONTEXT_ENGINE_HOST, diff --git a/src/agents/embedded-agent-runner/run/attempt-stream-runtime-prepare.test.ts b/src/agents/embedded-agent-runner/run/attempt-stream-runtime-prepare.test.ts index 80eefcd2c332..d7a06f6de66f 100644 --- a/src/agents/embedded-agent-runner/run/attempt-stream-runtime-prepare.test.ts +++ b/src/agents/embedded-agent-runner/run/attempt-stream-runtime-prepare.test.ts @@ -133,7 +133,7 @@ function createFixture(options: { aborted?: boolean } = {}) { trackPromptSettlePromise, compactionTimeoutMs: 1_000, guards: {}, - history: {}, + history: { sandboxed: false }, stream: {}, lifecycle: { isYieldDetected: () => false, diff --git a/src/agents/embedded-agent-runner/run/attempt-system-prompt-prepare.ts b/src/agents/embedded-agent-runner/run/attempt-system-prompt-prepare.ts index a899fe3572c9..9a46a33d59f5 100644 --- a/src/agents/embedded-agent-runner/run/attempt-system-prompt-prepare.ts +++ b/src/agents/embedded-agent-runner/run/attempt-system-prompt-prepare.ts @@ -23,6 +23,7 @@ import { } from "../../channel-tools.js"; import { resolveOpenClawReferencePaths } from "../../docs-path.js"; import { resolveHeartbeatPromptForSystemPrompt } from "../../heartbeat-system-prompt.js"; +import { prepareAgentMemoryPrompt } from "../../memory-prompt-prepare.js"; import { resolveDefaultModelForAgent } from "../../model-selection.js"; import { resolveAgentPromptSurfaceForSessionKey } from "../../prompt-surface.js"; import { collectRuntimeChannelCapabilities } from "../../runtime-capabilities.js"; @@ -220,6 +221,17 @@ export async function prepareEmbeddedAttemptSystemPrompt(params: { runtimeHandle: params.getProviderRuntimeHandle(), context: promptContributionContext, }); + const includeMemorySection = + !params.activeContextEngine || params.activeContextEngine.info.id === "legacy"; + const preparedMemoryPrompt = await prepareAgentMemoryPrompt({ + enabled: effectivePromptMode === "full" && includeMemorySection, + toolNames: params.effectiveTools.map((tool) => tool.name), + capabilityToolNames: params.capabilityToolNames, + citationsMode: attempt.config?.memory?.citations, + agentId: runtimeInfo.agentId, + agentSessionKey: runtimeInfo.sessionKey, + sandboxed: sandboxInfo?.enabled === true, + }); const attemptSystemPrompt = buildAttemptSystemPrompt({ isRawModelRun: params.isRawModelRun, @@ -271,8 +283,8 @@ export async function prepareEmbeddedAttemptSystemPrompt(params: { bootstrapTruncationNotice: buildBootstrapPromptWarningNotice( params.bootstrap.bootstrapPromptWarning.lines, ), - includeMemorySection: - !params.activeContextEngine || params.activeContextEngine.info.id === "legacy", + includeMemorySection, + preparedMemoryPrompt, promptContribution, }, providerTransform: { diff --git a/src/agents/embedded-agent-runner/system-prompt.ts b/src/agents/embedded-agent-runner/system-prompt.ts index 64a2e5bacd94..21b5b1d9e9a4 100644 --- a/src/agents/embedded-agent-runner/system-prompt.ts +++ b/src/agents/embedded-agent-runner/system-prompt.ts @@ -6,6 +6,7 @@ import type { ChatType } from "../../channels/chat-type.js"; import type { SubagentDelegationMode } from "../../config/types.agent-defaults.js"; import type { MemoryCitationsMode } from "../../config/types.memory.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; +import type { PreparedMemoryPromptSection } from "../../plugins/memory-state.js"; import type { AgentPromptSurfaceKind } from "../../plugins/types.js"; import type { ActiveProcessSessionReference } from "../bash-process-references.js"; import type { BootstrapMode } from "../bootstrap-mode.js"; @@ -90,6 +91,7 @@ export function buildEmbeddedSystemPrompt(params: { bootstrapTruncationNotice?: string; includeMemorySection?: boolean; memoryCitationsMode?: MemoryCitationsMode; + preparedMemoryPrompt?: PreparedMemoryPromptSection; promptContribution?: ProviderSystemPromptContribution; }): string { return buildConfiguredAgentSystemPrompt({ @@ -134,6 +136,7 @@ export function buildEmbeddedSystemPrompt(params: { bootstrapTruncationNotice: params.bootstrapTruncationNotice, includeMemorySection: params.includeMemorySection, memoryCitationsMode: params.memoryCitationsMode, + preparedMemoryPrompt: params.preparedMemoryPrompt, promptContribution: params.promptContribution, }); } diff --git a/src/agents/harness/context-engine-lifecycle.test.ts b/src/agents/harness/context-engine-lifecycle.test.ts index db1b05d3bcfc..b6bd8d41d6cf 100644 --- a/src/agents/harness/context-engine-lifecycle.test.ts +++ b/src/agents/harness/context-engine-lifecycle.test.ts @@ -1,6 +1,7 @@ // Covers context-engine message filtering, assemble validation, and turn finalization. import type { AgentMessage } from "openclaw/plugin-sdk/agent-core"; import { describe, expect, it, vi } from "vitest"; +import { buildMemorySystemPromptAddition } from "../../context-engine/delegate.js"; import { CODEX_APP_SERVER_CONTEXT_ENGINE_HOST, OPENCLAW_EMBEDDED_CONTEXT_ENGINE_HOST, @@ -12,6 +13,11 @@ import type { ContextEngineRuntimeContext, ContextEngineRuntimeSettings, } from "../../context-engine/types.js"; +import { + clearMemoryPluginState, + registerMemoryPromptPreparation, + registerMemoryPromptSection, +} from "../../plugins/memory-state.test-fixtures.js"; import { compactContextEngineWithSafetyTimeout } from "../embedded-agent-runner/compaction-safety-timeout.js"; import { OPENCLAW_RUNTIME_CONTEXT_CUSTOM_TYPE } from "../internal-runtime-context.js"; import { @@ -68,6 +74,52 @@ function uniqueConfiguredProofEngineId() { } describe("harness context engine lifecycle", () => { + it("scopes async memory preparation to non-legacy assembly with sandbox context", async () => { + const prepare = vi.fn(async ({ sandboxed }) => [ + "## Prepared Memory", + `sandboxed=${sandboxed}`, + "", + ]); + registerMemoryPromptSection(() => ["## Memory Recall", ""]); + registerMemoryPromptPreparation("memory-wiki", prepare); + const availableTools = new Set(["wiki_search"]); + const assemble = vi.fn(async (params: Parameters[0]) => ({ + messages: params.messages, + estimatedTokens: 0, + systemPromptAddition: buildMemorySystemPromptAddition({ + availableTools: params.availableTools ?? new Set(), + citationsMode: params.citationsMode, + }), + })); + + try { + const result = await assembleHarnessContextEngine({ + contextEngine: createContextEngine({ assemble }), + sessionId: sessionParams.sessionId, + sessionKey: "agent:support:main", + messages: [textMessage("user", "visible ask", 1)], + availableTools, + citationsMode: "on", + sandboxed: true, + modelId: "gpt-test", + }); + + expect(result?.systemPromptAddition).toBe( + "## Memory Recall\n\n## Prepared Memory\nsandboxed=true", + ); + expect(prepare).toHaveBeenCalledWith( + expect.objectContaining({ + agentId: "support", + agentSessionKey: "agent:support:main", + sandboxed: true, + }), + ); + expect(buildMemorySystemPromptAddition({ availableTools })).toBe("## Memory Recall"); + } finally { + clearMemoryPluginState(); + } + }); + it("keeps hidden runtime-context custom messages out of assemble hooks", async () => { const visibleUser = textMessage("user", "visible ask", 1); const hiddenRuntimeContext = runtimeContextMessage("hidden runtime context", 2); diff --git a/src/agents/harness/context-engine-lifecycle.ts b/src/agents/harness/context-engine-lifecycle.ts index e18b038a9ee8..04d36d2c5d52 100644 --- a/src/agents/harness/context-engine-lifecycle.ts +++ b/src/agents/harness/context-engine-lifecycle.ts @@ -14,6 +14,8 @@ import type { ContextEngineRuntimeSettings, ContextEngineSessionTarget, } from "../../context-engine/types.js"; +import { runWithPreparedMemoryPromptSection } from "../../plugins/memory-state.js"; +import { resolveAgentIdFromSessionKey } from "../../routing/session-key.js"; import { runContextEngineMaintenance } from "../embedded-agent-runner/context-engine-maintenance.js"; import { buildAfterTurnRuntimeContext, @@ -143,6 +145,7 @@ export async function assembleHarnessContextEngine(params: { tokenBudget?: number; availableTools?: Set; citationsMode?: MemoryCitationsMode; + sandboxed?: boolean; modelId: string; prompt?: string; runtimeSettings?: ContextEngineRuntimeSettings; @@ -159,20 +162,35 @@ export async function assembleHarnessContextEngine(params: { if (!params.contextEngine) { return undefined; } + const contextEngine = params.contextEngine; const messages = stripRuntimeContextCustomMessages(params.messages); const runtimeSettings = buildHarnessContextEngineRuntimeSettings(params); - const result = await params.contextEngine.assemble({ - sessionId: params.sessionId, - sessionKey: params.sessionKey, - messages, - tokenBudget: params.tokenBudget, - ...(params.availableTools ? { availableTools: params.availableTools } : {}), - ...(params.citationsMode ? { citationsMode: params.citationsMode } : {}), - model: params.modelId, - runtimeSettings, - ...(params.prompt !== undefined ? { prompt: params.prompt } : {}), - }); - return ensureAssembleResultShape(result, params.contextEngine.info.id); + const assemble = () => + contextEngine.assemble({ + sessionId: params.sessionId, + sessionKey: params.sessionKey, + messages, + tokenBudget: params.tokenBudget, + ...(params.availableTools ? { availableTools: params.availableTools } : {}), + ...(params.citationsMode ? { citationsMode: params.citationsMode } : {}), + model: params.modelId, + runtimeSettings, + ...(params.prompt !== undefined ? { prompt: params.prompt } : {}), + }); + const result = + contextEngine.info.id === "legacy" + ? await assemble() + : await runWithPreparedMemoryPromptSection( + { + availableTools: new Set(params.availableTools), + citationsMode: params.citationsMode, + agentId: resolveAgentIdFromSessionKey(params.sessionKey), + agentSessionKey: params.sessionKey, + sandboxed: params.sandboxed, + }, + assemble, + ); + return ensureAssembleResultShape(result, contextEngine.info.id); } /** diff --git a/src/agents/memory-prompt-prepare.ts b/src/agents/memory-prompt-prepare.ts new file mode 100644 index 000000000000..eecf77be6f73 --- /dev/null +++ b/src/agents/memory-prompt-prepare.ts @@ -0,0 +1,32 @@ +import type { MemoryCitationsMode } from "../config/types.memory.js"; +import { + prepareMemoryPromptSection, + type PreparedMemoryPromptSection, +} from "../plugins/memory-state.js"; + +/** Prepare memory prompt state with the same normalized tool context used by assembly. */ +export async function prepareAgentMemoryPrompt(params: { + enabled: boolean; + toolNames: Iterable; + capabilityToolNames?: Iterable; + citationsMode?: MemoryCitationsMode; + agentId?: string; + agentSessionKey?: string; + sandboxed?: boolean; +}): Promise { + if (!params.enabled) { + return undefined; + } + const availableTools = new Set( + [...params.toolNames, ...(params.capabilityToolNames ?? [])] + .map((tool) => tool.trim().toLowerCase()) + .filter(Boolean), + ); + return prepareMemoryPromptSection({ + availableTools, + citationsMode: params.citationsMode, + agentId: params.agentId, + agentSessionKey: params.agentSessionKey, + sandboxed: params.sandboxed, + }); +} diff --git a/src/agents/system-prompt.memory.test.ts b/src/agents/system-prompt.memory.test.ts index 38678a777509..21f2f83d0c91 100644 --- a/src/agents/system-prompt.memory.test.ts +++ b/src/agents/system-prompt.memory.test.ts @@ -1,10 +1,12 @@ // System prompt memory tests cover opt-out behavior when context engines own // memory prompt assembly for a run. -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { clearMemoryPluginState, + registerMemoryPromptPreparation, registerMemoryPromptSection, } from "../plugins/memory-state.test-fixtures.js"; +import { prepareAgentMemoryPrompt } from "./memory-prompt-prepare.js"; import { buildAgentSystemPrompt } from "./system-prompt.js"; describe("buildAgentSystemPrompt memory guidance", () => { @@ -59,4 +61,25 @@ describe("buildAgentSystemPrompt memory guidance", () => { "agent=marketing-agent session=agent:marketing-agent:main sandboxed=true", ); }); + + it("hands prepared memory lines to synchronous prompt assembly", async () => { + const prepare = vi.fn(async () => ["## Prepared Wiki", "Prepared before assembly.", ""]); + registerMemoryPromptPreparation("memory-wiki", prepare); + const preparedMemoryPrompt = await prepareAgentMemoryPrompt({ + enabled: true, + toolNames: ["WIKI_SEARCH"], + agentId: "main", + agentSessionKey: "agent:main:main", + }); + + const prompt = buildAgentSystemPrompt({ + workspaceDir: "/tmp/openclaw", + toolNames: ["WIKI_SEARCH"], + runtimeInfo: { agentId: "main", sessionKey: "agent:main:main" }, + preparedMemoryPrompt, + }); + + expect(prompt).toContain("## Prepared Wiki\nPrepared before assembly."); + expect(prepare).toHaveBeenCalledTimes(1); + }); }); diff --git a/src/agents/system-prompt.ts b/src/agents/system-prompt.ts index eb5cdbb7bb11..1de336b06dd3 100644 --- a/src/agents/system-prompt.ts +++ b/src/agents/system-prompt.ts @@ -28,7 +28,10 @@ import { } from "../channels/plugins/native-approval-prompt.js"; import type { SubagentDelegationMode } from "../config/types.agent-defaults.js"; import type { MemoryCitationsMode } from "../config/types.memory.js"; -import { buildMemoryPromptSection } from "../plugins/memory-state.js"; +import { + buildMemoryPromptSection, + type PreparedMemoryPromptSection, +} from "../plugins/memory-state.js"; import type { AgentPromptSurfaceKind } from "../plugins/types.js"; import { parseCronRunScopeSuffix } from "../sessions/session-key-utils.js"; import { listDeliverableMessageChannels } from "../utils/message-channel.js"; @@ -306,17 +309,21 @@ function buildMemorySection(params: { agentId?: string; agentSessionKey?: string; sandboxed?: boolean; + prepared?: PreparedMemoryPromptSection; }) { if (params.isMinimal || params.includeMemorySection === false) { return []; } - return buildMemoryPromptSection({ - availableTools: params.availableTools, - citationsMode: params.citationsMode, - agentId: params.agentId, - agentSessionKey: params.agentSessionKey, - sandboxed: params.sandboxed, - }); + return buildMemoryPromptSection( + { + availableTools: params.availableTools, + citationsMode: params.citationsMode, + agentId: params.agentId, + agentSessionKey: params.agentSessionKey, + sandboxed: params.sandboxed, + }, + params.prepared, + ); } function buildAgentBootstrapSystemContext(params: { @@ -757,6 +764,8 @@ export function buildAgentSystemPrompt(params: { }; includeMemorySection?: boolean; memoryCitationsMode?: MemoryCitationsMode; + /** Immutable memory state prepared before synchronous prompt assembly. */ + preparedMemoryPrompt?: PreparedMemoryPromptSection; promptContribution?: ProviderSystemPromptContribution; }) { const acpEnabled = params.acpEnabled === true; @@ -1002,6 +1011,7 @@ export function buildAgentSystemPrompt(params: { agentId: params.runtimeInfo?.agentId, agentSessionKey: params.runtimeInfo?.sessionKey, sandboxed: params.sandboxInfo?.enabled === true, + prepared: params.preparedMemoryPrompt, }); const docsSection = buildDocsSection({ docsPath: params.docsPath, diff --git a/src/context-engine/context-engine.test.ts b/src/context-engine/context-engine.test.ts index 11e158d79145..eba858bb3b87 100644 --- a/src/context-engine/context-engine.test.ts +++ b/src/context-engine/context-engine.test.ts @@ -5,13 +5,18 @@ import type { MemoryCitationsMode } from "../config/types.memory.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { clearMemoryPluginState, + registerMemoryPromptPreparation, registerMemoryPromptSection, } from "../plugins/memory-state.test-fixtures.js"; // --------------------------------------------------------------------------- // We dynamically import the registry so we can get a fresh module per test // group when needed. For most groups we use the shared singleton directly. // --------------------------------------------------------------------------- -import { buildMemorySystemPromptAddition, delegateCompactionToRuntime } from "./delegate.js"; +import { + buildMemorySystemPromptAddition, + delegateCompactionToRuntime, + prepareMemorySystemPromptAddition, +} from "./delegate.js"; import { LegacyContextEngine } from "./legacy.js"; import { registerLegacyContextEngine } from "./legacy.registration.js"; import { @@ -734,6 +739,20 @@ describe("Engine contract tests", () => { }), ).toBeUndefined(); }); + + it("prepares async memory state before context-engine prompt rendering", async () => { + const prepare = vi.fn(async () => ["## Prepared Memory", "loaded from sqlite", ""]); + registerMemoryPromptPreparation("memory-wiki", prepare); + + await expect( + prepareMemorySystemPromptAddition({ + availableTools: new Set(["wiki_search"]), + agentId: "main", + agentSessionKey: "agent:main:main", + }), + ).resolves.toBe("## Prepared Memory\nloaded from sqlite"); + expect(prepare).toHaveBeenCalledTimes(1); + }); }); // ═══════════════════════════════════════════════════════════════════════════ diff --git a/src/context-engine/delegate.ts b/src/context-engine/delegate.ts index c5f6de63873a..a64b09b2419e 100644 --- a/src/context-engine/delegate.ts +++ b/src/context-engine/delegate.ts @@ -1,8 +1,13 @@ // Context-engine delegates bridge custom engines to built-in compaction and memory prompt paths. import { normalizeStructuredPromptSection } from "@openclaw/ai/internal/shared"; import { parseSqliteSessionFileMarker } from "../config/sessions/sqlite-marker.js"; -import type { MemoryCitationsMode } from "../config/types.memory.js"; -import { buildMemoryPromptSection } from "../plugins/memory-state.js"; +import { + buildMemoryPromptSection, + getActivePreparedMemoryPromptSection, + prepareMemoryPromptSection, + type MemoryPromptSectionParams, + type PreparedMemoryPromptSection, +} from "../plugins/memory-state.js"; import { createLazyRuntimeModule } from "../shared/lazy-runtime.js"; import type { ContextEngine, @@ -130,23 +135,54 @@ export async function delegateCompactionToRuntime( * same memory/wiki guidance that the legacy engine gets via system prompt * assembly, without reimplementing memory prompt formatting. */ -export function buildMemorySystemPromptAddition(params: { - availableTools: Set; - citationsMode?: MemoryCitationsMode; - agentId?: string; - agentSessionKey?: string; - sandboxed?: boolean; -}): string | undefined { - const lines = buildMemoryPromptSection({ - availableTools: params.availableTools, - citationsMode: params.citationsMode, - agentId: params.agentId, - agentSessionKey: params.agentSessionKey, - sandboxed: params.sandboxed, - }); +function renderMemorySystemPromptAddition( + params: MemoryPromptSectionParams, + prepared?: PreparedMemoryPromptSection, +): string | undefined { + const lines = buildMemoryPromptSection( + { + availableTools: params.availableTools, + citationsMode: params.citationsMode, + agentId: params.agentId, + agentSessionKey: params.agentSessionKey, + sandboxed: params.sandboxed, + }, + prepared, + ); if (lines.length === 0) { return undefined; } const normalized = normalizeStructuredPromptSection(lines.join("\n")); return normalized || undefined; } + +export function buildMemorySystemPromptAddition( + params: MemoryPromptSectionParams, +): string | undefined { + const prepared = getActivePreparedMemoryPromptSection(); + if (!prepared) { + return renderMemorySystemPromptAddition(params); + } + const contextParams: MemoryPromptSectionParams = { + availableTools: params.availableTools, + citationsMode: params.citationsMode ?? prepared.context.citationsMode, + agentId: params.agentId ?? prepared.context.agentId, + agentSessionKey: params.agentSessionKey ?? prepared.context.agentSessionKey, + sandboxed: params.sandboxed ?? prepared.context.sandboxed, + }; + return renderMemorySystemPromptAddition(contextParams, prepared); +} + +/** Prepare memory state asynchronously, then render it without prompt-path I/O. */ +export async function prepareMemorySystemPromptAddition( + params: MemoryPromptSectionParams, +): Promise { + const prepared = await prepareMemoryPromptSection({ + availableTools: params.availableTools, + citationsMode: params.citationsMode, + agentId: params.agentId, + agentSessionKey: params.agentSessionKey, + sandboxed: params.sandboxed, + }); + return renderMemorySystemPromptAddition(params, prepared); +} diff --git a/src/plugin-sdk/core.ts b/src/plugin-sdk/core.ts index cbeb291d990f..f21df65991bc 100644 --- a/src/plugin-sdk/core.ts +++ b/src/plugin-sdk/core.ts @@ -227,6 +227,7 @@ export { resolveTailscalePublishedHost } from "../shared/tailscale-status.js"; export { buildMemorySystemPromptAddition, delegateCompactionToRuntime, + prepareMemorySystemPromptAddition, } from "../context-engine/delegate.js"; export { DEFAULT_ACCOUNT_ID, normalizeAccountId } from "../routing/session-key.js"; export { diff --git a/src/plugin-sdk/plugin-test-api.ts b/src/plugin-sdk/plugin-test-api.ts index 205f326bf33b..5256cce82960 100644 --- a/src/plugin-sdk/plugin-test-api.ts +++ b/src/plugin-sdk/plugin-test-api.ts @@ -86,6 +86,7 @@ export function createTestPluginApi(api: TestPluginApiInput = {}): OpenClawPlugi registerMemoryCapability() {}, registerMemoryPromptSection() {}, registerMemoryPromptSupplement() {}, + registerMemoryPromptPreparation() {}, registerMemoryCorpusSupplement() {}, registerMemoryFlushPlan() {}, registerMemoryRuntime() {}, diff --git a/src/plugins/api-builder.ts b/src/plugins/api-builder.ts index e3f3a4ef44d3..6227c5e9ec6f 100644 --- a/src/plugins/api-builder.ts +++ b/src/plugins/api-builder.ts @@ -82,6 +82,7 @@ type BuildPluginApiParams = { | "registerMemoryCapability" | "registerMemoryPromptSection" | "registerMemoryPromptSupplement" + | "registerMemoryPromptPreparation" | "registerMemoryCorpusSupplement" | "registerMemoryFlushPlan" | "registerMemoryRuntime" @@ -179,6 +180,8 @@ const noopRegisterMemoryCapability: OpenClawPluginApi["registerMemoryCapability" const noopRegisterMemoryPromptSection: OpenClawPluginApi["registerMemoryPromptSection"] = () => {}; const noopRegisterMemoryPromptSupplement: OpenClawPluginApi["registerMemoryPromptSupplement"] = () => {}; +const noopRegisterMemoryPromptPreparation: OpenClawPluginApi["registerMemoryPromptPreparation"] = + () => {}; const noopRegisterMemoryCorpusSupplement: OpenClawPluginApi["registerMemoryCorpusSupplement"] = () => {}; const noopRegisterMemoryFlushPlan: OpenClawPluginApi["registerMemoryFlushPlan"] = () => {}; @@ -293,6 +296,8 @@ export function buildPluginApi(params: BuildPluginApiParams): OpenClawPluginApi handlers.registerMemoryPromptSection ?? noopRegisterMemoryPromptSection, registerMemoryPromptSupplement: handlers.registerMemoryPromptSupplement ?? noopRegisterMemoryPromptSupplement, + registerMemoryPromptPreparation: + handlers.registerMemoryPromptPreparation ?? noopRegisterMemoryPromptPreparation, registerMemoryCorpusSupplement: handlers.registerMemoryCorpusSupplement ?? noopRegisterMemoryCorpusSupplement, registerMemoryFlushPlan: handlers.registerMemoryFlushPlan ?? noopRegisterMemoryFlushPlan, diff --git a/src/plugins/contracts/plugin-sdk-subpaths.test.ts b/src/plugins/contracts/plugin-sdk-subpaths.test.ts index 5fd845ef4fa2..31fa0fdcaf0d 100644 --- a/src/plugins/contracts/plugin-sdk-subpaths.test.ts +++ b/src/plugins/contracts/plugin-sdk-subpaths.test.ts @@ -1398,6 +1398,9 @@ describe("plugin-sdk subpath exports", () => { expect(coreSdk.definePluginEntry).toBe(pluginEntrySdk.definePluginEntry); expect(coreSdk.optionalStringEnum).toBe(coreDirectSdk.optionalStringEnum); + expect(coreSdk.prepareMemorySystemPromptAddition).toBe( + coreDirectSdk.prepareMemorySystemPromptAddition, + ); expect(channelActionsSdk.optionalStringEnum).toBe(channelActionsDirectSdk.optionalStringEnum); expect(channelActionsSdk.stringEnum).toBe(channelActionsDirectSdk.stringEnum); expect(globalSingletonSdk.resolveGlobalMap).toBe(globalSingletonDirectSdk.resolveGlobalMap); diff --git a/src/plugins/loader.activation.test-utils.ts b/src/plugins/loader.activation.test-utils.ts index 3b980315a36f..a59221e1ab8f 100644 --- a/src/plugins/loader.activation.test-utils.ts +++ b/src/plugins/loader.activation.test-utils.ts @@ -33,7 +33,10 @@ import { globalAfterEach0, globalAfterAll1, } from "./loader.test-harness.js"; -import { listMemoryPromptSupplements } from "./memory-state.test-fixtures.js"; +import { + listMemoryPromptPreparations, + listMemoryPromptSupplements, +} from "./memory-state.test-fixtures.js"; import type { PluginSdkResolutionPreference } from "./sdk-alias.js"; afterEach(globalAfterEach0); @@ -619,6 +622,21 @@ describe("loadOpenClawPlugins", () => { expect(listMemoryPromptSupplements()).toStrictEqual([]); }, }, + { + label: "rejects malformed memory prompt preparation registration", + pluginId: "memory-prompt-preparation-malformed", + body: `module.exports = { id: "memory-prompt-preparation-malformed", register(api) { + api.registerMemoryPromptPreparation({ id: "broken-memory-prompt" }); + } };`, + assert: (registry: ReturnType) => { + expectRegistryErrorDiagnostic({ + registry, + pluginId: "memory-prompt-preparation-malformed", + message: "memory prompt preparation registration missing prepare function", + }); + expect(listMemoryPromptPreparations()).toStrictEqual([]); + }, + }, { label: "requires plugin CLI registrars to declare explicit command roots", pluginId: "cli-missing-metadata", diff --git a/src/plugins/loader.registration.test-utils.ts b/src/plugins/loader.registration.test-utils.ts index df4e058de09b..a9004d042646 100644 --- a/src/plugins/loader.registration.test-utils.ts +++ b/src/plugins/loader.registration.test-utils.ts @@ -60,8 +60,10 @@ import { getMemoryRuntime, listActiveMemoryPublicArtifacts, listMemoryCorpusSupplements, + listMemoryPromptPreparations, registerMemoryCapability, registerMemoryCorpusSupplement, + registerMemoryPromptPreparation, registerMemoryPromptSupplement, resolveMemoryFlushPlan, } from "./memory-state.test-fixtures.js"; @@ -499,6 +501,7 @@ describe("loadOpenClawPlugins", () => { get: async () => null, }); registerMemoryPromptSupplement("memory-wiki", () => ["active wiki supplement"]); + registerMemoryPromptPreparation("memory-wiki", async () => ["active prepared wiki"]); const activeRuntime = { async getMemorySearchManager() { return { manager: null, error: "active" }; @@ -574,6 +577,7 @@ describe("loadOpenClawPlugins", () => { expect(resolveMemoryFlushPlan({})?.relativePath).toBe("memory/active.md"); expect(getMemoryRuntime()).toBe(activeRuntime); expect(listMemoryEmbeddingProviders().map((adapter) => adapter.id)).toEqual(["active"]); + expect(listMemoryPromptPreparations()).toHaveLength(1); }); it("does not replace active embedding providers during non-activating loads", () => { @@ -725,6 +729,7 @@ describe("loadOpenClawPlugins", () => { }); api.registerMemoryPromptSection(() => ["stale failure section"]); api.registerMemoryPromptSupplement(() => ["stale failure supplement"]); + api.registerMemoryPromptPreparation(async () => ["stale prepared supplement"]); api.registerMemoryCorpusSupplement({ search: async () => [], get: async () => null, @@ -766,6 +771,7 @@ describe("loadOpenClawPlugins", () => { expect(registry.plugins.find((entry) => entry.id === "failing-memory")?.status).toBe("error"); expect(buildMemoryPromptSection({ availableTools: new Set() })).toStrictEqual([]); expect(listMemoryCorpusSupplements()).toStrictEqual([]); + expect(listMemoryPromptPreparations()).toStrictEqual([]); expect(resolveMemoryFlushPlan({})).toBeNull(); expect(getMemoryRuntime()).toBeUndefined(); expect(listMemoryEmbeddingProviders()).toStrictEqual([]); diff --git a/src/plugins/memory-state.test.ts b/src/plugins/memory-state.test.ts index e24662aed23e..b63f29c254df 100644 --- a/src/plugins/memory-state.test.ts +++ b/src/plugins/memory-state.test.ts @@ -6,10 +6,13 @@ import { getMemoryCapabilityRegistration, getMemoryRuntime, listMemoryCorpusSupplements, + listMemoryPromptPreparations, listMemoryPromptSupplements, listActiveMemoryPublicArtifacts, + prepareMemoryPromptSection, registerMemoryCapability, registerMemoryCorpusSupplement, + registerMemoryPromptPreparation, registerMemoryPromptSupplement, registerMemoryPromptSection, resolveMemoryFlushPlan, @@ -52,6 +55,7 @@ function createMemoryStateSnapshot() { return { capability: getMemoryCapabilityRegistration(), corpusSupplements: listMemoryCorpusSupplements(), + promptPreparations: listMemoryPromptPreparations(), promptSupplements: listMemoryPromptSupplements(), }; } @@ -321,6 +325,65 @@ describe("memory plugin state", () => { expect(buildMemoryPromptSection({ availableTools: new Set() })).toEqual(["primary", "valid"]); }); + it("prepares immutable prompt lines once per run before synchronous assembly", async () => { + let compiledLines = ["compiled before"]; + const prepare = vi.fn(async () => [...compiledLines]); + registerMemoryPromptPreparation("memory-wiki", prepare); + const params = { + availableTools: new Set(["wiki_search"]), + agentId: "main", + agentSessionKey: "agent:main:main", + }; + + const preparedBefore = await prepareMemoryPromptSection(params); + compiledLines = ["compiled after"]; + + expect(Object.isFrozen(preparedBefore)).toBe(true); + expect(Object.isFrozen(preparedBefore.context)).toBe(true); + expect(Object.isFrozen(preparedBefore.context.availableTools)).toBe(true); + expect(Object.isFrozen(preparedBefore.lines)).toBe(true); + expect(buildMemoryPromptSection(params, preparedBefore)).toEqual(["compiled before"]); + expect(buildMemoryPromptSection(params, preparedBefore)).toEqual(["compiled before"]); + expect(prepare).toHaveBeenCalledTimes(1); + + const preparedAfter = await prepareMemoryPromptSection(params); + expect(buildMemoryPromptSection(params, preparedAfter)).toEqual(["compiled after"]); + expect(prepare).toHaveBeenCalledTimes(2); + }); + + it("rejects prepared state from a different run context", async () => { + registerMemoryPromptPreparation("memory-wiki", async () => ["private wiki state"]); + const prepared = await prepareMemoryPromptSection({ + availableTools: new Set(["wiki_search"]), + agentId: "first", + agentSessionKey: "agent:first:main", + }); + + expect(() => + buildMemoryPromptSection( + { + availableTools: new Set(["wiki_search"]), + agentId: "second", + agentSessionKey: "agent:second:main", + }, + prepared, + ), + ).toThrow("prepared memory prompt section does not match the current run"); + }); + + it("removes prompt preparations from future runs without mutating an in-flight snapshot", async () => { + registerMemoryPromptPreparation("memory-wiki", async () => ["prepared wiki"]); + const params = { availableTools: new Set() }; + const prepared = await prepareMemoryPromptSection(params); + + clearMemoryPluginState(); + + const afterRemoval = await prepareMemoryPromptSection(params); + expect(buildMemoryPromptSection(params, prepared)).toEqual(["prepared wiki"]); + expect(buildMemoryPromptSection(params, afterRemoval)).toEqual([]); + expect(listMemoryPromptPreparations()).toEqual([]); + }); + it("stores memory corpus supplements", async () => { const supplement = { search: async () => [{ corpus: "wiki", path: "sources/alpha.md", score: 1, snippet: "x" }], diff --git a/src/plugins/memory-state.ts b/src/plugins/memory-state.ts index 60964051efb7..f747b198b67a 100644 --- a/src/plugins/memory-state.ts +++ b/src/plugins/memory-state.ts @@ -1,4 +1,5 @@ /** Registry state for plugin memory runtimes, prompt supplements, and flush planning. */ +import { AsyncLocalStorage } from "node:async_hooks"; import type { MemoryCitationsMode } from "../config/types.memory.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; @@ -6,13 +7,35 @@ import type { MemorySearchManager } from "../memory-host-sdk/host/types.js"; const log = createSubsystemLogger("plugins/memory-state"); -export type MemoryPromptSectionBuilder = (params: { +export type MemoryPromptSectionParams = { availableTools: Set; citationsMode?: MemoryCitationsMode; agentId?: string; agentSessionKey?: string; sandboxed?: boolean; -}) => string[]; +}; + +export type MemoryPromptSectionBuilder = (params: MemoryPromptSectionParams) => string[]; + +/** + * Loads and renders prompt state before synchronous prompt assembly. + * Implementations may perform async state reads here, but must validate their + * owner instance before returning lines for the current run. + */ +type MemoryPromptSectionPreparer = ( + params: MemoryPromptSectionParams, +) => Promise; + +export type PreparedMemoryPromptSection = Readonly<{ + context: Readonly<{ + availableTools: readonly string[]; + citationsMode?: MemoryCitationsMode; + agentId?: string; + agentSessionKey?: string; + sandboxed: boolean; + }>; + lines: readonly string[]; +}>; export type MemoryCorpusSearchResult = { corpus: string; @@ -75,6 +98,11 @@ type MemoryPromptSupplementRegistration = { builder: MemoryPromptSectionBuilder; }; +type MemoryPromptPreparationRegistration = { + pluginId: string; + prepare: MemoryPromptSectionPreparer; +}; + export type MemoryFlushPlan = { softThresholdTokens: number; forceFlushTranscriptBytes: number; @@ -167,14 +195,19 @@ type MemoryPluginCapabilityRegistration = { type MemoryPluginState = { capability?: MemoryPluginCapabilityRegistration; corpusSupplements: MemoryCorpusSupplementRegistration[]; + promptPreparations: MemoryPromptPreparationRegistration[]; promptSupplements: MemoryPromptSupplementRegistration[]; }; const memoryPluginState: MemoryPluginState = { corpusSupplements: [], + promptPreparations: [], promptSupplements: [], }; +const preparedMemoryPromptSections = new WeakSet(); +const activePreparedMemoryPromptSection = new AsyncLocalStorage(); + export function registerMemoryCorpusSupplement( pluginId: string, supplement: MemoryCorpusSupplement, @@ -245,21 +278,134 @@ export function registerMemoryPromptSupplement( memoryPluginState.promptSupplements = next; } -export function buildMemoryPromptSection(params: { - availableTools: Set; - citationsMode?: MemoryCitationsMode; - agentId?: string; - agentSessionKey?: string; - sandboxed?: boolean; -}): string[] { +export function registerMemoryPromptPreparation( + pluginId: string, + prepare: MemoryPromptSectionPreparer, +): void { + const next = memoryPluginState.promptPreparations.filter( + (registration) => registration.pluginId !== pluginId, + ); + next.push({ pluginId, prepare }); + memoryPluginState.promptPreparations = next; +} + +function buildSynchronousMemoryPromptSection(params: MemoryPromptSectionParams): { + primary: string[]; + supplements: Array<{ pluginId: string; lines: string[] }>; +} { const primary = normalizeMemoryPromptLines( memoryPluginState.capability?.capability.promptBuilder?.(params) ?? [], ); const supplements = memoryPluginState.promptSupplements // Keep supplement order stable even if plugin registration order changes. .toSorted((left, right) => left.pluginId.localeCompare(right.pluginId)) - .flatMap((registration) => normalizeMemoryPromptLines(registration.builder(params))); - return [...primary, ...supplements]; + .map((registration) => ({ + pluginId: registration.pluginId, + lines: normalizeMemoryPromptLines(registration.builder(params)), + })); + return { primary, supplements }; +} + +function cloneMemoryPromptSectionParams( + params: MemoryPromptSectionParams, +): MemoryPromptSectionParams { + return { + availableTools: new Set(params.availableTools), + citationsMode: params.citationsMode, + agentId: params.agentId, + agentSessionKey: params.agentSessionKey, + sandboxed: params.sandboxed, + }; +} + +function snapshotMemoryPromptContext( + params: MemoryPromptSectionParams, +): PreparedMemoryPromptSection["context"] { + return Object.freeze({ + availableTools: Object.freeze([...params.availableTools].toSorted()), + citationsMode: params.citationsMode, + agentId: params.agentId, + agentSessionKey: params.agentSessionKey, + sandboxed: params.sandboxed === true, + }); +} + +function preparedMemoryPromptContextMatches( + prepared: PreparedMemoryPromptSection, + params: MemoryPromptSectionParams, +): boolean { + const current = snapshotMemoryPromptContext(params); + return ( + prepared.context.citationsMode === current.citationsMode && + prepared.context.agentId === current.agentId && + prepared.context.agentSessionKey === current.agentSessionKey && + prepared.context.sandboxed === current.sandboxed && + prepared.context.availableTools.length === current.availableTools.length && + prepared.context.availableTools.every((tool, index) => tool === current.availableTools[index]) + ); +} + +/** Prepare one immutable memory prompt snapshot for a run. */ +export async function prepareMemoryPromptSection( + params: MemoryPromptSectionParams, +): Promise { + const runParams = cloneMemoryPromptSectionParams(params); + const context = snapshotMemoryPromptContext(runParams); + const synchronous = buildSynchronousMemoryPromptSection( + cloneMemoryPromptSectionParams(runParams), + ); + const preparationRegistrations = [...memoryPluginState.promptPreparations]; + const preparedSupplements = await Promise.all( + preparationRegistrations.map(async (registration) => ({ + pluginId: registration.pluginId, + lines: normalizeMemoryPromptLines( + await registration.prepare(cloneMemoryPromptSectionParams(runParams)), + ), + })), + ); + const lines = Object.freeze([ + ...synchronous.primary, + ...[...synchronous.supplements, ...preparedSupplements] + .toSorted((left, right) => left.pluginId.localeCompare(right.pluginId)) + .flatMap((registration) => registration.lines), + ]); + const prepared = Object.freeze({ + context, + lines, + }); + preparedMemoryPromptSections.add(prepared); + return prepared; +} + +/** Keep async preparation run-scoped while a context engine assembles synchronously. */ +export async function runWithPreparedMemoryPromptSection( + params: MemoryPromptSectionParams, + run: () => Promise, +): Promise { + const prepared = await prepareMemoryPromptSection(params); + return activePreparedMemoryPromptSection.run(prepared, run); +} + +export function getActivePreparedMemoryPromptSection(): PreparedMemoryPromptSection | undefined { + return activePreparedMemoryPromptSection.getStore(); +} + +export function buildMemoryPromptSection( + params: MemoryPromptSectionParams, + prepared?: PreparedMemoryPromptSection, +): string[] { + if (prepared) { + // Run-scoped prompt state must never cross agent/session/tool boundaries. + if ( + !preparedMemoryPromptSections.has(prepared) || + !preparedMemoryPromptContextMatches(prepared, params) + ) { + throw new Error("prepared memory prompt section does not match the current run"); + } + return [...prepared.lines]; + } + const synchronous = buildSynchronousMemoryPromptSection(params); + return [...synchronous.primary, ...synchronous.supplements.flatMap((entry) => entry.lines)]; } function normalizeMemoryPromptLines(value: unknown): string[] { @@ -272,6 +418,9 @@ function normalizeMemoryPromptLines(value: unknown): string[] { export function listMemoryPromptSupplements(): MemoryPromptSupplementRegistration[] { return [...memoryPluginState.promptSupplements]; } +export function listMemoryPromptPreparations(): MemoryPromptPreparationRegistration[] { + return [...memoryPluginState.promptPreparations]; +} export function registerMemoryFlushPlanResolverForPlugin( pluginId: string, resolver: MemoryFlushPlanResolver, @@ -373,11 +522,13 @@ export function restoreMemoryPluginState(state: MemoryPluginState): void { } : undefined; memoryPluginState.corpusSupplements = [...state.corpusSupplements]; + memoryPluginState.promptPreparations = [...state.promptPreparations]; memoryPluginState.promptSupplements = [...state.promptSupplements]; } export function clearMemoryPluginState(): void { memoryPluginState.capability = undefined; memoryPluginState.corpusSupplements = []; + memoryPluginState.promptPreparations = []; memoryPluginState.promptSupplements = []; } diff --git a/src/plugins/plugin-api.types.ts b/src/plugins/plugin-api.types.ts index 3eb9e77630ba..4f42adbea525 100644 --- a/src/plugins/plugin-api.types.ts +++ b/src/plugins/plugin-api.types.ts @@ -449,6 +449,12 @@ export type OpenClawPluginApi = { registerMemoryPromptSupplement: ( builder: import("./memory-state.js").MemoryPromptSectionBuilder, ) => void; + /** Register an async memory prompt preparation step (non-exclusive). */ + registerMemoryPromptPreparation: ( + prepare: ( + params: import("./memory-state.js").MemoryPromptSectionParams, + ) => Promise, + ) => void; /** Register an additive memory-adjacent search/read corpus supplement (non-exclusive). */ registerMemoryCorpusSupplement: (supplement: MemoryCorpusSupplement) => void; /** diff --git a/src/plugins/plugin-registration-transaction.ts b/src/plugins/plugin-registration-transaction.ts index 610a19a4ff27..db26e7ea61db 100644 --- a/src/plugins/plugin-registration-transaction.ts +++ b/src/plugins/plugin-registration-transaction.ts @@ -27,6 +27,7 @@ import { import { getMemoryCapabilityRegistration, listMemoryCorpusSupplements, + listMemoryPromptPreparations, listMemoryPromptSupplements, restoreMemoryPluginState, } from "./memory-state.js"; @@ -42,6 +43,7 @@ export type PluginProcessGlobalState = { memoryCapability: ReturnType; memoryCorpusSupplements: ReturnType; memoryEmbeddingProviders: ReturnType; + memoryPromptPreparations: ReturnType; memoryPromptSupplements: ReturnType; }; @@ -56,6 +58,7 @@ export function snapshotPluginProcessGlobalState(): PluginProcessGlobalState { memoryCapability: getMemoryCapabilityRegistration(), memoryCorpusSupplements: listMemoryCorpusSupplements(), memoryEmbeddingProviders: listRegisteredMemoryEmbeddingProviders(), + memoryPromptPreparations: listMemoryPromptPreparations(), memoryPromptSupplements: listMemoryPromptSupplements(), }; } @@ -71,6 +74,7 @@ export function restorePluginProcessGlobalState(state: PluginProcessGlobalState) restoreMemoryPluginState({ capability: state.memoryCapability, corpusSupplements: state.memoryCorpusSupplements, + promptPreparations: state.memoryPromptPreparations, promptSupplements: state.memoryPromptSupplements, }); } diff --git a/src/plugins/registry-api.ts b/src/plugins/registry-api.ts index 4d1e0d67f92c..d348ffbc598c 100644 --- a/src/plugins/registry-api.ts +++ b/src/plugins/registry-api.ts @@ -104,6 +104,7 @@ export function createPluginApiFactory( registerMemoryCapability, registerMemoryPromptSection, registerMemoryPromptSupplement, + registerMemoryPromptPreparation, registerMemoryCorpusSupplement, registerMemoryFlushPlan, registerMemoryRuntime, @@ -361,6 +362,8 @@ export function createPluginApiFactory( registerMemoryPromptSection(record, builder), registerMemoryPromptSupplement: (builder) => registerMemoryPromptSupplement(record, builder), + registerMemoryPromptPreparation: (prepare) => + registerMemoryPromptPreparation(record, prepare), registerMemoryCorpusSupplement: (supplement) => registerMemoryCorpusSupplement(record, supplement), registerMemoryFlushPlan: (resolver) => registerMemoryFlushPlan(record, resolver), diff --git a/src/plugins/registry-registrars-memory.ts b/src/plugins/registry-registrars-memory.ts index b05274295803..cf4ab4ffee87 100644 --- a/src/plugins/registry-registrars-memory.ts +++ b/src/plugins/registry-registrars-memory.ts @@ -6,6 +6,7 @@ import { registerMemoryCapability as registerGlobalMemoryCapability, registerMemoryCorpusSupplement as registerGlobalMemoryCorpusSupplement, registerMemoryFlushPlanResolverForPlugin, + registerMemoryPromptPreparation as registerGlobalMemoryPromptPreparation, registerMemoryPromptSupplement as registerGlobalMemoryPromptSupplement, registerMemoryPromptSectionForPlugin, registerMemoryRuntimeForPlugin, @@ -68,6 +69,22 @@ export function createMemoryRegistrars(state: PluginRegistryState) { registerGlobalMemoryPromptSupplement(record.id, builder); }; + const registerMemoryPromptPreparation = ( + record: PluginRecord, + prepare: Parameters[0], + ) => { + if (typeof prepare !== "function") { + pushDiagnostic({ + level: "error", + pluginId: record.id, + source: record.source, + message: "memory prompt preparation registration missing prepare function", + }); + return; + } + registerGlobalMemoryPromptPreparation(record.id, prepare); + }; + const registerMemoryCorpusSupplement = ( record: PluginRecord, supplement: Parameters[0], @@ -135,6 +152,7 @@ export function createMemoryRegistrars(state: PluginRegistryState) { registerMemoryCapability, registerMemoryPromptSection, registerMemoryPromptSupplement, + registerMemoryPromptPreparation, registerMemoryCorpusSupplement, registerMemoryFlushPlan, registerMemoryRuntime, diff --git a/test/scripts/check-database-first-legacy-stores.test.ts b/test/scripts/check-database-first-legacy-stores.test.ts index 2818d351f4a9..ec9e4eaee2ed 100644 --- a/test/scripts/check-database-first-legacy-stores.test.ts +++ b/test/scripts/check-database-first-legacy-stores.test.ts @@ -8662,23 +8662,7 @@ describe("check-database-first-legacy-stores", () => { expect(violations).toEqual([{ kind: "legacy store filesystem write", line: 668 }]); }); - it("allows current legacy-debt writes after harmless line movement", () => { - const content = [ - `import { fsRoot } from "@openclaw/fs-safe/root";`, - `const relativePath = ".openclaw-wiki/cache/claims.jsonl";`, - `const root = await fsRoot(rootDir);`, - ...Array.from({ length: 8 }, () => ""), - `await root.write(relativePath, content);`, - ].join("\n"); - const violations = collectDatabaseFirstLegacyStoreViolations( - content, - "extensions/memory-wiki/src/compile.ts", - ); - - expect(violations).toEqual([]); - }); - - it("flags duplicate copies of current legacy-debt writes", () => { + it("flags duplicate copies when one current legacy-debt write is allowed", () => { const relativePath = "extensions/memory-wiki/src/compile.ts"; const allowedWrite = `fs.writeFileSync("sessions.json", "{}\\n")`; const currentLegacyWriteAllowances = new Map([ @@ -8693,18 +8677,6 @@ describe("check-database-first-legacy-stores", () => { expect(violations).toEqual([{ kind: "legacy store filesystem write", line: 3 }]); }); - it("flags stale current legacy-debt allowlist entries during full scans", () => { - const violations = collectDatabaseFirstLegacyStoreViolations( - ` - export const CLAIMS_DIGEST_PATH = ".openclaw-wiki/cache/claims.jsonl"; - `, - "extensions/memory-wiki/src/compile.ts", - { enforceCurrentLegacyAllowlist: true }, - ); - - expect(violations).toEqual([{ kind: "stale current legacy write allowlist", line: 1 }]); - }); - it("allows doctor and migration owners to import or archive legacy files", () => { const violations = collectDatabaseFirstLegacyStoreViolations( `