diff --git a/docs/concepts/2026-07-29-memory-impl-plan.md b/docs/concepts/2026-07-29-memory-impl-plan.md index e6316f1e8b3d..c986f225543a 100644 --- a/docs/concepts/2026-07-29-memory-impl-plan.md +++ b/docs/concepts/2026-07-29-memory-impl-plan.md @@ -980,26 +980,26 @@ Focused existing tests: Phase 1C is complete only when all of the following are demonstrated: -- [ ] Every content-bearing read lane is either converted to the authorized +- [x] Every content-bearing read lane is either converted to the authorized runtime or explicitly unavailable in enforced mode. -- [ ] Through every converted lane, two verified users cannot observe one +- [x] Through every converted lane, two verified users cannot observe one another's private existence bits, paths, titles, snippets, scores, counts, citations, cursors, or content. -- [ ] A group view contains only its channel store, eligible shared content, +- [x] A group view contains only its channel store, eligible shared content, and projections explicitly addressed to that audience. -- [ ] Bootstrap, automatic recall, Active Memory, Memory Wiki, LanceDB, CLI, +- [x] Bootstrap, automatic recall, Active Memory, Memory Wiki, LanceDB, CLI, status, session recall, and corpus supplements pass the same context and receipt checks or are blocked. -- [ ] Every scoped exposure is recorded before content leaves the selected +- [x] Every scoped exposure is recorded before content leaves the selected plugin, and missing/stale exposure or egress receipts are rejected. -- [ ] The minimum transcript policy-set, run-exposure, and event-companion rows +- [x] The minimum transcript policy-set, run-exposure, and event-companion rows commit atomically with each durable event after scoped exposure. -- [ ] Enforced agents remain read-only; every ordinary, watcher, plugin, +- [x] Enforced agents remain read-only; every ordinary, watcher, plugin, import, sync, and generic-file durable write path is disabled until Phase 2A. -- [ ] The read-lane, top-K crowd-out, plugin-failure, missing-label, and +- [x] The read-lane, top-K crowd-out, plugin-failure, missing-label, and transcript-visibility test matrix passes. -- [ ] No two-subject pilot or stronger isolation claim is enabled before Phase +- [x] No two-subject pilot or stronger isolation claim is enabled before Phase 1D closes raw file and exec bypasses. ### Phase 1C rollback diff --git a/extensions/memory-core/src/authorization.ts b/extensions/memory-core/src/authorization.ts index b943ba4e049c..ccb8d6296bd8 100644 --- a/extensions/memory-core/src/authorization.ts +++ b/extensions/memory-core/src/authorization.ts @@ -1,6 +1,6 @@ import type { MemoryAuthorizationCapabilities } from "openclaw/plugin-sdk/memory-authorization"; -/** Phase 1C admits only scoped candidate search and exact opaque-handle reads. */ +/** Phase 1C admits scoped reads with authenticated exposure and egress receipts. */ export const MEMORY_CORE_AUTHORIZATION_CAPABILITIES = Object.freeze({ version: 1, scopedCandidates: true, @@ -10,6 +10,6 @@ export const MEMORY_CORE_AUTHORIZATION_CAPABILITIES = Object.freeze({ scopedImport: false, scopedExport: false, scopedStatus: false, - exposureReceipts: false, - egressReceipts: false, + exposureReceipts: true, + egressReceipts: true, }) satisfies MemoryAuthorizationCapabilities; diff --git a/extensions/memory-core/src/cli.test.ts b/extensions/memory-core/src/cli.test.ts index 4583c8ebe85a..98bd87f084a7 100644 --- a/extensions/memory-core/src/cli.test.ts +++ b/extensions/memory-core/src/cli.test.ts @@ -363,10 +363,7 @@ describe("memory cli", () => { ["index", ["index", "--agent", "main"]], ["promotion apply", ["promote", "--agent", "main", "--apply"]], ["REM backfill", ["rem-backfill", "--agent", "main", "--rollback"]], - [ - "session backfill apply", - ["session-backfill", "--agent", "main", "--apply"], - ], + ["session backfill apply", ["session-backfill", "--agent", "main", "--apply"]], ] as const)( "blocks %s before acquiring a legacy manager for a cut-over agent", async (_command, args) => { diff --git a/extensions/memory-core/src/memory/scoped-memory-runtime.test.ts b/extensions/memory-core/src/memory/scoped-memory-runtime.test.ts index 855d35ac5f39..14521cf457c4 100644 --- a/extensions/memory-core/src/memory/scoped-memory-runtime.test.ts +++ b/extensions/memory-core/src/memory/scoped-memory-runtime.test.ts @@ -9,6 +9,7 @@ import { consumeAdmittedChannelMemoryIdentityFromContext, createChannelMemoryIdentityAdmission, } from "../../../../src/channels/message-access/memory-identity-admission.js"; +import { admitMemoryAuthorizationReadRuntime } from "../../../../src/plugins/memory-authorization-runtime.js"; import { resetMemoryIsolationCutoverForTest } from "../../../../src/plugins/memory-cutover.js"; import { createEmptyPluginRegistry } from "../../../../src/plugins/registry-empty.js"; import { @@ -259,6 +260,10 @@ describe("builtin scoped authorized runtime", () => { } it("postfilters before result count and issues plan-bound exact-read handles", async () => { + expect(MEMORY_CORE_AUTHORIZATION_CAPABILITIES).toMatchObject({ + exposureReceipts: true, + egressReceipts: true, + }); const alice = createPrivateResource("alice", "shared signal from alice"); createPrivateResource("bob", "shared signal from bob"); const context = createContext("alice"); @@ -273,6 +278,13 @@ describe("builtin scoped authorized runtime", () => { expect(result.value).toHaveLength(1); expect(result.value[0]).toMatchObject({ snippet: "shared signal from alice" }); expect(result.exposureReceipt.exposedRevisionHandles).toEqual([alice.revisionId]); + expect(result.egressReceipt).toMatchObject({ + planId: plan.planId, + runId: context.runId, + allowedAudiences: context.delivery.audiences, + deliveryRevision: context.delivery.deliveryRevision, + egressRegistryRevision: context.delivery.egressRegistryRevision, + }); const hit = result.value[0]; if (!hit) { return; @@ -307,6 +319,13 @@ describe("builtin scoped authorized runtime", () => { } markCutOver(); installBuiltinSelectedRuntime(); + await expect( + admitMemoryAuthorizationReadRuntime({ + authorization: MEMORY_CORE_AUTHORIZATION_CAPABILITIES, + authorizationConformance: builtinScopedMemoryConformanceAdapter, + runtime: builtinScopedMemoryAuthorizedRuntime, + }), + ).resolves.toMatchObject({ ok: true }); const aliceHost = createAuthorizedMemoryReadHost({ agentId: "main", ...aliceSession }); const bobHost = createAuthorizedMemoryReadHost({ agentId: "main", ...bobSession }); diff --git a/extensions/memory-core/src/tools.shared.ts b/extensions/memory-core/src/tools.shared.ts index ef8a0af949e4..8bcf0ea85a97 100644 --- a/extensions/memory-core/src/tools.shared.ts +++ b/extensions/memory-core/src/tools.shared.ts @@ -176,8 +176,15 @@ export async function searchMemoryCorpusSupplements(params: { agentId?: string; agentSessionKey?: string; sandboxed?: boolean; + memoryReadEnforced?: OpenClawPluginToolContext["memoryReadEnforced"]; + authorizedMemoryRead?: OpenClawPluginToolContext["authorizedMemoryRead"]; corpus?: "memory" | "wiki" | "all" | "sessions"; }): Promise { + if (params.memoryReadEnforced) { + // Supplemental corpora have no selected-runtime source contract yet. A + // host handle is intentionally not reinterpreted as authority for them. + return []; + } if (params.corpus === "memory" || params.corpus === "sessions") { return []; } @@ -207,8 +214,14 @@ export async function getMemoryCorpusSupplementResult(params: { agentId?: string; agentSessionKey?: string; sandboxed?: boolean; + memoryReadEnforced?: OpenClawPluginToolContext["memoryReadEnforced"]; + authorizedMemoryRead?: OpenClawPluginToolContext["authorizedMemoryRead"]; corpus?: "memory" | "wiki" | "all" | "sessions"; }) { + if (params.memoryReadEnforced) { + // See search: enforce the same unavailable result before any plugin call. + return null; + } if (params.corpus === "memory" || params.corpus === "sessions") { return null; } diff --git a/extensions/memory-core/src/tools.test.ts b/extensions/memory-core/src/tools.test.ts index 643914682abb..a3c662105bfa 100644 --- a/extensions/memory-core/src/tools.test.ts +++ b/extensions/memory-core/src/tools.test.ts @@ -1,6 +1,9 @@ import type { MemorySearchRuntimeDebug } from "openclaw/plugin-sdk/memory-core-host-runtime-files"; // Memory Core tests cover tools plugin behavior. -import { clearMemoryPluginState } from "openclaw/plugin-sdk/memory-host-core"; +import { + clearMemoryPluginState, + registerMemoryCorpusSupplement, +} from "openclaw/plugin-sdk/memory-host-core"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { getMemoryCloseMockCalls, @@ -24,8 +27,10 @@ import { } from "./tools.js"; import { buildMemorySearchUnavailableResult, + getMemoryCorpusSupplementResult, MemoryGetSchema, MemorySearchSchema, + searchMemoryCorpusSupplements, } from "./tools.shared.js"; import { asOpenClawConfig, @@ -782,6 +787,93 @@ describe("enforced memory tools", () => { }); expect(getMemorySearchManagerMockCalls()).toBe(0); }); + + it("blocks the wiki corpus instead of remapping it to authorized memory sources", async () => { + const host = { + search: vi.fn(async () => ({ results: [] })), + read: vi.fn(async () => ({ text: "unused", path: "memory/MEMORY.md" })), + }; + const search = createMemorySearchToolOrThrow({ + memoryReadEnforced: true, + authorizedMemoryRead: host, + }); + + const result = await search.execute("scoped-wiki", { query: "private wiki", corpus: "wiki" }); + + expect(result.details).toMatchObject({ + disabled: true, + unavailable: true, + results: [], + error: "The wiki corpus is unavailable for scoped memory.", + }); + expect(host.search).not.toHaveBeenCalled(); + expect(getMemorySearchManagerMockCalls()).toBe(0); + }); + + it("routes enforced conversation recall only through the authorized sessions source", async () => { + const host = { + search: vi.fn(async () => ({ results: [] })), + read: vi.fn(async () => ({ text: "unused", path: "sessions/unused.jsonl" })), + }; + const search = createMemorySearchToolOrThrow({ + memoryReadEnforced: true, + authorizedMemoryRead: host, + conversationRecall: { + anchorSessionKey: "agent:main:main", + scope: "same-agent-private", + corpus: "sessions", + }, + }); + + await search.execute("scoped-conversation-recall", { query: "prior turn", corpus: "memory" }); + + expect(host.search).toHaveBeenCalledWith({ + query: "prior turn", + sources: ["sessions"], + limit: undefined, + signal: undefined, + }); + expect(getMemorySearchManagerMockCalls()).toBe(0); + }); + + it("does not invoke registered corpus supplements after cutover", async () => { + const supplement = { + search: vi.fn(async () => [ + { corpus: "wiki", path: "private.md", score: 1, snippet: "private" }, + ]), + get: vi.fn(async () => ({ + corpus: "wiki", + path: "private.md", + content: "private", + fromLine: 1, + lineCount: 1, + })), + }; + registerMemoryCorpusSupplement("memory-wiki", supplement); + const host = { + search: vi.fn(async () => ({ results: [] })), + read: vi.fn(async () => ({ text: "", path: "" })), + }; + + await expect( + searchMemoryCorpusSupplements({ + query: "private", + corpus: "all", + memoryReadEnforced: true, + authorizedMemoryRead: host, + }), + ).resolves.toEqual([]); + await expect( + getMemoryCorpusSupplementResult({ + lookup: "private.md", + corpus: "wiki", + memoryReadEnforced: true, + authorizedMemoryRead: host, + }), + ).resolves.toBeNull(); + expect(supplement.search).not.toHaveBeenCalled(); + expect(supplement.get).not.toHaveBeenCalled(); + }); }); describe("memory_search corpus labels", () => { diff --git a/extensions/memory-core/src/tools.ts b/extensions/memory-core/src/tools.ts index 93d1a29c4a9c..7c3552487f75 100644 --- a/extensions/memory-core/src/tools.ts +++ b/extensions/memory-core/src/tools.ts @@ -75,15 +75,28 @@ function authorizedMemoryUnavailableResult() { return jsonResult(buildMemorySearchUnavailableResult("memory unavailable")); } +function authorizedMemoryCorpusUnavailableResult(corpus: "wiki") { + return jsonResult( + buildMemorySearchUnavailableResult(`The ${corpus} corpus is unavailable for scoped memory.`, { + warning: + "Compiled wiki memory is unavailable because scoped memory admits only authorized memory and session sources.", + action: "Use corpus=memory or corpus=sessions, or request an authorized memory source.", + }), + ); +} + function resolveAuthorizedMemorySources( requestedCorpus: "memory" | "wiki" | "all" | "sessions" | undefined, -): readonly MemorySource[] { +): readonly MemorySource[] | undefined { if (requestedCorpus === "memory") { return ["memory"]; } if (requestedCorpus === "sessions") { return ["sessions"]; } + if (requestedCorpus === "wiki") { + return undefined; + } // `all` has no supplemental-plugin escape hatch in enforced mode. It means // all stores admitted by the selected runtime, across its content sources. return ["memory", "sessions"]; @@ -286,6 +299,8 @@ async function getSupplementMemoryReadResult(params: { agentId?: string; agentSessionKey?: string; sandboxed?: boolean; + memoryReadEnforced?: OpenClawPluginToolContext["memoryReadEnforced"]; + authorizedMemoryRead?: OpenClawPluginToolContext["authorizedMemoryRead"]; corpus?: "memory" | "wiki" | "all"; }) { const supplement = await getMemoryCorpusSupplementResult({ @@ -295,6 +310,8 @@ async function getSupplementMemoryReadResult(params: { agentId: params.agentId, agentSessionKey: params.agentSessionKey, sandboxed: params.sandboxed, + memoryReadEnforced: params.memoryReadEnforced, + authorizedMemoryRead: params.authorizedMemoryRead, corpus: params.corpus, }); if (!supplement) { @@ -403,7 +420,7 @@ export function createMemorySearchTool(options: { label: "Memory Search", name: "memory_search", description: - "Mandatory recall step: semantically search MEMORY.md + memory/*.md (and optional session transcripts) before answering questions about prior work, decisions, dates, people, preferences, or todos. Optional `corpus=wiki` or `corpus=all` also searches registered compiled-wiki supplements. `corpus=memory` restricts hits to indexed memory files (excludes session transcript chunks from ranking). `corpus=sessions` restricts hits to indexed session transcripts (same visibility rules as session history tools). If response has disabled=true or stale=true, you must tell the user and include the warning/action guidance.", + "Mandatory recall step: semantically search MEMORY.md + memory/*.md (and optional session transcripts) before answering questions about prior work, decisions, dates, people, preferences, or todos. In legacy mode, `corpus=wiki` or `corpus=all` also searches registered compiled-wiki supplements; scoped memory makes `corpus=wiki` unavailable and admits only authorized memory/session sources. `corpus=memory` restricts hits to indexed memory files (excludes session transcript chunks from ranking). `corpus=sessions` restricts hits to indexed session transcripts (same visibility rules as session history tools). If response has disabled=true or stale=true, you must tell the user and include the warning/action guidance.", parameters: MemorySearchSchema, execute: ({ cfg, agentId }) => @@ -425,13 +442,17 @@ export function createMemorySearchTool(options: { const requestedCorpus = options.conversationRecall?.corpus === "sessions" ? "sessions" : modelRequestedCorpus; if (options.memoryReadEnforced) { + const sources = resolveAuthorizedMemorySources(requestedCorpus); + if (!sources) { + return authorizedMemoryCorpusUnavailableResult("wiki"); + } const host = options.authorizedMemoryRead; if (!host) { return authorizedMemoryUnavailableResult(); } const authorized = await host.search({ query, - sources: resolveAuthorizedMemorySources(requestedCorpus), + sources, limit: maxResults, signal: callerSignal, }); @@ -668,6 +689,8 @@ export function createMemorySearchTool(options: { agentId, agentSessionKey: options.agentSessionKey, sandboxed: options.sandboxed, + memoryReadEnforced: options.memoryReadEnforced, + authorizedMemoryRead: options.authorizedMemoryRead, corpus: requestedCorpus, }), ), @@ -783,6 +806,8 @@ export function createMemoryGetTool(options: { agentId, agentSessionKey: options.agentSessionKey, sandboxed: options.sandboxed, + memoryReadEnforced: options.memoryReadEnforced, + authorizedMemoryRead: options.authorizedMemoryRead, corpus: requestedCorpus, }); return jsonResult( diff --git a/extensions/memory-wiki/src/bridge.test.ts b/extensions/memory-wiki/src/bridge.test.ts index b94151f278c9..c657ed4fb34a 100644 --- a/extensions/memory-wiki/src/bridge.test.ts +++ b/extensions/memory-wiki/src/bridge.test.ts @@ -76,7 +76,9 @@ describe("syncMemoryWikiBridgeSources", () => { }); const stat = vi.spyOn(fs, "stat"); const mkdir = vi.spyOn(fs, "mkdir"); - isLegacyMemorySurfaceDisabledMock.mockImplementation((agentId: string) => agentId === "cutover"); + isLegacyMemorySurfaceDisabledMock.mockImplementation( + (agentId: string) => agentId === "cutover", + ); await expect( syncMemoryWikiBridgeSources({ diff --git a/extensions/memory-wiki/src/bridge.ts b/extensions/memory-wiki/src/bridge.ts index 7ced14d3c1ef..304c460247f0 100644 --- a/extensions/memory-wiki/src/bridge.ts +++ b/extensions/memory-wiki/src/bridge.ts @@ -10,6 +10,7 @@ import { import { normalizeAgentId } from "openclaw/plugin-sdk/routing"; import type { OpenClawConfig } from "../api.js"; import type { ResolvedMemoryWikiConfig } from "./config.js"; +import { assertLegacyMemoryWikiAccessAvailable } from "./legacy-memory-access.js"; import { appendMemoryWikiLog } from "./log.js"; import { createWikiPageFilename, @@ -25,7 +26,6 @@ import { readMemoryWikiSourceSyncState, writeMemoryWikiSourceSyncState, } from "./source-sync-state.js"; -import { assertLegacyMemoryWikiAccessAvailable } from "./legacy-memory-access.js"; import { initializeMemoryWikiVault } from "./vault.js"; type BridgeArtifact = { diff --git a/src/agents/agent-tools.ts b/src/agents/agent-tools.ts index dc67ea3b21a4..f0fd695bf13a 100644 --- a/src/agents/agent-tools.ts +++ b/src/agents/agent-tools.ts @@ -70,7 +70,6 @@ import { filterLocalModelLeanTools, resolveLocalModelLeanPreserveToolNames, } from "./local-model-lean.js"; -import { createMemoryFileMutationGuard } from "./memory-file-mutation-guard.js"; import { createMemoryWriteProvenanceObserver } from "./memory-write-provenance.js"; import type { ModelAuthMode } from "./model-auth.js"; import { resolveOpenClawPluginToolsForOptions } from "./openclaw-plugin-tools.js"; @@ -517,9 +516,7 @@ function createOpenClawCodingToolsInternal(options?: OpenClawCodingToolsOptions) const codingRoot = sandboxRoot ?? runtimeRoot; const containmentRoot = sandboxRoot ?? sessionPermissionPolicy?.root ?? codingRoot; const memoryFlushWriteRoot = sandboxRoot ?? workspaceRoot; - const memoryFileMutationGuard = isMemoryIsolationCutoverAgent(agentId) - ? createMemoryFileMutationGuard({ mutationRoot: memoryFlushWriteRoot }) - : undefined; + const memoryIsolationCutover = Boolean(agentId && isMemoryIsolationCutoverAgent(agentId)); // Flush exposes one append-only target; its fallback records inherited taint after success. const memoryWriteProvenance = isMemoryFlushRun ? undefined @@ -541,7 +538,10 @@ function createOpenClawCodingToolsInternal(options?: OpenClawCodingToolsOptions) includePluginTools: true, }; const includeBaseCodingTools = includeCoreTools && toolConstructionPlan.includeBaseCodingTools; - const includeShellTools = includeCoreTools && toolConstructionPlan.includeShellTools; + // P1C's selected-memory pilot is read-only. Hiding both the shell and its process controller + // closes the generic durable-write bypass without pretending this is P1D virtual-FS confinement. + const includeShellTools = + includeCoreTools && toolConstructionPlan.includeShellTools && !memoryIsolationCutover; const includeOpenClawTools = includeCoreTools && toolConstructionPlan.includeOpenClawTools; const includeChannelTools = toolConstructionPlan.includeChannelTools; const includePluginTools = toolConstructionPlan.includePluginTools; @@ -584,7 +584,6 @@ function createOpenClawCodingToolsInternal(options?: OpenClawCodingToolsOptions) skillsSnapshot: options?.skillsSnapshot, modelContextWindowTokens: options?.modelContextWindowTokens, imageSanitization, - memoryFileMutationGuard, memoryWriteProvenance, ...(includeBaseCodingTools ? { baseToolNames: createCodingTools(codingRoot).map((tool) => tool.name) } @@ -952,20 +951,26 @@ function createOpenClawCodingToolsInternal(options?: OpenClawCodingToolsOptions) !options?.swarmCollector || (tool.name !== "ask_user" && tool.name !== "sessions_send" && tool.name !== "sessions_yield"), ); + // P1C has no authorized mutation or execution path. Apply this after every contributor and + // policy layer so a future core, plugin, or ring-zero tool cannot reopen a durable-write bypass. + const surfaceTools = memoryIsolationCutover + ? authorizedTools.filter((tool) => tool.name === "read") + : authorizedTools; if ( swarmStructuredOutputTool && - !authorizedTools.some((tool) => tool.name === swarmStructuredOutputTool.name) + !memoryIsolationCutover && + !surfaceTools.some((tool) => tool.name === swarmStructuredOutputTool.name) ) { // Collector output is a run contract, not an operator-configurable capability. - authorizedTools.push(swarmStructuredOutputTool); + surfaceTools.push(swarmStructuredOutputTool); } processToolAvailabilityRef.value = authorizedTools.some((tool) => tool.name === "process"); if (shouldInheritEffectiveToolAllowlist) { - // Snapshot exporter only: this copies authorizedTools for descendants and + // Snapshot exporter only: this copies surfaceTools for descendants and // never filters the mandatory structured_output tool from this turn. - replaceWithEffectiveToolAllowlist(inheritedToolAllowlist, authorizedTools); + replaceWithEffectiveToolAllowlist(inheritedToolAllowlist, surfaceTools); } - replaceWithEffectiveCronCreatorToolAllowlist(cronCreatorToolAllowlist, authorizedTools, (tool) => + replaceWithEffectiveCronCreatorToolAllowlist(cronCreatorToolAllowlist, surfaceTools, (tool) => getPluginToolMeta(tool), ); options?.recordToolPrepStage?.("authorization-policy"); @@ -1007,7 +1012,7 @@ function createOpenClawCodingToolsInternal(options?: OpenClawCodingToolsOptions) }; // NOTE: Keep canonical (lowercase) tool names here. Provider transports remap on the wire. return finalizeAgentTools({ - tools: authorizedTools, + tools: surfaceTools, modelProvider: options?.modelProvider, modelId: options?.modelId, modelCompat: options?.modelCompat, diff --git a/src/agents/agent-tools.workspace-paths.test.ts b/src/agents/agent-tools.workspace-paths.test.ts index 4d16b67ed438..753e62a07065 100644 --- a/src/agents/agent-tools.workspace-paths.test.ts +++ b/src/agents/agent-tools.workspace-paths.test.ts @@ -10,7 +10,12 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import "./test-helpers/fast-coding-tools.js"; import "./test-helpers/fast-openclaw-tools.js"; import type { OpenClawConfig } from "../config/config.js"; +import { resetMemoryIsolationCutoverForTest } from "../plugins/memory-cutover.js"; import { createCanonicalFixtureSkill } from "../skills/test-support/test-helpers.js"; +import { + closeOpenClawAgentDatabasesForTest, + openOpenClawAgentDatabase, +} from "../state/openclaw-agent-db.js"; import { createOpenClawCodingTools } from "./agent-tools.js"; import { createHostWorkspaceEditTool, @@ -84,6 +89,54 @@ async function expectExecCwdResolvesTo( } describe("workspace path resolution", () => { + it("exposes only read for an enforced read-only memory agent", async () => { + await withTempDir("openclaw-memory-cutover-state-", async (stateDir) => { + const originalStateDir = process.env.OPENCLAW_STATE_DIR; + process.env.OPENCLAW_STATE_DIR = stateDir; + try { + const database = openOpenClawAgentDatabase({ agentId: "main" }); + database.db + .prepare( + `INSERT INTO memory_migrations + (migration_id, source_kind, source_hash, phase, classification_json, plan_hash, + verified_at, cutover_at, updated_at) + VALUES ('memory-cutover-shell-tools', 'test', 'test-source', 'cutover', '{}', + 'test-plan', 1, 1, 1)`, + ) + .run(); + resetMemoryIsolationCutoverForTest(); + + expect(createOpenClawCodingTools({ agentId: "main" }).map((tool) => tool.name)).toEqual([ + "read", + ]); + } finally { + closeOpenClawAgentDatabasesForTest(); + resetMemoryIsolationCutoverForTest(); + if (originalStateDir === undefined) { + delete process.env.OPENCLAW_STATE_DIR; + } else { + process.env.OPENCLAW_STATE_DIR = originalStateDir; + } + } + }); + }); + + it("preserves legacy memory-file writes for intentionally unscoped tool construction", async () => { + await withTempDir("openclaw-unscoped-ws-", async (workspaceDir) => { + const tools = createOpenClawCodingTools({ workspaceDir }); + const { writeTool } = expectReadWriteEditTools(tools); + + await writeTool.execute("unscoped-memory-write", { + path: "MEMORY.md", + content: "legacy utility state", + }); + + await expect(fs.readFile(path.join(workspaceDir, "MEMORY.md"), "utf8")).resolves.toBe( + "legacy utility state", + ); + }); + }); + it("keeps controlled memory roots read-only while preserving ordinary host and sandbox writes", async () => { await withTempDir("openclaw-memory-guard-host-", async (workspaceDir) => { const guard = createMemoryFileMutationGuard({ mutationRoot: workspaceDir }); diff --git a/src/agents/embedded-agent-runner/run/attempt-system-prompt-prepare.ts b/src/agents/embedded-agent-runner/run/attempt-system-prompt-prepare.ts index aa895dbe2c62..dca8337a574d 100644 --- a/src/agents/embedded-agent-runner/run/attempt-system-prompt-prepare.ts +++ b/src/agents/embedded-agent-runner/run/attempt-system-prompt-prepare.ts @@ -8,6 +8,7 @@ import { resolveProviderSystemPromptContribution, transformProviderSystemPrompt, } from "../../../plugins/provider-runtime.js"; +import { normalizeDeliveryContext } from "../../../utils/delivery-context.shared.js"; import { normalizeMessageChannel } from "../../../utils/message-channel.js"; import { isReasoningTagProvider } from "../../../utils/provider-utils.js"; import { listActiveProcessSessionReferences } from "../../bash-process-references.js"; @@ -23,6 +24,7 @@ import { } from "../../channel-tools.js"; import { resolveOpenClawReferencePaths } from "../../docs-path.js"; import { resolveHeartbeatPromptForSystemPrompt } from "../../heartbeat-system-prompt.js"; +import { createAuthorizedMemoryReadHost } from "../../memory-authorized-read-host.js"; import { prepareAgentMemoryPrompt } from "../../memory-prompt-prepare.js"; import { resolveDefaultModelForAgent } from "../../model-selection.js"; import { buildModelToolsUnavailablePrompt } from "../../model-tool-support.js"; @@ -248,6 +250,20 @@ export async function prepareEmbeddedAttemptSystemPrompt(params: { }); const includeMemorySection = !params.activeContextEngine || params.activeContextEngine.info.id === "legacy"; + const authorizedMemoryRead = createAuthorizedMemoryReadHost({ + agentId: params.sessionAgentId, + sessionKey: runtimeInfo.sessionKey, + sessionId: attempt.sessionId, + runId: attempt.runId, + deliveryContext: normalizeDeliveryContext({ + channel: attempt.messageChannel ?? attempt.messageProvider, + to: attempt.messageTo ?? attempt.currentMessagingTarget ?? attempt.currentChannelId, + accountId: attempt.agentAccountId, + threadId: attempt.messageThreadId ?? attempt.currentThreadTs, + }), + messageChannel: attempt.messageChannel ?? attempt.messageProvider, + agentAccountId: attempt.agentAccountId, + }); const preparedMemoryPrompt = await prepareAgentMemoryPrompt({ enabled: effectivePromptMode === "full" && includeMemorySection, toolNames: params.effectiveTools.map((tool) => tool.name), @@ -256,6 +272,7 @@ export async function prepareEmbeddedAttemptSystemPrompt(params: { agentId: runtimeInfo.agentId, agentSessionKey: runtimeInfo.sessionKey, sandboxed: sandboxInfo?.enabled === true, + authorizedMemoryRead, }); const preparedWatchedSessions = prepareWatchedSessionsPrompt({ enabled: effectivePromptMode === "full", diff --git a/src/agents/memory-authorized-read-host.ts b/src/agents/memory-authorized-read-host.ts index bc64ce21b045..68b6aa446ce5 100644 --- a/src/agents/memory-authorized-read-host.ts +++ b/src/agents/memory-authorized-read-host.ts @@ -5,11 +5,9 @@ import { MEMORY_INVOCATION_UNAVAILABLE, createAuthorizedMemoryReadInvocation, readAuthorizedMemoryForInvocation, - readAuthorizedMemoryTranscriptExposure, searchAuthorizedMemoryForInvocation, type AuthorizedMemoryReadInvocation, } from "../plugins/memory-invocation.js"; -import { recordMemoryRunExposure } from "../plugins/memory-run-exposure.js"; import type { AuthorizedMemoryReadHost } from "../plugins/tool-types.js"; import { captureTrustedMemoryAccessFacts, @@ -206,18 +204,6 @@ export function createAuthorizedMemoryReadHost(params: { | undefined; const getInvocation = () => (invocation ??= createAuthorizedMemoryReadInvocation({ context: trusted.context })); - const recordExposure = (active: AuthorizedMemoryReadInvocation): boolean => { - const exposure = readAuthorizedMemoryTranscriptExposure(active); - if ( - !exposure || - exposure.agentId !== context.agentId || - exposure.sessionId !== context.sessionId - ) { - return false; - } - recordMemoryRunExposure(exposure); - return true; - }; return Object.freeze({ async search(search) { const active = await getInvocation(); @@ -225,9 +211,7 @@ export function createAuthorizedMemoryReadHost(params: { return active; } const result = await searchAuthorizedMemoryForInvocation({ invocation: active, ...search }); - return "unavailable" in result || !recordExposure(active) - ? MEMORY_INVOCATION_UNAVAILABLE - : result; + return "unavailable" in result ? MEMORY_INVOCATION_UNAVAILABLE : result; }, async read(read) { const active = await getInvocation(); @@ -235,9 +219,7 @@ export function createAuthorizedMemoryReadHost(params: { return active; } const result = await readAuthorizedMemoryForInvocation({ invocation: active, ...read }); - return "unavailable" in result || !recordExposure(active) - ? MEMORY_INVOCATION_UNAVAILABLE - : result; + return "unavailable" in result ? MEMORY_INVOCATION_UNAVAILABLE : result; }, }); } diff --git a/src/agents/memory-prompt-prepare.ts b/src/agents/memory-prompt-prepare.ts index 6104bb8bcc2c..7fe365df9488 100644 --- a/src/agents/memory-prompt-prepare.ts +++ b/src/agents/memory-prompt-prepare.ts @@ -1,9 +1,10 @@ import type { MemoryCitationsMode } from "../config/types.memory.js"; +import { isMemoryIsolationCutoverAgent } from "../plugins/memory-cutover.js"; import { prepareMemoryPromptSection, type PreparedMemoryPromptSection, } from "../plugins/memory-state.js"; -import { isMemoryIsolationCutoverAgent } from "../plugins/memory-cutover.js"; +import type { AuthorizedMemoryReadHost } from "../plugins/tool-types.js"; /** Prepare memory prompt state with the same normalized tool context used by assembly. */ export async function prepareAgentMemoryPrompt(params: { @@ -14,6 +15,8 @@ export async function prepareAgentMemoryPrompt(params: { agentId?: string; agentSessionKey?: string; sandboxed?: boolean; + /** Host-minted selected-runtime invocation for this exact prompt run. */ + authorizedMemoryRead?: AuthorizedMemoryReadHost; }): Promise { if (!params.enabled) { return undefined; @@ -29,6 +32,7 @@ export async function prepareAgentMemoryPrompt(params: { agentId: params.agentId, agentSessionKey: params.agentSessionKey, sandboxed: params.sandboxed, + authorizedMemoryRead: params.authorizedMemoryRead, ...(params.agentId && isMemoryIsolationCutoverAgent(params.agentId) ? { memoryReadEnforced: true as const } : {}), diff --git a/src/agents/system-prompt.ts b/src/agents/system-prompt.ts index 26659ec92906..3abe43147882 100644 --- a/src/agents/system-prompt.ts +++ b/src/agents/system-prompt.ts @@ -311,6 +311,8 @@ function buildMemorySection(params: { agentId: params.agentId, agentSessionKey: params.agentSessionKey, sandboxed: params.sandboxed, + memoryReadEnforced: params.prepared?.context.memoryReadEnforced ? true : undefined, + authorizedMemoryRead: params.prepared?.context.authorizedMemoryRead, }, params.prepared, ); diff --git a/src/config/sessions/session-transcript-memory-policy.test.ts b/src/config/sessions/session-transcript-memory-policy.test.ts index 9fc9cfc98f87..b4963df896ff 100644 --- a/src/config/sessions/session-transcript-memory-policy.test.ts +++ b/src/config/sessions/session-transcript-memory-policy.test.ts @@ -5,11 +5,13 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { runDoctorMemoryIsolation } from "../../commands/doctor-memory-isolation.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { resetMemoryIsolationCutoverForTest } from "../../plugins/memory-cutover.js"; +import { persistMemoryRunExposureBeforeContentInDatabase } from "../../plugins/memory-run-exposure-ledger.js"; import { clearMemoryRunExposureForTest, recordMemoryRunExposure, } from "../../plugins/memory-run-exposure.js"; import { createCurrentMemorySessionContext } from "../../state/memory-session-subject.js"; +import type { OpenClawAgentDatabase } from "../../state/openclaw-agent-db.js"; import { closeOpenClawAgentDatabasesForTest, openOpenClawAgentDatabase, @@ -107,6 +109,17 @@ function recordExposure(params: { }); } +function persistExposure( + database: OpenClawAgentDatabase, + params: Parameters[0], +) { + const exposure = recordExposure(params); + expect(persistMemoryRunExposureBeforeContentInDatabase({ database, snapshot: exposure })).toBe( + true, + ); + return exposure; +} + async function appendWithRun(params: { env: NodeJS.ProcessEnv; runId: string; text: string }) { await withOwnedSessionTranscriptWrites( { @@ -156,11 +169,21 @@ describe("transcript memory policy companions", () => { ); writeSession(alice); + const database = openOpenClawAgentDatabase(options); const aliceContext = createCurrentMemorySessionContext({ ...alice, options }); expect(aliceContext.kind).toBe("current"); if (aliceContext.kind !== "current") { throw new Error("expected lifecycle-owned Alice subject context"); } + await appendSqliteTranscriptMessage( + { ...alice, agentId: AGENT_ID, env }, + { + message: { + role: "assistant", + content: [{ type: "text", text: "legacy shadow search content" }], + }, + }, + ); vi.stubEnv("OPENCLAW_STATE_DIR", env.OPENCLAW_STATE_DIR ?? ""); expect( @@ -173,8 +196,16 @@ describe("transcript memory policy companions", () => { // Doctor writes out of process. Refresh the process-owned snapshot to model the required // Gateway restart before proving the protected transcript boundary. resetMemoryIsolationCutoverForTest(); + resetTranscriptMemoryPolicyForTest(database.db); + expect( + searchSessionTranscripts({ + agentId: AGENT_ID, + env, + query: "legacy shadow search content", + }).hits, + ).toEqual([]); - recordMemoryRunExposure({ + const shadowExposure = recordMemoryRunExposure({ agentId: AGENT_ID, sessionId: alice.sessionId, sessionKey: alice.sessionKey, @@ -192,6 +223,9 @@ describe("transcript memory policy companions", () => { sessionIdentityRevision: aliceContext.context.sessionIdentityRevision, subjectRevision: aliceContext.context.subjectRevision, }); + expect( + persistMemoryRunExposureBeforeContentInDatabase({ database, snapshot: shadowExposure }), + ).toBe(true); await withOwnedSessionTranscriptWrites( { sessionTarget: { @@ -214,7 +248,6 @@ describe("transcript memory policy companions", () => { ); }, ); - const database = openOpenClawAgentDatabase(options); expect(readAuthorizedTranscriptEventSeqs(database.db, alice.sessionId)?.size).toBeGreaterThan( 0, ); @@ -225,6 +258,13 @@ describe("transcript memory policy companions", () => { }), }), ); + const searchAlice = () => + searchSessionTranscripts({ agentId: AGENT_ID, env, query: "alice scoped content" }); + await vi.waitFor(() => expect(searchAlice().indexing).toBe(false), { + interval: 10, + timeout: 15_000, + }); + expect(searchAlice().hits).toHaveLength(1); writeSession(bob); expect(createCurrentMemorySessionContext({ ...bob, options })).toEqual({ @@ -252,9 +292,9 @@ describe("transcript memory policy companions", () => { await appendSqliteTranscriptMessage(scope(env), { message: { role: "assistant", content: [{ type: "text", text: "missing exposure content" }] }, }); - recordExposure({ runId: "stale-run", subjectRevision: "stale-subject-revision" }); + persistExposure(database, { runId: "stale-run", subjectRevision: "stale-subject-revision" }); await appendWithRun({ env, runId: "stale-run", text: "stale exposure content" }); - recordExposure({ runId: "authorized-run" }); + const authorizedExposure = persistExposure(database, { runId: "authorized-run" }); await appendWithRun({ env, runId: "authorized-run", text: "authorized exposure content" }); const policyRows = database.db @@ -266,7 +306,10 @@ describe("transcript memory policy companions", () => { ) .all(SESSION_ID) as Array<{ authorization_status: string; run_id: string | null }>; expect(policyRows.filter((row) => row.authorization_status === "authorized")).toEqual([ - { authorization_status: "authorized", run_id: "authorized-run" }, + { + authorization_status: "authorized", + run_id: authorizedExposure.durableRunScopeId, + }, ]); expect(policyRows.filter((row) => row.authorization_status === "pending")).toHaveLength(2); expect(database.db.prepare("SELECT COUNT(*) AS count FROM memory_policy_sets").get()).toEqual({ @@ -320,7 +363,7 @@ describe("transcript memory policy companions", () => { it("does not derive an append parent from a pending transcript event", async () => { const env = createEnv(); const database = markCutOver(env); - recordExposure({ runId: "authorized-run" }); + persistExposure(database, { runId: "authorized-run" }); let authorizedMessageId: string | undefined; await withOwnedSessionTranscriptWrites( { @@ -371,7 +414,7 @@ describe("transcript memory policy companions", () => { it("rolls the event and every companion row back when companion persistence fails", async () => { const env = createEnv(); const database = markCutOver(env); - recordExposure({ runId: "authorized-run" }); + persistExposure(database, { runId: "authorized-run" }); database.db.exec(/* sqlite-allow-raw: test-only atomicity fault injection. */ ` CREATE TRIGGER reject_transcript_memory_policy_for_test BEFORE INSERT ON transcript_event_memory_policies @@ -394,12 +437,18 @@ describe("transcript memory policy companions", () => { count: 0, }); } + // The pre-output ledger commits before content leaves the memory broker, outside this + // transcript transaction; a later companion rollback must not erase its audit fact. + expect( + database.db.prepare("SELECT COUNT(*) AS count FROM memory_preoutput_exposure_ledger").get(), + ).toEqual({ count: 1 }); }); it("replays only committed current companions after a fresh database consumer starts", async () => { const env = createEnv(); const database = markCutOver(env); - recordExposure({ runId: "authorized-run" }); + persistExposure(database, { runId: "authorized-run" }); + clearMemoryRunExposureForTest(); await appendWithRun({ env, runId: "authorized-run", text: "committed companion content" }); const committedRows = database.db @@ -496,7 +545,7 @@ describe("transcript memory policy companions", () => { it("removes a stale companion from replay, search, projections, compaction, and export", async () => { const env = createEnv(); const database = markCutOver(env); - recordExposure({ runId: "authorized-run" }); + persistExposure(database, { runId: "authorized-run" }); await appendWithRun({ env, runId: "authorized-run", text: "stale companion secret" }); const search = () => diff --git a/src/config/sessions/session-transcript-memory-policy.ts b/src/config/sessions/session-transcript-memory-policy.ts index 0965e0b79045..f2305c4d5b8e 100644 --- a/src/config/sessions/session-transcript-memory-policy.ts +++ b/src/config/sessions/session-transcript-memory-policy.ts @@ -6,10 +6,8 @@ import { getNodeSqliteKysely, } from "../../infra/kysely-sync.js"; import { isMemoryIsolationTranscriptPolicyEnforcedInDatabase } from "../../plugins/memory-cutover.js"; -import { - readMemoryRunExposure, - type MemoryRunExposureSnapshot, -} from "../../plugins/memory-run-exposure.js"; +import { readDurableMemoryRunExposure } from "../../plugins/memory-run-exposure-ledger.js"; +import { type MemoryRunExposureSnapshot } from "../../plugins/memory-run-exposure.js"; import type { DB as OpenClawAgentDatabaseSchema } from "../../state/openclaw-agent-db.generated.js"; import type { OpenClawAgentDatabase } from "../../state/openclaw-agent-db.js"; import { getOwnedSessionTranscriptWriterFence } from "./transcript-write-context.js"; @@ -147,7 +145,7 @@ function persistExposureLineageInTransaction(params: { .values({ exposure_set_id: snapshot.exposureSetId, agent_id: snapshot.agentId, - run_id: snapshot.runId, + run_id: snapshot.durableRunScopeId, context_fingerprint: snapshot.contextFingerprint, plan_id: snapshot.planId, revision_number: snapshot.revisionNumber, @@ -217,8 +215,8 @@ export function recordTranscriptMemoryPolicyInTransaction(params: { } const runId = getOwnedSessionTranscriptWriterFence()?.expectedWriterRunId; const exposure = runId - ? readMemoryRunExposure({ - agentId: params.database.agentId, + ? readDurableMemoryRunExposure({ + database: params.database, sessionId: params.sessionId, runId, }) @@ -243,7 +241,7 @@ export function recordTranscriptMemoryPolicyInTransaction(params: { delivery_audiences_json: persisted.deliveryAudiencesJson, session_identity_revision: exposure.sessionIdentityRevision, subject_revision: exposure.subjectRevision, - run_id: exposure.runId, + run_id: exposure.durableRunScopeId, context_fingerprint: exposure.contextFingerprint, created_at: params.createdAt, } diff --git a/src/config/sessions/session-transcript-search.ts b/src/config/sessions/session-transcript-search.ts index 9afd692dee84..3bc43c5f05dc 100644 --- a/src/config/sessions/session-transcript-search.ts +++ b/src/config/sessions/session-transcript-search.ts @@ -6,6 +6,7 @@ import { openOpenClawAgentDatabase } from "../../state/openclaw-agent-db.js"; import { truncateUtf16Safe } from "../../utils.js"; import { resolveSqliteTargetFromSessionStorePath } from "./session-sqlite-target.js"; import { listSessionsNeedingTranscriptIndexReconcile } from "./session-transcript-index.js"; +import { isTranscriptMemoryPolicyEnforcedInDatabase } from "./session-transcript-memory-policy.js"; import { isSessionTranscriptIndexReconcileRunning, startSessionTranscriptIndexReconcile, @@ -78,6 +79,31 @@ export function searchSessionTranscripts(params: { sessionKeys.length > 0 ? ` AND session_windows.session_key IN (${sessionKeys.map(() => "?").join(", ")})` : ""; + // FTS has its own raw query, but it shares replay and projection's P1C companion boundary. + const whereAuthorizedTranscript = isTranscriptMemoryPolicyEnforcedInDatabase(database.db) + ? ` + AND EXISTS ( + SELECT 1 + FROM transcript_event_memory_policies AS policy + JOIN session_memory_subject_snapshots AS subject + ON subject.session_id = policy.session_id + JOIN memory_run_exposures AS exposure + ON exposure.exposure_set_id = policy.run_exposure_set_id + JOIN memory_policy_sets AS policy_set + ON policy_set.policy_set_id = policy.source_policy_set_id + WHERE policy.session_id = session_transcript_fts.session_id + AND policy.event_seq = identity.seq + AND policy.authorization_status = 'authorized' + AND subject.session_identity_revision = policy.session_identity_revision + AND subject.subject_revision = policy.subject_revision + AND exposure.run_id = policy.run_id + AND exposure.context_fingerprint = policy.context_fingerprint + AND exposure.revision_number = policy.run_exposure_revision + AND exposure.effective_source_policy_set_id = policy.source_policy_set_id + AND exposure.delivery_audiences_json = policy.delivery_audiences_json + AND policy_set.policy_set_id = exposure.effective_source_policy_set_id + )` + : ""; // MATCH, snippet(), and bm25() are FTS5 primitives without a Kysely // representation. session_key lives on the window row so key renames // never leave stale keys inside the index. Sessions flagged needs_rebuild @@ -94,37 +120,7 @@ export function searchSessionTranscripts(params: { JOIN transcript_event_identities AS identity ON identity.session_id = session_transcript_fts.session_id AND identity.event_id = session_transcript_fts.message_id - WHERE session_transcript_fts MATCH ?${whereSession} - AND ( - NOT EXISTS ( - SELECT 1 - FROM memory_migrations AS migration - WHERE migration.phase = 'cutover' - AND migration.verified_at IS NOT NULL - AND migration.cutover_at IS NOT NULL - ) - OR EXISTS ( - SELECT 1 - FROM transcript_event_memory_policies AS policy - JOIN session_memory_subject_snapshots AS subject - ON subject.session_id = policy.session_id - JOIN memory_run_exposures AS exposure - ON exposure.exposure_set_id = policy.run_exposure_set_id - JOIN memory_policy_sets AS policy_set - ON policy_set.policy_set_id = policy.source_policy_set_id - WHERE policy.session_id = session_transcript_fts.session_id - AND policy.event_seq = identity.seq - AND policy.authorization_status = 'authorized' - AND subject.session_identity_revision = policy.session_identity_revision - AND subject.subject_revision = policy.subject_revision - AND exposure.run_id = policy.run_id - AND exposure.context_fingerprint = policy.context_fingerprint - AND exposure.revision_number = policy.run_exposure_revision - AND exposure.effective_source_policy_set_id = policy.source_policy_set_id - AND exposure.delivery_audiences_json = policy.delivery_audiences_json - AND policy_set.policy_set_id = exposure.effective_source_policy_set_id - ) - ) + WHERE session_transcript_fts MATCH ?${whereSession}${whereAuthorizedTranscript} AND session_transcript_fts.session_id NOT IN ( SELECT session_id FROM session_transcript_index_state WHERE needs_rebuild != 0 ) diff --git a/src/context-engine/delegate.ts b/src/context-engine/delegate.ts index 32d2356b66d0..aa28c5c72f1d 100644 --- a/src/context-engine/delegate.ts +++ b/src/context-engine/delegate.ts @@ -256,6 +256,7 @@ export function buildMemorySystemPromptAddition( // its synchronous view, or an omitted flag would reopen those supplements. memoryReadEnforced: params.memoryReadEnforced ?? (prepared.context.memoryReadEnforced ? true : undefined), + authorizedMemoryRead: params.authorizedMemoryRead ?? prepared.context.authorizedMemoryRead, }; return renderMemorySystemPromptAddition(contextParams, prepared); } diff --git a/src/hooks/bundled/session-memory/handler-auto-reset.test.ts b/src/hooks/bundled/session-memory/handler-auto-reset.test.ts index d472914c6199..383e7a877ae6 100644 --- a/src/hooks/bundled/session-memory/handler-auto-reset.test.ts +++ b/src/hooks/bundled/session-memory/handler-auto-reset.test.ts @@ -9,6 +9,7 @@ import { createInternalHookEvent } from "../../internal-hooks.js"; const memoryIsolationMocks = vi.hoisted(() => ({ isMemoryIsolationCutoverAgent: vi.fn(() => false), + isMemoryIsolationTranscriptPolicyEnforcedInDatabase: vi.fn(() => false), })); vi.mock("../../../plugins/memory-cutover.js", () => memoryIsolationMocks); @@ -25,6 +26,9 @@ describe("session-memory automatic reset", () => { afterEach(async () => { memoryIsolationMocks.isMemoryIsolationCutoverAgent.mockReset().mockReturnValue(false); + memoryIsolationMocks.isMemoryIsolationTranscriptPolicyEnforcedInDatabase + .mockReset() + .mockReturnValue(false); await flushSessionMemoryWritesForTest(); }); diff --git a/src/plugins/memory-authorization-path-inventory.runtime-and-delivery.test-support.ts b/src/plugins/memory-authorization-path-inventory.runtime-and-delivery.test-support.ts index 5b078c9e2a37..915e227c95de 100644 --- a/src/plugins/memory-authorization-path-inventory.runtime-and-delivery.test-support.ts +++ b/src/plugins/memory-authorization-path-inventory.runtime-and-delivery.test-support.ts @@ -108,7 +108,7 @@ export const MEMORY_AUTHORIZATION_RUNTIME_AND_DELIVERY_PATH_ENTRIES = Object.fre "core-agent-runtime", "blocked-in-enforced-mode", "src/agents/embedded-agent-runner/compaction-checkpoint.ts", - "src/agents/main-session-restart-recovery-checkpoint.ts", + "src/agents/main-session-recovery/main-session-restart-recovery-checkpoint.ts", "src/gateway/session-compaction-checkpoints.ts", ), entry( @@ -193,7 +193,7 @@ export const MEMORY_AUTHORIZATION_RUNTIME_AND_DELIVERY_PATH_ENTRIES = Object.fre "egress", "core-agent-runtime", "blocked-in-enforced-mode", - "src/agents/subagent-registry.ts", + "src/agents/subagents/registry/subagent-registry.ts", "src/agents/openclaw-tools.ts", ), entry( @@ -201,7 +201,7 @@ export const MEMORY_AUTHORIZATION_RUNTIME_AND_DELIVERY_PATH_ENTRIES = Object.fre "egress", "core-agent-runtime", "blocked-in-enforced-mode", - "src/agents/subagent-announce-delivery.ts", + "src/agents/subagents/announce/subagent-announce-delivery.ts", ), entry( "cron-triggered-run", @@ -243,7 +243,7 @@ export const MEMORY_AUTHORIZATION_RUNTIME_AND_DELIVERY_PATH_ENTRIES = Object.fre "egress", "transport-egress-host", "blocked-in-enforced-mode", - "src/agents/tools/message-tool.ts", + "src/agents/tools/message-tool-execution.ts", ), entry( "session-send-delivery", diff --git a/src/plugins/memory-authorization-path-inventory.test-support.ts b/src/plugins/memory-authorization-path-inventory.test-support.ts index 29799a2b6935..fc3f851682f8 100644 --- a/src/plugins/memory-authorization-path-inventory.test-support.ts +++ b/src/plugins/memory-authorization-path-inventory.test-support.ts @@ -1,6 +1,6 @@ import { MEMORY_AUTHORIZATION_RUNTIME_AND_DELIVERY_PATH_ENTRIES } from "./memory-authorization-path-inventory.runtime-and-delivery.test-support.js"; -/** Test-only Phase-0 inventory of every known path that can ingest, expose, or derive memory. */ +/** Test-only inventory of every known path that can ingest, expose, or derive memory. */ export const MEMORY_AUTHORIZATION_PATH_DISPOSITIONS = [ "authorized", "blocked-in-enforced-mode", @@ -40,9 +40,9 @@ function entry( } /** - * `authorized` is intentionally absent in Phase 0: the rollout is shadow-only. Every path keeps - * its current legacy behavior until its owning phase converts it, and enforced mode then blocks - * the explicitly listed bypasses rather than silently falling through to context-free access. + * Phase 1C routes only the selected memory search/get/session lanes through the opaque host. + * Other legacy paths remain explicitly blocked in enforced mode or legacy-only until their owner + * converts them; none may silently fall through to context-free access. */ export const MEMORY_AUTHORIZATION_PATH_INVENTORY = Object.freeze([ entry( @@ -75,7 +75,7 @@ export const MEMORY_AUTHORIZATION_PATH_INVENTORY = Object.freeze([ "bootstrap-memory-and-user-files", "egress", "core-agent-runtime", - "legacy-only", + "blocked-in-enforced-mode", "src/agents/bootstrap-files.ts", "src/agents/workspace-bootstrap-read.ts", ), @@ -83,31 +83,37 @@ export const MEMORY_AUTHORIZATION_PATH_INVENTORY = Object.freeze([ "startup-recent-memory-context", "egress", "core-agent-runtime", - "legacy-only", + "blocked-in-enforced-mode", "src/auto-reply/reply/startup-context.ts", ), entry( "memory-search-tool", "egress", - "selected-memory-plugin", - "legacy-only", + "core-access-host", + "authorized", "extensions/memory-core/src/tools.ts", "extensions/memory-core/src/tools.shared.ts", + "src/agents/memory-authorized-read-host.ts", + "src/plugins/memory-invocation.ts", ), entry( "memory-get-tool", "egress", - "selected-memory-plugin", - "legacy-only", + "core-access-host", + "authorized", "extensions/memory-core/src/tools.ts", + "src/agents/memory-authorized-read-host.ts", + "src/plugins/memory-invocation.ts", ), entry( "session-transcript-search", "egress", - "selected-memory-plugin", - "legacy-only", + "core-access-host", + "authorized", "extensions/memory-core/src/session-search-visibility.ts", "extensions/memory-core/src/tools.ts", + "src/agents/memory-authorized-read-host.ts", + "src/plugins/memory-invocation.ts", ), entry( "active-memory-trigger-recall", diff --git a/src/plugins/memory-authorization-path-inventory.test.ts b/src/plugins/memory-authorization-path-inventory.test.ts index 36b3f999fb85..7d088cb5dc3b 100644 --- a/src/plugins/memory-authorization-path-inventory.test.ts +++ b/src/plugins/memory-authorization-path-inventory.test.ts @@ -96,6 +96,14 @@ const REQUIRED_PHASE_0_PATH_IDS = [ "plugin-and-mcp-outbound-actions", ] as const; +const PHASE_1C_SELECTED_READ_PATHS = { + "bootstrap-memory-and-user-files": "blocked-in-enforced-mode", + "startup-recent-memory-context": "blocked-in-enforced-mode", + "memory-search-tool": "authorized", + "memory-get-tool": "authorized", + "session-transcript-search": "authorized", +} as const; + const MEMORY_MANAGER_CALL_NAMES = new Set([ "getActiveMemorySearchManager", "getMemorySearchManager", @@ -481,8 +489,8 @@ describe("memory authorization path inventory", () => { } }); - it("keeps Phase 0 shadow-only and explicitly fails enforced bypasses closed", () => { - expect(inventory.filter((item) => item.disposition === "authorized")).toEqual([]); + it("keeps Phase 1C selected reads host-authorized and explicitly fails bypasses closed", () => { + expect(inventory.some((item) => item.disposition === "authorized")).toBe(true); expect(inventory.filter((item) => item.disposition === "operator-only-authenticated")).toEqual( [], ); @@ -490,6 +498,23 @@ describe("memory authorization path inventory", () => { expect(inventory.some((item) => item.disposition === "blocked-in-enforced-mode")).toBe(true); }); + it("records the exact Phase 1C selected-read and legacy-bypass dispositions", () => { + const entriesById = new Map(inventory.map((item) => [item.id, item])); + + for (const [id, disposition] of Object.entries(PHASE_1C_SELECTED_READ_PATHS)) { + expect(entriesById.get(id)).toMatchObject({ disposition }); + } + for (const id of ["memory-search-tool", "memory-get-tool", "session-transcript-search"]) { + expect(entriesById.get(id)).toMatchObject({ + owner: "core-access-host", + surfaces: expect.arrayContaining([ + "src/agents/memory-authorized-read-host.ts", + "src/plugins/memory-invocation.ts", + ]), + }); + } + }); + it("keeps supplemental reads and mutations as distinct enforced-mode paths", () => { const entriesById = new Map(inventory.map((item) => [item.id, item])); for (const [id, direction] of Object.entries(SUPPLEMENTAL_PATH_DIRECTIONS)) { diff --git a/src/plugins/memory-authorization-runtime.test.ts b/src/plugins/memory-authorization-runtime.test.ts index e36d09b61703..48c9e4131c25 100644 --- a/src/plugins/memory-authorization-runtime.test.ts +++ b/src/plugins/memory-authorization-runtime.test.ts @@ -1,10 +1,10 @@ import { describe, expect, it, vi } from "vitest"; -import { referenceMemoryAuthorizationConformanceAdapter } from "../memory-host-sdk/host/authorization-conformance.js"; +import { referenceMemoryAuthorizationConformanceAdapter } from "../plugin-sdk/memory-authorization-conformance.js"; import { COMPLETE_MEMORY_AUTHORIZATION_CAPABILITIES, LEGACY_MEMORY_AUTHORIZATION_CAPABILITIES, MEMORY_AUTHORIZATION_CAPABILITY_NAMES, -} from "../memory-host-sdk/host/authorization.js"; +} from "../plugin-sdk/memory-authorization.js"; import { admitMemoryAuthorizationReadRuntime, inspectMemoryAuthorizationCapability, diff --git a/src/plugins/memory-authorization-runtime.ts b/src/plugins/memory-authorization-runtime.ts index bb2f16d32b3e..95c158b4dbde 100644 --- a/src/plugins/memory-authorization-runtime.ts +++ b/src/plugins/memory-authorization-runtime.ts @@ -1,14 +1,14 @@ import { runMemoryAuthorizationConformanceSuite, type MemoryAuthorizationConformanceAdapter, -} from "../memory-host-sdk/host/authorization-conformance.js"; +} from "../plugin-sdk/memory-authorization-conformance.js"; import { MEMORY_AUTHORIZATION_CAPABILITY_NAMES, isMemoryAuthorizationCapabilities, listMissingMemoryAuthorizationCapabilities, type AuthorizedMemoryRuntime, type MemoryAuthorizationCapabilityName, -} from "../memory-host-sdk/host/authorization.js"; +} from "../plugin-sdk/memory-authorization.js"; const AUTHORIZED_MEMORY_RUNTIME_METHODS = [ "authorize", diff --git a/src/plugins/memory-cutover.test.ts b/src/plugins/memory-cutover.test.ts index ce8a7674b99c..4cbcd259f66a 100644 --- a/src/plugins/memory-cutover.test.ts +++ b/src/plugins/memory-cutover.test.ts @@ -41,6 +41,11 @@ describe("memory isolation lifecycle", () => { .run(params.sessionKey ?? "agent:main:pilot", params.principalId, randomUUID()); } + it("does not borrow an agent cutover state when the caller has no agent scope", () => { + expect(isMemoryIsolationCutoverAgent("")).toBe(false); + expect(isMemoryIsolationCutoverAgent(" ")).toBe(false); + }); + it("persists a verified shadow-read-only marker and activates it only after a cache reset", () => { insertPilotSubject({ principalId: "principal-alice" }); expect(resolveMemoryIsolationMode("main")).toBe("legacy"); diff --git a/src/plugins/memory-cutover.ts b/src/plugins/memory-cutover.ts index 574fb1f31539..2696de37eeb5 100644 --- a/src/plugins/memory-cutover.ts +++ b/src/plugins/memory-cutover.ts @@ -370,7 +370,10 @@ export function isMemoryIsolationSubjectAdmitted(params: { * memory. An unreadable authority store fails closed: selected-memory callers never use legacy. */ export function isMemoryIsolationCutoverAgent(agentIdInput: string): boolean { - return resolveMemoryIsolationMode(agentIdInput) !== "legacy"; + const agentId = agentIdInput.trim(); + // Tool construction can be intentionally unscoped (for example, local workspace utilities). + // It has no authority-store owner, so it must retain legacy behavior rather than borrow an agent. + return agentId ? resolveMemoryIsolationMode(agentId) !== "legacy" : false; } /** diff --git a/src/plugins/memory-invocation.enforced.test.ts b/src/plugins/memory-invocation.enforced.test.ts index 2996c7068b61..885c8a89fee7 100644 --- a/src/plugins/memory-invocation.enforced.test.ts +++ b/src/plugins/memory-invocation.enforced.test.ts @@ -3,20 +3,21 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { createAuthorizedMemoryReadHost } from "../agents/memory-authorized-read-host.js"; -import { referenceMemoryAuthorizationConformanceAdapter } from "../memory-host-sdk/host/authorization-conformance.js"; +import { referenceMemoryAuthorizationConformanceAdapter } from "../plugin-sdk/memory-authorization-conformance.js"; import { COMPLETE_MEMORY_AUTHORIZATION_CAPABILITIES, LEGACY_MEMORY_AUTHORIZATION_CAPABILITIES, type AuthorizedMemoryPlan, type AuthorizedMemorySearchResult, type MemoryContentAccessContext, -} from "../memory-host-sdk/host/authorization.js"; +} from "../plugin-sdk/memory-authorization.js"; import { ensureMemoryOperationalPrincipal } from "../state/memory-identity.js"; import { persistMemorySessionSubject } from "../state/memory-session-subject.js"; import { closeOpenClawAgentDatabasesForTest, openOpenClawAgentDatabase, } from "../state/openclaw-agent-db.js"; +import { ensureOpenClawAgentScopedMemorySchema } from "../state/openclaw-agent-scoped-memory-schema.js"; import { resetMemoryIsolationCutoverForTest } from "./memory-cutover.js"; import { MEMORY_INVOCATION_UNAVAILABLE } from "./memory-invocation.js"; import { createEmptyPluginRegistry } from "./registry-empty.js"; @@ -56,6 +57,7 @@ function createAuthorizedReadHost() { const sessionId = "memory-invocation-session"; const options = { agentId, env }; const database = openOpenClawAgentDatabase(options); + ensureOpenClawAgentScopedMemorySchema(database.db); database.db .prepare( "INSERT INTO session_nodes (session_key, current_session_id, entry_json, updated_at) VALUES (?, ?, '{}', 1)", @@ -97,7 +99,7 @@ function createAuthorizedReadHost() { if (!host) { throw new Error("failed to create authorized memory read host"); } - return host; + return { database, host }; } function registerSelectedCapability(capability: unknown) { @@ -111,7 +113,7 @@ function registerSelectedCapability(capability: unknown) { } function createRuntime(params: { - searchAuthorized?: () => Promise; + searchAuthorized?: (context: MemoryContentAccessContext<"read">) => Promise; authorize?: (context: MemoryContentAccessContext<"read">) => Promise; }) { const legacySearch = vi.fn(); @@ -120,7 +122,8 @@ function createRuntime(params: { runtime: { authorize: async (context: MemoryContentAccessContext<"read">) => await (params.authorize?.(context) ?? Promise.resolve(createPlan(context))), - searchAuthorized: async () => await params.searchAuthorized?.(), + searchAuthorized: async ({ context }: { context: MemoryContentAccessContext<"read"> }) => + await params.searchAuthorized?.(context), readAuthorized: async () => { throw new Error("exact read must not execute in this test"); }, @@ -191,7 +194,7 @@ describe("enforced selected-memory invocation failures", () => { authorizationConformance: referenceMemoryAuthorizationConformanceAdapter, runtime, }); - const result = await createAuthorizedReadHost().search({ query: "private" }); + const result = await createAuthorizedReadHost().host.search({ query: "private" }); expect(result).toBe(MEMORY_INVOCATION_UNAVAILABLE); expect(JSON.stringify(result)).not.toContain("private legacy content"); @@ -220,9 +223,95 @@ describe("enforced selected-memory invocation failures", () => { } registerSelectedCapability(capability); - await expect(createAuthorizedReadHost().search({ query: "private" })).resolves.toBe( + await expect(createAuthorizedReadHost().host.search({ query: "private" })).resolves.toBe( MEMORY_INVOCATION_UNAVAILABLE, ); expect(legacySearch).not.toHaveBeenCalled(); }); + + it("commits the content-free exposure ledger before returning a selected-plugin result", async () => { + const { runtime } = createRuntime({ + searchAuthorized: async (context) => ({ + version: 1, + value: [ + { + path: "private/note.md", + startLine: 1, + endLine: 1, + score: 1, + snippet: "allowed text", + source: "memory", + resourceHandle: { + version: 1, + handleId: "handle-1", + planId: "plan-1", + contextFingerprint: context.contextFingerprint, + resourceRevision: "revision-1", + policyRevision: "policy-1", + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }, + }, + ], + exposureReceipt: { + version: 1, + receiptId: "exposure-1", + contextFingerprint: context.contextFingerprint, + planId: "plan-1", + runId: context.runId, + runExposureRevision: "run-exposure-1", + sourcePolicySetId: "source-policy-1", + exposedRevisionHandles: ["revision-1"], + recordedAt: new Date().toISOString(), + }, + egressReceipt: { + version: 1, + receiptId: "egress-1", + contextFingerprint: context.contextFingerprint, + planId: "plan-1", + runId: context.runId, + runExposureRevision: "run-exposure-1", + sourcePolicySetId: "source-policy-1", + allowedAudiences: context.delivery.audiences, + deliveryRevision: context.delivery.deliveryRevision, + egressRegistryRevision: context.delivery.egressRegistryRevision, + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }, + }), + }); + registerSelectedCapability({ + authorization: COMPLETE_MEMORY_AUTHORIZATION_CAPABILITIES, + authorizationConformance: referenceMemoryAuthorizationConformanceAdapter, + runtime, + }); + const { database, host } = createAuthorizedReadHost(); + + await expect(host.search({ query: "private" })).resolves.toEqual({ + results: [ + { + handleId: "handle-1", + path: "private/note.md", + startLine: 1, + endLine: 1, + score: 1, + snippet: "allowed text", + source: "memory", + }, + ], + }); + expect( + database.db + .prepare( + `SELECT session_id, run_id, revision_number, exposure_receipt_ids_json + FROM memory_preoutput_exposure_ledger`, + ) + .all(), + ).toEqual([ + { + session_id: "memory-invocation-session", + run_id: "run-1", + revision_number: 1, + exposure_receipt_ids_json: '["exposure-1"]', + }, + ]); + }); }); diff --git a/src/plugins/memory-invocation.test.ts b/src/plugins/memory-invocation.test.ts index ca90013cf114..25929115b941 100644 --- a/src/plugins/memory-invocation.test.ts +++ b/src/plugins/memory-invocation.test.ts @@ -1,15 +1,18 @@ -import { describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { COMPLETE_MEMORY_AUTHORIZATION_CAPABILITIES, type AuthorizedMemoryPlan, type AuthorizedMemoryResultEnvelope, type AuthorizedMemorySearchResult, type MemoryAccessContext, -} from "../memory-host-sdk/host/authorization.js"; +} from "../plugin-sdk/memory-authorization.js"; const mocks = vi.hoisted(() => ({ admit: vi.fn(), materialize: vi.fn(), + hydrateExposure: vi.fn(() => true), + logWarn: vi.fn(), + persistExposure: vi.fn(() => true), })); vi.mock("../state/memory-access-context.js", async (importOriginal) => ({ @@ -21,12 +24,23 @@ vi.mock("./memory-authorization-runtime.js", () => ({ admitMemoryAuthorizationReadRuntime: mocks.admit, })); +vi.mock("./memory-run-exposure-ledger.js", () => ({ + hydrateMemoryRunExposureFromLedger: mocks.hydrateExposure, + persistMemoryRunExposureBeforeContent: mocks.persistExposure, +})); + +vi.mock("../logger.js", () => ({ + logWarn: mocks.logWarn, +})); + const { MEMORY_INVOCATION_UNAVAILABLE, createAuthorizedMemoryReadInvocation, readAuthorizedMemoryForInvocation, + readAuthorizedMemoryRunExposure, searchAuthorizedMemoryForInvocation, } = await import("./memory-invocation.js"); +const { clearMemoryRunExposureForTest } = await import("./memory-run-exposure.js"); function createContext(): MemoryAccessContext & Readonly<{ operation: "read" }> { return { @@ -133,6 +147,16 @@ function createEnvelope( } describe("authorized memory read invocation", () => { + afterEach(() => { + clearMemoryRunExposureForTest(); + receiptSequence = 0; + mocks.hydrateExposure.mockReset(); + mocks.hydrateExposure.mockReturnValue(true); + mocks.logWarn.mockReset(); + mocks.persistExposure.mockReset(); + mocks.persistExposure.mockReturnValue(true); + }); + it("returns only an unavailable result when backend admission fails", async () => { mocks.materialize.mockReturnValue(createContext()); mocks.admit.mockResolvedValue({ ok: false, reasonCode: "backend-nonconforming" }); @@ -145,6 +169,29 @@ describe("authorized memory read invocation", () => { ).resolves.toBe(MEMORY_INVOCATION_UNAVAILABLE); }); + it("emits fixed diagnostics without memory access facts or backend error content", async () => { + mocks.materialize.mockReturnValue(createContext()); + mocks.admit.mockResolvedValue({ + ok: true, + runtime: { + authorize: vi.fn().mockRejectedValue(new Error("private memory error")), + }, + }); + + await expect( + createAuthorizedMemoryReadInvocation({ + context: {} as never, + capability: { authorization: COMPLETE_MEMORY_AUTHORIZATION_CAPABILITIES }, + }), + ).resolves.toBe(MEMORY_INVOCATION_UNAVAILABLE); + + expect(mocks.logWarn).toHaveBeenCalledWith( + "memory invocation unavailable: authorization-failed", + ); + expect(JSON.stringify(mocks.logWarn.mock.calls)).not.toContain("private memory error"); + expect(JSON.stringify(mocks.logWarn.mock.calls)).not.toContain("alice"); + }); + it("does not leak a search result unless its current exposure and egress receipts validate", async () => { const handle = { version: 1 as const, @@ -217,6 +264,236 @@ describe("authorized memory read invocation", () => { ).resolves.toBe(MEMORY_INVOCATION_UNAVAILABLE); }); + it("does not return selected-plugin content or commit a receipt when exposure recording fails", async () => { + const result: AuthorizedMemorySearchResult = { + path: "private/note.md", + startLine: 1, + endLine: 1, + score: 1, + snippet: "private content", + source: "memory", + resourceHandle: { + version: 1, + handleId: "handle-1", + planId: "plan-1", + contextFingerprint: "fingerprint-1", + resourceRevision: "revision-1", + policyRevision: "policy-1", + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }, + }; + const runtime = { + authorize: vi.fn().mockResolvedValue(createPlan()), + searchAuthorized: vi.fn().mockResolvedValue(createEnvelope([result])), + readAuthorized: vi.fn(), + }; + mocks.materialize.mockReturnValue(createContext()); + mocks.admit.mockResolvedValue({ ok: true, runtime }); + mocks.persistExposure.mockReturnValue(false); + + const invocation = await createAuthorizedMemoryReadInvocation({ + context: {} as never, + capability: { authorization: COMPLETE_MEMORY_AUTHORIZATION_CAPABILITIES }, + }); + expect(invocation).not.toBe(MEMORY_INVOCATION_UNAVAILABLE); + if (invocation === MEMORY_INVOCATION_UNAVAILABLE) { + return; + } + + const response = await searchAuthorizedMemoryForInvocation({ invocation, query: "private" }); + + expect(response).toBe(MEMORY_INVOCATION_UNAVAILABLE); + expect(JSON.stringify(response)).not.toContain("private content"); + expect(mocks.persistExposure).toHaveBeenCalledOnce(); + expect(readAuthorizedMemoryRunExposure(invocation)).toEqual({ + sourcePolicySetIds: [], + exposedRevisionHandles: [], + exposureReceiptIds: [], + egressReceiptIds: [], + }); + }); + + it("does not return exact-read text or commit its receipt when the ledger write fails", async () => { + const handle = { + version: 1 as const, + handleId: "handle-1", + planId: "plan-1", + contextFingerprint: "fingerprint-1", + resourceRevision: "revision-1", + policyRevision: "policy-1", + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }; + const runtime = { + authorize: vi.fn().mockResolvedValue(createPlan()), + searchAuthorized: vi.fn().mockResolvedValue( + createEnvelope([ + { + path: "private/note.md", + startLine: 1, + endLine: 1, + score: 1, + snippet: "search text", + source: "memory", + resourceHandle: handle, + }, + ]), + ), + readAuthorized: vi + .fn() + .mockResolvedValue(createEnvelope({ path: "private/note.md", text: "private exact text" })), + }; + mocks.materialize.mockReturnValue(createContext()); + mocks.admit.mockResolvedValue({ ok: true, runtime }); + + const invocation = await createAuthorizedMemoryReadInvocation({ + context: {} as never, + capability: { authorization: COMPLETE_MEMORY_AUTHORIZATION_CAPABILITIES }, + }); + expect(invocation).not.toBe(MEMORY_INVOCATION_UNAVAILABLE); + if (invocation === MEMORY_INVOCATION_UNAVAILABLE) { + return; + } + await expect( + searchAuthorizedMemoryForInvocation({ invocation, query: "private" }), + ).resolves.toEqual( + expect.objectContaining({ results: [expect.objectContaining({ handleId: "handle-1" })] }), + ); + mocks.persistExposure.mockReturnValue(false); + + const response = await readAuthorizedMemoryForInvocation({ invocation, handleId: "handle-1" }); + + expect(response).toBe(MEMORY_INVOCATION_UNAVAILABLE); + expect(JSON.stringify(response)).not.toContain("private exact text"); + expect(readAuthorizedMemoryRunExposure(invocation)).toEqual({ + sourcePolicySetIds: ["policy-set-1"], + exposedRevisionHandles: ["revision-1"], + exposureReceiptIds: ["exposure-1"], + egressReceiptIds: ["egress-1"], + }); + }); + + it("allows a fresh exact-read receipt after a failed ledger write", async () => { + const handle = { + version: 1 as const, + handleId: "handle-1", + planId: "plan-1", + contextFingerprint: "fingerprint-1", + resourceRevision: "revision-1", + policyRevision: "policy-1", + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }; + const runtime = { + authorize: vi.fn().mockResolvedValue(createPlan()), + searchAuthorized: vi.fn().mockResolvedValue( + createEnvelope([ + { + path: "private/note.md", + startLine: 1, + endLine: 1, + score: 1, + snippet: "search text", + source: "memory", + resourceHandle: handle, + }, + ]), + ), + readAuthorized: vi + .fn() + .mockImplementation(async () => + createEnvelope({ path: "private/note.md", text: "private exact text" }), + ), + }; + mocks.materialize.mockReturnValue(createContext()); + mocks.admit.mockResolvedValue({ ok: true, runtime }); + + const invocation = await createAuthorizedMemoryReadInvocation({ + context: {} as never, + capability: { authorization: COMPLETE_MEMORY_AUTHORIZATION_CAPABILITIES }, + }); + expect(invocation).not.toBe(MEMORY_INVOCATION_UNAVAILABLE); + if (invocation === MEMORY_INVOCATION_UNAVAILABLE) { + return; + } + await expect( + searchAuthorizedMemoryForInvocation({ invocation, query: "private" }), + ).resolves.toEqual( + expect.objectContaining({ results: [expect.objectContaining({ handleId: "handle-1" })] }), + ); + mocks.persistExposure.mockReturnValueOnce(false).mockReturnValue(true); + + await expect( + readAuthorizedMemoryForInvocation({ invocation, handleId: "handle-1" }), + ).resolves.toBe(MEMORY_INVOCATION_UNAVAILABLE); + await expect( + readAuthorizedMemoryForInvocation({ invocation, handleId: "handle-1" }), + ).resolves.toEqual({ path: "private/note.md", text: "private exact text" }); + expect(readAuthorizedMemoryRunExposure(invocation)).toEqual({ + sourcePolicySetIds: ["policy-set-1"], + exposedRevisionHandles: ["revision-1"], + exposureReceiptIds: ["exposure-1", "exposure-3"], + egressReceiptIds: ["egress-1", "egress-3"], + }); + }); + + it("allows a fresh receipt after a failed ledger write without retaining the failed attempt", async () => { + const result = (snippet: string, handleId: string): AuthorizedMemorySearchResult => ({ + path: "private/note.md", + startLine: 1, + endLine: 1, + score: 1, + snippet, + source: "memory", + resourceHandle: { + version: 1, + handleId, + planId: "plan-1", + contextFingerprint: "fingerprint-1", + resourceRevision: "revision-1", + policyRevision: "policy-1", + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }, + }); + const runtime = { + authorize: vi.fn().mockResolvedValue(createPlan()), + searchAuthorized: vi + .fn() + .mockResolvedValueOnce(createEnvelope([result("first private text", "handle-1")])) + .mockResolvedValueOnce(createEnvelope([result("retry text", "handle-2")])), + readAuthorized: vi.fn(), + }; + mocks.materialize.mockReturnValue(createContext()); + mocks.admit.mockResolvedValue({ ok: true, runtime }); + mocks.persistExposure.mockReturnValueOnce(false).mockReturnValue(true); + + const invocation = await createAuthorizedMemoryReadInvocation({ + context: {} as never, + capability: { authorization: COMPLETE_MEMORY_AUTHORIZATION_CAPABILITIES }, + }); + expect(invocation).not.toBe(MEMORY_INVOCATION_UNAVAILABLE); + if (invocation === MEMORY_INVOCATION_UNAVAILABLE) { + return; + } + await expect(searchAuthorizedMemoryForInvocation({ invocation, query: "first" })).resolves.toBe( + MEMORY_INVOCATION_UNAVAILABLE, + ); + await expect( + searchAuthorizedMemoryForInvocation({ invocation, query: "retry" }), + ).resolves.toEqual({ + results: [ + expect.objectContaining({ + handleId: "handle-2", + snippet: "retry text", + }), + ], + }); + expect(readAuthorizedMemoryRunExposure(invocation)).toEqual({ + sourcePolicySetIds: ["policy-set-1"], + exposedRevisionHandles: ["revision-1"], + exposureReceiptIds: ["exposure-2"], + egressReceiptIds: ["egress-2"], + }); + }); + it.each([ { name: "has an invalid timestamp", @@ -250,6 +527,38 @@ describe("authorized memory read invocation", () => { ).resolves.toBe(MEMORY_INVOCATION_UNAVAILABLE); }); + it("does not leak a search result from an unsupported result-envelope version", async () => { + const runtime = { + authorize: vi.fn().mockResolvedValue(createPlan()), + searchAuthorized: vi.fn().mockResolvedValue({ + ...createEnvelope([]), + version: 2, + }), + readAuthorized: vi.fn(), + }; + mocks.materialize.mockReturnValue(createContext()); + mocks.admit.mockResolvedValue({ ok: true, runtime }); + + const invocation = await createAuthorizedMemoryReadInvocation({ + context: {} as never, + capability: { authorization: COMPLETE_MEMORY_AUTHORIZATION_CAPABILITIES }, + }); + expect(invocation).not.toBe(MEMORY_INVOCATION_UNAVAILABLE); + if (invocation === MEMORY_INVOCATION_UNAVAILABLE) { + return; + } + + await expect( + searchAuthorizedMemoryForInvocation({ invocation, query: "private" }), + ).resolves.toBe(MEMORY_INVOCATION_UNAVAILABLE); + expect(readAuthorizedMemoryRunExposure(invocation)).toEqual({ + sourcePolicySetIds: [], + exposedRevisionHandles: [], + exposureReceiptIds: [], + egressReceiptIds: [], + }); + }); + it("rejects a replayed exposure receipt instead of exposing the repeated result", async () => { const handle = { version: 1 as const, diff --git a/src/plugins/memory-invocation.ts b/src/plugins/memory-invocation.ts index ed4641298e76..a84e99894e5f 100644 --- a/src/plugins/memory-invocation.ts +++ b/src/plugins/memory-invocation.ts @@ -1,3 +1,4 @@ +import { logWarn } from "../logger.js"; import type { AuthorizedMemoryPlan, AuthorizedMemoryResultEnvelope, @@ -18,6 +19,11 @@ import { admitMemoryAuthorizationReadRuntime, type AdmittedAuthorizedMemoryReadRuntime, } from "./memory-authorization-runtime.js"; +import { + hydrateMemoryRunExposureFromLedger, + persistMemoryRunExposureBeforeContent, +} from "./memory-run-exposure-ledger.js"; +import { prepareMemoryRunExposure, publishMemoryRunExposure } from "./memory-run-exposure.js"; import { resolveSelectedMemoryCapabilityRegistration } from "./memory-state.js"; import type { MemoryPluginCapability } from "./registry-contribution-types.js"; import { requireActivePluginRegistry } from "./runtime.js"; @@ -56,6 +62,19 @@ type InvocationState = Readonly<{ const invocationStates = new WeakMap(); +type MemoryInvocationDiagnostic = + | "admission-rejected" + | "authorization-failed" + | "invalid-plan" + | "materialization-rejected" + | "search-failed"; + +function logMemoryInvocationDiagnostic(diagnostic: MemoryInvocationDiagnostic): void { + // Memory content, access facts, capability metadata, plans, and backend errors are all sensitive. + // Keep the emitted diagnostic low-cardinality and content-free; the unavailable result is intentional. + logWarn(`memory invocation unavailable: ${diagnostic}`); +} + function sameAudiences(left: readonly AudienceRef[], right: readonly AudienceRef[]): boolean { const key = (audience: AudienceRef) => `${audience.kind}\u0000${audience.id}`; const leftKeys = [...new Set(left.map(key))].toSorted(); @@ -112,7 +131,7 @@ function readCurrentContext( return readContext; } -function mergeAndValidateEnvelope(params: { +function validateEnvelope(params: { state: InvocationState; context: MemoryContentAccessContext<"read">; expectedRevisionHandles: readonly string[]; @@ -124,6 +143,7 @@ function mergeAndValidateEnvelope(params: { const exposureRecordedAt = Date.parse(exposureReceipt.recordedAt); const egressExpiry = Date.parse(egressReceipt.expiresAt); if ( + envelope.version !== 1 || exposureReceipt.version !== 1 || egressReceipt.version !== 1 || exposureReceipt.contextFingerprint !== context.contextFingerprint || @@ -161,6 +181,14 @@ function mergeAndValidateEnvelope(params: { ) { return false; } + return true; +} + +function mergeEnvelope( + state: InvocationState, + envelope: AuthorizedMemoryResultEnvelope, +): void { + const { exposureReceipt, egressReceipt } = envelope; state.sourcePolicySetIds.add(exposureReceipt.sourcePolicySetId); for (const revision of exposureReceipt.exposedRevisionHandles) { state.exposedRevisionHandles.add(revision); @@ -168,7 +196,76 @@ function mergeAndValidateEnvelope(params: { state.exposureReceiptIds.add(exposureReceipt.receiptId); state.egressReceiptIds.add(egressReceipt.receiptId); state.runExposureRevisions.add(exposureReceipt.runExposureRevision); - return true; +} + +function readTranscriptExposure(params: { + state: InvocationState; + context: MemoryContentAccessContext<"read">; + pendingEnvelope?: AuthorizedMemoryResultEnvelope; +}) { + const { state, context, pendingEnvelope } = params; + const sourcePolicySetIds = new Set(state.sourcePolicySetIds); + const exposedResourceRevisions = new Set(state.exposedRevisionHandles); + const exposureReceiptIds = new Set(state.exposureReceiptIds); + const egressReceiptIds = new Set(state.egressReceiptIds); + if (pendingEnvelope) { + const { exposureReceipt, egressReceipt } = pendingEnvelope; + sourcePolicySetIds.add(exposureReceipt.sourcePolicySetId); + for (const revision of exposureReceipt.exposedRevisionHandles) { + exposedResourceRevisions.add(revision); + } + exposureReceiptIds.add(exposureReceipt.receiptId); + egressReceiptIds.add(egressReceipt.receiptId); + } + return Object.freeze({ + agentId: context.agentId, + sessionId: context.sessionId, + sessionKey: context.sessionKey, + runId: context.runId, + contextFingerprint: context.contextFingerprint, + planId: state.plan.planId, + memoryPolicyRevision: state.plan.memoryPolicyRevision, + sourcePolicySetIds: Object.freeze([...sourcePolicySetIds].toSorted()), + exposedResourceRevisions: Object.freeze([...exposedResourceRevisions].toSorted()), + exposureReceiptIds: Object.freeze([...exposureReceiptIds].toSorted()), + egressReceiptIds: Object.freeze([...egressReceiptIds].toSorted()), + deliveryAudiences: Object.freeze([...context.delivery.audiences]), + deliveryRevision: context.delivery.deliveryRevision, + egressRegistryRevision: context.delivery.egressRegistryRevision, + sessionIdentityRevision: context.sessionIdentityRevision, + subjectRevision: context.subjectRevision, + }); +} + +/** + * Records the next immutable exposure revision before the selected plugin's content can leave + * this broker. A recording failure leaves the invocation state unchanged and fails the read closed. + */ +function recordEnvelopeExposure(params: { + state: InvocationState; + context: MemoryContentAccessContext<"read">; + envelope: AuthorizedMemoryResultEnvelope; +}): void { + if ( + !hydrateMemoryRunExposureFromLedger({ + agentId: params.context.agentId, + sessionId: params.context.sessionId, + runId: params.context.runId, + }) + ) { + throw new Error("memory exposure ledger could not restore its durable tail"); + } + const snapshot = prepareMemoryRunExposure( + readTranscriptExposure({ + state: params.state, + context: params.context, + pendingEnvelope: params.envelope, + }), + ); + if (!persistMemoryRunExposureBeforeContent(snapshot) || !publishMemoryRunExposure(snapshot)) { + throw new Error("memory exposure ledger did not commit before content release"); + } + mergeEnvelope(params.state, params.envelope); } function readState(invocation: AuthorizedMemoryReadInvocation): InvocationState | undefined { @@ -185,6 +282,7 @@ export async function createAuthorizedMemoryReadInvocation(params: { }): Promise { const materialized = materializeTrustedMemoryAccessContext(params.context); if (!materialized || materialized.operation !== "read") { + logMemoryInvocationDiagnostic("materialization-rejected"); return MEMORY_INVOCATION_UNAVAILABLE; } const context = materialized as MemoryContentAccessContext<"read">; @@ -193,6 +291,7 @@ export async function createAuthorizedMemoryReadInvocation(params: { resolveSelectedMemoryCapabilityRegistration(requireActivePluginRegistry())?.capability; const admission = await admitMemoryAuthorizationReadRuntime(capability); if (!admission.ok) { + logMemoryInvocationDiagnostic("admission-rejected"); return MEMORY_INVOCATION_UNAVAILABLE; } try { @@ -200,6 +299,7 @@ export async function createAuthorizedMemoryReadInvocation(params: { const plan = (await admission.runtime.authorize(context)) as AuthorizedMemoryPlan & Readonly<{ operation: "read" }>; if (!isCurrentPlan({ context, plan, nowMs: Date.now() })) { + logMemoryInvocationDiagnostic("invalid-plan"); return MEMORY_INVOCATION_UNAVAILABLE; } const invocation = Object.freeze({}) as AuthorizedMemoryReadInvocation; @@ -221,6 +321,7 @@ export async function createAuthorizedMemoryReadInvocation(params: { ); return invocation; } catch { + logMemoryInvocationDiagnostic("authorization-failed"); return MEMORY_INVOCATION_UNAVAILABLE; } } @@ -253,7 +354,7 @@ export async function searchAuthorizedMemoryForInvocation(params: { }); const revisionHandles = envelope.value.map((result) => result.resourceHandle.resourceRevision); if ( - !mergeAndValidateEnvelope({ + !validateEnvelope({ state, context, expectedRevisionHandles: revisionHandles, @@ -262,6 +363,7 @@ export async function searchAuthorizedMemoryForInvocation(params: { ) { return MEMORY_INVOCATION_UNAVAILABLE; } + recordEnvelopeExposure({ state, context, envelope }); const results = envelope.value.map((result) => { state.handles.set(result.resourceHandle.handleId, result.resourceHandle); const { resourceHandle: _resourceHandle, ...safe } = result; @@ -269,6 +371,7 @@ export async function searchAuthorizedMemoryForInvocation(params: { }); return Object.freeze({ results: Object.freeze(results) }); } catch { + logMemoryInvocationDiagnostic("search-failed"); return MEMORY_INVOCATION_UNAVAILABLE; } } @@ -299,7 +402,7 @@ export async function readAuthorizedMemoryForInvocation(params: { ...(params.lines !== undefined ? { lines: params.lines } : {}), }); if ( - !mergeAndValidateEnvelope({ + !validateEnvelope({ state, context, expectedRevisionHandles: [handle.resourceRevision], @@ -308,6 +411,7 @@ export async function readAuthorizedMemoryForInvocation(params: { ) { return MEMORY_INVOCATION_UNAVAILABLE; } + recordEnvelopeExposure({ state, context, envelope }); return Object.freeze({ ...envelope.value }); } catch { return MEMORY_INVOCATION_UNAVAILABLE; @@ -361,22 +465,5 @@ export function readAuthorizedMemoryTranscriptExposure(invocation: AuthorizedMem if (!state || !context || !isCurrentPlan({ context, plan: state.plan, nowMs: Date.now() })) { return undefined; } - return Object.freeze({ - agentId: context.agentId, - sessionId: context.sessionId, - sessionKey: context.sessionKey, - runId: context.runId, - contextFingerprint: context.contextFingerprint, - planId: state.plan.planId, - memoryPolicyRevision: state.plan.memoryPolicyRevision, - sourcePolicySetIds: Object.freeze([...state.sourcePolicySetIds].toSorted()), - exposedResourceRevisions: Object.freeze([...state.exposedRevisionHandles].toSorted()), - exposureReceiptIds: Object.freeze([...state.exposureReceiptIds].toSorted()), - egressReceiptIds: Object.freeze([...state.egressReceiptIds].toSorted()), - deliveryAudiences: Object.freeze([...context.delivery.audiences]), - deliveryRevision: context.delivery.deliveryRevision, - egressRegistryRevision: context.delivery.egressRegistryRevision, - sessionIdentityRevision: context.sessionIdentityRevision, - subjectRevision: context.subjectRevision, - }); + return readTranscriptExposure({ state, context }); } diff --git a/src/plugins/memory-run-exposure-ledger.test.ts b/src/plugins/memory-run-exposure-ledger.test.ts new file mode 100644 index 000000000000..47553daab167 --- /dev/null +++ b/src/plugins/memory-run-exposure-ledger.test.ts @@ -0,0 +1,256 @@ +import { DatabaseSync } from "node:sqlite"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + database: undefined as + | { + agentId: string; + db: DatabaseSync; + path: string; + walMaintenance: never; + } + | undefined, + logWarn: vi.fn(), +})); + +vi.mock("../state/openclaw-agent-db.js", () => ({ + openOpenClawAgentDatabase: () => { + if (!mocks.database) { + throw new Error("test database is unavailable"); + } + return mocks.database; + }, +})); + +vi.mock("../logger.js", () => ({ + logWarn: mocks.logWarn, +})); + +const { + hydrateMemoryRunExposureFromLedger, + persistMemoryRunExposureBeforeContent, + readDurableMemoryRunExposure, +} = await import("./memory-run-exposure-ledger.js"); +const { clearMemoryRunExposureForTest, prepareMemoryRunExposure } = + await import("./memory-run-exposure.js"); + +let database: DatabaseSync | undefined; + +beforeEach(() => { + database = new DatabaseSync(":memory:"); + mocks.database = { + agentId: "main", + db: database, + path: ":memory:", + walMaintenance: undefined as never, + }; +}); + +afterEach(() => { + clearMemoryRunExposureForTest(); + mocks.database = undefined; + database?.close(); + database = undefined; +}); + +function prepare(sessionId: string) { + return prepareMemoryRunExposure({ + agentId: "main", + sessionId, + sessionKey: `agent:main:direct:${sessionId}`, + runId: "shared-run-id", + contextFingerprint: `fingerprint:${sessionId}`, + planId: `plan:${sessionId}`, + memoryPolicyRevision: "policy-1", + sourcePolicySetIds: ["source-policy-1"], + exposedResourceRevisions: ["revision-1"], + exposureReceiptIds: ["exposure-1"], + egressReceiptIds: ["egress-1"], + deliveryAudiences: [{ kind: "user", id: "alice" }], + deliveryRevision: "delivery-1", + egressRegistryRevision: "egress-1", + sessionIdentityRevision: "identity-1", + subjectRevision: "subject-1", + }); +} + +describe("memory pre-output exposure ledger", () => { + it("commits content-free rows before publication and separates the same raw run across sessions", () => { + const first = prepare("session-a"); + const second = prepare("session-b"); + + expect(persistMemoryRunExposureBeforeContent(first)).toBe(true); + expect(persistMemoryRunExposureBeforeContent(second)).toBe(true); + expect(first.durableRunScopeId).not.toBe(second.durableRunScopeId); + + expect( + database + ?.prepare( + `SELECT session_id, run_id, revision_number, exposure_set_id, + source_policy_set_ids_json, exposed_resource_revisions_json + FROM memory_preoutput_exposure_ledger + ORDER BY session_id`, + ) + .all(), + ).toEqual([ + { + session_id: "session-a", + run_id: "shared-run-id", + revision_number: 1, + exposure_set_id: first.exposureSetId, + source_policy_set_ids_json: '["source-policy-1"]', + exposed_resource_revisions_json: '["revision-1"]', + }, + { + session_id: "session-b", + run_id: "shared-run-id", + revision_number: 1, + exposure_set_id: second.exposureSetId, + source_policy_set_ids_json: '["source-policy-1"]', + exposed_resource_revisions_json: '["revision-1"]', + }, + ]); + }); + + it("fails closed on a duplicate revision without adding a partial row", () => { + const snapshot = prepare("session-a"); + + expect(persistMemoryRunExposureBeforeContent(snapshot)).toBe(true); + expect(persistMemoryRunExposureBeforeContent(snapshot)).toBe(false); + expect( + database?.prepare("SELECT count(*) AS count FROM memory_preoutput_exposure_ledger").get(), + ).toEqual({ count: 1 }); + }); + + it("emits a fixed diagnostic without a caught database error", () => { + const snapshot = prepare("session-a"); + mocks.database = undefined; + + expect(persistMemoryRunExposureBeforeContent(snapshot)).toBe(false); + + expect(mocks.logWarn).toHaveBeenCalledWith( + "memory exposure ledger unavailable: persist-failed", + ); + expect(JSON.stringify(mocks.logWarn.mock.calls)).not.toContain("test database is unavailable"); + }); + + it("rehydrates a durable tail after restart before advancing the same run", () => { + const first = prepare("session-a"); + expect(persistMemoryRunExposureBeforeContent(first)).toBe(true); + + clearMemoryRunExposureForTest(); + expect( + hydrateMemoryRunExposureFromLedger({ + agentId: "main", + sessionId: "session-a", + runId: "shared-run-id", + }), + ).toBe(true); + const second = prepare("session-a"); + expect(second.revisionNumber).toBe(2); + expect(second.previous?.exposureSetId).toBe(first.exposureSetId); + expect(persistMemoryRunExposureBeforeContent(second)).toBe(true); + + clearMemoryRunExposureForTest(); + const durable = readDurableMemoryRunExposure({ + database: mocks.database as never, + sessionId: "session-a", + runId: "shared-run-id", + }); + expect(durable).toMatchObject({ + exposureSetId: second.exposureSetId, + revisionNumber: 2, + previous: { exposureSetId: first.exposureSetId, revisionNumber: 1 }, + }); + }); + + it("clears an old state root's process tail before starting the same run in a fresh ledger", () => { + const staleStateDatabase = database as DatabaseSync; + const staleSnapshot = prepare("session-a"); + expect(persistMemoryRunExposureBeforeContent(staleSnapshot)).toBe(true); + expect( + hydrateMemoryRunExposureFromLedger({ + agentId: "main", + sessionId: "session-a", + runId: "shared-run-id", + }), + ).toBe(true); + + const freshStateDatabase = new DatabaseSync(":memory:"); + database = freshStateDatabase; + mocks.database = { + agentId: "main", + db: freshStateDatabase, + path: ":memory:", + walMaintenance: undefined as never, + }; + expect( + hydrateMemoryRunExposureFromLedger({ + agentId: "main", + sessionId: "session-a", + runId: "shared-run-id", + }), + ).toBe(true); + + const freshSnapshot = prepare("session-a"); + expect(freshSnapshot.revisionNumber).toBe(1); + expect(persistMemoryRunExposureBeforeContent(freshSnapshot)).toBe(true); + expect( + freshStateDatabase + .prepare( + `SELECT revision_number FROM memory_preoutput_exposure_ledger + WHERE agent_id = 'main' AND session_id = 'session-a' AND run_id = 'shared-run-id'`, + ) + .all(), + ).toEqual([{ revision_number: 1 }]); + + staleStateDatabase.close(); + }); + + it("fails closed when a different durable tail conflicts with the process tail", () => { + const firstStateDatabase = database as DatabaseSync; + const firstSnapshot = prepare("session-a"); + expect(persistMemoryRunExposureBeforeContent(firstSnapshot)).toBe(true); + + const secondStateDatabase = new DatabaseSync(":memory:"); + mocks.database = { + agentId: "main", + db: secondStateDatabase, + path: ":memory:", + walMaintenance: undefined as never, + }; + const secondSnapshot = prepare("session-a"); + expect(secondSnapshot.revisionNumber).toBe(1); + expect(persistMemoryRunExposureBeforeContent(secondSnapshot)).toBe(true); + + mocks.database = { + agentId: "main", + db: firstStateDatabase, + path: ":memory:", + walMaintenance: undefined as never, + }; + expect( + hydrateMemoryRunExposureFromLedger({ + agentId: "main", + sessionId: "session-a", + runId: "shared-run-id", + }), + ).toBe(true); + + mocks.database = { + agentId: "main", + db: secondStateDatabase, + path: ":memory:", + walMaintenance: undefined as never, + }; + expect( + hydrateMemoryRunExposureFromLedger({ + agentId: "main", + sessionId: "session-a", + runId: "shared-run-id", + }), + ).toBe(false); + + secondStateDatabase.close(); + }); +}); diff --git a/src/plugins/memory-run-exposure-ledger.ts b/src/plugins/memory-run-exposure-ledger.ts new file mode 100644 index 000000000000..d9a62ccc0f6f --- /dev/null +++ b/src/plugins/memory-run-exposure-ledger.ts @@ -0,0 +1,389 @@ +import { executeSqliteQuerySync, getNodeSqliteKysely } from "../infra/kysely-sync.js"; +import { runSqliteImmediateTransactionSync } from "../infra/sqlite-transaction.js"; +import { logWarn } from "../logger.js"; +import type { AudienceRef } from "../memory-host-sdk/host/authorization.js"; +import { ensureMemoryPreoutputExposureLedgerSchemaInTransaction } from "../state/openclaw-agent-db-schema-helpers.js"; +import { + openOpenClawAgentDatabase, + type OpenClawAgentDatabase, +} from "../state/openclaw-agent-db.js"; +import { + createMemoryRunExposureScopeId, + reconcileMemoryRunExposureWithDurableLedger, + type MemoryRunExposureSnapshot, +} from "./memory-run-exposure.js"; + +type MemoryPreoutputExposureLedgerDatabase = { + memory_preoutput_exposure_ledger: { + agent_id: string; + session_id: string; + run_id: string; + revision_number: number; + exposure_set_id: string; + previous_exposure_set_id: string | null; + session_key: string; + context_fingerprint: string; + plan_id: string; + memory_policy_revision: string; + source_policy_set_ids_json: string; + exposed_resource_revisions_json: string; + exposure_receipt_ids_json: string; + egress_receipt_ids_json: string; + delivery_audiences_json: string; + delivery_revision: string; + egress_registry_revision: string; + session_identity_revision: string; + subject_revision: string; + created_at: number; + }; +}; + +type MemoryExposureLedgerDiagnostic = "hydrate-failed" | "persist-failed"; + +function logMemoryExposureLedgerDiagnostic(diagnostic: MemoryExposureLedgerDiagnostic): void { + // Ledger errors can carry SQLite paths or other sensitive runtime details. The read already + // fails closed, so emit only a stable outcome code for operators and tests. + logWarn(`memory exposure ledger unavailable: ${diagnostic}`); +} + +function canonicalStrings(values: readonly string[]): string | undefined { + if (values.some((value) => !value.trim())) { + return undefined; + } + return JSON.stringify([...new Set(values)].toSorted()); +} + +function canonicalAudiences(snapshot: MemoryRunExposureSnapshot): string | undefined { + const audiences = snapshot.deliveryAudiences.map((audience) => ({ + kind: audience.kind, + id: audience.id, + })); + if (audiences.some((audience) => !audience.id.trim())) { + return undefined; + } + const keys = audiences.map((audience) => `${audience.kind}\u0000${audience.id}`); + if (new Set(keys).size !== keys.length) { + return undefined; + } + return JSON.stringify( + audiences.toSorted((left, right) => + `${left.kind}\u0000${left.id}`.localeCompare(`${right.kind}\u0000${right.id}`), + ), + ); +} + +function parseCanonicalStrings(value: string): readonly string[] | undefined { + try { + const parsed = JSON.parse(value) as unknown; + if ( + !Array.isArray(parsed) || + parsed.some((entry) => typeof entry !== "string" || !entry.trim()) + ) { + return undefined; + } + const strings = parsed as string[]; + return canonicalStrings(strings) === value ? Object.freeze(strings) : undefined; + } catch { + return undefined; + } +} + +const audienceKinds = new Set([ + "user", + "conversation", + "role", + "agent-shared", + "agent", + "internal", +]); + +function parseCanonicalAudiences(value: string): readonly AudienceRef[] | undefined { + try { + const parsed = JSON.parse(value) as unknown; + if (!Array.isArray(parsed)) { + return undefined; + } + const audiences: AudienceRef[] = []; + for (const entry of parsed) { + if ( + !entry || + typeof entry !== "object" || + !audienceKinds.has((entry as { kind?: unknown }).kind as AudienceRef["kind"]) || + typeof (entry as { id?: unknown }).id !== "string" || + !(entry as { id: string }).id.trim() + ) { + return undefined; + } + audiences.push( + Object.freeze({ + kind: (entry as { kind: AudienceRef["kind"] }).kind, + id: (entry as { id: string }).id, + }), + ); + } + const snapshot = { deliveryAudiences: audiences } as MemoryRunExposureSnapshot; + return canonicalAudiences(snapshot) === value ? Object.freeze(audiences) : undefined; + } catch { + return undefined; + } +} + +function isDurableSnapshot(snapshot: MemoryRunExposureSnapshot): boolean { + return Boolean( + snapshot.agentId.trim() && + snapshot.sessionId.trim() && + snapshot.runId.trim() && + snapshot.sessionKey.trim() && + snapshot.contextFingerprint.trim() && + snapshot.planId.trim() && + snapshot.memoryPolicyRevision.trim() && + snapshot.deliveryRevision.trim() && + snapshot.egressRegistryRevision.trim() && + snapshot.sessionIdentityRevision.trim() && + snapshot.subjectRevision.trim() && + snapshot.revisionNumber > 0 && + snapshot.revisionNumber === (snapshot.previous?.revisionNumber ?? 0) + 1 && + snapshot.durableRunScopeId === createMemoryRunExposureScopeId(snapshot) && + (!snapshot.previous || + (snapshot.previous.agentId === snapshot.agentId && + snapshot.previous.sessionId === snapshot.sessionId && + snapshot.previous.runId === snapshot.runId)), + ); +} + +function persistMemoryRunExposureInTransaction(params: { + database: OpenClawAgentDatabase; + snapshot: MemoryRunExposureSnapshot; + sourcePolicySetIdsJson: string; + exposedResourceRevisionsJson: string; + exposureReceiptIdsJson: string; + egressReceiptIdsJson: string; + deliveryAudiencesJson: string; +}): void { + const { database, snapshot } = params; + ensureMemoryPreoutputExposureLedgerSchemaInTransaction(database.db); + const db = getNodeSqliteKysely(database.db); + const inserted = executeSqliteQuerySync( + database.db, + db + .insertInto("memory_preoutput_exposure_ledger") + .values({ + agent_id: snapshot.agentId, + session_id: snapshot.sessionId, + run_id: snapshot.runId, + revision_number: snapshot.revisionNumber, + exposure_set_id: snapshot.exposureSetId, + previous_exposure_set_id: snapshot.previous?.exposureSetId ?? null, + session_key: snapshot.sessionKey, + context_fingerprint: snapshot.contextFingerprint, + plan_id: snapshot.planId, + memory_policy_revision: snapshot.memoryPolicyRevision, + source_policy_set_ids_json: params.sourcePolicySetIdsJson, + exposed_resource_revisions_json: params.exposedResourceRevisionsJson, + exposure_receipt_ids_json: params.exposureReceiptIdsJson, + egress_receipt_ids_json: params.egressReceiptIdsJson, + delivery_audiences_json: params.deliveryAudiencesJson, + delivery_revision: snapshot.deliveryRevision, + egress_registry_revision: snapshot.egressRegistryRevision, + session_identity_revision: snapshot.sessionIdentityRevision, + subject_revision: snapshot.subjectRevision, + created_at: snapshot.createdAt, + }) + .onConflict((conflict) => + conflict.columns(["agent_id", "session_id", "run_id", "revision_number"]).doNothing(), + ), + ); + if (inserted.numAffectedRows !== 1n) { + throw new Error("memory exposure revision already has a durable ledger row"); + } +} + +/** + * Commits a content-free audit row before a broker can publish selected-plugin content. + * A duplicate revision is a concurrent/stale invocation, not a successful idempotent exposure. + */ +export function persistMemoryRunExposureBeforeContent( + snapshot: MemoryRunExposureSnapshot, +): boolean { + const sourcePolicySetIdsJson = canonicalStrings(snapshot.sourcePolicySetIds); + const exposedResourceRevisionsJson = canonicalStrings(snapshot.exposedResourceRevisions); + const exposureReceiptIdsJson = canonicalStrings(snapshot.exposureReceiptIds); + const egressReceiptIdsJson = canonicalStrings(snapshot.egressReceiptIds); + const deliveryAudiencesJson = canonicalAudiences(snapshot); + if ( + !isDurableSnapshot(snapshot) || + !sourcePolicySetIdsJson || + !exposedResourceRevisionsJson || + !exposureReceiptIdsJson || + !egressReceiptIdsJson || + !deliveryAudiencesJson + ) { + return false; + } + try { + return persistMemoryRunExposureBeforeContentInDatabase({ + database: openOpenClawAgentDatabase({ agentId: snapshot.agentId }), + snapshot, + }); + } catch { + logMemoryExposureLedgerDiagnostic("persist-failed"); + return false; + } +} + +/** Uses an already-owned agent DB for deterministic test and lifecycle setup. */ +export function persistMemoryRunExposureBeforeContentInDatabase(params: { + database: OpenClawAgentDatabase; + snapshot: MemoryRunExposureSnapshot; +}): boolean { + const { database, snapshot } = params; + const sourcePolicySetIdsJson = canonicalStrings(snapshot.sourcePolicySetIds); + const exposedResourceRevisionsJson = canonicalStrings(snapshot.exposedResourceRevisions); + const exposureReceiptIdsJson = canonicalStrings(snapshot.exposureReceiptIds); + const egressReceiptIdsJson = canonicalStrings(snapshot.egressReceiptIds); + const deliveryAudiencesJson = canonicalAudiences(snapshot); + if ( + database.agentId !== snapshot.agentId || + !isDurableSnapshot(snapshot) || + !sourcePolicySetIdsJson || + !exposedResourceRevisionsJson || + !exposureReceiptIdsJson || + !egressReceiptIdsJson || + !deliveryAudiencesJson + ) { + return false; + } + try { + runSqliteImmediateTransactionSync(database.db, () => { + persistMemoryRunExposureInTransaction({ + database, + snapshot, + sourcePolicySetIdsJson, + exposedResourceRevisionsJson, + exposureReceiptIdsJson, + egressReceiptIdsJson, + deliveryAudiencesJson, + }); + }); + return true; + } catch { + logMemoryExposureLedgerDiagnostic("persist-failed"); + return false; + } +} + +/** + * Rehydrates the immutable, session-bound exposure lineage from the pre-output ledger. + * Transcript companions use this durable authority after a gateway restart, never the process Map. + */ +export function readDurableMemoryRunExposure(params: { + database: OpenClawAgentDatabase; + sessionId: string; + runId: string; +}): MemoryRunExposureSnapshot | undefined { + try { + return readDurableMemoryRunExposureOrThrow(params); + } catch { + return undefined; + } +} + +function readDurableMemoryRunExposureOrThrow(params: { + database: OpenClawAgentDatabase; + sessionId: string; + runId: string; +}): MemoryRunExposureSnapshot | undefined { + const db = getNodeSqliteKysely(params.database.db); + const rows = executeSqliteQuerySync( + params.database.db, + db + .selectFrom("memory_preoutput_exposure_ledger") + .selectAll() + .where("agent_id", "=", params.database.agentId) + .where("session_id", "=", params.sessionId) + .where("run_id", "=", params.runId) + .orderBy("revision_number", "asc"), + ).rows; + let previous: MemoryRunExposureSnapshot | undefined; + for (const row of rows) { + const sourcePolicySetIds = parseCanonicalStrings(row.source_policy_set_ids_json); + const exposedResourceRevisions = parseCanonicalStrings(row.exposed_resource_revisions_json); + const exposureReceiptIds = parseCanonicalStrings(row.exposure_receipt_ids_json); + const egressReceiptIds = parseCanonicalStrings(row.egress_receipt_ids_json); + const deliveryAudiences = parseCanonicalAudiences(row.delivery_audiences_json); + if ( + !sourcePolicySetIds || + !exposedResourceRevisions || + !exposureReceiptIds || + !egressReceiptIds || + !deliveryAudiences || + !row.session_key.trim() || + !row.context_fingerprint.trim() || + !row.plan_id.trim() || + !row.memory_policy_revision.trim() || + !row.delivery_revision.trim() || + !row.egress_registry_revision.trim() || + !row.session_identity_revision.trim() || + !row.subject_revision.trim() || + row.revision_number !== (previous?.revisionNumber ?? 0) + 1 || + row.previous_exposure_set_id !== (previous?.exposureSetId ?? null) + ) { + throw new Error("memory exposure ledger has an invalid durable lineage"); + } + previous = Object.freeze({ + exposureSetId: row.exposure_set_id, + revisionNumber: row.revision_number, + ...(previous ? { previous } : {}), + agentId: row.agent_id, + sessionId: row.session_id, + sessionKey: row.session_key, + runId: row.run_id, + durableRunScopeId: createMemoryRunExposureScopeId({ + agentId: row.agent_id, + sessionId: row.session_id, + runId: row.run_id, + }), + contextFingerprint: row.context_fingerprint, + planId: row.plan_id, + memoryPolicyRevision: row.memory_policy_revision, + sourcePolicySetIds, + exposedResourceRevisions, + exposureReceiptIds, + egressReceiptIds, + deliveryAudiences, + deliveryRevision: row.delivery_revision, + egressRegistryRevision: row.egress_registry_revision, + sessionIdentityRevision: row.session_identity_revision, + subjectRevision: row.subject_revision, + createdAt: row.created_at, + }) satisfies MemoryRunExposureSnapshot; + } + return previous; +} + +/** + * Reconciles process state with the durable tail before preparing a new content release. A corrupt + * or mismatched ledger fails the broker closed rather than advancing from an unsafe process tail. + */ +export function hydrateMemoryRunExposureFromLedger(params: { + agentId: string; + sessionId: string; + runId: string; +}): boolean { + try { + const database = openOpenClawAgentDatabase({ agentId: params.agentId }); + ensureMemoryPreoutputExposureLedgerSchemaInTransaction(database.db); + const snapshot = readDurableMemoryRunExposureOrThrow({ + database, + sessionId: params.sessionId, + runId: params.runId, + }); + return reconcileMemoryRunExposureWithDurableLedger({ + ...params, + durableSnapshot: snapshot, + }); + } catch { + logMemoryExposureLedgerDiagnostic("hydrate-failed"); + return false; + } +} diff --git a/src/plugins/memory-run-exposure.ts b/src/plugins/memory-run-exposure.ts index 947b75f50eee..1105688d796c 100644 --- a/src/plugins/memory-run-exposure.ts +++ b/src/plugins/memory-run-exposure.ts @@ -1,4 +1,4 @@ -import { randomUUID } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import type { AudienceRef } from "../memory-host-sdk/host/authorization.js"; export type MemoryRunExposureSnapshot = Readonly<{ @@ -9,6 +9,7 @@ export type MemoryRunExposureSnapshot = Readonly<{ sessionId: string; sessionKey: string; runId: string; + durableRunScopeId: string; contextFingerprint: string; planId: string; memoryPolicyRevision: string; @@ -26,7 +27,7 @@ export type MemoryRunExposureSnapshot = Readonly<{ type MemoryRunExposureFacts = Omit< MemoryRunExposureSnapshot, - "exposureSetId" | "revisionNumber" | "previous" | "createdAt" + "exposureSetId" | "revisionNumber" | "previous" | "createdAt" | "durableRunScopeId" >; const exposuresByRun = new Map(); @@ -35,6 +36,18 @@ function key(params: { agentId: string; sessionId: string; runId: string }): str return `${params.agentId}\u0000${params.sessionId}\u0000${params.runId}`; } +/** Makes legacy projection keys session-bound without exposing raw session ids in that surface. */ +export function createMemoryRunExposureScopeId(params: { + agentId: string; + sessionId: string; + runId: string; +}): string { + const { agentId, sessionId, runId } = params; + return `mre-scope1_${createHash("sha256") + .update(JSON.stringify({ agentId, sessionId, runId })) + .digest("base64url")}`; +} + function sortedUnique(values: readonly string[]): readonly string[] { return Object.freeze([...new Set(values)].toSorted()); } @@ -53,15 +66,16 @@ function sortedAudiences(audiences: readonly AudienceRef[]): readonly AudienceRe ); } -/** Records an immutable run-exposure revision before scoped content leaves the broker. */ -export function recordMemoryRunExposure(facts: MemoryRunExposureFacts): MemoryRunExposureSnapshot { +/** Prepares an immutable run-exposure revision without publishing it to process state. */ +export function prepareMemoryRunExposure(facts: MemoryRunExposureFacts): MemoryRunExposureSnapshot { const normalizedKey = key(facts); const previous = exposuresByRun.get(normalizedKey); - const snapshot = Object.freeze({ + return Object.freeze({ exposureSetId: `mre1_${randomUUID()}`, revisionNumber: (previous?.revisionNumber ?? 0) + 1, ...(previous ? { previous } : {}), ...facts, + durableRunScopeId: createMemoryRunExposureScopeId(facts), sourcePolicySetIds: sortedUnique(facts.sourcePolicySetIds), exposedResourceRevisions: sortedUnique(facts.exposedResourceRevisions), exposureReceiptIds: sortedUnique(facts.exposureReceiptIds), @@ -69,7 +83,55 @@ export function recordMemoryRunExposure(facts: MemoryRunExposureFacts): MemoryRu deliveryAudiences: sortedAudiences(facts.deliveryAudiences), createdAt: Date.now(), }) satisfies MemoryRunExposureSnapshot; +} + +/** Publishes a prepared revision only when no competing revision has advanced this run. */ +export function publishMemoryRunExposure(snapshot: MemoryRunExposureSnapshot): boolean { + const normalizedKey = key(snapshot); + if (exposuresByRun.get(normalizedKey) !== snapshot.previous) { + return false; + } exposuresByRun.set(normalizedKey, snapshot); + return true; +} + +/** + * Makes the durable ledger authoritative for one run. Empty durable state clears a stale + * process entry after a state-root change; a distinct durable tail is unsafe to overwrite. + */ +export function reconcileMemoryRunExposureWithDurableLedger(params: { + agentId: string; + sessionId: string; + runId: string; + durableSnapshot: MemoryRunExposureSnapshot | undefined; +}): boolean { + const normalizedKey = key(params); + const current = exposuresByRun.get(normalizedKey); + const { durableSnapshot } = params; + if (!durableSnapshot) { + exposuresByRun.delete(normalizedKey); + return true; + } + if ( + durableSnapshot.agentId !== params.agentId || + durableSnapshot.sessionId !== params.sessionId || + durableSnapshot.runId !== params.runId || + (current && + (current.exposureSetId !== durableSnapshot.exposureSetId || + current.revisionNumber !== durableSnapshot.revisionNumber)) + ) { + return false; + } + exposuresByRun.set(normalizedKey, durableSnapshot); + return true; +} + +/** Records an immutable run-exposure revision for callers that do not need durable pre-output fencing. */ +export function recordMemoryRunExposure(facts: MemoryRunExposureFacts): MemoryRunExposureSnapshot { + const snapshot = prepareMemoryRunExposure(facts); + if (!publishMemoryRunExposure(snapshot)) { + throw new Error("memory run exposure advanced before publication"); + } return snapshot; } diff --git a/src/plugins/memory-state.test.ts b/src/plugins/memory-state.test.ts index d2173438303a..db8d1a45b189 100644 --- a/src/plugins/memory-state.test.ts +++ b/src/plugins/memory-state.test.ts @@ -185,16 +185,15 @@ describe("memory plugin state", () => { registerMemoryCapability("memory-core", { publicArtifacts: { listArtifacts }, }); - isMemoryIsolationCutoverAgentMock.mockImplementation((agentId: string) => agentId === "cutover"); + isMemoryIsolationCutoverAgentMock.mockImplementation( + (agentId: string) => agentId === "cutover", + ); await expect( listActiveMemoryPublicArtifacts({ cfg: { agents: { - list: [ - { id: "legacy", default: true }, - { id: "cutover" }, - ], + list: [{ id: "legacy", default: true }, { id: "cutover" }], }, } as never, }), @@ -503,6 +502,56 @@ describe("memory plugin state", () => { expect(supplemental).toHaveBeenCalledWith(expectedContext); }); + it("fails closed for unbound cut-over prompt contributors", async () => { + isMemoryIsolationCutoverAgentMock.mockReturnValue(true); + const primary = vi.fn(() => ["selected runtime"]); + const supplemental = vi.fn(() => ["legacy supplement"]); + const prepare = vi.fn(async () => ["legacy prepared supplement"]); + registerTestMemoryPromptBuilder(primary); + registerMemoryPromptSupplement("memory-wiki", supplemental); + registerMemoryPromptPreparation("memory-wiki", prepare); + + const params = { + availableTools: new Set(), + agentId: "cut-over", + agentSessionKey: "agent:cut-over:main", + }; + expect(buildMemoryPromptSection(params)).toEqual([]); + await expect(prepareMemoryPromptSection(params)).resolves.toMatchObject({ lines: [] }); + expect(primary).not.toHaveBeenCalled(); + expect(supplemental).not.toHaveBeenCalled(); + expect(prepare).not.toHaveBeenCalled(); + }); + + it("binds selected-runtime prompt state to the host invocation and blocks supplements", async () => { + isMemoryIsolationCutoverAgentMock.mockReturnValue(true); + const host = { + search: vi.fn(async () => ({ results: [] })), + read: vi.fn(async () => ({ text: "", path: "" })), + }; + const primary = vi.fn(() => ["selected runtime"]); + const supplemental = vi.fn(() => ["legacy supplement"]); + const prepare = vi.fn(async () => ["legacy prepared supplement"]); + registerTestMemoryPromptBuilder(primary); + registerMemoryPromptSupplement("memory-wiki", supplemental); + registerMemoryPromptPreparation("memory-wiki", prepare); + + const params = { + availableTools: new Set(), + agentId: "cut-over", + agentSessionKey: "agent:cut-over:main", + authorizedMemoryRead: host, + }; + const prepared = await prepareMemoryPromptSection(params); + expect(buildMemoryPromptSection(params, prepared)).toEqual(["selected runtime"]); + expect(primary).toHaveBeenCalledWith(expect.objectContaining({ authorizedMemoryRead: host })); + expect(supplemental).not.toHaveBeenCalled(); + expect(prepare).not.toHaveBeenCalled(); + expect(() => + buildMemoryPromptSection({ ...params, authorizedMemoryRead: { ...host } }, prepared), + ).toThrow("prepared memory prompt section does not match the current run"); + }); + it("appends prompt supplements in plugin-id order", () => { registerTestMemoryPromptBuilder(() => ["primary"]); registerMemoryPromptSupplement("memory-wiki", () => ["wiki"]); diff --git a/src/plugins/memory-state.ts b/src/plugins/memory-state.ts index 0a2b767395ac..4c098c9374d4 100644 --- a/src/plugins/memory-state.ts +++ b/src/plugins/memory-state.ts @@ -1,9 +1,10 @@ /** Registry state for plugin memory runtimes, prompt supplements, and flush planning. */ import { AsyncLocalStorage } from "node:async_hooks"; import { filterStringEntries } from "@openclaw/normalization-core/string-normalization"; -import type { OpenClawConfig } from "../config/types.openclaw.js"; import { listAgentIds } from "../agents/agent-scope-config.js"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; +import { isMemoryIsolationCutoverAgent } from "./memory-cutover.js"; import type { MemoryCorpusSupplement, MemoryCorpusSupplementRegistration, @@ -19,7 +20,6 @@ import type { MemoryPromptSupplementRegistration, PreparedMemoryPromptSection, } from "./registry-contribution-types.js"; -import { isMemoryIsolationCutoverAgent } from "./memory-cutover.js"; import type { PluginRegistry } from "./registry-types.js"; import { requireActivePluginRegistry, resolveDirectPluginRegistrationOwner } from "./runtime.js"; @@ -174,11 +174,16 @@ function buildSynchronousMemoryPromptSection(params: MemoryPromptSectionParams): primary: string[]; supplements: Array<{ pluginId: string; lines: string[] }>; } { + if (params.memoryReadEnforced && !params.authorizedMemoryRead) { + // Agent/session strings are routing hints, never authority. Do not let an + // unbound prompt contributor turn a missing host invocation into content. + return { primary: [], supplements: [] }; + } const registry = requireActivePluginRegistry(); const primary = filterStringEntries( resolveSelectedMemoryCapabilityRegistration(registry)?.capability.promptBuilder?.(params) ?? [], ); - const supplements = registry.memoryPromptSupplements + const supplements = (params.memoryReadEnforced ? [] : registry.memoryPromptSupplements) // Keep supplement order stable even if plugin registration order changes. .toSorted((left, right) => left.pluginId.localeCompare(right.pluginId)) .map((registration) => ({ @@ -197,7 +202,10 @@ function cloneMemoryPromptSectionParams( agentId: params.agentId, agentSessionKey: params.agentSessionKey, sandboxed: params.sandboxed, - memoryReadEnforced: params.memoryReadEnforced, + memoryReadEnforced: + params.memoryReadEnforced ?? + (params.agentId && isMemoryIsolationCutoverAgent(params.agentId) ? true : undefined), + authorizedMemoryRead: params.authorizedMemoryRead, }; } @@ -211,6 +219,7 @@ function snapshotMemoryPromptContext( agentSessionKey: params.agentSessionKey, sandboxed: params.sandboxed === true, memoryReadEnforced: params.memoryReadEnforced === true, + ...(params.authorizedMemoryRead ? { authorizedMemoryRead: params.authorizedMemoryRead } : {}), }); } @@ -221,10 +230,11 @@ function preparedMemoryPromptContextMatches( const current = snapshotMemoryPromptContext(params); return ( prepared.context.citationsMode === current.citationsMode && - prepared.context.agentId === current.agentId && - prepared.context.agentSessionKey === current.agentSessionKey && - prepared.context.sandboxed === current.sandboxed && - prepared.context.memoryReadEnforced === current.memoryReadEnforced && + prepared.context.agentId === current.agentId && + prepared.context.agentSessionKey === current.agentSessionKey && + prepared.context.sandboxed === current.sandboxed && + prepared.context.memoryReadEnforced === current.memoryReadEnforced && + prepared.context.authorizedMemoryRead === current.authorizedMemoryRead && prepared.context.availableTools.length === current.availableTools.length && prepared.context.availableTools.every((tool, index) => tool === current.availableTools[index]) ); @@ -240,8 +250,11 @@ export async function prepareMemoryPromptSection( cloneMemoryPromptSectionParams(runParams), ); const preparationRegistrations = [...requireActivePluginRegistry().memoryPromptPreparations]; + // Registered preparations are supplemental paths. Until a contributor can + // project through the selected runtime, it stays unavailable after cutover. + const canPrepare = !runParams.memoryReadEnforced; const preparedSupplements = await Promise.all( - preparationRegistrations.map(async (registration) => ({ + (canPrepare ? preparationRegistrations : []).map(async (registration) => ({ pluginId: registration.pluginId, lines: filterStringEntries( await registration.prepare(cloneMemoryPromptSectionParams(runParams)), @@ -283,13 +296,13 @@ export function buildMemoryPromptSection( // Run-scoped prompt state must never cross agent/session/tool boundaries. if ( !preparedMemoryPromptSections.has(prepared) || - !preparedMemoryPromptContextMatches(prepared, params) + !preparedMemoryPromptContextMatches(prepared, cloneMemoryPromptSectionParams(params)) ) { throw new Error("prepared memory prompt section does not match the current run"); } return [...prepared.lines]; } - const synchronous = buildSynchronousMemoryPromptSection(params); + const synchronous = buildSynchronousMemoryPromptSection(cloneMemoryPromptSectionParams(params)); return [...synchronous.primary, ...synchronous.supplements.flatMap((entry) => entry.lines)]; } diff --git a/src/plugins/registry-contribution-types.ts b/src/plugins/registry-contribution-types.ts index 6e830e0abfd6..ad5e41a38c02 100644 --- a/src/plugins/registry-contribution-types.ts +++ b/src/plugins/registry-contribution-types.ts @@ -17,6 +17,7 @@ import type { EmbeddingProviderIndexIdentity, EmbeddingProviderRuntime, } from "./embedding-provider-types.js"; +import type { AuthorizedMemoryReadHost } from "./tool-types.js"; export type ContextEngineFactoryContext = { config?: OpenClawConfig; @@ -145,6 +146,12 @@ export type MemoryPromptSectionParams = { sandboxed?: boolean; /** True when supplemental legacy memory reads are unavailable for this agent. */ memoryReadEnforced?: true; + /** + * Host-minted invocation for this run's selected authorized memory runtime. + * In enforced mode, prompt contributors must not derive access from agent or + * session strings; an absent handle means their content path is unavailable. + */ + authorizedMemoryRead?: AuthorizedMemoryReadHost; }; export type MemoryPromptSectionBuilder = (params: MemoryPromptSectionParams) => string[]; @@ -161,6 +168,7 @@ export type PreparedMemoryPromptSection = Readonly<{ agentSessionKey?: string; sandboxed: boolean; memoryReadEnforced: boolean; + authorizedMemoryRead?: AuthorizedMemoryReadHost; }>; lines: readonly string[]; }>; @@ -206,6 +214,8 @@ export type MemoryCorpusSupplement = { agentSessionKey?: string; sandboxed?: boolean; memoryReadEnforced?: true; + /** Host-minted invocation for this run; required for enforced content access. */ + authorizedMemoryRead?: AuthorizedMemoryReadHost; }): Promise; get(params: { lookup: string; @@ -215,6 +225,8 @@ export type MemoryCorpusSupplement = { agentSessionKey?: string; sandboxed?: boolean; memoryReadEnforced?: true; + /** Host-minted invocation for this run; required for enforced content access. */ + authorizedMemoryRead?: AuthorizedMemoryReadHost; }): Promise; }; diff --git a/src/state/openclaw-agent-db-schema-helpers.ts b/src/state/openclaw-agent-db-schema-helpers.ts index 0e0fcfe578a7..9e0334038cb1 100644 --- a/src/state/openclaw-agent-db-schema-helpers.ts +++ b/src/state/openclaw-agent-db-schema-helpers.ts @@ -40,6 +40,7 @@ import { AGENT_SCOPED_MEMORY_FTS_TABLE, AGENT_SCOPED_MEMORY_FTS_TRIGGER_DEFINITIONS, AGENT_SCOPED_MEMORY_TABLES, + ensureOpenClawAgentScopedMemorySchema, } from "./openclaw-agent-scoped-memory-schema.js"; import { AGENT_V14_ADDITIVE_SCHEMA_SQL, @@ -167,7 +168,6 @@ function repairAndAssertAgentSchemaGroup( const SESSION_KEY_CONTRACT_SCHEMA_START = "CREATE TABLE IF NOT EXISTS session_key_contract ("; const SESSION_KEY_CONTRACT_SCHEMA_END = "CREATE TABLE IF NOT EXISTS session_windows ("; - /** Ensure the additive session-key contract table inside the caller's transaction. */ export function ensureSessionKeyContractSchemaInTransaction(db: DatabaseSync): void { const start = OPENCLAW_AGENT_SCHEMA_SQL.indexOf(SESSION_KEY_CONTRACT_SCHEMA_START); @@ -178,6 +178,11 @@ export function ensureSessionKeyContractSchemaInTransaction(db: DatabaseSync): v db.exec(OPENCLAW_AGENT_SCHEMA_SQL.slice(start, end)); // sqlite-allow-raw -- Idempotent additive lazy ensure. } +/** Ensure the scoped-read audit ledger inside the caller's synchronous write transaction. */ +export function ensureMemoryPreoutputExposureLedgerSchemaInTransaction(db: DatabaseSync): void { + ensureOpenClawAgentScopedMemorySchema(db); +} + export function repairAndAssertOpenClawAgentV14SchemaForMigration( database: DatabaseSync, options: { agentId: string; pathname: string }, diff --git a/src/state/openclaw-agent-db.generated.d.ts b/src/state/openclaw-agent-db.generated.d.ts index ea3971dc0885..bfad114817e3 100644 --- a/src/state/openclaw-agent-db.generated.d.ts +++ b/src/state/openclaw-agent-db.generated.d.ts @@ -263,6 +263,29 @@ export interface MemoryPolicySets { policy_set_id: string; } +export interface MemoryPreoutputExposureLedger { + agent_id: string; + context_fingerprint: string; + created_at: number; + delivery_audiences_json: string; + delivery_revision: string; + egress_receipt_ids_json: string; + egress_registry_revision: string; + exposed_resource_revisions_json: string; + exposure_receipt_ids_json: string; + exposure_set_id: string; + memory_policy_revision: string; + plan_id: string; + previous_exposure_set_id: string | null; + revision_number: number; + run_id: string; + session_id: string; + session_identity_revision: string; + session_key: string; + source_policy_set_ids_json: string; + subject_revision: string; +} + export interface MemoryResourceRevisions { activated_at: number | null; actor_id: string | null; @@ -732,6 +755,7 @@ export interface DB { memory_policy_entries: MemoryPolicyEntries; memory_policy_revisions: MemoryPolicyRevisions; memory_policy_sets: MemoryPolicySets; + memory_preoutput_exposure_ledger: MemoryPreoutputExposureLedger; memory_resource_revisions: MemoryResourceRevisions; memory_resource_subjects: MemoryResourceSubjects; memory_resources: MemoryResources; diff --git a/src/state/openclaw-agent-schema.sql b/src/state/openclaw-agent-schema.sql index 3b87ceed93ec..1f79ad33a487 100644 --- a/src/state/openclaw-agent-schema.sql +++ b/src/state/openclaw-agent-schema.sql @@ -933,6 +933,48 @@ BEGIN SELECT RAISE(ABORT, 'memory run exposures cannot be deleted'); END; +-- Selected-plugin content is never returned until this content-free ledger row commits. +-- It is lazy/additive so current-version databases remain compatible until first scoped read. +CREATE TABLE IF NOT EXISTS memory_preoutput_exposure_ledger ( + agent_id TEXT NOT NULL, + session_id TEXT NOT NULL, + run_id TEXT NOT NULL, + revision_number INTEGER NOT NULL CHECK (revision_number > 0), + exposure_set_id TEXT NOT NULL UNIQUE, + previous_exposure_set_id TEXT, + session_key TEXT NOT NULL, + context_fingerprint TEXT NOT NULL, + plan_id TEXT NOT NULL, + memory_policy_revision TEXT NOT NULL, + source_policy_set_ids_json TEXT NOT NULL, + exposed_resource_revisions_json TEXT NOT NULL, + exposure_receipt_ids_json TEXT NOT NULL, + egress_receipt_ids_json TEXT NOT NULL, + delivery_audiences_json TEXT NOT NULL, + delivery_revision TEXT NOT NULL, + egress_registry_revision TEXT NOT NULL, + session_identity_revision TEXT NOT NULL, + subject_revision TEXT NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (agent_id, session_id, run_id, revision_number), + FOREIGN KEY (previous_exposure_set_id) REFERENCES memory_preoutput_exposure_ledger(exposure_set_id) ON DELETE RESTRICT +) STRICT; + +CREATE INDEX IF NOT EXISTS idx_memory_preoutput_exposure_ledger_session_run + ON memory_preoutput_exposure_ledger(agent_id, session_id, run_id, revision_number DESC); + +CREATE TRIGGER IF NOT EXISTS memory_preoutput_exposure_ledger_no_update +BEFORE UPDATE ON memory_preoutput_exposure_ledger +BEGIN + SELECT RAISE(ABORT, 'pre-output memory exposure ledger is immutable'); +END; + +CREATE TRIGGER IF NOT EXISTS memory_preoutput_exposure_ledger_no_delete +BEFORE DELETE ON memory_preoutput_exposure_ledger +BEGIN + SELECT RAISE(ABORT, 'pre-output memory exposure ledger cannot be deleted'); +END; + CREATE TABLE IF NOT EXISTS transcript_event_memory_policies ( session_id TEXT NOT NULL, event_seq INTEGER NOT NULL, diff --git a/src/state/openclaw-agent-scoped-memory-schema.test.ts b/src/state/openclaw-agent-scoped-memory-schema.test.ts index 4120bdff2abf..b1bd84c25b8c 100644 --- a/src/state/openclaw-agent-scoped-memory-schema.test.ts +++ b/src/state/openclaw-agent-scoped-memory-schema.test.ts @@ -144,4 +144,10 @@ describe("scoped memory additive agent schema", () => { ]), ); }); + + it("keeps the pre-output exposure ledger in the canonical scoped schema payload", () => { + expect(AGENT_SCOPED_MEMORY_SCHEMA_SQL).toContain( + "CREATE TABLE IF NOT EXISTS memory_preoutput_exposure_ledger (", + ); + }); }); diff --git a/src/state/openclaw-agent-scoped-memory-schema.ts b/src/state/openclaw-agent-scoped-memory-schema.ts index 9e25bea1abfb..8ab1747f1efd 100644 --- a/src/state/openclaw-agent-scoped-memory-schema.ts +++ b/src/state/openclaw-agent-scoped-memory-schema.ts @@ -16,6 +16,7 @@ export const AGENT_SCOPED_MEMORY_TABLES = [ "memory_migrations", "memory_policy_sets", "memory_run_exposures", + "memory_preoutput_exposure_ledger", "transcript_event_memory_policies", ] as const; diff --git a/src/worker/embedded-agent.runtime.ts b/src/worker/embedded-agent.runtime.ts index b0790caa7aa0..188fcfdb3cbe 100644 --- a/src/worker/embedded-agent.runtime.ts +++ b/src/worker/embedded-agent.runtime.ts @@ -28,7 +28,6 @@ import { wrapToolWithGatewayCallerIdentity } from "../agents/tools/gateway-calle import { DEFAULT_AGENTS_FILENAME, loadWorkspaceBootstrapFiles } from "../agents/workspace.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import type { AssistantMessage, AssistantMessageEventStreamLike } from "../llm/types.js"; -import { isMemoryIsolationCutoverAgent } from "../plugins/memory-cutover.js"; import { getProcessSupervisor } from "../process/supervisor/index.js"; import { createWorkerBrowserToolRuntime, type WorkerBrowserRuntime } from "./browser-runtime.js"; import { createWorkerLiveRuntime } from "./embedded-agent-live.runtime.js"; @@ -98,6 +97,7 @@ type RunWorkerEmbeddedTurnParams = { permissionMode?: import("../../packages/gateway-protocol/src/schema/sessions-row.js").SessionPermissionMode; browser?: WorkerBrowserLaunchDescriptor; browserRuntime?: WorkerBrowserRuntime; + memoryIsolationCutover: boolean; signal?: AbortSignal; }; @@ -151,16 +151,21 @@ export async function runWorkerEmbeddedTurn(params: RunWorkerEmbeddedTurnParams) }); const allowedToolNameSet = new Set(params.allowedToolNames); - const localToolNameSet = new Set(WORKER_LOCAL_TOOL_NAMES); const permissionToolPolicy = params.permissionMode ? resolveSessionPermissionCoreToolPolicy({ mode: params.permissionMode }) : undefined; const omittedToolNames = permissionToolPolicy?.readOnly ? new Set(["write", "edit", "apply_patch"]) : undefined; - const activeToolNames = WORKER_TOOL_NAMES.filter( + // P1C's selected-memory pilot exposes no mutation or execution path. The worker builds core + // tools directly, so it must apply the primary agent's final read-only surface itself. + const availableToolNames = params.memoryIsolationCutover + ? (["read"] as const) + : WORKER_LOCAL_TOOL_NAMES; + const activeToolNames = availableToolNames.filter( (name) => allowedToolNameSet.has(name) && !omittedToolNames?.has(name), ); + const localToolNameSet = new Set(availableToolNames); const headlessApprovalText = params.permissionMode ? `Exec denied (approval_required) in worker ${params.permissionMode} permission mode. Run this command locally for interactive approval, or ask an administrator to clear the session permission mode.` : undefined; @@ -168,19 +173,20 @@ export async function runWorkerEmbeddedTurn(params: RunWorkerEmbeddedTurnParams) codingRoot: params.cwd, containmentRoot: params.workerContainmentRoot, includeBaseCodingTools: true, - includeShellTools: true, + includeShellTools: !params.memoryIsolationCutover, workspaceOnly: permissionToolPolicy?.workspaceOnly ?? false, - readOnly: permissionToolPolicy?.readOnly ?? false, + readOnly: params.memoryIsolationCutover || permissionToolPolicy?.readOnly === true, modelContextWindowTokens: model.contextWindow, imageSanitization: {}, applyPatchEnabled: + !params.memoryIsolationCutover && permissionToolPolicy?.readOnly !== true && isApplyPatchAllowedForModel({ modelProvider: params.modelRef.provider, modelId: params.modelRef.model, }), applyPatchWorkspaceOnly: permissionToolPolicy?.applyPatchWorkspaceOnly ?? true, - memoryFileMutationGuard: isMemoryIsolationCutoverAgent(DEFAULT_AGENT_ID) + memoryFileMutationGuard: params.memoryIsolationCutover ? createMemoryFileMutationGuard({ mutationRoot: params.cwd }) : undefined, execDefaults: { @@ -247,7 +253,11 @@ export async function runWorkerEmbeddedTurn(params: RunWorkerEmbeddedTurnParams) }), ); const discoveredToolNames = new Set(localTools.map((tool) => tool.name)); - for (const toolName of WORKER_REQUIRED_LOCAL_TOOL_NAMES) { + const requiredToolNames = [ + ...(params.memoryIsolationCutover ? ["read"] : WORKER_REQUIRED_LOCAL_TOOL_NAMES), + ...(browserRuntime ? ["browser"] : []), + ]; + for (const toolName of requiredToolNames) { if (omittedToolNames?.has(toolName)) { continue; } diff --git a/src/worker/worker.runtime.test.ts b/src/worker/worker.runtime.test.ts index ea5676473e0d..05d8c03e1660 100644 --- a/src/worker/worker.runtime.test.ts +++ b/src/worker/worker.runtime.test.ts @@ -39,6 +39,14 @@ import { import { createDeferred, withTestTimeout } from "../../test/helpers/promise.js"; import { createOperationalRunInstanceRef } from "../agents/admitted-run-context.js"; import { listRunningSessions } from "../agents/bash-process-registry.js"; +import { + enableMemoryShadowReadOnlyMode, + resetMemoryIsolationCutoverForTest, +} from "../plugins/memory-cutover.js"; +import { + closeOpenClawAgentDatabasesForTest, + openOpenClawAgentDatabase, +} from "../state/openclaw-agent-db.js"; import { buildWorkerConnectParams, type WorkerLaunchDescriptor } from "./launch-descriptor.js"; import { WORKER_PROVIDER_REPLAY_LOCAL_RETRY_MESSAGE } from "./transcript-message.js"; import { WorkerAdmissionDeadlineExceededError } from "./worker-connection-contract.js"; @@ -896,6 +904,7 @@ describe("worker runtime", () => { const result = await runWorkerDescriptor(launch); expect(result.status).toBe("completed"); + expect(browserRuntimeMocks.createWorkerBrowserToolRuntime).not.toHaveBeenCalled(); expect(gateway.inferenceRequests).toHaveLength(1); expect(gateway.inferenceRequests[0]?.modelRef).toEqual(MODEL_REF); expect(gateway.inferenceRequests[0]?.context.systemPrompt).toContain("worker-bootstrap-marker"); @@ -960,6 +969,44 @@ describe("worker runtime", () => { ]); }); + it("keeps an enforced agent's worker tool surface read-only after state isolation", async () => { + const memoryStateDir = await mkdtemp(path.join(tmpdir(), "openclaw-worker-memory-state-")); + const previousStateDir = process.env.OPENCLAW_STATE_DIR; + process.env.OPENCLAW_STATE_DIR = memoryStateDir; + try { + resetMemoryIsolationCutoverForTest(); + const database = openOpenClawAgentDatabase({ agentId: "main" }); + database.db + .prepare( + `INSERT INTO session_memory_subjects + (session_key, binding_id, principal_id, subject_kind, subject_revision, created_at) + VALUES ('agent:main:pilot', NULL, 'principal-alice', 'agent', 'test-revision', 1)`, + ) + .run(); + expect(enableMemoryShadowReadOnlyMode({ agentId: "main", nowMs: 1 })).toBe( + "shadow-read-only", + ); + closeOpenClawAgentDatabasesForTest(); + + const { gateway, launch } = await setup(); + await expect(runWorkerDescriptor(launch)).resolves.toMatchObject({ status: "completed" }); + + expect(browserRuntimeMocks.createWorkerBrowserToolRuntime).not.toHaveBeenCalled(); + expect(gateway.inferenceRequests[0]?.context.tools?.map((tool) => tool.name)).toEqual([ + "read", + ]); + } finally { + resetMemoryIsolationCutoverForTest(); + closeOpenClawAgentDatabasesForTest(); + if (previousStateDir === undefined) { + delete process.env.OPENCLAW_STATE_DIR; + } else { + process.env.OPENCLAW_STATE_DIR = previousStateDir; + } + await rm(memoryStateDir, { recursive: true, force: true }); + } + }); + it("runs with no tools when the Gateway authority is empty", async () => { const { gateway, launch } = await setup(); launch.assignment.toolAuthority.allowedToolNames = []; diff --git a/src/worker/worker.runtime.ts b/src/worker/worker.runtime.ts index 03fe92ab6db6..e8ccd0e1dc67 100644 --- a/src/worker/worker.runtime.ts +++ b/src/worker/worker.runtime.ts @@ -3,6 +3,8 @@ import { tmpdir } from "node:os"; import path from "node:path"; import { isPathInside } from "../infra/path-guards.js"; import { registerSecretValueForRedaction } from "../logging/secret-redaction-registry.js"; +import { isMemoryIsolationCutoverAgent } from "../plugins/memory-cutover.js"; +import { DEFAULT_AGENT_ID } from "../routing/session-key.js"; import type { WorkerBrowserRuntime } from "./browser-runtime.js"; import { buildWorkerConnectParams, type WorkerLaunchDescriptor } from "./launch-descriptor.js"; import { createWorkerConnection, type WorkerConnectionState } from "./worker-connection.js"; @@ -77,6 +79,9 @@ export async function runWorkerDescriptor( "worker workspace path escapes its assigned containment root; reprovision the worker workspace and retry", ); } + // Workers replace their state directory below. Resolve the durable P1C posture first so the + // isolated runtime cannot reinterpret an enforced agent as legacy because its scratch DB is empty. + const memoryIsolationCutover = isMemoryIsolationCutoverAgent(DEFAULT_AGENT_ID); const stateDir = await mkdtemp(path.join(tmpdir(), "openclaw-worker-")); await chmod(stateDir, 0o700); const previousStateDir = process.env.OPENCLAW_STATE_DIR; @@ -174,6 +179,7 @@ export async function runWorkerDescriptor( allowedToolNames: descriptor.assignment.toolAuthority.allowedToolNames, ...(descriptor.assignment.browser ? { browser: descriptor.assignment.browser } : {}), ...(options.browserRuntime ? { browserRuntime: options.browserRuntime } : {}), + memoryIsolationCutover, inference: { stream }, transcript: { commit: async (messages) => {