From 93d50ddee890d64c6f7f95f45ca884d2b5fc6b98 Mon Sep 17 00:00:00 2001 From: Galin Iliev Date: Fri, 14 Aug 2026 04:14:21 -0700 Subject: [PATCH] feat(memory): seal derived artifact lineage --- docs/concepts/2026-07-29-memory-impl-plan.md | 18 +- extensions/memory-core/index.ts | 1 + .../src/dreaming-consolidation-candidates.ts | 60 +- .../memory-core/src/dreaming-phases.test.ts | 66 ++ extensions/memory-core/src/dreaming-phases.ts | 25 +- .../src/memory/scoped-memory-db.ts | 85 ++ .../src/memory/scoped-memory-resources.ts | 138 ++++ .../src/memory/scoped-memory-runtime.test.ts | 278 ++++++- .../src/memory/scoped-memory-runtime.ts | 723 +++++++++++++++++- .../src/short-term-promotion-apply.ts | 50 +- .../src/short-term-promotion-metadata.ts | 13 + .../src/short-term-promotion-record.ts | 46 +- .../src/short-term-promotion-types.ts | 14 + .../src/short-term-promotion-utils.ts | 29 + .../src/short-term-promotion.test.ts | 161 ++++ .../memory-core/src/short-term-promotion.ts | 5 + .../memory-host-sdk/src/host/authorization.ts | 60 +- src/agents/agent-tools.ts | 29 +- .../agent-tools.workspace-paths.test.ts | 39 + .../compact.hooks.harness.ts | 128 ++++ .../compact.hooks.test.ts | 117 +++ .../embedded-agent-runner/compact.queued.ts | 17 + .../compaction-sealed.test.ts | 81 ++ .../compaction-session-execution.ts | 168 +++- .../context-engine-maintenance.test.ts | 34 + .../context-engine-maintenance.ts | 18 + .../prepared-compaction-runtime.ts | 50 +- .../run/attempt-tool-prepare.ts | 4 +- .../embedded-agent-runner/run/attempt.ts | 1 + .../embedded-agent-runner/run/params.ts | 2 + .../run/run-attempt-dispatch.ts | 1 + .../memory-authorized-read-host.test.ts | 201 +++++ src/agents/memory-authorized-read-host.ts | 147 +++- .../memory-autonomous-run-policy.test.ts | 34 + src/agents/memory-autonomous-run-policy.ts | 19 + src/agents/memory-egress-admission.ts | 1 - src/agents/openclaw-tools.ts | 3 + .../sessions/agent-session-compaction.ts | 121 ++- .../agent-session-loop-correctness.test.ts | 22 + .../sessions/session-manager-entries.ts | 64 +- src/agents/sessions/session-manager.test.ts | 31 + .../subagents/spawn/subagent-spawn-context.ts | 21 +- .../subagents/spawn/subagent-spawn-deps.ts | 3 + .../spawn/subagent-spawn.context.test.ts | 19 + .../subagents/spawn/subagent-spawn.runtime.ts | 1 + .../spawn/subagent-spawn.test-helpers.ts | 3 + src/auto-reply/dispatch.memory-egress.test.ts | 8 + src/auto-reply/dispatch.ts | 2 - .../reply/agent-runner-memory.test.ts | 60 +- src/auto-reply/reply/agent-runner-memory.ts | 99 ++- .../sessions/session-accessor.sqlite-scope.ts | 1 + ...ession-accessor.sqlite-transcript-store.ts | 40 +- ...ession-accessor.sqlite-transcript-write.ts | 148 +++- .../session-transcript-memory-policy.test.ts | 313 +++++++- .../session-transcript-memory-policy.ts | 247 ++++++ src/cron/isolated-agent/run-prepare.ts | 6 + src/infra/heartbeat-runner-execution.ts | 52 +- src/infra/heartbeat-runner-session.ts | 10 + ...eat-runner.isolated-session-mirror.test.ts | 69 +- src/plugins/memory-authorization-runtime.ts | 8 + src/plugins/memory-invocation.test.ts | 36 + src/plugins/memory-invocation.ts | 118 ++- src/plugins/memory-state.test.ts | 11 + src/plugins/memory-state.ts | 3 - src/plugins/registry-contribution-types.ts | 1 + src/state/openclaw-agent-db.generated.d.ts | 29 + src/state/openclaw-agent-schema.sql | 87 +++ .../openclaw-agent-scoped-memory-schema.ts | 3 + 68 files changed, 4291 insertions(+), 211 deletions(-) create mode 100644 src/agents/embedded-agent-runner/compaction-sealed.test.ts create mode 100644 src/agents/memory-authorized-read-host.test.ts create mode 100644 src/agents/memory-autonomous-run-policy.test.ts create mode 100644 src/agents/memory-autonomous-run-policy.ts diff --git a/docs/concepts/2026-07-29-memory-impl-plan.md b/docs/concepts/2026-07-29-memory-impl-plan.md index 92b29f5000d9..1bf6fb75f66e 100644 --- a/docs/concepts/2026-07-29-memory-impl-plan.md +++ b/docs/concepts/2026-07-29-memory-impl-plan.md @@ -1528,25 +1528,25 @@ Focused existing tests: Phase 2C is complete only when all of the following are demonstrated: -- [ ] Every compaction, checkpoint, memory flush, dreaming output, promotion, +- [x] Every compaction, checkpoint, memory flush, dreaming output, promotion, export, and child-produced durable artifact identifies immutable parents and records lineage. -- [ ] `derive` authority is checked before source content enters a model +- [x] `derive` authority is checked before source content enters a model context. -- [ ] No unlabeled or policy-unrepresentable derived artifact is readable. -- [ ] Group compaction and flush remain channel-scoped; private compaction and +- [x] No unlabeled or policy-unrepresentable derived artifact is readable. +- [x] Group compaction and flush remain channel-scoped; private compaction and flush remain user-scoped; autonomous work remains agent-scoped or writes nothing. -- [ ] Mixed audiences are partitioned or denied and can never be widened by +- [x] Mixed audiences are partitioned or denied and can never be widened by model wording. -- [ ] Tombstoning/revoking an ancestor denies descendants immediately; any +- [x] Tombstoning/revoking an ancestor denies descendants immediately; any recomputation creates a new reviewed immutable revision. -- [ ] Dreaming and promotion run one authorized store at a time, and postbox or +- [x] Dreaming and promotion run one authorized store at a time, and postbox or quarantine content cannot auto-promote. -- [ ] Child agents receive only the intersection of parent view, task +- [x] Child agents receive only the intersection of parent view, task capability, session visibility, and current authority; cron, heartbeat, webhook, and system runs cannot recover private access from a session key. -- [ ] Compaction, flush, dreaming, lineage, revocation, delegation, and +- [x] Compaction, flush, dreaming, lineage, revocation, delegation, and interruption tests pass, including any dependency-specific contract checks required by the selected harness. diff --git a/extensions/memory-core/index.ts b/extensions/memory-core/index.ts index e880f0d6de79..caaf8c4fdd90 100644 --- a/extensions/memory-core/index.ts +++ b/extensions/memory-core/index.ts @@ -306,6 +306,7 @@ function createLazyMemoryRuntime(host: MemoryCoreRuntimeHost): MemoryPluginRunti searchAuthorized: builtinScopedMemoryAuthorizedRuntime.searchAuthorized, readAuthorized: builtinScopedMemoryAuthorizedRuntime.readAuthorized, writeAuthorized: builtinScopedMemoryAuthorizedRuntime.writeAuthorized, + stageSealedCompaction: builtinScopedMemoryAuthorizedRuntime.stageSealedCompaction, importAuthorized: builtinScopedMemoryAuthorizedRuntime.importAuthorized, syncAuthorized: builtinScopedMemoryAuthorizedRuntime.syncAuthorized, exportAuthorized: builtinScopedMemoryAuthorizedRuntime.exportAuthorized, diff --git a/extensions/memory-core/src/dreaming-consolidation-candidates.ts b/extensions/memory-core/src/dreaming-consolidation-candidates.ts index bfb55c138e4e..785edb1076a1 100644 --- a/extensions/memory-core/src/dreaming-consolidation-candidates.ts +++ b/extensions/memory-core/src/dreaming-consolidation-candidates.ts @@ -1,4 +1,7 @@ -import type { PromotionCandidate } from "./short-term-promotion-types.js"; +import type { + PromotionCandidate, + ShortTermPromotionAuthorizedView, +} from "./short-term-promotion-types.js"; import { isShortTermSessionCorpusPath } from "./short-term-promotion-utils.js"; export function filterConsolidationCandidates( @@ -13,6 +16,61 @@ export function isPromotionOriginBlocked(candidate: PromotionCandidate): boolean return originClass === "untrusted" || originClass === "system"; } +function authorizedViewIdentity(view: ShortTermPromotionAuthorizedView): string { + return `${view.storeId}\u0000${view.viewId}`; +} + +/** A scoped candidate can promote only from one active, immutable authorized view. */ +export function isPromotionAuthorizedViewBlocked( + candidate: Pick, +): boolean { + const view = candidate.authorizedView; + return Boolean( + view && + (typeof view.storeId !== "string" || + typeof view.viewId !== "string" || + typeof view.resourceRevision !== "string" || + !view.storeId.trim() || + !view.viewId.trim() || + !view.resourceRevision.trim() || + view.lifecycle !== "active"), + ); +} + +/** Legacy records lack a scoped view; if one is present, never mix it with another view or legacy data. */ +export function hasOnePromotionAuthorizedView(candidates: readonly PromotionCandidate[]): boolean { + const scoped = candidates.filter((candidate) => candidate.authorizedView); + if (scoped.length === 0) { + return true; + } + if (scoped.length !== candidates.length || scoped.some(isPromotionAuthorizedViewBlocked)) { + return false; + } + return new Set(scoped.map((candidate) => authorizedViewIdentity(candidate.authorizedView!))).size === 1; +} + +/** + * Dreaming may process legacy workspace records together, but a scoped record + * changes the boundary: a phase gets one active authorized view or no input. + * That prevents a narrative/consolidation model context from joining stores. + */ +export function filterToOnePromotionAuthorizedView< + Candidate extends Pick, +>(candidates: readonly Candidate[]): Candidate[] { + const eligible = candidates.filter((candidate) => !isPromotionAuthorizedViewBlocked(candidate)); + const scoped = eligible.filter((candidate) => candidate.authorizedView); + if (scoped.length === 0) { + return eligible; + } + if (scoped.length !== eligible.length) { + return []; + } + const identities = new Set( + scoped.map((candidate) => authorizedViewIdentity(candidate.authorizedView!)), + ); + return identities.size === 1 ? eligible : []; +} + export function isConsolidationCandidateEligible(candidate: PromotionCandidate): boolean { const trustedOrigin = candidate.provenance?.originClass === "owner" || candidate.provenance?.originClass === "agent"; diff --git a/extensions/memory-core/src/dreaming-phases.test.ts b/extensions/memory-core/src/dreaming-phases.test.ts index ca59c3c55268..ebe92a3a5e1a 100644 --- a/extensions/memory-core/src/dreaming-phases.test.ts +++ b/extensions/memory-core/src/dreaming-phases.test.ts @@ -23,6 +23,7 @@ import { DREAMING_DAILY_PROVENANCE_NAMESPACE, writeMemoryCoreWorkspaceEntry, } from "./dreaming-state.js"; +import { filterToOnePromotionAuthorizedView } from "./dreaming-consolidation-candidates.js"; import { previewRemHarness } from "./rem-harness.js"; import { writeSessionIngestionState } from "./session-ingestion.js"; import { @@ -130,6 +131,71 @@ function expectNotIncludesSubstring(values: readonly string[], expected: string) expect(values.join("\n")).not.toContain(expected); } +function createAuthorizedRecallEntry(params: { + key: string; + storeId: string; + viewId: string; + resourceRevision: string; + lifecycle?: "active" | "postbox" | "quarantine"; +}): ShortTermRecallEntry { + return { + key: params.key, + path: `memory/${params.key}.md`, + startLine: 1, + endLine: 1, + source: "memory", + snippet: `Dreaming source ${params.key}.`, + recallCount: 1, + dailyCount: 0, + groundedCount: 0, + totalScore: 0.9, + maxScore: 0.9, + firstRecalledAt: DREAMING_TEST_BASE_TIME.toISOString(), + lastRecalledAt: DREAMING_TEST_BASE_TIME.toISOString(), + queryHashes: [params.key], + recallDays: [DREAMING_TEST_DAY], + conceptTags: [], + authorizedView: { + storeId: params.storeId, + viewId: params.viewId, + resourceRevision: params.resourceRevision, + lifecycle: params.lifecycle ?? "active", + }, + }; +} + +it("keeps dreaming input inside one authorized view", () => { + const first = createAuthorizedRecallEntry({ + key: "first", + storeId: "store-a", + viewId: "view-a", + resourceRevision: "revision-a-1", + }); + const sameView = createAuthorizedRecallEntry({ + key: "same-view", + storeId: "store-a", + viewId: "view-a", + resourceRevision: "revision-a-2", + }); + const otherView = createAuthorizedRecallEntry({ + key: "other-view", + storeId: "store-b", + viewId: "view-b", + resourceRevision: "revision-b-1", + }); + const postbox = createAuthorizedRecallEntry({ + key: "postbox", + storeId: "store-a", + viewId: "view-a", + resourceRevision: "revision-postbox", + lifecycle: "postbox", + }); + + expect(filterToOnePromotionAuthorizedView([first, sameView])).toEqual([first, sameView]); + expect(filterToOnePromotionAuthorizedView([first, otherView])).toEqual([]); + expect(filterToOnePromotionAuthorizedView([first, postbox])).toEqual([first]); +}); + async function expectPathMissing(targetPath: string): Promise { try { await fs.access(targetPath); diff --git a/extensions/memory-core/src/dreaming-phases.ts b/extensions/memory-core/src/dreaming-phases.ts index 652eceaf369b..d7332990c37b 100644 --- a/extensions/memory-core/src/dreaming-phases.ts +++ b/extensions/memory-core/src/dreaming-phases.ts @@ -17,6 +17,7 @@ import { import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry"; import { normalizeStringEntries, uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime"; import { appendFailedDreamingEvent } from "./dreaming-events.js"; +import { filterToOnePromotionAuthorizedView } from "./dreaming-consolidation-candidates.js"; import { normalizeDailyIngestionState, normalizeMemoryDay, @@ -1311,11 +1312,13 @@ async function runLightDreaming(params: { entries: await filterFreshLightDreamingEntries({ workspaceDir: params.workspaceDir, nowMs, - entries: filterRecallEntriesWithinLookback({ - entries: await readShortTermRecallEntries({ workspaceDir: params.workspaceDir, nowMs }), - nowMs, - lookbackDays: params.config.lookbackDays, - }), + entries: filterToOnePromotionAuthorizedView( + filterRecallEntriesWithinLookback({ + entries: await readShortTermRecallEntries({ workspaceDir: params.workspaceDir, nowMs }), + nowMs, + lookbackDays: params.config.lookbackDays, + }), + ), }), }); const rankedEntries = dedupeEntries( @@ -1409,11 +1412,13 @@ async function runRemDreaming(params: { }); const allEntries = await filterLiveShortTermRecallEntries({ workspaceDir: params.workspaceDir, - entries: filterRecallEntriesWithinLookback({ - entries: await readShortTermRecallEntries({ workspaceDir: params.workspaceDir, nowMs }), - nowMs, - lookbackDays: params.config.lookbackDays, - }), + entries: filterToOnePromotionAuthorizedView( + filterRecallEntriesWithinLookback({ + entries: await readShortTermRecallEntries({ workspaceDir: params.workspaceDir, nowMs }), + nowMs, + lookbackDays: params.config.lookbackDays, + }), + ), }); // Prefer entries staged by light sleep so REM synthesises from the // sequential light→REM pipeline instead of rescanning the full store. diff --git a/extensions/memory-core/src/memory/scoped-memory-db.ts b/extensions/memory-core/src/memory/scoped-memory-db.ts index 1eb16ae0e7da..f27038070db2 100644 --- a/extensions/memory-core/src/memory/scoped-memory-db.ts +++ b/extensions/memory-core/src/memory/scoped-memory-db.ts @@ -107,6 +107,83 @@ export type MemoryResourceRevisionRow = { retired_at: number | null; }; +export type MemoryRevisionPolicyRequirementRow = { + revision_id: string; + policy_id: string; + expected_revision_id: string; + expected_revocation_epoch: number; + requirement_kind: "output-policy" | "source-policy"; + created_at: number; +}; + +export type MemoryLineageEdgeRow = { + child_revision_id: string; + parent_kind: + | "resource-revision" + | "transcript-policy-set" + | "compaction-policy" + | "checkpoint" + | "export" + | "child-artifact"; + parent_id: string; + relation_kind: + | "derived-from" + | "compacted-from" + | "flushed-from" + | "dreamed-from" + | "promoted-from" + | "exported-from" + | "child-produced"; + created_at: number; +}; + +type MemoryPolicySetMemberRow = { + policy_set_id: string; + policy_id: string; + expected_revision_id: string; + expected_revocation_epoch: number; + audience_intersection_json: string; + retention_state: string; + created_at: number; +}; + +type SessionMemorySubjectSnapshotRow = { + session_id: string; + session_identity_revision: string; + subject_revision: string; +}; + +type TranscriptEventMemoryPolicyRow = { + session_id: string; + event_seq: number; + authorization_status: string; + source_policy_set_id: string | null; + run_exposure_set_id: string | null; + delivery_audiences_json: string | null; + session_identity_revision: string | null; + subject_revision: string | null; +}; + +type TranscriptEventMemoryPolicyDetailRow = { + session_id: string; + event_seq: number; + retention_state: string; + normalized_audience_intersection_json: string; + finalized_delivery_audiences_json: string; +}; + +type TranscriptEventRow = { + session_id: string; + seq: number; +}; + +type MemoryRunExposureResourceRow = { + exposure_set_id: string; + resource_revision_id: string; + policy_set_id: string; + created_at: number; +}; + export type MemoryResourceSubjectRow = { revision_id: string; subject_kind: "person" | "project" | "conversation" | "topic"; @@ -203,6 +280,14 @@ export type ScopedMemoryDatabase = { memory_policy_entries: MemoryPolicyEntryRow; memory_resources: MemoryResourceRow; memory_resource_revisions: MemoryResourceRevisionRow; + memory_revision_policy_requirements: MemoryRevisionPolicyRequirementRow; + memory_lineage_edges: MemoryLineageEdgeRow; + memory_policy_set_members: MemoryPolicySetMemberRow; + session_memory_subject_snapshots: SessionMemorySubjectSnapshotRow; + transcript_event_memory_policies: TranscriptEventMemoryPolicyRow; + transcript_event_memory_policy_details: TranscriptEventMemoryPolicyDetailRow; + transcript_events: TranscriptEventRow; + memory_run_exposure_resources: MemoryRunExposureResourceRow; memory_resource_subjects: MemoryResourceSubjectRow; memory_scoped_chunks: MemoryScopedChunkRow; memory_scoped_chunk_vectors: MemoryScopedChunkVectorRow; diff --git a/extensions/memory-core/src/memory/scoped-memory-resources.ts b/extensions/memory-core/src/memory/scoped-memory-resources.ts index 4c17998150aa..72821fa165ef 100644 --- a/extensions/memory-core/src/memory/scoped-memory-resources.ts +++ b/extensions/memory-core/src/memory/scoped-memory-resources.ts @@ -143,6 +143,131 @@ function removeArtifact(pathname: string): void { } catch {} } +type RevisionPolicyRequirement = Readonly<{ + policyId: string; + expectedRevisionId: string; + expectedRevocationEpoch: number; +}>; + +function readRevisionPolicyRequirements(params: { + database: Parameters>[0]; + revisionId: string; +}): readonly RevisionPolicyRequirement[] { + const db = getNodeSqliteKysely(params.database); + return Object.freeze( + executeSqliteQuerySync( + params.database, + db + .selectFrom("memory_revision_policy_requirements") + .select(["policy_id", "expected_revision_id", "expected_revocation_epoch"]) + .where("revision_id", "=", params.revisionId) + .orderBy("policy_id"), + ).rows.map((row) => + Object.freeze({ + policyId: row.policy_id, + expectedRevisionId: row.expected_revision_id, + expectedRevocationEpoch: row.expected_revocation_epoch, + }), + ), + ); +} + +/** Requirements and parent revisions are checked for every future exposure, not only at write time. */ +function isRevisionLineageCurrent(params: { + database: Parameters>[0]; + revisionId: string; + visited: Set; +}): boolean { + if (params.visited.has(params.revisionId)) { + return false; + } + params.visited.add(params.revisionId); + const db = getNodeSqliteKysely(params.database); + const requirements = readRevisionPolicyRequirements(params); + if (requirements.length === 0) { + return false; + } + for (const requirement of requirements) { + const current = executeSqliteQueryTakeFirstSync( + params.database, + db + .selectFrom("memory_policies as policy") + .innerJoin( + "memory_policy_revisions as revision", + "revision.revision_id", + "policy.current_revision_id", + ) + .select([ + "policy.current_revision_id", + "policy.revocation_epoch", + "policy.lifecycle_state as policy_lifecycle_state", + "revision.lifecycle_state as revision_lifecycle_state", + ]) + .where("policy.policy_id", "=", requirement.policyId), + ); + if ( + !current || + current.policy_lifecycle_state !== "active" || + current.revision_lifecycle_state !== "active" || + current.current_revision_id !== requirement.expectedRevisionId || + current.revocation_epoch !== requirement.expectedRevocationEpoch + ) { + return false; + } + } + const parents = executeSqliteQuerySync( + params.database, + db + .selectFrom("memory_lineage_edges") + .select(["parent_kind", "parent_id"]) + .where("child_revision_id", "=", params.revisionId) + .orderBy("parent_kind") + .orderBy("parent_id"), + ).rows; + for (const parent of parents) { + if (parent.parent_kind === "transcript-policy-set") { + // Transcript sources materialize their stable policy requirements and every exposed resource + // parent on the child revision. Those checks above and below are the durable invalidation path. + continue; + } + // Other Phase 2C producers add their own immutable parent types. A resource parent is already + // selectable today, so it must recurse rather than merely checking its direct lifecycle row. + if (parent.parent_kind !== "resource-revision") { + return false; + } + const revision = executeSqliteQueryTakeFirstSync( + params.database, + db + .selectFrom("memory_resource_revisions") + .select("lifecycle_state") + .where("revision_id", "=", parent.parent_id), + ); + if ( + revision?.lifecycle_state !== "active" || + !isRevisionLineageCurrent({ + database: params.database, + revisionId: parent.parent_id, + visited: params.visited, + }) + ) { + return false; + } + } + return true; +} + +/** Recovery paths use the same recursive check as normal reads before activating a pending revision. */ +export function isBuiltinScopedMemoryRevisionLineageCurrent(params: { + agentId: string; + revisionId: string; +}): boolean { + const agentId = normalizeAgentId(params.agentId); + const revisionId = normalizeScopedMemoryRequiredText(params.revisionId, "revisionId"); + return withScopedMemoryDatabase(agentId, (database) => + isRevisionLineageCurrent({ database, revisionId, visited: new Set() }), + ); +} + /** * Resolve a revision only while its immutable catalog evidence is current. * The Phase 1C runtime supplies the authorized store view; this foundation @@ -220,6 +345,7 @@ export function readBuiltinScopedMemoryRevisionSnapshot(params: { revision.current_policy_revocation_epoch !== revision.revocation_epoch || revision.source_policy_set_id !== createScopedMemorySourcePolicySetId(revision.current_revision_id) || + !isRevisionLineageCurrent({ database, revisionId, visited: new Set() }) || (revision.expires_at !== null && revision.expires_at <= nowMs) ) { return undefined; @@ -324,6 +450,7 @@ function createRevision(params: { ) .select([ "resource.resource_id", + "policy.policy_id", "policy.current_revision_id", "policy.revocation_epoch", "policy_revision.revision_number", @@ -379,6 +506,17 @@ function createRevision(params: { retired_at: null, }), ); + executeSqliteQuerySync( + database, + db.insertInto("memory_revision_policy_requirements").values({ + revision_id: revisionId, + policy_id: current.policy_id, + expected_revision_id: current.current_revision_id, + expected_revocation_epoch: current.revocation_epoch, + requirement_kind: "output-policy", + created_at: params.nowMs, + }), + ); const chunks = chunkScopedMemoryMarkdown(content); if (chunks.length > 0) { executeSqliteQuerySync( diff --git a/extensions/memory-core/src/memory/scoped-memory-runtime.test.ts b/extensions/memory-core/src/memory/scoped-memory-runtime.test.ts index 0580c42982cf..6f239a6f131f 100644 --- a/extensions/memory-core/src/memory/scoped-memory-runtime.test.ts +++ b/extensions/memory-core/src/memory/scoped-memory-runtime.test.ts @@ -20,9 +20,17 @@ import { consumeAdmittedChannelMemoryIdentityFromContext, createChannelMemoryIdentityAdmission, } from "../../../../src/channels/message-access/memory-identity-admission.js"; +import { appendSqliteTranscriptMessage } from "../../../../src/config/sessions/session-accessor.sqlite-transcript-write.js"; +import { readAuthorizedTranscriptDerivation } from "../../../../src/config/sessions/session-transcript-memory-policy.js"; +import { withOwnedSessionTranscriptWrites } from "../../../../src/config/sessions/transcript-write-context.js"; import { admitMemoryAuthorizationReadRuntime } from "../../../../src/plugins/memory-authorization-runtime.js"; import { resetMemoryIsolationCutoverForTest } from "../../../../src/plugins/memory-cutover.js"; -import { readLatestDurableMemoryRunExposure } from "../../../../src/plugins/memory-run-exposure-ledger.js"; +import { registerAgentRunContext, resetAgentRunRegistryForTest } from "../../../../src/infra/agent-run-registry.js"; +import { + persistMemoryRunExposureBeforeContentInDatabase, + readLatestDurableMemoryRunExposure, +} from "../../../../src/plugins/memory-run-exposure-ledger.js"; +import { prepareMemoryRunExposure } from "../../../../src/plugins/memory-run-exposure.js"; import { createEmptyPluginRegistry } from "../../../../src/plugins/registry-empty.js"; import { resetPluginRuntimeStateForTest, @@ -49,6 +57,7 @@ import { builtinScopedMemoryConformanceAdapter } from "./scoped-memory-policy.js import { createBuiltinScopedMemoryResource, readBuiltinScopedMemoryRevisionSnapshot, + setBuiltinScopedMemoryRevisionLifecycle, } from "./scoped-memory-resources.js"; import { builtinScopedMemoryAuthorizedRuntime, @@ -77,6 +86,7 @@ describe("builtin scoped authorized runtime", () => { dispatchReplyFromConfig.mockReset(); resetBuiltinScopedMemoryAuthorizedRuntimeForTest(); resetMemoryIsolationCutoverForTest(); + resetAgentRunRegistryForTest(); resetPluginRuntimeStateForTest(); closeOpenClawAgentDatabasesForTest(); closeOpenClawStateDatabaseForTest(); @@ -386,6 +396,20 @@ describe("builtin scoped authorized runtime", () => { now, revisionLifecycleState === "active" ? now : null, ); + database + .prepare( + `INSERT INTO memory_revision_policy_requirements + (revision_id, policy_id, expected_revision_id, expected_revocation_epoch, + requirement_kind, created_at) + VALUES (?, ?, ?, ?, 'output-policy', ?)`, + ) + .run( + revisionId, + row.policy_id, + params.policyRevisionId ?? row.current_revision_id, + row.revocation_epoch, + now, + ); database .prepare( `INSERT INTO memory_write_intents @@ -476,6 +500,253 @@ describe("builtin scoped authorized runtime", () => { ).rejects.toThrow("unavailable"); }); + it("authorizes compaction sources with derive rather than downgrading them to read", async () => { + const principalId = "derive-owner"; + const store = createBuiltinScopedMemoryStore({ + agentId: "main", + scopeKind: "user", + audienceKind: "user", + audienceId: principalId, + authorityKind: "user", + authorityOwnerId: principalId, + defaultCapabilities: ["retrieve", "read", "derive"], + actor: { kind: "human", id: principalId }, + reason: "derive source fixture", + }); + createBuiltinScopedMemoryResource({ + agentId: "main", + store, + logicalLocator: "MEMORY.md", + content: "DERIVE_ONLY_SOURCE_SENTINEL", + actor: { kind: "human", id: principalId }, + }); + const context = { + ...createContext(principalId), + operation: "derive" as const, + } satisfies MemoryContentAccessContext<"derive">; + const plan = await builtinScopedMemoryAuthorizedRuntime.authorize(context); + + const searched = await builtinScopedMemoryAuthorizedRuntime.searchAuthorized({ + context, + plan, + query: "DERIVE_ONLY_SOURCE_SENTINEL", + limit: 10, + }); + expect(searched).toMatchObject({ value: [{ snippet: "DERIVE_ONLY_SOURCE_SENTINEL" }] }); + const source = searched.value[0]; + if (!source) { + throw new Error("fixture expected a derivation source"); + } + const derived = await builtinScopedMemoryAuthorizedRuntime.writeAuthorized({ + context, + plan, + mutation: { + version: 1, + kind: "derive", + derivationPurpose: "flush", + mutationId: "derive-output", + idempotencyKey: "derive-output-request", + content: "DERIVED_OUTPUT_SENTINEL", + contentType: "markdown", + sourceHandles: [source.resourceHandle], + sourcePolicySetId: searched.exposureReceipt.sourcePolicySetId, + }, + }); + const derivedRevisionId = derived.resourceHandle?.resourceRevision; + if (!derivedRevisionId) { + throw new Error("fixture expected a derived revision"); + } + withScopedMemoryDatabase("main", (database) => { + expect( + database + .prepare( + "SELECT parent_kind, parent_id, relation_kind FROM memory_lineage_edges WHERE child_revision_id = ?", + ) + .all(derivedRevisionId), + ).toEqual([ + { + parent_kind: "resource-revision", + parent_id: source.resourceHandle.resourceRevision, + relation_kind: "derived-from", + }, + ]); + expect( + database + .prepare( + "SELECT count(*) AS count FROM memory_revision_policy_requirements WHERE revision_id = ?", + ) + .get(derivedRevisionId), + ).toEqual({ count: 1 }); + }); + setBuiltinScopedMemoryRevisionLifecycle({ + agentId: "main", + revisionId: source.resourceHandle.resourceRevision, + lifecycleState: "tombstoned", + }); + expect( + readBuiltinScopedMemoryRevisionSnapshot({ + agentId: "main", + storeIds: [store.storeId], + revisionId: derivedRevisionId, + }), + ).toBeUndefined(); + expect(plan.mounts[0]?.capabilities).toEqual(["retrieve", "read", "derive"]); + }); + + it("records transcript policy-set lineage for an authorized compaction derivation", async () => { + const principalId = "compaction-owner"; + const sourceStore = createBuiltinScopedMemoryStore({ + agentId: "main", + scopeKind: "user", + audienceKind: "user", + audienceId: principalId, + authorityKind: "user", + authorityOwnerId: principalId, + defaultCapabilities: ["retrieve", "read", "derive"], + actor: { kind: "human", id: principalId }, + reason: "compaction transcript fixture", + }); + const source = createBuiltinScopedMemoryResource({ + agentId: "main", + store: sourceStore, + logicalLocator: "MEMORY.md", + content: "COMPACTION_TRANSCRIPT_SOURCE_SENTINEL", + actor: { kind: "human", id: principalId }, + }); + const context = { + ...createContext(principalId), + operation: "derive" as const, + } satisfies MemoryContentAccessContext<"derive">; + const plan = await builtinScopedMemoryAuthorizedRuntime.authorize(context); + const database = openOpenClawAgentDatabase({ agentId: "main" }); + markCutOver(); + database.db + .prepare( + `INSERT INTO session_memory_subjects + (session_key, subject_kind, binding_id, principal_id, subject_revision, created_at) + VALUES (?, 'user', ?, ?, ?, 1)`, + ) + .run(context.sessionKey, `binding-${principalId}`, principalId, context.subjectRevision); + database.db + .prepare( + `INSERT INTO session_memory_subject_snapshots + (session_id, session_key, subject_revision, session_identity_revision, created_at) + VALUES (?, ?, ?, ?, 1)`, + ) + .run( + context.sessionId, + context.sessionKey, + context.subjectRevision, + context.sessionIdentityRevision, + ); + const exposure = prepareMemoryRunExposure({ + agentId: context.agentId, + sessionId: context.sessionId, + sessionKey: context.sessionKey, + runId: context.runId, + contextFingerprint: context.contextFingerprint, + planId: plan.planId, + memoryPolicyRevision: plan.memoryPolicyRevision, + sourcePolicySetIds: [source.sourcePolicySetId], + exposedResourceRevisions: [source.revisionId], + exposureReceiptIds: ["compaction-exposure-receipt"], + egressReceiptIds: ["compaction-egress-receipt"], + deliveryAudiences: context.delivery.audiences, + deliveryRevision: context.delivery.deliveryRevision, + egressRegistryRevision: context.delivery.egressRegistryRevision, + sessionIdentityRevision: context.sessionIdentityRevision, + subjectRevision: context.subjectRevision, + actorEvidence: { + version: 1, + kind: "principal", + actorKind: "human", + principalId, + assurance: "gateway-profile", + evidenceRevision: `binding-${principalId}`, + }, + delegationSnapshot: { version: 1, kind: "none" }, + hostFactsRevision: context.hostFactsRevision, + }); + expect(persistMemoryRunExposureBeforeContentInDatabase({ database, snapshot: exposure })).toBe( + true, + ); + await withOwnedSessionTranscriptWrites( + { + sessionTarget: { + agentId: context.agentId, + expectedWriterRunId: context.runId, + sessionId: context.sessionId, + sessionKey: context.sessionKey, + }, + withTranscriptWrite: async (run) => await run(), + }, + async () => { + await appendSqliteTranscriptMessage( + { agentId: context.agentId, sessionId: context.sessionId, sessionKey: context.sessionKey }, + { + message: { + role: "assistant", + content: [{ type: "text", text: "COMPACTION_TRANSCRIPT_EVENT_SENTINEL" }], + }, + }, + ); + }, + ); + const transcript = readAuthorizedTranscriptDerivation(database.db, context.sessionId); + if (!transcript) { + throw new Error("fixture expected an authorized transcript derivation"); + } + + const derived = await builtinScopedMemoryAuthorizedRuntime.writeAuthorized({ + context, + plan, + mutation: { + version: 1, + kind: "derive", + derivationPurpose: "compaction", + mutationId: "compaction-derived-output", + idempotencyKey: "compaction-derived-output-request", + content: "COMPACTION_DERIVED_OUTPUT_SENTINEL", + contentType: "markdown", + sourcePolicySetId: transcript.sourcePolicySetId, + transcriptSource: { + kind: "transcript", + sessionId: context.sessionId, + eventSeqs: transcript.eventSeqs, + sourcePolicySetId: transcript.sourcePolicySetId, + deliveryAudiencesJson: transcript.deliveryAudiencesJson, + }, + }, + }); + const derivedRevisionId = derived.resourceHandle?.resourceRevision; + if (!derivedRevisionId) { + throw new Error("fixture expected a derived revision"); + } + withScopedMemoryDatabase("main", (scopedDatabase) => { + expect( + scopedDatabase + .prepare( + `SELECT parent_kind, parent_id, relation_kind + FROM memory_lineage_edges + WHERE child_revision_id = ? + ORDER BY parent_kind, parent_id, relation_kind`, + ) + .all(derivedRevisionId), + ).toEqual([ + { + parent_kind: "resource-revision", + parent_id: source.revisionId, + relation_kind: "derived-from", + }, + { + parent_kind: "transcript-policy-set", + parent_id: transcript.sourcePolicySetId, + relation_kind: "compacted-from", + }, + ]); + }); + }); + it("keeps verified private stores isolated through the actual host and selected runtime", async () => { const aliceSession = { sessionKey: "agent:main:direct:alice", sessionId: "alice-session" }; const bobSession = { sessionKey: "agent:main:direct:bob", sessionId: "bob-session" }; @@ -599,6 +870,11 @@ describe("builtin scoped authorized runtime", () => { createPrivateResource(alicePrincipalId, "ALICE_FINAL_REPLY_AUTHORIZED_CONTENT"); markCutOver(); installBuiltinSelectedRuntime(); + registerAgentRunContext(runId, { + agentId: "main", + sessionId: session.sessionId, + sessionKey: session.sessionKey, + }); const host = createAuthorizedMemoryReadHost({ agentId: "main", diff --git a/extensions/memory-core/src/memory/scoped-memory-runtime.ts b/extensions/memory-core/src/memory/scoped-memory-runtime.ts index bec2147c6506..dd468acc1b2e 100644 --- a/extensions/memory-core/src/memory/scoped-memory-runtime.ts +++ b/extensions/memory-core/src/memory/scoped-memory-runtime.ts @@ -5,9 +5,12 @@ import type { AudienceRef, AuthorizedMemoryMutation, AuthorizedMemoryPlan, + AuthorizedTranscriptDerivationSource, AuthorizedMemoryReadParams, AuthorizedMemoryResultEnvelope, AuthorizedMemoryRuntime, + AuthorizedSealedCompactionArtifact, + AuthorizedSealedCompactionStageParams, AuthorizedMemorySearchParams, AuthorizedMemorySearchResult, AuthorizedMemoryStatus, @@ -38,6 +41,7 @@ import { } from "./scoped-memory-db.js"; import { evaluateBuiltinScopedMemoryPolicy } from "./scoped-memory-policy.js"; import { + isBuiltinScopedMemoryRevisionLineageCurrent, readBuiltinScopedMemoryRevisionSnapshot, resolveBuiltinScopedMemoryArtifactPath, } from "./scoped-memory-resources.js"; @@ -234,6 +238,8 @@ function createPlan(context: MemoryAccessContext): PlanState { capabilities: Object.freeze( context.operation === "read" ? (["retrieve", "read"] as const) + : context.operation === "derive" + ? (["retrieve", "read", "derive"] as const) : ([context.operation] as const), ), audienceRevision: store.audienceRevision, @@ -530,6 +536,27 @@ type MutableScopedRevision = Readonly<{ contentBytes: number; }>; +type DerivationSource = Readonly<{ + revisionId: string; + policyRevisionId: string; + audience: AudienceRef; + policyRequirements: readonly Readonly<{ + policyId: string; + expectedRevisionId: string; + expectedRevocationEpoch: number; + }>[]; +}>; + +type TranscriptDerivationSource = Readonly<{ + sourcePolicySetId: string; + sourceRevisionIds: readonly string[]; + policyRequirements: readonly Readonly<{ + policyId: string; + expectedRevisionId: string; + expectedRevocationEpoch: number; + }>[]; +}>; + function contentHash(content: string): string { return createHash("sha256").update(content).digest("hex"); } @@ -671,11 +698,22 @@ function defaultAudience(context: MemoryAccessContext): AudienceRef | undefined } } +function sameAudience(left: AudienceRef, right: AudienceRef): boolean { + return left.kind === right.kind && left.id === right.id; +} + function selectWriteStore(params: { database: Parameters>[0]; agentId: string; state: PlanState; -}): { storeId: string; pathKey: string; policyRevisionId: string; policyRevocationEpoch: number } { +}): { + storeId: string; + pathKey: string; + policyId: string; + policyRevisionId: string; + policyRevocationEpoch: number; + audience: AudienceRef; +} { const audience = defaultAudience(params.state.context); if (!audience) { throw new Error("authorized memory mutation is unavailable"); @@ -689,7 +727,10 @@ function selectWriteStore(params: { .innerJoin("memory_policies as policy", "policy.policy_id", "store.policy_id") .select([ "store.store_id", + "store.audience_kind", + "store.audience_id", "root.path_key", + "policy.policy_id", "policy.current_revision_id", "policy.revocation_epoch", ]) @@ -708,11 +749,272 @@ function selectWriteStore(params: { return { storeId: row.store_id, pathKey: row.path_key, + policyId: row.policy_id, policyRevisionId: row.current_revision_id, policyRevocationEpoch: row.revocation_epoch, + audience: { kind: row.audience_kind, id: row.audience_id }, }; } +/** A derive mutation can only retain exact same-audience sources; mixed sets are denied, not widened. */ +function resolveDerivationSources(params: { + database: Parameters>[0]; + agentId: string; + state: PlanState; + targetAudience: AudienceRef; + sourceHandles: readonly AuthorizedResourceHandle[]; + sourcePolicySetId: string; +}): readonly DerivationSource[] { + if (params.sourceHandles.length === 0 || !params.sourcePolicySetId.trim()) { + throw new Error("authorized memory derivation is unavailable"); + } + const db = getNodeSqliteKysely(params.database); + const sources: DerivationSource[] = []; + for (const handle of params.sourceHandles) { + const stored = params.state.handles.get(handle.handleId); + if ( + !stored || + stored.planId !== params.state.plan.planId || + stored.contextFingerprint !== params.state.contextFingerprint || + stored.resourceRevision !== handle.resourceRevision || + stored.policyRevision !== handle.policyRevision || + stored.expiresAt !== handle.expiresAt + ) { + throw new Error("authorized memory derivation is unavailable"); + } + const snapshot = readBuiltinScopedMemoryRevisionSnapshot({ + agentId: params.agentId, + storeIds: params.state.stores.map((store) => store.storeId), + revisionId: stored.resourceRevision, + }); + if (!snapshot || snapshot.policyRevisionId !== stored.policyRevision) { + throw new Error("authorized memory derivation is unavailable"); + } + const source = executeSqliteQueryTakeFirstSync( + params.database, + db + .selectFrom("memory_resource_revisions as revision") + .innerJoin("memory_resources as resource", "resource.resource_id", "revision.resource_id") + .innerJoin("memory_stores as store", "store.store_id", "resource.store_id") + .select(["store.audience_kind", "store.audience_id"]) + .where("revision.revision_id", "=", stored.resourceRevision) + .where("resource.agent_id", "=", params.agentId), + ); + const audience = source && { kind: source.audience_kind, id: source.audience_id }; + if (!audience || !sameAudience(audience, params.targetAudience)) { + throw new Error("authorized memory derivation has no representable audience"); + } + const policyRequirements = executeSqliteQuerySync( + params.database, + db + .selectFrom("memory_revision_policy_requirements") + .select(["policy_id", "expected_revision_id", "expected_revocation_epoch"]) + .where("revision_id", "=", stored.resourceRevision) + .orderBy("policy_id"), + ).rows.map((requirement) => + Object.freeze({ + policyId: requirement.policy_id, + expectedRevisionId: requirement.expected_revision_id, + expectedRevocationEpoch: requirement.expected_revocation_epoch, + }), + ); + if (policyRequirements.length === 0) { + throw new Error("authorized memory derivation is unavailable"); + } + sources.push( + Object.freeze({ + revisionId: stored.resourceRevision, + policyRevisionId: snapshot.policyRevisionId, + audience, + policyRequirements: Object.freeze(policyRequirements), + }), + ); + } + if (new Set(sources.map((source) => source.revisionId)).size !== sources.length) { + throw new Error("authorized memory derivation is unavailable"); + } + const expectedPolicySetId = `mpset1_${hash( + sources.map((source) => `mps1_${source.policyRevisionId}`).toSorted(), + )}`; + if (params.sourcePolicySetId !== expectedPolicySetId) { + throw new Error("authorized memory derivation is unavailable"); + } + return Object.freeze(sources); +} + +/** Transcript companions are an immutable source set, not a mutable session-path permission. */ +function resolveTranscriptDerivationSource(params: { + database: Parameters>[0]; + source: AuthorizedTranscriptDerivationSource; + sourcePolicySetId: string; + targetAudience: AudienceRef; +}): TranscriptDerivationSource { + const source = params.source; + if ( + !source || + source.kind !== "transcript" || + source.sourcePolicySetId !== params.sourcePolicySetId || + !source.sessionId.trim() || + source.eventSeqs.length === 0 + ) { + throw new Error("authorized transcript derivation is unavailable"); + } + const eventSeqs = [...source.eventSeqs]; + if ( + eventSeqs.some((eventSeq) => !Number.isSafeInteger(eventSeq) || eventSeq < 0) || + new Set(eventSeqs).size !== eventSeqs.length || + eventSeqs.some((eventSeq, index) => index > 0 && eventSeqs[index - 1]! >= eventSeq) + ) { + throw new Error("authorized transcript derivation is unavailable"); + } + let audiences: unknown; + try { + audiences = JSON.parse(source.deliveryAudiencesJson); + } catch { + throw new Error("authorized transcript derivation is unavailable"); + } + if ( + !Array.isArray(audiences) || + audiences.length !== 1 || + typeof audiences[0] !== "object" || + audiences[0] === null || + (audiences[0] as AudienceRef).kind !== params.targetAudience.kind || + (audiences[0] as AudienceRef).id !== params.targetAudience.id + ) { + throw new Error("authorized memory derivation has no representable audience"); + } + const db = getNodeSqliteKysely(params.database); + const subject = executeSqliteQueryTakeFirstSync( + params.database, + db + .selectFrom("session_memory_subject_snapshots") + .select(["session_identity_revision", "subject_revision"]) + .where("session_id", "=", source.sessionId) + .limit(1), + ); + if (!subject) { + throw new Error("authorized transcript derivation is unavailable"); + } + const events = executeSqliteQuerySync( + params.database, + db + .selectFrom("transcript_events as event") + .innerJoin("transcript_event_memory_policies as policy", (join) => + join + .onRef("policy.session_id", "=", "event.session_id") + .onRef("policy.event_seq", "=", "event.seq"), + ) + .innerJoin("transcript_event_memory_policy_details as detail", (join) => + join + .onRef("detail.session_id", "=", "policy.session_id") + .onRef("detail.event_seq", "=", "policy.event_seq"), + ) + .select([ + "event.seq", + "policy.delivery_audiences_json", + "policy.run_exposure_set_id", + "policy.session_identity_revision", + "policy.subject_revision", + ]) + .where("event.session_id", "=", source.sessionId) + .where("event.seq", "in", eventSeqs) + .where("policy.authorization_status", "=", "authorized") + .where("policy.source_policy_set_id", "=", source.sourcePolicySetId) + .where("policy.delivery_audiences_json", "=", source.deliveryAudiencesJson) + .where("detail.retention_state", "=", "retained") + .where("detail.normalized_audience_intersection_json", "=", source.deliveryAudiencesJson) + .where("detail.finalized_delivery_audiences_json", "=", source.deliveryAudiencesJson) + .orderBy("event.seq"), + ).rows; + if ( + events.length !== eventSeqs.length || + events.some( + (event, index) => + event.seq !== eventSeqs[index] || + event.run_exposure_set_id === null || + event.session_identity_revision !== subject.session_identity_revision || + event.subject_revision !== subject.subject_revision, + ) + ) { + throw new Error("authorized transcript derivation is unavailable"); + } + const requirements = executeSqliteQuerySync( + params.database, + db + .selectFrom("memory_policy_set_members") + .select(["policy_id", "expected_revision_id", "expected_revocation_epoch", "retention_state"]) + .where("policy_set_id", "=", source.sourcePolicySetId) + .orderBy("policy_id"), + ).rows; + if (requirements.length === 0 || requirements.some((requirement) => requirement.retention_state !== "retained")) { + throw new Error("authorized transcript derivation is unavailable"); + } + for (const requirement of requirements) { + const current = executeSqliteQueryTakeFirstSync( + params.database, + db + .selectFrom("memory_policies as policy") + .innerJoin( + "memory_policy_revisions as revision", + "revision.revision_id", + "policy.current_revision_id", + ) + .select(["policy.lifecycle_state", "policy.revocation_epoch", "revision.lifecycle_state as revision_state"]) + .where("policy.policy_id", "=", requirement.policy_id) + .where("policy.current_revision_id", "=", requirement.expected_revision_id) + .where("policy.revocation_epoch", "=", requirement.expected_revocation_epoch) + .limit(1), + ); + if (current?.lifecycle_state !== "active" || current.revision_state !== "active") { + throw new Error("authorized transcript derivation is unavailable"); + } + } + const exposureSetIds = [...new Set(events.map((event) => event.run_exposure_set_id!))]; + const exposures = executeSqliteQuerySync( + params.database, + db + .selectFrom("memory_run_exposure_resources as exposure") + .innerJoin( + "memory_resource_revisions as revision", + "revision.revision_id", + "exposure.resource_revision_id", + ) + .select([ + "exposure.exposure_set_id", + "exposure.resource_revision_id", + "revision.expires_at", + "revision.lifecycle_state", + ]) + .where("exposure.exposure_set_id", "in", exposureSetIds), + ).rows; + const nowMs = Date.now(); + if ( + exposureSetIds.some((exposureSetId) => !exposures.some((row) => row.exposure_set_id === exposureSetId)) || + exposures.some( + (exposure) => + exposure.lifecycle_state !== "active" || + (exposure.expires_at !== null && exposure.expires_at <= nowMs), + ) + ) { + throw new Error("authorized transcript derivation is unavailable"); + } + return Object.freeze({ + sourcePolicySetId: source.sourcePolicySetId, + sourceRevisionIds: Object.freeze( + [...new Set(exposures.map((exposure) => exposure.resource_revision_id))].toSorted(), + ), + policyRequirements: Object.freeze( + requirements.map((requirement) => + Object.freeze({ + policyId: requirement.policy_id, + expectedRevisionId: requirement.expected_revision_id, + expectedRevocationEpoch: requirement.expected_revocation_epoch, + }), + ), + ), + }); +} + function resolveWriteTarget(params: { database: Parameters>[0]; agentId: string; @@ -1005,6 +1307,9 @@ function recoverPendingWrites(agentId: string): void { }); continue; } + // The recovery transaction closes over this value. Keep the validated + // revision immutable so it cannot become an absent intent reference. + const revisionId = intent.pending_revision_id; const directory = path.join(resolveScopedMemoryArtifactBase(databasePath), intent.path_key); const known = knownByDirectory.get(directory) ?? new Set(); known.add(intent.final_locator); @@ -1052,7 +1357,7 @@ function recoverPendingWrites(agentId: string): void { quarantineWriteIntent({ database, intentId: intent.intent_id, - revisionId: intent.pending_revision_id, + revisionId, nowMs: Date.now(), reasonCode: "recovery-artifact-mismatch", }); @@ -1083,7 +1388,7 @@ function recoverPendingWrites(agentId: string): void { "policy.current_revision_id", "policy.revocation_epoch", ]) - .where("revision.revision_id", "=", intent.pending_revision_id) + .where("revision.revision_id", "=", revisionId) .where("resource.agent_id", "=", agentId) .where("store.lifecycle_state", "=", "active") .where("policy.lifecycle_state", "=", "active"), @@ -1091,7 +1396,11 @@ function recoverPendingWrites(agentId: string): void { if ( !revision || revision.policy_revision_id !== revision.current_revision_id || - revision.policy_revocation_epoch !== revision.revocation_epoch + revision.policy_revocation_epoch !== revision.revocation_epoch || + !isBuiltinScopedMemoryRevisionLineageCurrent({ + agentId, + revisionId, + }) ) { policyChanged = true; requireExactlyOneAffected( @@ -1100,7 +1409,7 @@ function recoverPendingWrites(agentId: string): void { db .updateTable("memory_resource_revisions") .set({ lifecycle_state: "quarantined", retired_at: Date.now() }) - .where("revision_id", "=", intent.pending_revision_id) + .where("revision_id", "=", revisionId) .where("lifecycle_state", "=", "pending"), ), "recovery policy quarantine revision", @@ -1136,7 +1445,7 @@ function recoverPendingWrites(agentId: string): void { db .updateTable("memory_resource_revisions") .set({ lifecycle_state: "active", activated_at: Date.now() }) - .where("revision_id", "=", intent.pending_revision_id) + .where("revision_id", "=", revisionId) .where("lifecycle_state", "=", "pending"), ), "recovery activation revision", @@ -1174,7 +1483,7 @@ function recoverPendingWrites(agentId: string): void { indexRecoveredRevision({ database, intentId: intent.intent_id, - revisionId: intent.pending_revision_id, + revisionId, content, nowMs: Date.now(), }); @@ -1233,6 +1542,30 @@ async function writeAuthorizedMutation(params: { const result = withScopedMemoryDatabase(agentId, (database, databasePath) => { const store = selectWriteStore({ database, agentId, state }); const db = getNodeSqliteKysely(database); + const derivationSources = + params.mutation.kind === "derive" && "sourceHandles" in params.mutation + ? resolveDerivationSources({ + database, + agentId, + state, + targetAudience: store.audience, + sourceHandles: params.mutation.sourceHandles, + sourcePolicySetId: params.mutation.sourcePolicySetId, + }) + : []; + const transcriptDerivationSource = + params.mutation.kind === "derive" && "transcriptSource" in params.mutation + ? resolveTranscriptDerivationSource({ + database, + source: params.mutation.transcriptSource, + sourcePolicySetId: params.mutation.sourcePolicySetId, + targetAudience: store.audience, + }) + : undefined; + const transcriptDerivationPurpose = + params.mutation.kind === "derive" && "transcriptSource" in params.mutation + ? params.mutation.derivationPurpose + : undefined; const existing = "target" in params.mutation ? resolveWriteTarget({ @@ -1503,6 +1836,90 @@ async function writeAuthorizedMutation(params: { retired_at: null, }), ); + executeSqliteQuerySync( + database, + db + .insertInto("memory_revision_policy_requirements") + .values({ + revision_id: revisionId, + policy_id: store.policyId, + expected_revision_id: store.policyRevisionId, + expected_revocation_epoch: store.policyRevocationEpoch, + requirement_kind: "output-policy", + created_at: nowMs, + }), + ); + for (const source of derivationSources) { + for (const requirement of source.policyRequirements) { + executeSqliteQuerySync( + database, + db + .insertInto("memory_revision_policy_requirements") + .values({ + revision_id: revisionId, + policy_id: requirement.policyId, + expected_revision_id: requirement.expectedRevisionId, + expected_revocation_epoch: requirement.expectedRevocationEpoch, + requirement_kind: "source-policy", + created_at: nowMs, + }) + .onConflict((conflict) => conflict.columns(["revision_id", "policy_id"]).doNothing()), + ); + } + executeSqliteQuerySync( + database, + db.insertInto("memory_lineage_edges").values({ + child_revision_id: revisionId, + parent_kind: "resource-revision", + parent_id: source.revisionId, + relation_kind: "derived-from", + created_at: nowMs, + }), + ); + } + if (transcriptDerivationSource && transcriptDerivationPurpose) { + for (const requirement of transcriptDerivationSource.policyRequirements) { + executeSqliteQuerySync( + database, + db + .insertInto("memory_revision_policy_requirements") + .values({ + revision_id: revisionId, + policy_id: requirement.policyId, + expected_revision_id: requirement.expectedRevisionId, + expected_revocation_epoch: requirement.expectedRevocationEpoch, + requirement_kind: "source-policy", + created_at: nowMs, + }) + .onConflict((conflict) => conflict.columns(["revision_id", "policy_id"]).doNothing()), + ); + } + for (const sourceRevisionId of transcriptDerivationSource.sourceRevisionIds) { + executeSqliteQuerySync( + database, + db.insertInto("memory_lineage_edges").values({ + child_revision_id: revisionId, + parent_kind: "resource-revision", + parent_id: sourceRevisionId, + relation_kind: "derived-from", + created_at: nowMs, + }), + ); + } + executeSqliteQuerySync( + database, + db.insertInto("memory_lineage_edges").values({ + child_revision_id: revisionId, + parent_kind: "transcript-policy-set", + parent_id: transcriptDerivationSource.sourcePolicySetId, + relation_kind: + transcriptDerivationPurpose === "compaction" + ? "compacted-from" + : "flushed-from", + created_at: nowMs, + }), + ); + } executeSqliteQuerySync( database, db.insertInto("memory_write_intents").values({ @@ -1577,6 +1994,29 @@ async function writeAuthorizedMutation(params: { ) { throw new Error("authorized memory mutation is unavailable"); } + if (params.mutation.kind === "derive") { + // The model work and artifact rename already happened; this final synchronous reread is + // the authority boundary that prevents a revoke or parent tombstone from activating it. + if ("sourceHandles" in params.mutation) { + resolveDerivationSources({ + database, + agentId, + state, + targetAudience: currentStore.audience, + sourceHandles: params.mutation.sourceHandles, + sourcePolicySetId: params.mutation.sourcePolicySetId, + }); + } else if ("transcriptSource" in params.mutation) { + resolveTranscriptDerivationSource({ + database, + source: params.mutation.transcriptSource, + sourcePolicySetId: params.mutation.sourcePolicySetId, + targetAudience: currentStore.audience, + }); + } else { + throw new Error("authorized memory derivation is unavailable"); + } + } if (existing) { executeSqliteQuerySync( database, @@ -1669,6 +2109,263 @@ async function writeAuthorizedMutation(params: { return result; } +/** + * Files are finalized before the core transaction starts. A crash before commit + * leaves only an uncatalogued artifact, which recovery quarantines; a committed + * revision never needs a filesystem operation to become readable. + */ +async function stageSealedCompaction( + params: AuthorizedSealedCompactionStageParams, +): Promise { + if (!params.content.trim()) { + throw new Error("sealed compaction content is unavailable"); + } + const state = readPlan({ context: params.context, plan: params.plan }); + if (!state || state.expiresAtMs <= Date.now()) { + throw new Error("sealed compaction authorization is unavailable"); + } + const agentId = params.context.agentId; + return withScopedMemoryDatabase(agentId, (database, databasePath) => { + const store = selectWriteStore({ database, agentId, state }); + const source = resolveTranscriptDerivationSource({ + database, + source: params.transcriptSource, + sourcePolicySetId: params.transcriptSource.sourcePolicySetId, + targetAudience: store.audience, + }); + const revisionId = randomUUID(); + const resourceId = randomUUID(); + const intentId = randomUUID(); + const finalLocator = `r1_${revisionId}.md`; + const stageLocator = `scst1_${intentId}.tmp`; + const directory = path.join(resolveScopedMemoryArtifactBase(databasePath), store.pathKey); + const finalPath = resolveBuiltinScopedMemoryArtifactPath({ + databasePath, + pathKey: store.pathKey, + artifactLocator: finalLocator, + }); + const stagePath = path.join(directory, stageLocator); + const hash = contentHash(params.content); + const bytes = Buffer.byteLength(params.content); + fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); + const descriptor = fs.openSync(stagePath, "wx", 0o600); + try { + fs.writeFileSync(descriptor, params.content, "utf8"); + fs.fchmodSync(descriptor, 0o600); + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } + syncDirectory(directory); + fs.renameSync(stagePath, finalPath); + syncDirectory(directory); + const verified = readVerifiedFile({ + pathname: finalPath, + contentHash: hash, + contentBytes: bytes, + }); + if (verified === undefined) { + throw new Error("sealed compaction artifact is unavailable"); + } + return Object.freeze({ + resourceRevisionId: revisionId, + commitInTransaction({ database: transactionDatabase, compactionPolicyId, eventSeq }) { + if (state.expiresAtMs <= Date.now()) { + throw new Error("sealed compaction authorization is unavailable"); + } + const currentStore = selectWriteStore({ + database: transactionDatabase, + agentId, + state, + }); + if ( + currentStore.storeId !== store.storeId || + currentStore.policyRevisionId !== store.policyRevisionId || + currentStore.policyRevocationEpoch !== store.policyRevocationEpoch + ) { + throw new Error("sealed compaction authorization is unavailable"); + } + const currentSource = resolveTranscriptDerivationSource({ + database: transactionDatabase, + source: params.transcriptSource, + sourcePolicySetId: params.transcriptSource.sourcePolicySetId, + targetAudience: currentStore.audience, + }); + if ( + currentSource.sourcePolicySetId !== source.sourcePolicySetId || + currentSource.sourceRevisionIds.length !== source.sourceRevisionIds.length || + currentSource.sourceRevisionIds.some( + (revision, index) => revision !== source.sourceRevisionIds[index], + ) + ) { + throw new Error("sealed compaction source is unavailable"); + } + const db = getNodeSqliteKysely(transactionDatabase); + const nowMs = Date.now(); + executeSqliteQuerySync( + transactionDatabase, + db.insertInto("memory_resources").values({ + resource_id: resourceId, + agent_id: agentId, + store_id: store.storeId, + logical_locator: `memory/${revisionId}.md`, + source: "memory", + created_at: nowMs, + }), + ); + executeSqliteQuerySync( + transactionDatabase, + db.insertInto("memory_resource_revisions").values({ + revision_id: revisionId, + resource_id: resourceId, + revision_number: 1, + artifact_locator: finalLocator, + content_hash: hash, + content_bytes: bytes, + policy_revision_id: store.policyRevisionId, + policy_revocation_epoch: store.policyRevocationEpoch, + source_policy_set_id: createScopedMemorySourcePolicySetId(store.policyRevisionId), + lifecycle_state: "active", + actor_kind: + params.context.actor.kind === "principal" + ? params.context.actor.actorKind + : "unattributed", + actor_id: + params.context.actor.kind === "principal" ? params.context.actor.principalId : null, + expires_at: null, + created_at: nowMs, + activated_at: nowMs, + retired_at: null, + }), + ); + executeSqliteQuerySync( + transactionDatabase, + db.insertInto("memory_revision_policy_requirements").values({ + revision_id: revisionId, + policy_id: store.policyId, + expected_revision_id: store.policyRevisionId, + expected_revocation_epoch: store.policyRevocationEpoch, + requirement_kind: "output-policy", + created_at: nowMs, + }), + ); + for (const requirement of source.policyRequirements) { + executeSqliteQuerySync( + transactionDatabase, + db + .insertInto("memory_revision_policy_requirements") + .values({ + revision_id: revisionId, + policy_id: requirement.policyId, + expected_revision_id: requirement.expectedRevisionId, + expected_revocation_epoch: requirement.expectedRevocationEpoch, + requirement_kind: "source-policy", + created_at: nowMs, + }) + .onConflict((conflict) => + conflict.columns(["revision_id", "policy_id"]).doNothing(), + ), + ); + } + const lineage = [ + ...source.sourceRevisionIds.map((parentId) => ({ + parent_kind: "resource-revision" as const, + parent_id: parentId, + relation_kind: "derived-from" as const, + })), + { + parent_kind: "transcript-policy-set" as const, + parent_id: source.sourcePolicySetId, + relation_kind: "compacted-from" as const, + }, + { + parent_kind: "compaction-policy" as const, + parent_id: compactionPolicyId, + relation_kind: "compacted-from" as const, + }, + ]; + for (const parent of lineage) { + executeSqliteQuerySync( + transactionDatabase, + db.insertInto("memory_lineage_edges").values({ + child_revision_id: revisionId, + ...parent, + created_at: nowMs, + }), + ); + } + const chunks = chunkContent(verified); + if (chunks.length > 0) { + executeSqliteQuerySync( + transactionDatabase, + db.insertInto("memory_scoped_chunks").values( + chunks.map((chunk) => ({ + chunk_id: randomUUID(), + revision_id: revisionId, + chunk_ordinal: chunk.ordinal, + start_line: chunk.startLine, + end_line: chunk.endLine, + text: chunk.text, + content_hash: contentHash(chunk.text), + model: "builtin-markdown-v1", + updated_at: nowMs, + })), + ), + ); + } + executeSqliteQuerySync( + transactionDatabase, + db.insertInto("memory_write_intents").values({ + intent_id: intentId, + idempotency_key: `sealed-compaction:${compactionPolicyId}`, + mutation_id: compactionPolicyId, + agent_id: agentId, + request_id: params.context.requestId, + run_id: params.context.runId, + context_fingerprint: params.context.contextFingerprint, + plan_id: params.plan.planId, + mutation_kind: "derive", + store_id: store.storeId, + resource_id: resourceId, + pending_revision_id: revisionId, + staged_locator: stageLocator, + final_locator: finalLocator, + content_hash: hash, + content_bytes: bytes, + state: "active", + created_at: nowMs, + updated_at: nowMs, + activated_at: nowMs, + indexed_at: nowMs, + }), + ); + executeSqliteQuerySync( + transactionDatabase, + db.insertInto("memory_audit_outbox").values({ + event_id: randomUUID(), + intent_id: intentId, + agent_id: agentId, + request_id: params.context.requestId, + run_id: params.context.runId, + actor_ref: auditActorRef(params.context), + subject_ref: auditSubjectRef(params.context), + operation: params.context.operation, + resource_revision_id: revisionId, + content_hash: hash, + decision: "committed", + reason_code: `sealed-compaction:${eventSeq}`, + state: "pending", + attempts: 0, + created_at: nowMs, + updated_at: nowMs, + delivered_at: null, + }), + ); + }, + }); + }); +} + const builtinScopedMemoryRuntime = { async authorize(context: MemoryAccessContext): Promise { recoverPendingWrites(context.agentId); @@ -1679,9 +2376,9 @@ const builtinScopedMemoryRuntime = { }, async searchAuthorized( - params: AuthorizedMemorySearchParams<"read">, + params: AuthorizedMemorySearchParams<"read"> | AuthorizedMemorySearchParams<"derive">, ): Promise> { - if (params.context.operation !== "read") { + if (params.context.operation !== "read" && params.context.operation !== "derive") { throw new Error("authorized memory search is unavailable"); } const state = readPlan(params); @@ -1738,9 +2435,9 @@ const builtinScopedMemoryRuntime = { }, async readAuthorized( - params: AuthorizedMemoryReadParams<"read">, + params: AuthorizedMemoryReadParams<"read"> | AuthorizedMemoryReadParams<"derive">, ): Promise> { - if (params.context.operation !== "read") { + if (params.context.operation !== "read" && params.context.operation !== "derive") { throw new Error("authorized memory read is unavailable"); } const state = readPlan(params); @@ -1797,6 +2494,10 @@ const builtinScopedMemoryRuntime = { return await writeAuthorizedMutation(params); }, + async stageSealedCompaction(params: AuthorizedSealedCompactionStageParams) { + return await stageSealedCompaction(params); + }, + async importAuthorized(params: { context: MemoryAccessContext; plan: AuthorizedMemoryPlan; diff --git a/extensions/memory-core/src/short-term-promotion-apply.ts b/extensions/memory-core/src/short-term-promotion-apply.ts index ebdfe9a882e3..1e1d53cde14e 100644 --- a/extensions/memory-core/src/short-term-promotion-apply.ts +++ b/extensions/memory-core/src/short-term-promotion-apply.ts @@ -15,6 +15,8 @@ import { storeMemoryPreimage, } from "./dreaming-consolidation-artifacts.js"; import { + hasOnePromotionAuthorizedView, + isPromotionAuthorizedViewBlocked, isConsolidationCandidateEligible, isPromotionOriginBlocked, } from "./dreaming-consolidation-candidates.js"; @@ -33,6 +35,7 @@ import { writeMemoryContent, } from "./short-term-promotion-memory-write.js"; import { + buildPromotionAuthorizedViewAnnotation, buildPromotionRecallAnnotations, groupPromotionCandidatesByProjectKey, } from "./short-term-promotion-metadata.js"; @@ -86,7 +89,7 @@ function buildPromotionSection( // rehydrated snippet so ranking, provenance, and dream narratives remain // tied to the source entry instead of this presentation budget. lines.push( - `- ${formatPromotedSnippetForMemory(candidate.snippet, maxPromotedSnippetTokens)} ${metadata} ${buildPromotionRecallAnnotations(candidate)}`, + `- ${formatPromotedSnippetForMemory(candidate.snippet, maxPromotedSnippetTokens)} ${metadata} ${buildPromotionRecallAnnotations(candidate)}${buildPromotionAuthorizedViewAnnotation(candidate)}`, ); } if (projectGroups.length > 1) { @@ -165,6 +168,7 @@ function consolidationCandidateFingerprint(candidate: PromotionCandidate): strin endLine: candidate.endLine, snippet: candidate.snippet, provenance: candidate.provenance, + authorizedView: candidate.authorizedView, projectKey: candidate.projectKey, }); } @@ -277,6 +281,7 @@ export async function applyShortTermPromotions( startLine: entry.startLine, endLine: entry.endLine, snippet: entry.snippet, + ...(entry.authorizedView ? { authorizedView: entry.authorizedView } : {}), }, entry.provenance, ) @@ -295,21 +300,23 @@ export async function applyShortTermPromotions( // legitimate daily-note candidates stay eligible. const reason = isPromotionOriginBlocked(candidate) ? `origin filter (${candidate.provenance?.originClass})` - : options.consolidation && (!latest || !isConsolidationCandidateEligible(candidate)) - ? "consolidation origin/session filter" - : isContaminatedDreamingSnippet(candidate.snippet) - ? "contamination filter" - : candidate.promotedAt || latest?.promotedAt - ? "already promoted" - : candidate.score < minScore - ? `score threshold (${candidate.score.toFixed(3)} < ${minScore})` - : candidate.signalCount < minRecallCount - ? `signal threshold (${candidate.signalCount} < ${minRecallCount})` - : queryCount < minUniqueQueries - ? `query threshold (${queryCount} < ${minUniqueQueries})` - : maxAgeDays >= 0 && candidate.ageDays > maxAgeDays - ? `age threshold (${candidate.ageDays.toFixed(1)}d > ${maxAgeDays}d)` - : undefined; + : isPromotionAuthorizedViewBlocked(candidate) + ? "authorized view filter" + : options.consolidation && (!latest || !isConsolidationCandidateEligible(candidate)) + ? "consolidation origin/session filter" + : isContaminatedDreamingSnippet(candidate.snippet) + ? "contamination filter" + : candidate.promotedAt || latest?.promotedAt + ? "already promoted" + : candidate.score < minScore + ? `score threshold (${candidate.score.toFixed(3)} < ${minScore})` + : candidate.signalCount < minRecallCount + ? `signal threshold (${candidate.signalCount} < ${minRecallCount})` + : queryCount < minUniqueQueries + ? `query threshold (${queryCount} < ${minUniqueQueries})` + : maxAgeDays >= 0 && candidate.ageDays > maxAgeDays + ? `age threshold (${candidate.ageDays.toFixed(1)}d > ${maxAgeDays}d)` + : undefined; if (reason) { rejectionReasons.set(candidate.key, reason); } @@ -319,10 +326,16 @@ export async function applyShortTermPromotions( for (const candidate of eligible.slice(limit)) { rejectionReasons.set(candidate.key, `selection limit (${limit})`); } + const selectedInOneAuthorizedView = hasOnePromotionAuthorizedView(selected) ? selected : []; + if (selectedInOneAuthorizedView.length === 0 && selected.length > 0) { + for (const candidate of selected) { + rejectionReasons.set(candidate.key, "authorized view boundary"); + } + } const rehydratedSelected: PromotionCandidate[] = []; const plannedSourceFingerprints = new Map(); - for (const candidate of selected) { + for (const candidate of selectedInOneAuthorizedView) { const sourceFingerprintBefore = await promotionSourceFingerprint(workspaceDir, candidate); const rehydrated = await rehydratePromotionCandidate(workspaceDir, candidate); const sourceFingerprintAfter = await promotionSourceFingerprint(workspaceDir, candidate); @@ -454,6 +467,9 @@ export async function applyShortTermPromotions( continue; } const currentCandidate = withAuthoritativeProvenance(candidate, entry.provenance); + if (isPromotionAuthorizedViewBlocked(currentCandidate)) { + continue; + } if (options.consolidation && !isConsolidationCandidateEligible(currentCandidate)) { continue; } diff --git a/extensions/memory-core/src/short-term-promotion-metadata.ts b/extensions/memory-core/src/short-term-promotion-metadata.ts index c21ca5cc702e..b9d23f0ca452 100644 --- a/extensions/memory-core/src/short-term-promotion-metadata.ts +++ b/extensions/memory-core/src/short-term-promotion-metadata.ts @@ -67,3 +67,16 @@ export function buildPromotionRecallAnnotations( const project = projectKey ? ` ` : ""; return ` ${project}`; } + +export function buildPromotionAuthorizedViewAnnotation( + candidate: Pick, +): string { + const view = candidate.authorizedView; + if (!view) { + return ""; + } + // Keep opaque authority identifiers inert in markdown while recording the + // immutable source revision beside the durable promotion. + const encode = (value: string) => encodeURIComponent(value); + return ` `; +} diff --git a/extensions/memory-core/src/short-term-promotion-record.ts b/extensions/memory-core/src/short-term-promotion-record.ts index e045fcf15d87..fdb8327d110f 100644 --- a/extensions/memory-core/src/short-term-promotion-record.ts +++ b/extensions/memory-core/src/short-term-promotion-record.ts @@ -9,7 +9,11 @@ import { appendMemoryHostEvent } from "openclaw/plugin-sdk/memory-host-events"; import pLimit from "p-limit"; import { deriveConceptTags } from "./concept-vocabulary.js"; import { readStore, withShortTermLock, writeStore } from "./short-term-promotion-store.js"; -import type { ShortTermRecallEntry, ShortTermRecallStore } from "./short-term-promotion-types.js"; +import type { + ShortTermPromotionAuthorizedView, + ShortTermRecallEntry, + ShortTermRecallStore, +} from "./short-term-promotion-types.js"; import { buildDailyClaimEntryKey, buildClaimHash, @@ -25,6 +29,7 @@ import { mergeRecentDistinct, normalizeIsoDay, normalizeMemoryPath, + normalizeShortTermPromotionAuthorizedView, normalizeSnippet, truncateShortTermSnippet, } from "./short-term-promotion-utils.js"; @@ -33,6 +38,21 @@ import { resolveMemoryCoreNowMs, resolveMemoryCoreTimestamp } from "./time.js"; // One recall batch can inspect every retained entry; cap filesystem pressure. const SHORT_TERM_SOURCE_FILE_CHECK_CONCURRENCY = 32; +function promotionAuthorityKey(view: ShortTermPromotionAuthorizedView): string { + return hashQuery(`${view.storeId}\u0000${view.viewId}\u0000${view.resourceRevision}`); +} + +function samePromotionAuthority( + left: ShortTermPromotionAuthorizedView | undefined, + right: ShortTermPromotionAuthorizedView | undefined, +): boolean { + return ( + left?.storeId === right?.storeId && + left?.viewId === right?.viewId && + left?.resourceRevision === right?.resourceRevision + ); +} + function mergeRecallProvenance( existing: MemoryEntryProvenance | undefined, incoming: MemoryEntryProvenance | undefined, @@ -151,7 +171,12 @@ async function updateShortTermRecallStore( export async function recordShortTermRecalls(params: { workspaceDir?: string; query: string; - results: Array; + results: Array< + MemorySearchResult & { + identitySnippet?: string; + authorizedView?: ShortTermPromotionAuthorizedView; + } + >; signalType?: "recall" | "daily"; dedupeByQueryPerDay?: boolean; dayBucket?: string; @@ -196,6 +221,10 @@ export async function recordShortTermRecalls(params: { nowIso, (store) => { for (const result of relevant) { + const authorizedView = normalizeShortTermPromotionAuthorizedView(result.authorizedView); + if (result.authorizedView !== undefined && !authorizedView) { + continue; + } const normalizedPath = normalizeMemoryPath(result.path); const rawSnippet = normalizeSnippet(result.snippet); const snippet = truncateShortTermSnippet(rawSnippet); @@ -220,7 +249,10 @@ export async function recordShortTermRecalls(params: { Math.max(0, Math.floor(entry.recallCount ?? 0)) + Math.max(0, Math.floor(entry.groundedCount ?? 0)) > 0 && - entry.claimHash === claimHash, + entry.claimHash === claimHash && + // Do not merge evidence from different scoped views or immutable revisions. + // A review state transition keeps the same identity and may update lifecycle. + samePromotionAuthority(entry.authorizedView, authorizedView), ) : undefined; // Interactive/grounded writers retain their path-qualified identity. Do @@ -240,13 +272,18 @@ export async function recordShortTermRecalls(params: { }) : null; const baseKey = buildEntryKey(result); - const key = + const unscopedKey = nonDailyEntry?.key ?? (signalType === "daily" && groundedKey ? groundedKey : groundedKey && store.entries[groundedKey] ? groundedKey : baseKey); + const key = nonDailyEntry?.key + ? nonDailyEntry.key + : authorizedView + ? `${unscopedKey}:view:${promotionAuthorityKey(authorizedView)}` + : unscopedKey; const existing = store.entries[key]; const score = clampScore(result.score); const recallDaysBase = existing?.recallDays ?? []; @@ -307,6 +344,7 @@ export async function recordShortTermRecalls(params: { recallDays, conceptTags: conceptTags.length > 0 ? conceptTags : (existing?.conceptTags ?? []), provenance, + ...(authorizedView ? { authorizedView } : {}), claimHash, ...(projectKey ? { projectKey } : {}), ...(existing?.promotedAt ? { promotedAt: existing.promotedAt } : {}), diff --git a/extensions/memory-core/src/short-term-promotion-types.ts b/extensions/memory-core/src/short-term-promotion-types.ts index 6f9e04152d70..a3a6721b6ad9 100644 --- a/extensions/memory-core/src/short-term-promotion-types.ts +++ b/extensions/memory-core/src/short-term-promotion-types.ts @@ -30,6 +30,18 @@ export type PromotionWeights = { conceptual: number; }; +/** + * Opaque authority carried from a scoped read. `resourceRevision` is the + * immutable source record; promotion must never treat a workspace path as a + * substitute for that provenance. + */ +export type ShortTermPromotionAuthorizedView = { + storeId: string; + viewId: string; + resourceRevision: string; + lifecycle: "active" | "postbox" | "quarantine"; +}; + export type ShortTermRecallEntry = { key: string; path: string; @@ -51,6 +63,7 @@ export type ShortTermRecallEntry = { projectKey?: string; promotedAt?: string; provenance?: MemoryEntryProvenance; + authorizedView?: ShortTermPromotionAuthorizedView; }; export type ShortTermRecallStore = { @@ -117,6 +130,7 @@ export type PromotionCandidate = { conceptTags: string[]; components: PromotionComponents; provenance?: MemoryEntryProvenance; + authorizedView?: ShortTermPromotionAuthorizedView; }; export type ShortTermAuditIssue = { diff --git a/extensions/memory-core/src/short-term-promotion-utils.ts b/extensions/memory-core/src/short-term-promotion-utils.ts index 2d53139515da..40dd3cd0b77a 100644 --- a/extensions/memory-core/src/short-term-promotion-utils.ts +++ b/extensions/memory-core/src/short-term-promotion-utils.ts @@ -7,6 +7,7 @@ import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; import { deriveConceptTags, MAX_CONCEPT_TAGS } from "./concept-vocabulary.js"; import type { PromotionWeights, + ShortTermPromotionAuthorizedView, ShortTermRecallEntry, ShortTermRecallStore, } from "./short-term-promotion-types.js"; @@ -40,6 +41,29 @@ const DEFAULT_PROMOTION_WEIGHTS: PromotionWeights = { conceptual: 0.06, }; +export function normalizeShortTermPromotionAuthorizedView( + value: unknown, +): ShortTermPromotionAuthorizedView | undefined { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return undefined; + } + const record = value as Record; + const storeId = typeof record.storeId === "string" ? record.storeId.trim() : ""; + const viewId = typeof record.viewId === "string" ? record.viewId.trim() : ""; + const resourceRevision = + typeof record.resourceRevision === "string" ? record.resourceRevision.trim() : ""; + const lifecycle = record.lifecycle; + if ( + !storeId || + !viewId || + !resourceRevision || + (lifecycle !== "active" && lifecycle !== "postbox" && lifecycle !== "quarantine") + ) { + return undefined; + } + return { storeId, viewId, resourceRevision, lifecycle }; +} + export function clampScore(value: number): number { if (!Number.isFinite(value)) { return 0; @@ -406,6 +430,10 @@ export function normalizeShortTermRecallStore(raw: unknown, nowIso: string): Sho : {}), } : undefined; + const authorizedView = normalizeShortTermPromotionAuthorizedView(entry.authorizedView); + if (entry.authorizedView !== undefined && !authorizedView) { + continue; + } const normalizedKey = key || buildEntryKey({ path: entryPath, startLine, endLine, source, claimHash }); @@ -427,6 +455,7 @@ export function normalizeShortTermRecallStore(raw: unknown, nowIso: string): Sho recallDays: recallDays.slice(-MAX_RECALL_DAYS), conceptTags, ...(provenance ? { provenance } : {}), + ...(authorizedView ? { authorizedView } : {}), ...(claimHash ? { claimHash } : {}), ...(projectKey ? { projectKey } : {}), ...(promotedAt ? { promotedAt } : {}), diff --git a/extensions/memory-core/src/short-term-promotion.test.ts b/extensions/memory-core/src/short-term-promotion.test.ts index ccacb77de465..413bcdad1a4c 100644 --- a/extensions/memory-core/src/short-term-promotion.test.ts +++ b/extensions/memory-core/src/short-term-promotion.test.ts @@ -2165,6 +2165,167 @@ describe("short-term promotion", () => { }); }); + it("does not combine promotion candidates from different authorized views", async () => { + await withTempWorkspace(async (workspaceDir) => { + const notes = [ + ["2026-04-01", "Personal store promotion candidate."], + ["2026-04-02", "Channel store promotion candidate."], + ] as const; + for (const [date, snippet] of notes) { + await writeDailyMemoryNote(workspaceDir, date, [snippet]); + } + await recordShortTermRecalls({ + workspaceDir, + query: "authorized promotion views", + results: notes.map(([date, snippet], index) => ({ + path: `memory/${date}.md`, + startLine: 1, + endLine: 1, + score: 0.92, + snippet, + source: "memory" as const, + authorizedView: { + storeId: index === 0 ? "personal-store" : "channel-store", + viewId: index === 0 ? "personal-view" : "channel-view", + resourceRevision: `revision-${index + 1}`, + lifecycle: "active" as const, + }, + })), + }); + + const ranked = await rankShortTermPromotionCandidates({ + workspaceDir, + minScore: 0, + minRecallCount: 0, + minUniqueQueries: 0, + }); + expect(ranked).toHaveLength(2); + + const applied = await applyShortTermPromotions({ + workspaceDir, + candidates: ranked, + minScore: 0, + minRecallCount: 0, + minUniqueQueries: 0, + }); + expect(applied.applied).toBe(0); + await expectEnoent(fs.readFile(path.join(workspaceDir, "MEMORY.md"), "utf-8")); + }); + }); + + it.each(["postbox", "quarantine"] as const)( + "does not rank %s content for automatic promotion", + async (lifecycle) => { + await withTempWorkspace(async (workspaceDir) => { + const snippet = `${lifecycle} content must remain review-only.`; + await writeDailyMemoryNote(workspaceDir, "2026-04-01", [snippet]); + await recordShortTermRecalls({ + workspaceDir, + query: `${lifecycle} promotion`, + results: [ + { + path: "memory/2026-04-01.md", + startLine: 1, + endLine: 1, + score: 0.92, + snippet, + source: "memory", + authorizedView: { + storeId: "review-store", + viewId: "review-view", + resourceRevision: "revision-review", + lifecycle, + }, + }, + ], + }); + + await expect( + rankShortTermPromotionCandidates({ + workspaceDir, + minScore: 0, + minRecallCount: 0, + minUniqueQueries: 0, + }), + ).resolves.toEqual([]); + }); + }, + ); + + it("requires an immutable source revision and records it beside an authorized promotion", async () => { + await withTempWorkspace(async (workspaceDir) => { + const snippet = "Authorized source retains its immutable revision."; + await writeDailyMemoryNote(workspaceDir, "2026-04-01", [snippet]); + const result = { + path: "memory/2026-04-01.md", + startLine: 1, + endLine: 1, + score: 0.92, + snippet, + source: "memory" as const, + }; + await recordShortTermRecalls({ + workspaceDir, + query: "missing immutable revision", + results: [ + { + ...result, + authorizedView: { + storeId: "personal-store", + viewId: "personal-view", + resourceRevision: "", + lifecycle: "active", + }, + }, + ], + }); + await expect( + rankShortTermPromotionCandidates({ + workspaceDir, + minScore: 0, + minRecallCount: 0, + minUniqueQueries: 0, + }), + ).resolves.toEqual([]); + + await recordShortTermRecalls({ + workspaceDir, + query: "immutable revision", + results: [ + { + ...result, + authorizedView: { + storeId: "personal-store", + viewId: "personal-view", + resourceRevision: "revision-personal-1", + lifecycle: "active", + }, + }, + ], + }); + const ranked = await rankShortTermPromotionCandidates({ + workspaceDir, + minScore: 0, + minRecallCount: 0, + minUniqueQueries: 0, + }); + expect(ranked).toHaveLength(1); + + await expect( + applyShortTermPromotions({ + workspaceDir, + candidates: ranked, + minScore: 0, + minRecallCount: 0, + minUniqueQueries: 0, + }), + ).resolves.toMatchObject({ applied: 1 }); + await expect(fs.readFile(path.join(workspaceDir, "MEMORY.md"), "utf-8")).resolves.toContain( + "", + ); + }); + }); + it("does not double-prefix promoted snippets that are already markdown bullets", async () => { await withTempWorkspace(async (workspaceDir) => { await writeDailyMemoryNote(workspaceDir, "2026-04-01", [ diff --git a/extensions/memory-core/src/short-term-promotion.ts b/extensions/memory-core/src/short-term-promotion.ts index ae57147d2c3a..80281ad40d6e 100644 --- a/extensions/memory-core/src/short-term-promotion.ts +++ b/extensions/memory-core/src/short-term-promotion.ts @@ -1,5 +1,6 @@ // Stable public surface for short-term promotion behavior. import { expectDefined } from "openclaw/plugin-sdk/expect-runtime"; +import { isPromotionAuthorizedViewBlocked } from "./dreaming-consolidation-candidates.js"; import { readPhaseSignalStore, readStore } from "./short-term-promotion-store.js"; import { DEFAULT_PROMOTION_MIN_RECALL_COUNT, @@ -136,6 +137,9 @@ export async function rankShortTermPromotionCandidates( if (!includePromoted && entry.promotedAt) { continue; } + if (isPromotionAuthorizedViewBlocked(entry)) { + continue; + } const recallCount = Math.max(0, Math.floor(entry.recallCount ?? 0)); const dailyCount = Math.max(0, Math.floor(entry.dailyCount ?? 0)); const groundedCount = Math.max(0, Math.floor(entry.groundedCount ?? 0)); @@ -217,6 +221,7 @@ export async function rankShortTermPromotionCandidates( conceptual, }, provenance: entry.provenance, + authorizedView: entry.authorizedView, }); } diff --git a/packages/memory-host-sdk/src/host/authorization.ts b/packages/memory-host-sdk/src/host/authorization.ts index c2f7dd14864f..28851ec03c20 100644 --- a/packages/memory-host-sdk/src/host/authorization.ts +++ b/packages/memory-host-sdk/src/host/authorization.ts @@ -1,3 +1,4 @@ +import type { DatabaseSync } from "node:sqlite"; import type { MemoryReadResult, MemorySearchResult, MemorySource } from "./types.js"; /** Version shared by every serializable multiplayer-memory authorization shape. */ @@ -404,6 +405,43 @@ type AuthorizedMemoryContentMutation = Readonly<{ contentType: "markdown" | "text" | "json"; }>; +/** + * Opaque transcript provenance prepared by core before a derivation model sees + * the transcript. The selected runtime validates the durable companion rows; + * tool arguments never choose this source. + */ +export type AuthorizedTranscriptDerivationSource = Readonly<{ + kind: "transcript"; + sessionId: string; + eventSeqs: readonly number[]; + sourcePolicySetId: string; + deliveryAudiencesJson: string; +}>; + +/** The immutable transcript-policy edge names the durable artifact it produced. */ +export type AuthorizedTranscriptDerivationPurpose = "flush" | "compaction"; + +/** + * A staged sealed artifact is intentionally opaque to core. The selected + * memory runtime owns its bytes and catalog rows; core owns the surrounding + * transcript/checkpoint transaction and supplies the same SQLite connection. + */ +export type AuthorizedSealedCompactionArtifact = Readonly<{ + resourceRevisionId: string; + commitInTransaction(params: Readonly<{ + database: DatabaseSync; + compactionPolicyId: string; + eventSeq: number; + }>): void; +}>; + +export type AuthorizedSealedCompactionStageParams = Readonly<{ + context: MemoryAccessContext & Readonly<{ operation: "derive" }>; + plan: AuthorizedMemoryPlan & Readonly<{ operation: "derive" }>; + content: string; + transcriptSource: AuthorizedTranscriptDerivationSource; +}>; + export type AuthorizedMemoryMutation = | (AuthorizedMemoryContentMutation & Readonly<{ @@ -422,9 +460,18 @@ export type AuthorizedMemoryMutation = | (AuthorizedMemoryContentMutation & Readonly<{ kind: "derive"; - sourceHandles: readonly AuthorizedResourceHandle[]; - sourcePolicySetId: string; - }>) + derivationPurpose: AuthorizedTranscriptDerivationPurpose; + }> & + ( + | Readonly<{ + sourceHandles: readonly AuthorizedResourceHandle[]; + sourcePolicySetId: string; + }> + | Readonly<{ + transcriptSource: AuthorizedTranscriptDerivationSource; + sourcePolicySetId: string; + }> + )) | (AuthorizedMemoryContentMutation & Readonly<{ kind: "project" | "publish"; @@ -555,6 +602,13 @@ export interface AuthorizedMemoryRuntime { params: AuthorizedMemoryReadParams<"derive">, ): Promise>; writeAuthorized(params: AuthorizedMemoryWriteParams): Promise; + /** + * Optional because only a runtime that can commit its catalog against core's + * transaction may support cutover compaction. Absence is a fail-closed deny. + */ + stageSealedCompaction?( + params: AuthorizedSealedCompactionStageParams, + ): Promise; importAuthorized( params: AuthorizedMemoryOperationParams<"import"> & Readonly<{ diff --git a/src/agents/agent-tools.ts b/src/agents/agent-tools.ts index 595a35140e3b..c1cbdf32f9c1 100644 --- a/src/agents/agent-tools.ts +++ b/src/agents/agent-tools.ts @@ -26,7 +26,7 @@ import type { import { isMemoryIsolationCutoverAgent } from "../plugins/memory-cutover.js"; import { resolveMemoryFlushPlan } from "../plugins/memory-state.js"; import { appendRuntimePluginToolGrant } from "../plugins/tool-grant-allowlist.js"; -import type { AuthorizedMemoryReadHost } from "../plugins/tool-types.js"; +import type { AuthorizedMemoryReadHost, AuthorizedMemoryWriteHost } from "../plugins/tool-types.js"; import { getPluginToolMeta } from "../plugins/tools.js"; import { getActiveSecretsRuntimeConfigSnapshot } from "../secrets/runtime-state.js"; import { GATEWAY_OWNER_ONLY_CORE_TOOLS } from "../security/dangerous-tools.js"; @@ -121,6 +121,7 @@ import type { CronToolOptions } from "./tools/cron-tool.types.js"; import { wrapToolWithGatewayCallerIdentity } from "./tools/gateway-caller-context.js"; const MEMORY_FLUSH_ALLOWED_TOOL_NAMES = new Set(["read", "write"]); +const AUTHORIZED_MEMORY_FLUSH_ALLOWED_TOOL_NAMES = new Set(["memory_remember"]); const MEMORY_ISOLATION_READ_TOOL_NAMES = new Set(["memory_search", "memory_get"]); const AUTHORIZED_MEMORY_VIEW_TOOL_NAMES = new Set(["memory_search", "memory_get", "read"]); @@ -216,6 +217,8 @@ type OpenClawCodingToolsOptions = { * share this exact host rather than minting authority from routing strings. */ authorizedMemoryRead?: AuthorizedMemoryReadHost; + /** Host-prepared one-mutation derivation for an authorized memory flush. */ + authorizedMemoryWrite?: AuthorizedMemoryWriteHost; /** Admission-prepared closed filesystem policy; never derived from tool input. */ fsPolicy?: ToolFsPolicy; /** Core-private opaque broker for the admitted memory view. */ @@ -394,7 +397,7 @@ type OpenClawCodingToolsOptions = { function createOpenClawCodingToolsInternal(options?: OpenClawCodingToolsOptions): AnyAgentTool[] { const sandbox = options?.sandbox?.enabled ? options.sandbox : undefined; const isMemoryFlushRun = options?.trigger === "memory"; - if (isMemoryFlushRun && !options?.memoryFlushWritePath) { + if (isMemoryFlushRun && !options?.memoryFlushWritePath && !options?.authorizedMemoryWrite) { throw new Error("memoryFlushWritePath required for memory-triggered tool runs"); } const memoryFlushWritePath = isMemoryFlushRun ? options.memoryFlushWritePath : undefined; @@ -803,6 +806,7 @@ function createOpenClawCodingToolsInternal(options?: OpenClawCodingToolsOptions) allowHostBrowserControl: sandbox ? sandbox.browserAllowHostControl : true, agentSessionKey: options?.sessionKey, runId: options?.runId, + authorizedMemoryWrite: options?.authorizedMemoryWrite, runSessionKey: options?.runSessionKey, agentChannel: resolveGatewayMessageChannel( options?.messageChannel ?? options?.messageProvider, @@ -896,13 +900,16 @@ function createOpenClawCodingToolsInternal(options?: OpenClawCodingToolsOptions) options?.swarmCollector && options.swarmOutputSchema ? tools.find((tool) => tool.name === "structured_output") : undefined; - const toolsForMemoryFlush: AnyAgentTool[] = isMemoryFlushRun && memoryFlushWritePath ? [] : tools; - if (isMemoryFlushRun && memoryFlushWritePath) { + const toolsForMemoryFlush: AnyAgentTool[] = isMemoryFlushRun ? [] : tools; + if (isMemoryFlushRun) { for (const tool of tools) { - if (!MEMORY_FLUSH_ALLOWED_TOOL_NAMES.has(tool.name)) { + const allowedToolNames = memoryIsolationCutover + ? AUTHORIZED_MEMORY_FLUSH_ALLOWED_TOOL_NAMES + : MEMORY_FLUSH_ALLOWED_TOOL_NAMES; + if (!allowedToolNames.has(tool.name)) { continue; } - if (tool.name === "write") { + if (tool.name === "write" && memoryFlushWritePath) { toolsForMemoryFlush.push( wrapToolMemoryFlushAppendOnlyWrite(tool, { root: memoryFlushWriteRoot, @@ -920,8 +927,10 @@ function createOpenClawCodingToolsInternal(options?: OpenClawCodingToolsOptions) } } const unavailableCoreToolReason = - isMemoryFlushRun && memoryFlushWritePath - ? "memory-triggered compaction runs expose only read and append-only write" + isMemoryFlushRun + ? memoryIsolationCutover + ? "memory-triggered compaction runs expose only subject-scoped memory_remember" + : "memory-triggered compaction runs expose only read and append-only write" : undefined; const toolsForMessageProvider = filterToolsByMessageProvider( toolsForMemoryFlush, @@ -984,7 +993,9 @@ function createOpenClawCodingToolsInternal(options?: OpenClawCodingToolsOptions) const surfaceTools = authorizedMemoryView ? authorizedTools.filter((tool) => AUTHORIZED_MEMORY_VIEW_TOOL_NAMES.has(tool.name)) : memoryIsolationCutover - ? authorizedTools.filter((tool) => MEMORY_ISOLATION_READ_TOOL_NAMES.has(tool.name)) + ? isMemoryFlushRun + ? authorizedTools.filter((tool) => AUTHORIZED_MEMORY_FLUSH_ALLOWED_TOOL_NAMES.has(tool.name)) + : authorizedTools.filter((tool) => MEMORY_ISOLATION_READ_TOOL_NAMES.has(tool.name)) : authorizedTools; if ( swarmStructuredOutputTool && diff --git a/src/agents/agent-tools.workspace-paths.test.ts b/src/agents/agent-tools.workspace-paths.test.ts index 839c04c6846e..b29846ce98c6 100644 --- a/src/agents/agent-tools.workspace-paths.test.ts +++ b/src/agents/agent-tools.workspace-paths.test.ts @@ -126,6 +126,45 @@ describe("workspace path resolution", () => { }); }); + it("uses memory_remember instead of a workspace write for a cut-over memory flush", async () => { + await withTempDir("openclaw-memory-flush-cutover-state-", async (stateDir) => { + const originalStateDir = process.env.OPENCLAW_STATE_DIR; + process.env.OPENCLAW_STATE_DIR = stateDir; + try { + const database = openOpenClawAgentDatabase({ agentId: "main" }); + database.db + .prepare( + `INSERT INTO memory_migrations + (migration_id, source_kind, source_hash, phase, classification_json, plan_hash, + verified_at, cutover_at, updated_at) + VALUES ('memory-cutover-flush-tools', 'test', 'test-source', 'cutover', '{}', + 'test-plan', 1, 1, 1)`, + ) + .run(); + resetMemoryIsolationCutoverForTest(); + vi.mocked(createOpenClawTools).mockImplementationOnce(() => [ + { name: "memory_remember" } as never, + ]); + + expect( + createOpenClawCodingTools({ + agentId: "main", + trigger: "memory", + memoryFlushWritePath: "memory/2026-07-29.md", + }).map((tool) => tool.name), + ).toEqual(["memory_remember"]); + } finally { + closeOpenClawAgentDatabasesForTest(); + resetMemoryIsolationCutoverForTest(); + if (originalStateDir === undefined) { + delete process.env.OPENCLAW_STATE_DIR; + } else { + process.env.OPENCLAW_STATE_DIR = originalStateDir; + } + } + }); + }); + it("preserves legacy memory-file writes for intentionally unscoped tool construction", async () => { await withTempDir("openclaw-unscoped-ws-", async (workspaceDir) => { const tools = createOpenClawCodingTools({ workspaceDir }); diff --git a/src/agents/embedded-agent-runner/compact.hooks.harness.ts b/src/agents/embedded-agent-runner/compact.hooks.harness.ts index f8c40b157f88..e46c86739a7f 100644 --- a/src/agents/embedded-agent-runner/compact.hooks.harness.ts +++ b/src/agents/embedded-agent-runner/compact.hooks.harness.ts @@ -41,6 +41,12 @@ export const contextEngineCompactMock: Mock<() => Promise> = vi.f result: { summary: "engine-summary", tokensBefore: 120, tokensAfter: 50 }, })); +export const isMemoryIsolationCutoverAgentMock = vi.fn(() => false); +export const readAuthorizedTranscriptDerivationMock = vi.fn(() => undefined); +export const admitAuthorizedMemoryDerivationMock = vi.fn(async () => true); +export const createAuthorizedMemoryDerivationHostMock = vi.fn(() => undefined); +export const prepareAuthorizedSealedCompactionHostMock = vi.fn(async () => undefined); + export const hookRunner = { hasHooks: vi.fn<(hookName?: string) => boolean>(), runBeforeCompaction: vi.fn(async () => undefined), @@ -80,6 +86,34 @@ export const sessionAutomaticCompactionMock = vi.fn(); export const attemptServerEndpointCompactionMock: Mock<(_params?: unknown) => Promise> = vi.fn(async () => undefined); export const triggerInternalHookMock: Mock<(event?: unknown) => void> = vi.fn(); +export const sessionDeferredCompactionMock = vi.fn(); +export const sessionApplyDeferredCompactionMock = vi.fn(); +export const sessionDiscardDeferredCompactionMock = vi.fn(); +export const sealedCompactionStageMock = vi.fn(); +export const sealedCompactionCommitMock = vi.fn(); +export const commitSealedSqliteTranscriptCompactionMock = vi.fn(); + +type SealedTranscriptCommitInput = { + compactionPolicyId: string; + source: { + eventSeqs: readonly number[]; + sourcePolicySetId: string; + deliveryAudiencesJson: string; + }; + commitDerivedState: (params: { + database: { db: unknown }; + compactionPolicy: { + compactionPolicyId: string; + sessionId: string; + sourcePolicySetId: string; + deliveryAudiencesJson: string; + eventSeqs: readonly number[]; + retentionState: "retained"; + createdAt: number; + }; + eventSeq: number; + }) => void; +}; const sanitizeSessionHistoryMock = vi.fn( async (params: { messages: unknown[] }) => params.messages, ); @@ -177,6 +211,31 @@ function createMockCompactionSession() { session.messages.splice(1); return await sessionCompactImpl(); }), + compactDeferred: vi.fn(async () => { + sessionDeferredCompactionMock(); + const result = await sessionCompactImpl(); + return { + entry: { + type: "compaction", + id: "sealed-compaction-entry", + parentId: "entry-1", + timestamp: new Date(1).toISOString(), + summary: result.summary, + firstKeptEntryId: result.firstKeptEntryId, + tokensBefore: result.tokensBefore, + details: result.details, + }, + fromExtension: false, + result, + }; + }), + applyDeferredCompaction: vi.fn(async () => { + sessionApplyDeferredCompactionMock(); + session.messages.splice(1); + }), + discardDeferredCompaction: vi.fn((error: unknown) => { + sessionDiscardDeferredCompactionMock(error); + }), setActiveToolsByName: vi.fn(), setBaseSystemPrompt: vi.fn((systemPrompt: string) => { session.agent.state.systemPrompt = systemPrompt; @@ -493,6 +552,34 @@ export function resetCompactSessionStateMocks(): void { sessionAbortCompactionMock.mockReset(); sessionManualCompactionMock.mockReset(); sessionAutomaticCompactionMock.mockReset(); + sessionDeferredCompactionMock.mockReset(); + sessionApplyDeferredCompactionMock.mockReset(); + sessionDiscardDeferredCompactionMock.mockReset(); + sealedCompactionStageMock.mockReset(); + sealedCompactionCommitMock.mockReset(); + commitSealedSqliteTranscriptCompactionMock.mockReset(); + sealedCompactionStageMock.mockResolvedValue({ + resourceRevisionId: "sealed-resource-revision", + commitInTransaction: sealedCompactionCommitMock, + }); + commitSealedSqliteTranscriptCompactionMock.mockImplementation( + async (input: SealedTranscriptCommitInput) => { + input.commitDerivedState({ + database: { db: {} }, + compactionPolicy: { + compactionPolicyId: input.compactionPolicyId, + sessionId: "session-1", + sourcePolicySetId: input.source.sourcePolicySetId, + deliveryAudiencesJson: input.source.deliveryAudiencesJson, + eventSeqs: input.source.eventSeqs, + retentionState: "retained", + createdAt: 1, + }, + eventSeq: 7, + }); + return { compactionPolicy: { compactionPolicyId: input.compactionPolicyId }, eventSeq: 7 }; + }, + ); attemptServerEndpointCompactionMock.mockReset(); attemptServerEndpointCompactionMock.mockResolvedValue(undefined); resolveEffectiveCompactionModeMock.mockReset(); @@ -576,6 +663,16 @@ export function resetCompactHooksHarnessMocks(): void { hookRunner.runBeforeCompaction.mockResolvedValue(undefined); hookRunner.runAfterCompaction.mockReset(); hookRunner.runAfterCompaction.mockResolvedValue(undefined); + isMemoryIsolationCutoverAgentMock.mockReset(); + isMemoryIsolationCutoverAgentMock.mockReturnValue(false); + readAuthorizedTranscriptDerivationMock.mockReset(); + readAuthorizedTranscriptDerivationMock.mockReturnValue(undefined); + admitAuthorizedMemoryDerivationMock.mockReset(); + admitAuthorizedMemoryDerivationMock.mockResolvedValue(true); + createAuthorizedMemoryDerivationHostMock.mockReset(); + createAuthorizedMemoryDerivationHostMock.mockReturnValue(undefined); + prepareAuthorizedSealedCompactionHostMock.mockReset(); + prepareAuthorizedSealedCompactionHostMock.mockResolvedValue(undefined); acquireAgentRunPreparedModelRuntimeMock.mockClear(); resolveDefaultAgentDirMock.mockReset(); @@ -672,6 +769,37 @@ export async function loadCompactHooksHarness(): Promise<{ runGlobalGatewayStopSafely: vi.fn(async () => undefined), })); + vi.doMock("../../plugins/memory-cutover.js", () => ({ + isMemoryIsolationCutoverAgent: isMemoryIsolationCutoverAgentMock, + })); + + vi.doMock("../../config/sessions/session-transcript-memory-policy.js", () => ({ + readAuthorizedTranscriptDerivation: readAuthorizedTranscriptDerivationMock, + })); + + vi.doMock("../../config/sessions/session-accessor.sqlite-transcript-write.js", () => ({ + commitSealedSqliteTranscriptCompaction: commitSealedSqliteTranscriptCompactionMock, + })); + + vi.doMock("../memory-authorized-read-host.js", () => ({ + admitAuthorizedMemoryDerivation: admitAuthorizedMemoryDerivationMock, + createAuthorizedMemoryDerivationHost: createAuthorizedMemoryDerivationHostMock, + createAuthorizedMemoryReadHost: vi.fn(() => undefined), + prepareAuthorizedSealedCompactionHost: prepareAuthorizedSealedCompactionHostMock, + })); + + vi.doMock("./compaction-checkpoint.js", () => ({ + compactionCheckpointStore: { + captureSnapshot: vi.fn(async () => ({ + sessionId: "session-1", + leafId: "entry-1", + entryId: "entry-1", + })), + cleanupSnapshot: vi.fn(async () => undefined), + }, + persistCompactionCheckpoint: vi.fn(async () => true), + })); + vi.doMock("../../plugins/current-plugin-metadata-snapshot.js", () => ({ getCurrentPluginMetadataSnapshot: getCurrentPluginMetadataSnapshotMock, resolvePluginMetadataControlPlaneFingerprint: vi.fn(() => "test-plugin-fingerprint"), diff --git a/src/agents/embedded-agent-runner/compact.hooks.test.ts b/src/agents/embedded-agent-runner/compact.hooks.test.ts index 649bbec60e6d..76a68b6ed76b 100644 --- a/src/agents/embedded-agent-runner/compact.hooks.test.ts +++ b/src/agents/embedded-agent-runner/compact.hooks.test.ts @@ -18,6 +18,7 @@ import { getCurrentPluginMetadataSnapshotMock, applyExtraParamsToAgentMock, applyAgentCompactionSettingsFromConfigMock, + admitAuthorizedMemoryDerivationMock, buildEmbeddedExtensionFactoriesMock, buildAgentRuntimePlanMock, buildEmbeddedSystemPromptMock, @@ -51,6 +52,8 @@ import { resolveSandboxContextMock, resolveSessionAgentIdMock, resolveSessionAgentIdsMock, + readAuthorizedTranscriptDerivationMock, + prepareAuthorizedSealedCompactionHostMock, rotateTranscriptAfterCompactionMock, runCliAgentMock, selectAgentHarnessForPreparedModelProvidersMock, @@ -59,10 +62,18 @@ import { resetCompactHooksHarnessMocks, resetCompactSessionStateMocks, sessionAbortCompactionMock, + sessionApplyDeferredCompactionMock, sessionAutomaticCompactionMock, + sessionDeferredCompactionMock, + sessionDiscardDeferredCompactionMock, sessionMessages, sessionCompactImpl, sessionManualCompactionMock, + sealedCompactionCommitMock, + sealedCompactionStageMock, + commitSealedSqliteTranscriptCompactionMock, + createAuthorizedMemoryDerivationHostMock, + isMemoryIsolationCutoverAgentMock, triggerInternalHookMock, } from "./compact.hooks.harness.js"; import { @@ -348,6 +359,112 @@ describe("compactEmbeddedAgentSessionDirect hooks", () => { resetCompactSessionStateMocks(); }); + it("rechecks transcript derivation after opening the native session before summarizing", async () => { + isMemoryIsolationCutoverAgentMock.mockReturnValue(true); + admitAuthorizedMemoryDerivationMock.mockResolvedValue(true); + createAuthorizedMemoryDerivationHostMock.mockReturnValue(undefined); + prepareAuthorizedSealedCompactionHostMock.mockResolvedValue({ + source: { + kind: "transcript", + sessionId: TEST_SESSION_ID, + eventSeqs: [0, 1], + sourcePolicySetId: "policy-set-1", + deliveryAudiencesJson: '[{"kind":"user","id":"alice"}]', + }, + stage: sealedCompactionStageMock, + }); + readAuthorizedTranscriptDerivationMock + .mockReturnValueOnce({ + eventSeqs: [0, 1], + sourcePolicySetId: "policy-set-1", + deliveryAudiencesJson: '[{"kind":"user","id":"alice"}]', + }) + // A revocation or a new pending event after preparation must stop before + // native compaction turns transcript content into a prompt. + .mockReturnValueOnce(undefined); + + const result = await compactEmbeddedAgentSessionDirect({ + ...wrappedCompactionArgs(), + provider: "openai", + model: "gpt-5.5", + }); + + expect(result).toMatchObject({ + ok: false, + compacted: false, + reason: expect.stringContaining("transcript derivation authorization unavailable"), + }); + expect(readAuthorizedTranscriptDerivationMock).toHaveBeenCalledTimes(2); + expect(sessionCompactImpl).not.toHaveBeenCalled(); + }); + + it("does not route an authorized cutover transcript through the legacy compaction writer", async () => { + isMemoryIsolationCutoverAgentMock.mockReturnValue(true); + admitAuthorizedMemoryDerivationMock.mockResolvedValue(true); + createAuthorizedMemoryDerivationHostMock.mockReturnValue(undefined); + readAuthorizedTranscriptDerivationMock.mockReturnValue({ + eventSeqs: [0, 1], + sourcePolicySetId: "policy-set-1", + deliveryAudiencesJson: '[{"kind":"user","id":"alice"}]', + }); + + const result = await compactEmbeddedAgentSessionDirect({ + ...wrappedCompactionArgs(), + provider: "openai", + model: "gpt-5.5", + }); + + expect(result).toMatchObject({ + ok: false, + compacted: false, + reason: expect.stringContaining("sealed compaction authorization unavailable"), + }); + expect(readAuthorizedTranscriptDerivationMock).toHaveBeenCalledTimes(1); + expect(sessionCompactImpl).not.toHaveBeenCalled(); + }); + + it("stages and commits a cutover summary before applying its in-memory compaction", async () => { + const source = { + eventSeqs: [0, 1], + sourcePolicySetId: "policy-set-1", + deliveryAudiencesJson: '[{"kind":"user","id":"alice"}]', + }; + isMemoryIsolationCutoverAgentMock.mockReturnValue(true); + admitAuthorizedMemoryDerivationMock.mockResolvedValue(true); + readAuthorizedTranscriptDerivationMock.mockReturnValue(source); + prepareAuthorizedSealedCompactionHostMock.mockResolvedValue({ + source: { kind: "transcript", sessionId: TEST_SESSION_ID, ...source }, + stage: sealedCompactionStageMock, + }); + + const result = await compactEmbeddedAgentSessionDirect({ + ...wrappedCompactionArgs(), + provider: "openai", + model: "gpt-5.5", + }); + + expect(result).toMatchObject({ ok: true, compacted: true }); + expect(sessionDeferredCompactionMock).toHaveBeenCalledOnce(); + expect(sessionManualCompactionMock).not.toHaveBeenCalled(); + expect(sessionAutomaticCompactionMock).not.toHaveBeenCalled(); + expect(sealedCompactionStageMock).toHaveBeenCalledWith("summary"); + expect(commitSealedSqliteTranscriptCompactionMock).toHaveBeenCalledWith( + expect.objectContaining({ + event: expect.objectContaining({ id: "sealed-compaction-entry", summary: "summary" }), + source, + checkpoint: expect.objectContaining({ + preCompaction: expect.objectContaining({ entryId: "entry-1" }), + postCompaction: expect.objectContaining({ entryId: "sealed-compaction-entry" }), + }), + }), + ); + expect(sealedCompactionCommitMock).toHaveBeenCalledWith( + expect.objectContaining({ compactionPolicyId: expect.any(String), eventSeq: 7 }), + ); + expect(sessionApplyDeferredCompactionMock).toHaveBeenCalledOnce(); + expect(sessionDiscardDeferredCompactionMock).not.toHaveBeenCalled(); + }); + it("returns a summaryless xAI manual endpoint result", async () => { mockResolvedModel(); attemptServerEndpointCompactionMock.mockResolvedValueOnce({ diff --git a/src/agents/embedded-agent-runner/compact.queued.ts b/src/agents/embedded-agent-runner/compact.queued.ts index 4ac49dbedef5..734bbcbc57c0 100644 --- a/src/agents/embedded-agent-runner/compact.queued.ts +++ b/src/agents/embedded-agent-runner/compact.queued.ts @@ -18,6 +18,7 @@ import type { import type { CapturedCompactionCheckpointSnapshot } from "../../gateway/session-compaction-checkpoints.js"; import { formatErrorMessage } from "../../infra/errors.js"; import { getGlobalHookRunner } from "../../plugins/hook-runner-global.js"; +import { isMemoryIsolationCutoverAgent } from "../../plugins/memory-cutover.js"; import type { ProviderRuntimeModel } from "../../plugins/provider-runtime-model.types.js"; import { requireActivePluginRegistry } from "../../plugins/runtime.js"; import { withPluginRuntimeGenerationScope } from "../../plugins/runtime/generation-scope.js"; @@ -443,6 +444,22 @@ async function compactResolvedContextEngine( ...params, missingSessionKey: "resolve-existing", }); + const { sessionAgentId } = resolveSessionAgentIds({ + sessionKey: runtimeTarget.sessionKey, + config: params.config, + agentId: runtimeTarget.agentId, + }); + // Context engines own their prompt assembly and can invoke a model without the + // prepared native-compaction host. Until their public contract carries an opaque + // derive plan, letting one compact a cutover transcript would launder raw history. + if (contextEngine.info.ownsCompaction === true && isMemoryIsolationCutoverAgent(sessionAgentId)) { + return { + ok: false, + compacted: false, + reason: "memory derivation authorization unavailable for context-engine compaction", + failure: { reason: "memory_derivation_unavailable" }, + }; + } const lockedHarnessRuntime = params.modelSelectionLocked === true ? normalizeOptionalAgentRuntimeId(params.agentHarnessId) diff --git a/src/agents/embedded-agent-runner/compaction-sealed.test.ts b/src/agents/embedded-agent-runner/compaction-sealed.test.ts new file mode 100644 index 000000000000..054ef6617d64 --- /dev/null +++ b/src/agents/embedded-agent-runner/compaction-sealed.test.ts @@ -0,0 +1,81 @@ +import { beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { + admitAuthorizedMemoryDerivationMock, + commitSealedSqliteTranscriptCompactionMock, + isMemoryIsolationCutoverAgentMock, + loadCompactHooksHarness, + prepareAuthorizedSealedCompactionHostMock, + readAuthorizedTranscriptDerivationMock, + resetCompactHooksHarnessMocks, + sealedCompactionCommitMock, + sealedCompactionStageMock, + sessionApplyDeferredCompactionMock, + sessionAutomaticCompactionMock, + sessionDeferredCompactionMock, + sessionDiscardDeferredCompactionMock, + sessionManualCompactionMock, +} from "./compact.hooks.harness.js"; + +let compactEmbeddedAgentSessionDirect: typeof import("./compact.js").compactEmbeddedAgentSessionDirect; + +beforeAll(async () => { + ({ compactEmbeddedAgentSessionDirect } = await loadCompactHooksHarness()); +}); + +beforeEach(() => { + resetCompactHooksHarnessMocks(); +}); + +describe("sealed embedded compaction", () => { + it("stages and commits the summary before applying its in-memory transcript entry", async () => { + const source = { + eventSeqs: [0, 1], + sourcePolicySetId: "policy-set-1", + deliveryAudiencesJson: '[{"kind":"user","id":"alice"}]', + }; + isMemoryIsolationCutoverAgentMock.mockReturnValue(true); + admitAuthorizedMemoryDerivationMock.mockResolvedValue(true); + readAuthorizedTranscriptDerivationMock.mockReturnValue(source); + prepareAuthorizedSealedCompactionHostMock.mockResolvedValue({ + source: { kind: "transcript", sessionId: "session-1", ...source }, + stage: sealedCompactionStageMock, + }); + + const result = await compactEmbeddedAgentSessionDirect({ + sessionId: "session-1", + sessionKey: "agent:main:session-1", + sessionFile: "agent:main:session-1", + sessionTarget: { + agentId: "main", + sessionId: "session-1", + sessionKey: "agent:main:session-1", + storePath: "/tmp/sessions.json", + }, + workspaceDir: "/tmp", + provider: "openai", + model: "gpt-5.5", + enqueue: async (task: () => Promise | T) => await task(), + }); + + expect(result).toMatchObject({ ok: true, compacted: true }); + expect(sessionDeferredCompactionMock).toHaveBeenCalledOnce(); + expect(sessionManualCompactionMock).not.toHaveBeenCalled(); + expect(sessionAutomaticCompactionMock).not.toHaveBeenCalled(); + expect(sealedCompactionStageMock).toHaveBeenCalledWith("summary"); + expect(commitSealedSqliteTranscriptCompactionMock).toHaveBeenCalledWith( + expect.objectContaining({ + event: expect.objectContaining({ id: "sealed-compaction-entry", summary: "summary" }), + source, + checkpoint: expect.objectContaining({ + preCompaction: expect.objectContaining({ entryId: "entry-1" }), + postCompaction: expect.objectContaining({ entryId: "sealed-compaction-entry" }), + }), + }), + ); + expect(sealedCompactionCommitMock).toHaveBeenCalledWith( + expect.objectContaining({ compactionPolicyId: expect.any(String), eventSeq: 7 }), + ); + expect(sessionApplyDeferredCompactionMock).toHaveBeenCalledOnce(); + expect(sessionDiscardDeferredCompactionMock).not.toHaveBeenCalled(); + }); +}); diff --git a/src/agents/embedded-agent-runner/compaction-session-execution.ts b/src/agents/embedded-agent-runner/compaction-session-execution.ts index e609a01de48b..48abdbb125db 100644 --- a/src/agents/embedded-agent-runner/compaction-session-execution.ts +++ b/src/agents/embedded-agent-runner/compaction-session-execution.ts @@ -2,8 +2,15 @@ * Executes compaction while owning the transcript lock, session lifecycle, * hooks, checkpoint, and optional successor transcript rotation. */ +import { randomUUID } from "node:crypto"; +import { SESSION_TOTAL_TOKENS_VERSION } from "../../config/sessions.js"; import { formatSqliteSessionFileMarker } from "../../config/sessions/legacy-sqlite-marker.js"; -import type { CapturedCompactionCheckpointSnapshot } from "../../gateway/session-compaction-checkpoints.js"; +import { commitSealedSqliteTranscriptCompaction } from "../../config/sessions/session-accessor.sqlite-transcript-write.js"; +import { readAuthorizedTranscriptDerivation } from "../../config/sessions/session-transcript-memory-policy.js"; +import { + resolveSessionCompactionCheckpointReason, + type CapturedCompactionCheckpointSnapshot, +} from "../../gateway/session-compaction-checkpoints.js"; import { resolveDiagnosticModelContentCapturePolicy } from "../../infra/diagnostic-llm-content.js"; import { formatErrorMessage } from "../../infra/errors.js"; import { @@ -14,6 +21,8 @@ import { } from "../../logging/diagnostic-run-activity.js"; import { getCurrentPluginMetadataSnapshot } from "../../plugins/current-plugin-metadata-snapshot.js"; import { getGlobalHookRunner } from "../../plugins/hook-runner-global.js"; +import { isMemoryIsolationCutoverAgent } from "../../plugins/memory-cutover.js"; +import { openOpenClawAgentDatabase } from "../../state/openclaw-agent-db.js"; import { consumeCompactionSafeguardCancelReason, setCompactionSafeguardCancelReason, @@ -101,6 +110,7 @@ export async function executePreparedCompactionSession(runtime: PreparedCompacti effectiveTools, allowedToolNames, buildSystemPromptText, + authorizedSealedCompaction, resolvedMessageProvider, sessionAgentId, } = runtime; @@ -136,6 +146,20 @@ export async function executePreparedCompactionSession(runtime: PreparedCompacti : undefined, allowedToolNames, }); + // Preparation admits this source, but a revoke or transcript append can + // race it. Re-read after opening the session and before prompt assembly. + if ( + isMemoryIsolationCutoverAgent(sessionAgentId) && + !readAuthorizedTranscriptDerivation( + openOpenClawAgentDatabase({ agentId: sessionAgentId }).db, + sessionTarget.sessionId, + ) + ) { + throw new Error("compaction transcript derivation authorization unavailable"); + } + if (isMemoryIsolationCutoverAgent(sessionAgentId) && !authorizedSealedCompaction) { + throw new Error("scoped compaction derived commit unavailable"); + } checkpointSnapshot = await compactionCheckpointStore.captureSnapshot({ sessionManager, sessionFile: params.sessionFile, @@ -411,45 +435,65 @@ export async function executePreparedCompactionSession(runtime: PreparedCompacti } const compactStartedAt = Date.now(); - const serverResult = await attemptServerEndpointCompaction({ - trigger, - streamFn: session.agent.streamFn, - model: effectiveModel, - context: { systemPrompt: systemPromptText, messages: session.messages }, - sessionManager, - extraParams: effectiveExtraParams, - customInstructions: params.customInstructions, - requestOptions: { - apiKey: transportApiKey, - sessionId: params.sessionId, - authProfileId: runtimePlan.auth.forwardedAuthProfileId, - timeoutMs: compactionTimeoutMs, - signal: params.abortSignal, - }, - }); const activeSession = session; - const clientResult = serverResult - ? undefined - : await compactWithSafetyTimeout( - () => { - setCompactionSafeguardCancelReason(compactionSessionManager, undefined); - return resolveEffectiveCompactionMode(params.config) === "default" && - trigger !== "manual" - ? activeSession[agentSessionAutomaticCompaction](params.customInstructions) - : activeSession.compact(params.customInstructions); - }, + const cutoverCompaction = isMemoryIsolationCutoverAgent(sessionAgentId); + const deferred = cutoverCompaction + ? await compactWithSafetyTimeout( + () => + activeSession.compactDeferred( + params.customInstructions, + resolveEffectiveCompactionMode(params.config) === "default" && + trigger !== "manual" + ? "retry-invalid-once" + : "none", + ), compactionTimeoutMs, { abortSignal: params.abortSignal, - onCancel: () => { - activeSession.abortCompaction(); - }, + onCancel: () => activeSession.abortCompaction(), }, - ); + ) + : undefined; + const serverResult = cutoverCompaction + ? undefined + : await attemptServerEndpointCompaction({ + trigger, + streamFn: session.agent.streamFn, + model: effectiveModel, + context: { systemPrompt: systemPromptText, messages: session.messages }, + sessionManager, + extraParams: effectiveExtraParams, + customInstructions: params.customInstructions, + requestOptions: { + apiKey: transportApiKey, + sessionId: params.sessionId, + authProfileId: runtimePlan.auth.forwardedAuthProfileId, + timeoutMs: compactionTimeoutMs, + signal: params.abortSignal, + }, + }); + const clientResult = deferred + ? deferred.result + : serverResult + ? undefined + : await compactWithSafetyTimeout( + () => { + setCompactionSafeguardCancelReason(compactionSessionManager, undefined); + return resolveEffectiveCompactionMode(params.config) === "default" && + trigger !== "manual" + ? activeSession[agentSessionAutomaticCompaction](params.customInstructions) + : activeSession.compact(params.customInstructions); + }, + compactionTimeoutMs, + { + abortSignal: params.abortSignal, + onCancel: () => activeSession.abortCompaction(), + }, + ); const effectiveFirstKeptEntryId = clientResult?.firstKeptEntryId; const tokensBefore = serverResult?.usage.input_tokens ?? clientResult!.tokensBefore; // Estimate tokens after compaction by summing token estimates for remaining messages - const tokensAfter = + let tokensAfter = serverResult?.usage.output_tokens ?? estimateTokensAfterCompaction({ messagesAfter: session.messages, @@ -463,6 +507,64 @@ export async function executePreparedCompactionSession(runtime: PreparedCompacti ...sessionTarget, sessionId: params.sessionId, }); + if (deferred && authorizedSealedCompaction) { + try { + if (!checkpointSnapshot) { + throw new Error("sealed compaction checkpoint is unavailable"); + } + const staged = await authorizedSealedCompaction.stage(clientResult.summary); + if ("unavailable" in staged) { + throw new Error("scoped compaction derived commit unavailable"); + } + const compactionPolicyId = randomUUID(); + await commitSealedSqliteTranscriptCompaction({ + scope: sessionTarget, + event: deferred.entry, + compactionPolicyId, + source: { + eventSeqs: authorizedSealedCompaction.source.eventSeqs, + sourcePolicySetId: authorizedSealedCompaction.source.sourcePolicySetId, + deliveryAudiencesJson: authorizedSealedCompaction.source.deliveryAudiencesJson, + }, + checkpoint: { + checkpointId: randomUUID(), + sessionKey: sessionTarget.sessionKey, + sessionId: params.sessionId, + createdAt: compactStartedAt, + reason: resolveSessionCompactionCheckpointReason({ trigger: params.trigger }), + tokensVersion: SESSION_TOTAL_TOKENS_VERSION, + tokensBefore: observedTokenCount ?? clientResult.tokensBefore, + tokensAfter, + summary: clientResult.summary, + firstKeptEntryId: effectiveFirstKeptEntryId, + preCompaction: { + sessionId: checkpointSnapshot.sessionId, + leafId: checkpointSnapshot.leafId, + ...(checkpointSnapshot.entryId ? { entryId: checkpointSnapshot.entryId } : {}), + }, + postCompaction: { sessionId: params.sessionId, entryId: deferred.entry.id }, + }, + commitDerivedState({ database, eventSeq }) { + staged.commitInTransaction({ + database: database.db, + compactionPolicyId, + eventSeq, + }); + }, + }); + await activeSession.applyDeferredCompaction(deferred); + tokensAfter = estimateTokensAfterCompaction({ + messagesAfter: session.messages, + observedTokenCount, + fullSessionTokensBefore: limitedTranscriptTokensBefore ?? 0, + estimateTokensFn: estimateTokens, + }); + checkpointSnapshotRetained = true; + } catch (error) { + activeSession.discardDeferredCompaction(error); + throw error; + } + } await runPostCompactionSideEffects({ config: params.config, sessionKey: params.sessionKey, @@ -470,7 +572,7 @@ export async function executePreparedCompactionSession(runtime: PreparedCompacti agentId: sessionAgentId, sessionFile: activeSessionFile, }); - if (clientResult) { + if (clientResult && !deferred) { checkpointSnapshotRetained = await persistCompactionCheckpoint({ config: params.config, sessionKey: params.sessionKey, diff --git a/src/agents/embedded-agent-runner/context-engine-maintenance.test.ts b/src/agents/embedded-agent-runner/context-engine-maintenance.test.ts index fb89c6930f40..836961cf50e8 100644 --- a/src/agents/embedded-agent-runner/context-engine-maintenance.test.ts +++ b/src/agents/embedded-agent-runner/context-engine-maintenance.test.ts @@ -35,6 +35,7 @@ const resolveRuntimeTranscriptReadTargetMock = vi.fn(async (scope: Record vi.fn(() => false)); let createDeferredTurnMaintenanceAbortSignal: typeof import("./context-engine-maintenance.test-support.js").createDeferredTurnMaintenanceAbortSignal; let resetDeferredTurnMaintenanceStateForTest: typeof import("./context-engine-maintenance.test-support.js").resetDeferredTurnMaintenanceStateForTest; let waitForDeferredTurnMaintenanceForSession: typeof import("./context-engine-maintenance.js").waitForDeferredTurnMaintenanceForSession; @@ -103,6 +104,10 @@ vi.mock("./context-engine-capabilities.js", () => ({ resolveContextEngineCapabilities: () => ({ llm: undefined }), })); +vi.mock("../../plugins/memory-cutover.js", () => ({ + isMemoryIsolationCutoverAgent: isMemoryIsolationCutoverAgentMock, +})); + vi.mock("./transcript-rewrite.js", () => ({ rewriteTranscriptEntriesInSessionManager: (params: unknown) => rewriteTranscriptEntriesInSessionManagerMock(params), @@ -183,9 +188,38 @@ describe("runContextEngineMaintenance", () => { rewriteTranscriptEntriesInSessionManagerMock.mockClear(); sessionManagerOpenMock.mockClear(); resolveRuntimeTranscriptReadTargetMock.mockClear(); + isMemoryIsolationCutoverAgentMock.mockReset(); + isMemoryIsolationCutoverAgentMock.mockReturnValue(false); await loadFreshContextEngineMaintenanceModuleForTest(); }); + it("does not let cutover maintenance route transcript content through an owning engine", async () => { + isMemoryIsolationCutoverAgentMock.mockReturnValue(true); + const maintain = vi.fn(async () => ({ + changed: true, + bytesFreed: 10, + rewrittenEntries: 1, + })); + + const result = await runContextEngineMaintenance({ + contextEngine: { + info: { id: "test", name: "Test Engine", ownsCompaction: true }, + ingest: async () => ({ ingested: true }), + assemble: async ({ messages }) => ({ messages, estimatedTokens: 0 }), + compact: async () => ({ ok: true, compacted: false }), + maintain, + }, + agentId: "main", + sessionId: "session-1", + sessionKey: "agent:main:session-1", + sessionFile: "/tmp/session.jsonl", + reason: "turn", + }); + + expect(result).toBeUndefined(); + expect(maintain).not.toHaveBeenCalled(); + }); + it("passes a rewrite-capable runtime context into maintain()", async () => { const sessionTarget = { agentId: "main", diff --git a/src/agents/embedded-agent-runner/context-engine-maintenance.ts b/src/agents/embedded-agent-runner/context-engine-maintenance.ts index 3359c1bbf328..c66bbb12549b 100644 --- a/src/agents/embedded-agent-runner/context-engine-maintenance.ts +++ b/src/agents/embedded-agent-runner/context-engine-maintenance.ts @@ -15,6 +15,7 @@ import type { ContextEngineSessionTarget, } from "../../context-engine/types.js"; import { formatErrorMessage } from "../../infra/errors.js"; +import { isMemoryIsolationCutoverAgent } from "../../plugins/memory-cutover.js"; import { enqueueCommandInLane, GatewayDrainingError, @@ -33,6 +34,7 @@ import { updateTaskNotifyPolicyForOwner, } from "../../tasks/task-owner-access.js"; import { findActiveSessionTask } from "../session-async-task-status.js"; +import { resolveSessionAgentIds } from "../agent-scope.js"; import { SessionManager } from "../sessions/index.js"; import { resolveContextEngineCapabilities } from "./context-engine-capabilities.js"; import { log } from "./logger.js"; @@ -560,6 +562,22 @@ export async function runContextEngineMaintenance( return undefined; } + const { sessionAgentId } = resolveSessionAgentIds({ + sessionKey: params.sessionTarget?.sessionKey ?? params.sessionKey, + config: params.config, + agentId: params.sessionTarget?.agentId ?? params.agentId, + }); + // An owning engine can compact during routine maintenance. Its current plugin + // contract has no opaque derive plan, so cutover sessions must not let that + // background path assemble transcript content outside the authorized host. + if (contextEngine.info.ownsCompaction === true && isMemoryIsolationCutoverAgent(sessionAgentId)) { + log.warn("skipping context-engine maintenance without memory derivation authority", { + reason: params.reason, + sessionId: params.sessionId, + }); + return undefined; + } + const executionMode = params.executionMode ?? "foreground"; const shouldDefer = params.reason === "turn" && diff --git a/src/agents/embedded-agent-runner/prepared-compaction-runtime.ts b/src/agents/embedded-agent-runner/prepared-compaction-runtime.ts index 25a07582cb54..60b8706a4b61 100644 --- a/src/agents/embedded-agent-runner/prepared-compaction-runtime.ts +++ b/src/agents/embedded-agent-runner/prepared-compaction-runtime.ts @@ -6,6 +6,7 @@ import os from "node:os"; import path from "node:path"; import { isAcpRuntimeSpawnAvailable } from "../../acp/runtime/availability.js"; import type { ThinkLevel } from "../../auto-reply/thinking.js"; +import { readAuthorizedTranscriptDerivation } from "../../config/sessions/session-transcript-memory-policy.js"; import { formatActiveNodeContextLabel, getCurrentActiveNodeContext, @@ -13,6 +14,7 @@ import { import { getMachineDisplayName } from "../../infra/machine-name.js"; import { resolveRuntimeOsLabel } from "../../infra/os-summary.js"; import { listRegisteredPluginAgentPromptGuidance } from "../../plugins/command-registry-state.js"; +import { isMemoryIsolationCutoverAgent } from "../../plugins/memory-cutover.js"; import { extractModelCompat } from "../../plugins/provider-model-compat.js"; import type { ProviderRuntimeModel } from "../../plugins/provider-runtime-model.types.js"; import { transformProviderSystemPrompt } from "../../plugins/provider-runtime.js"; @@ -23,6 +25,7 @@ import { applySkillEnvOverrides, applySkillEnvOverridesFromSnapshot, } from "../../skills/runtime/env-overrides.js"; +import { openOpenClawAgentDatabase } from "../../state/openclaw-agent-db.js"; import { normalizeMessageChannel } from "../../utils/message-channel.js"; import { isReasoningTagProvider } from "../../utils/provider-utils.js"; import { createBundleLspToolRuntime } from "../agent-bundle-lsp-runtime.js"; @@ -45,7 +48,12 @@ import { resolveConversationCapabilityProfile } from "../conversation-capability import { formatDateStamp, resolveUserTimezone } from "../date-time.js"; import { resolveOpenClawReferencePaths } from "../docs-path.js"; import { resolveHeartbeatPromptForSystemPrompt } from "../heartbeat-system-prompt.js"; -import { createAuthorizedMemoryReadHost } from "../memory-authorized-read-host.js"; +import { + admitAuthorizedMemoryDerivation, + createAuthorizedMemoryDerivationHost, + createAuthorizedMemoryReadHost, + prepareAuthorizedSealedCompactionHost, +} from "../memory-authorized-read-host.js"; import { prepareAgentMemoryPrompt } from "../memory-prompt-prepare.js"; import { applyAuthHeaderOverride, @@ -276,16 +284,51 @@ export async function buildPreparedCompactionRuntime(prepared: DirectCompactionP config: params.config, agentId: params.agentId, }); + // Compaction turns transcript and injected memory into a durable summary. + // Admit derivation before prompt construction so a revoked source cannot + // reach the summary model through a stale read host. + const cutoverCompaction = isMemoryIsolationCutoverAgent(sessionAgentId); + if ( + cutoverCompaction && + !(await admitAuthorizedMemoryDerivation({ + agentId: sessionAgentId, + sessionKey: sandboxSessionKey, + sessionId: params.sessionId, + runId: params.runId, + messageChannel: resolvedMessageProvider, + agentAccountId: params.agentAccountId, + })) + ) { + throw new Error("memory derivation authorization unavailable for compaction"); + } + if ( + cutoverCompaction && + !readAuthorizedTranscriptDerivation( + openOpenClawAgentDatabase({ agentId: sessionAgentId }).db, + params.sessionId, + ) + ) { + throw new Error("compaction transcript derivation authorization unavailable"); + } // Compaction shares the live run's admission discipline: prepare one host // before tools, then reuse it for the prompt snapshot. - const authorizedMemoryRead = createAuthorizedMemoryReadHost({ + const memoryHostParams = { agentId: sessionAgentId, sessionKey: sandboxSessionKey, sessionId: params.sessionId, runId: params.runId, messageChannel: resolvedMessageProvider, agentAccountId: params.agentAccountId, - }); + }; + const authorizedMemoryRead = cutoverCompaction + ? createAuthorizedMemoryDerivationHost(memoryHostParams) + : createAuthorizedMemoryReadHost(memoryHostParams); + const authorizedSealedCompaction = cutoverCompaction + ? await prepareAuthorizedSealedCompactionHost(memoryHostParams) + : undefined; + if (cutoverCompaction && !authorizedSealedCompaction) { + throw new Error("sealed compaction authorization unavailable"); + } const runtimeCapabilityProfile = resolveConversationCapabilityProfile({ config: params.config, sessionKey: sandboxSessionKey, @@ -655,6 +698,7 @@ export async function buildPreparedCompactionRuntime(prepared: DirectCompactionP effectiveTools, allowedToolNames, buildSystemPromptText, + authorizedSealedCompaction, resolvedMessageProvider, sessionAgentId, disposeToolRuntimes, diff --git a/src/agents/embedded-agent-runner/run/attempt-tool-prepare.ts b/src/agents/embedded-agent-runner/run/attempt-tool-prepare.ts index 302d4b8e1c74..89282754bee0 100644 --- a/src/agents/embedded-agent-runner/run/attempt-tool-prepare.ts +++ b/src/agents/embedded-agent-runner/run/attempt-tool-prepare.ts @@ -9,7 +9,7 @@ import { logCodeModeDiagnostic, } from "../../../logging/code-mode-diagnostic.js"; import { extractModelCompat } from "../../../plugins/provider-model-compat.js"; -import type { AuthorizedMemoryReadHost } from "../../../plugins/tool-types.js"; +import type { AuthorizedMemoryReadHost, AuthorizedMemoryWriteHost } from "../../../plugins/tool-types.js"; import { getPluginToolMeta } from "../../../plugins/tools.js"; import { isSubagentSessionKey } from "../../../routing/session-key.js"; import { normalizeDeliveryContext } from "../../../utils/delivery-context.shared.js"; @@ -65,6 +65,7 @@ type SkillUsagePaths = OpenClawCodingToolsOptions["skillUsagePaths"]; export async function prepareEmbeddedAttemptToolBase(params: { agentDir: string; authorizedMemoryRead?: AuthorizedMemoryReadHost; + authorizedMemoryWrite?: AuthorizedMemoryWriteHost; authorizedMemoryVirtualBroker?: AuthorizedMemoryVirtualFileBroker; attempt: EmbeddedRunAttemptParams; effectiveCwd: string; @@ -326,6 +327,7 @@ export async function prepareEmbeddedAttemptToolBase(params: { runId: attempt.runId, operationalRunInstance: attempt.admittedRunContext.operationalRunInstance, authorizedMemoryRead, + authorizedMemoryWrite: params.authorizedMemoryWrite, ...(fsPolicy ? { fsPolicy } : {}), ...(authorizedMemoryVirtualBroker ? { diff --git a/src/agents/embedded-agent-runner/run/attempt.ts b/src/agents/embedded-agent-runner/run/attempt.ts index 7f442f37d91c..53d0af7f644c 100644 --- a/src/agents/embedded-agent-runner/run/attempt.ts +++ b/src/agents/embedded-agent-runner/run/attempt.ts @@ -198,6 +198,7 @@ export async function runEmbeddedAttempt( prepareEmbeddedAttemptToolBase({ agentDir, authorizedMemoryRead, + authorizedMemoryWrite: params.authorizedMemoryWrite, authorizedMemoryVirtualBroker, attempt: params, effectiveCwd, diff --git a/src/agents/embedded-agent-runner/run/params.ts b/src/agents/embedded-agent-runner/run/params.ts index 67f6120e90bf..4ccd46fc5bab 100644 --- a/src/agents/embedded-agent-runner/run/params.ts +++ b/src/agents/embedded-agent-runner/run/params.ts @@ -119,6 +119,8 @@ export type RunEmbeddedAgentParams = { scheduledRuntimeAuthorityRecoveryRequired?: boolean; /** Relative workspace path that memory-triggered writes are allowed to append to. */ memoryFlushWritePath?: string; + /** Pre-admitted host-owned mutation for a transcript-derived memory flush. */ + authorizedMemoryWrite?: import("../../../plugins/tool-types.js").AuthorizedMemoryWriteHost; /** Delivery target for topic/thread routing. */ messageTo?: string; /** Thread/topic identifier for routing replies to the originating thread. */ diff --git a/src/agents/embedded-agent-runner/run/run-attempt-dispatch.ts b/src/agents/embedded-agent-runner/run/run-attempt-dispatch.ts index 468938e40a86..5a7bc73eee6f 100644 --- a/src/agents/embedded-agent-runner/run/run-attempt-dispatch.ts +++ b/src/agents/embedded-agent-runner/run/run-attempt-dispatch.ts @@ -275,6 +275,7 @@ export async function dispatchEmbeddedRunAttempt(input: { sandboxSessionKey: params.sandboxSessionKey, trigger: params.trigger, memoryFlushWritePath: params.memoryFlushWritePath, + authorizedMemoryWrite: params.authorizedMemoryWrite, messageChannel: params.messageChannel, messageProvider: params.messageProvider, clientCaps: params.clientCaps, diff --git a/src/agents/memory-authorized-read-host.test.ts b/src/agents/memory-authorized-read-host.test.ts new file mode 100644 index 000000000000..3dccee51ff2f --- /dev/null +++ b/src/agents/memory-authorized-read-host.test.ts @@ -0,0 +1,201 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + createDeriveInvocation: vi.fn(), + createWriteInvocation: vi.fn(), + createTrustedContext: vi.fn(), + readTranscriptDerivation: vi.fn(), + search: vi.fn(), + write: vi.fn(), +})); + +vi.mock("../plugins/memory-cutover.js", () => ({ + isMemoryIsolationCutoverAgent: () => true, +})); +vi.mock("../plugins/memory-invocation.js", () => ({ + MEMORY_INVOCATION_UNAVAILABLE: { unavailable: true }, + createAuthorizedMemoryDeriveInvocation: mocks.createDeriveInvocation, + createAuthorizedMemoryReadInvocation: vi.fn(), + createAuthorizedMemoryWriteInvocation: mocks.createWriteInvocation, + materializeAuthorizedMemoryVirtualView: vi.fn(), + readAuthorizedMemoryVirtualFile: vi.fn(), + readAuthorizedMemoryForInvocation: vi.fn(), + searchAuthorizedMemoryForInvocation: mocks.search, + writeAuthorizedMemoryForInvocation: mocks.write, +})); +vi.mock("../config/sessions/session-transcript-memory-policy.js", () => ({ + readAuthorizedTranscriptDerivation: mocks.readTranscriptDerivation, +})); +vi.mock("../state/openclaw-agent-db.js", () => ({ + openOpenClawAgentDatabase: () => ({ db: { kind: "agent-db" } }), +})); +vi.mock("../state/memory-access-context.js", () => ({ + captureTrustedMemoryAccessFacts: (facts: unknown) => facts, + createTrustedMemoryAccessContext: mocks.createTrustedContext, +})); +vi.mock("../state/memory-identity.js", () => ({ + recheckMemoryIdentityBinding: () => true, +})); +vi.mock("../state/memory-session-subject.js", () => ({ + createCurrentMemorySessionContext: () => ({ + kind: "current", + context: { + agentId: "main", + fingerprint: "session-fingerprint", + principalId: "service:main", + sessionId: "session-1", + sessionKey: "agent:main:session-1", + authorityRevision: "authority-1", + subject: { kind: "service", principalId: "service:main" }, + }, + }), +})); +vi.mock("./memory-egress-admission.js", () => ({ + resolveMemoryEgressDeliveryFacts: () => ({ + sink: "internal", + audiences: [{ kind: "agent", id: "main" }], + deliveryRevision: "delivery-1", + egressRegistryRevision: "egress-1", + }), +})); + +import { + admitAuthorizedMemoryDerivation, + createAuthorizedMemoryDerivationHost, + prepareAuthorizedTranscriptDerivationHost, +} from "./memory-authorized-read-host.js"; + +describe("admitAuthorizedMemoryDerivation", () => { + beforeEach(() => { + mocks.createDeriveInvocation.mockReset(); + mocks.createWriteInvocation.mockReset(); + mocks.createTrustedContext.mockReset(); + mocks.readTranscriptDerivation.mockReset(); + mocks.search.mockReset(); + mocks.write.mockReset(); + mocks.createTrustedContext.mockReturnValue({ kind: "current", context: { trusted: true } }); + }); + + it("admits a host-minted derive context before a model can receive derived content", async () => { + mocks.createDeriveInvocation.mockResolvedValue({}); + + await expect( + admitAuthorizedMemoryDerivation({ + agentId: "main", + sessionKey: "agent:main:session-1", + sessionId: "session-1", + runId: "run-1", + }), + ).resolves.toBe(true); + + expect(mocks.createDeriveInvocation).toHaveBeenCalledWith({ context: { trusted: true } }); + expect(mocks.createTrustedContext).toHaveBeenCalledWith( + expect.objectContaining({ facts: expect.objectContaining({ operation: "derive" }) }), + ); + }); + + it("fails closed when derive admission is unavailable", async () => { + mocks.createDeriveInvocation.mockResolvedValue({ unavailable: true }); + + await expect( + admitAuthorizedMemoryDerivation({ + agentId: "main", + sessionKey: "agent:main:session-1", + sessionId: "session-1", + runId: "run-1", + }), + ).resolves.toBe(false); + }); + + it("keeps source reads on the derive invocation after admission", async () => { + const invocation = {}; + mocks.createDeriveInvocation.mockResolvedValue(invocation); + mocks.search.mockResolvedValue({ results: [] }); + + const host = createAuthorizedMemoryDerivationHost({ + agentId: "main", + sessionKey: "agent:main:session-1", + sessionId: "session-1", + runId: "run-1", + }); + + await host?.search({ query: "compaction source" }); + + expect(mocks.createDeriveInvocation).toHaveBeenCalledWith({ context: { trusted: true } }); + }); + + it("binds a flush mutation to the host-read transcript policy set before the model can write", async () => { + const invocation = {}; + mocks.readTranscriptDerivation.mockReturnValue({ + eventSeqs: [0, 1], + sourcePolicySetId: "policy-set-1", + deliveryAudiencesJson: '[{"kind":"user","id":"alice"}]', + }); + mocks.createWriteInvocation.mockResolvedValue(invocation); + mocks.write.mockResolvedValue({ version: 1, mutationId: "mutation", status: "committed" }); + + const host = await prepareAuthorizedTranscriptDerivationHost({ + agentId: "main", + sessionKey: "agent:main:session-1", + sessionId: "session-1", + runId: "run-1", + }); + + await host?.remember({ content: "durable fact" }); + + expect(mocks.readTranscriptDerivation).toHaveBeenCalledWith({ kind: "agent-db" }, "session-1"); + expect(mocks.createWriteInvocation).toHaveBeenCalledWith({ context: { trusted: true } }); + expect(mocks.readTranscriptDerivation.mock.invocationCallOrder[0]).toBeLessThan( + mocks.createWriteInvocation.mock.invocationCallOrder[0] ?? Infinity, + ); + expect(mocks.write).toHaveBeenCalledWith( + expect.objectContaining({ + invocation, + mutation: expect.objectContaining({ + kind: "derive", + derivationPurpose: "flush", + sourcePolicySetId: "policy-set-1", + transcriptSource: { + kind: "transcript", + sessionId: "session-1", + eventSeqs: [0, 1], + sourcePolicySetId: "policy-set-1", + deliveryAudiencesJson: '[{"kind":"user","id":"alice"}]', + }, + }), + }), + ); + }); + + it("names a compacted summary without letting the caller select transcript lineage", async () => { + const invocation = {}; + mocks.readTranscriptDerivation.mockReturnValue({ + eventSeqs: [0, 1], + sourcePolicySetId: "policy-set-1", + deliveryAudiencesJson: '[{"kind":"user","id":"alice"}]', + }); + mocks.createWriteInvocation.mockResolvedValue(invocation); + mocks.write.mockResolvedValue({ version: 1, mutationId: "mutation", status: "committed" }); + + const host = await prepareAuthorizedTranscriptDerivationHost({ + agentId: "main", + sessionKey: "agent:main:session-1", + sessionId: "session-1", + runId: "run-1", + derivationPurpose: "compaction", + }); + + await host?.remember({ content: "summary" }); + + expect(mocks.write).toHaveBeenCalledWith( + expect.objectContaining({ + invocation, + mutation: expect.objectContaining({ + kind: "derive", + derivationPurpose: "compaction", + sourcePolicySetId: "policy-set-1", + }), + }), + ); + }); +}); diff --git a/src/agents/memory-authorized-read-host.ts b/src/agents/memory-authorized-read-host.ts index 8205404045cf..90b4c8dcdeaa 100644 --- a/src/agents/memory-authorized-read-host.ts +++ b/src/agents/memory-authorized-read-host.ts @@ -1,18 +1,24 @@ import { createHash, randomUUID } from "node:crypto"; import type { AuthorizedMemoryVirtualView, + AuthorizedSealedCompactionArtifact, + AuthorizedTranscriptDerivationSource, + AuthorizedTranscriptDerivationPurpose, MemoryAccessContext, MemoryActorEvidence, } from "../memory-host-sdk/host/authorization.js"; +import { readAuthorizedTranscriptDerivation } from "../config/sessions/session-transcript-memory-policy.js"; import { isMemoryIsolationCutoverAgent } from "../plugins/memory-cutover.js"; import { MEMORY_INVOCATION_UNAVAILABLE, + createAuthorizedMemoryDeriveInvocation, createAuthorizedMemoryReadInvocation, createAuthorizedMemoryWriteInvocation, materializeAuthorizedMemoryVirtualView, readAuthorizedMemoryVirtualFile, readAuthorizedMemoryForInvocation, searchAuthorizedMemoryForInvocation, + stageAuthorizedMemorySealedCompactionForInvocation, writeAuthorizedMemoryForInvocation, type AuthorizedMemoryReadInvocation, } from "../plugins/memory-invocation.js"; @@ -27,6 +33,7 @@ import { createCurrentMemorySessionContext, type CurrentMemorySessionContext, } from "../state/memory-session-subject.js"; +import { openOpenClawAgentDatabase } from "../state/openclaw-agent-db.js"; import type { DeliveryContext } from "../utils/delivery-context.types.js"; import { resolveMemoryEgressDeliveryFacts } from "./memory-egress-admission.js"; @@ -40,6 +47,12 @@ export type AuthorizedMemoryVirtualFileBroker = Readonly<{ readFile: (virtualPath: string) => Promise; }>; +/** Core-private sealed compaction capability; plugins never receive this host. */ +export type AuthorizedSealedCompactionHost = Readonly<{ + source: AuthorizedTranscriptDerivationSource; + stage: (content: string) => Promise; +}>; + type AuthorizedMemoryReadHostWithVirtualBroker = AuthorizedMemoryReadHost & Readonly<{ [authorizedMemoryVirtualBroker]: () => Promise; @@ -220,10 +233,11 @@ function createTrustedMemoryHostContext( * Builds the sole tool-facing read handle for a cut-over run. Session identity and delivery facts * are reread from their owners; sender IDs, `toolsBySender`, and paths never name a memory subject. */ -export function createAuthorizedMemoryReadHost( +function createAuthorizedMemoryContentHost( params: AuthorizedMemoryHostParams, + operation: "read" | "derive", ): AuthorizedMemoryReadHost | undefined { - const trusted = createTrustedMemoryHostContext({ ...params, operation: "read" }); + const trusted = createTrustedMemoryHostContext({ ...params, operation }); if (!trusted) { return undefined; } @@ -231,7 +245,10 @@ export function createAuthorizedMemoryReadHost( | Promise | undefined; const getInvocation = () => - (invocation ??= createAuthorizedMemoryReadInvocation({ context: trusted })); + (invocation ??= + operation === "derive" + ? createAuthorizedMemoryDeriveInvocation({ context: trusted }) + : createAuthorizedMemoryReadInvocation({ context: trusted })); let virtualBroker: Promise | undefined; const getVirtualBroker = () => (virtualBroker ??= (async () => { @@ -276,6 +293,130 @@ export function createAuthorizedMemoryReadHost( }) as AuthorizedMemoryReadHost; } +export function createAuthorizedMemoryReadHost( + params: AuthorizedMemoryHostParams, +): AuthorizedMemoryReadHost | undefined { + return createAuthorizedMemoryContentHost(params, "read"); +} + +/** + * Builds a content host whose every source read is authorized as a derivation. A caller cannot + * turn an admitted derive plan into a weaker read plan after the source reaches model context. + */ +export function createAuthorizedMemoryDerivationHost( + params: AuthorizedMemoryHostParams, +): AuthorizedMemoryReadHost | undefined { + return createAuthorizedMemoryContentHost(params, "derive"); +} + +/** + * Rechecks the separate derive capability before a runtime can place memory-derived + * material in a model context. Read admission alone intentionally never implies this. + */ +export async function admitAuthorizedMemoryDerivation( + params: AuthorizedMemoryHostParams, +): Promise { + const trusted = createTrustedMemoryHostContext({ ...params, operation: "derive" }); + if (!trusted) { + return false; + } + const invocation = await createAuthorizedMemoryDeriveInvocation({ context: trusted }); + return !("unavailable" in invocation); +} + +/** + * Admits one transcript-backed mutation before the flush model sees history. + * The opaque source stays host-owned; neither the model nor a plugin tool can + * substitute a session, event list, policy set, or delivery audience. + */ +export async function prepareAuthorizedTranscriptDerivationHost( + params: AuthorizedMemoryHostParams & Readonly<{ derivationPurpose?: AuthorizedTranscriptDerivationPurpose }>, +): Promise { + const sessionId = params.sessionId?.trim(); + const trusted = createTrustedMemoryHostContext({ ...params, operation: "derive" }); + if (!trusted || !sessionId) { + return undefined; + } + const transcriptSource = readAuthorizedTranscriptDerivation( + openOpenClawAgentDatabase({ agentId: params.agentId }).db, + sessionId, + ); + if (!transcriptSource) { + return undefined; + } + const invocation = await createAuthorizedMemoryWriteInvocation({ context: trusted }); + if ("unavailable" in invocation) { + return undefined; + } + return Object.freeze({ + async remember({ content, contentType = "markdown" }) { + const result = await writeAuthorizedMemoryForInvocation({ + invocation, + mutation: { + version: 1, + kind: "derive", + derivationPurpose: params.derivationPurpose ?? "flush", + mutationId: randomUUID(), + idempotencyKey: randomUUID(), + content, + contentType, + sourcePolicySetId: transcriptSource.sourcePolicySetId, + transcriptSource: { + kind: "transcript", + sessionId, + eventSeqs: transcriptSource.eventSeqs, + sourcePolicySetId: transcriptSource.sourcePolicySetId, + deliveryAudiencesJson: transcriptSource.deliveryAudiencesJson, + }, + }, + }); + return "unavailable" in result ? MEMORY_INVOCATION_UNAVAILABLE : result; + }, + }); +} + +/** + * Captures the exact transcript source before model work. The returned staging + * capability has no caller-selectable session, policy, audience, or store. + */ +export async function prepareAuthorizedSealedCompactionHost( + params: AuthorizedMemoryHostParams, +): Promise { + const sessionId = params.sessionId?.trim(); + const trusted = createTrustedMemoryHostContext({ ...params, operation: "derive" }); + if (!trusted || !sessionId) { + return undefined; + } + const transcriptSource = readAuthorizedTranscriptDerivation( + openOpenClawAgentDatabase({ agentId: params.agentId }).db, + sessionId, + ); + if (!transcriptSource) { + return undefined; + } + const invocation = await createAuthorizedMemoryWriteInvocation({ context: trusted }); + if ("unavailable" in invocation) { + return undefined; + } + const sealedSource = Object.freeze({ + kind: "transcript", + sessionId, + eventSeqs: transcriptSource.eventSeqs, + sourcePolicySetId: transcriptSource.sourcePolicySetId, + deliveryAudiencesJson: transcriptSource.deliveryAudiencesJson, + }); + return Object.freeze({ + source: sealedSource, + async stage(content) { + return await stageAuthorizedMemorySealedCompactionForInvocation({ + invocation, + content, + transcriptSource: sealedSource, + }); + }, + }); +} + /** * Builds a one-mutation append host for a cut-over run. The model supplies only content; the host * reissues append facts and the selected runtime chooses the subject-owned store and audience. diff --git a/src/agents/memory-autonomous-run-policy.test.ts b/src/agents/memory-autonomous-run-policy.test.ts new file mode 100644 index 000000000000..4d4708087377 --- /dev/null +++ b/src/agents/memory-autonomous-run-policy.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vitest"; +import { + mayInjectAutonomousSourceTranscript, + resolveSubagentMemoryContextMode, +} from "./memory-autonomous-run-policy.js"; + +describe("memory autonomous-run policy", () => { + it("removes raw fork context while memory isolation is active", () => { + expect( + resolveSubagentMemoryContextMode({ requested: "fork", memoryIsolationActive: true }), + ).toBe("isolated"); + expect( + resolveSubagentMemoryContextMode({ requested: "isolated", memoryIsolationActive: true }), + ).toBe("isolated"); + }); + + it("does not inject a current session transcript into an isolated service run", () => { + expect( + mayInjectAutonomousSourceTranscript({ sessionTarget: "current", memoryIsolationActive: true }), + ).toBe(false); + expect( + mayInjectAutonomousSourceTranscript({ sessionTarget: "isolated", memoryIsolationActive: true }), + ).toBe(true); + }); + + it("preserves the legacy path until isolation is enabled", () => { + expect( + resolveSubagentMemoryContextMode({ requested: "fork", memoryIsolationActive: false }), + ).toBe("fork"); + expect( + mayInjectAutonomousSourceTranscript({ sessionTarget: "current", memoryIsolationActive: false }), + ).toBe(true); + }); +}); diff --git a/src/agents/memory-autonomous-run-policy.ts b/src/agents/memory-autonomous-run-policy.ts new file mode 100644 index 000000000000..a74705d77001 --- /dev/null +++ b/src/agents/memory-autonomous-run-policy.ts @@ -0,0 +1,19 @@ +/** + * Memory isolation narrows autonomous work before any session transcript is + * selected. A service/child run may gain an explicit plugin capability later, + * but it must never inherit raw user history as a substitute for one. + */ +export function resolveSubagentMemoryContextMode(params: { + requested: "fork" | "isolated"; + memoryIsolationActive: boolean; +}): "fork" | "isolated" { + return params.memoryIsolationActive && params.requested === "fork" ? "isolated" : params.requested; +} + +/** Current-bound cron work is autonomous under memory isolation, not a replay of its old target. */ +export function mayInjectAutonomousSourceTranscript(params: { + sessionTarget?: string; + memoryIsolationActive: boolean; +}): boolean { + return !(params.memoryIsolationActive && params.sessionTarget === "current"); +} diff --git a/src/agents/memory-egress-admission.ts b/src/agents/memory-egress-admission.ts index 164a6487382e..b644a3ccc499 100644 --- a/src/agents/memory-egress-admission.ts +++ b/src/agents/memory-egress-admission.ts @@ -128,7 +128,6 @@ export function resolveMemoryEgressDeliveryFacts(params: { channel, accountId, recipientId: to, - options: { agentId: params.agentId }, }).kind === "current" ? { sink: "private" as const, diff --git a/src/agents/openclaw-tools.ts b/src/agents/openclaw-tools.ts index 064ffa224b40..eba9b4a07c60 100644 --- a/src/agents/openclaw-tools.ts +++ b/src/agents/openclaw-tools.ts @@ -25,6 +25,7 @@ import { wrapToolWithBeforeToolCallHook, } from "./agent-tools.before-tool-call.js"; import type { ConversationRecallContext } from "./conversation-recall.types.js"; +import type { AuthorizedMemoryWriteHost } from "../plugins/tool-types.js"; import { resolveOpenClawPluginToolsForOptions } from "./openclaw-plugin-tools.js"; import { filterToolsByClientCaps } from "./openclaw-tools.client-caps.js"; import { @@ -104,6 +105,8 @@ export function createOpenClawTools( sandboxBrowserBridgeUrl?: string; allowHostBrowserControl?: boolean; agentSessionKey?: string; + /** Pre-admitted host-owned mutation for a transcript-derived flush. */ + authorizedMemoryWrite?: AuthorizedMemoryWriteHost; toolBindings?: Readonly>; /** Durable store key when it differs from the sandbox/policy session key. */ runSessionKey?: string; diff --git a/src/agents/sessions/agent-session-compaction.ts b/src/agents/sessions/agent-session-compaction.ts index 576bf9987b4b..fa3ace0737d2 100644 --- a/src/agents/sessions/agent-session-compaction.ts +++ b/src/agents/sessions/agent-session-compaction.ts @@ -22,8 +22,18 @@ import type { SettingsManager } from "./settings-manager.js"; type CompactionReason = "manual" | "threshold" | "overflow"; type SummaryOutputPolicy = "none" | "retry-invalid-once"; +export type DeferredSessionCompaction = Readonly<{ + entry: CompactionEntry; + fromExtension: boolean; + result: CompactionResult; +}>; type CompactionWorkOutcome = - | { status: "completed"; result: CompactionResult; tokensAfter: number } + | { + status: "completed"; + result: CompactionResult; + tokensAfter: number; + deferred?: DeferredSessionCompaction; + } | { status: "aborted" } | { status: "skipped"; reason: string }; @@ -49,21 +59,83 @@ export abstract class AgentSessionCompaction extends AgentSessionInspection { */ async compact(customInstructions?: string): Promise { return await this.runWithSessionWriteSettlement( - async () => await this.compactWithSessionWriteSettlement(customInstructions, "none"), + async () => (await this.compactWithSessionWriteSettlement(customInstructions, "none")).result, ); } async [agentSessionAutomaticCompaction](customInstructions?: string): Promise { return await this.runWithSessionWriteSettlement( async () => - await this.compactWithSessionWriteSettlement(customInstructions, "retry-invalid-once"), + (await this.compactWithSessionWriteSettlement(customInstructions, "retry-invalid-once")) + .result, ); } + /** + * Produces a summary without publishing it. The caller must durably commit + * this exact entry before applying it to the in-memory session. + */ + async compactDeferred( + customInstructions?: string, + summaryOutputPolicy: SummaryOutputPolicy = "none", + ): Promise { + return await this.runWithSessionWriteSettlement(async () => { + const outcome = await this.compactWithSessionWriteSettlement( + customInstructions, + summaryOutputPolicy, + true, + ); + if (!outcome.deferred) { + throw new Error("Deferred compaction entry is unavailable"); + } + return outcome.deferred; + }); + } + + /** Apply only an entry committed by the sealed transcript owner. */ + async applyDeferredCompaction(params: DeferredSessionCompaction): Promise { + this.sessionManager.applyPersistedCompaction(params.entry); + const sessionContext = this.sessionManager.buildSessionContext(); + // Compaction replaces the request prefix, invalidating retained usage and thinking signatures. + // Keep the deferred path replay-safe just like the direct append path. + this.agent.state.messages = sanitizeCompactionReplayMessages(sessionContext.messages); + if (this.currentExtensionRunner) { + await this.currentExtensionRunner.emit({ + type: "session_compact", + compactionEntry: params.entry, + fromExtension: params.fromExtension, + }); + } + const tokensAfter = estimateContextTokens(this.agent.state.messages).tokens; + this.emit({ + type: "compaction_end", + reason: "manual", + outcome: { + status: "completed", + tokensBefore: params.result.tokensBefore, + tokensAfter, + willRetry: false, + }, + }); + } + + /** The sealed owner calls this if staging or durable commit fails. */ + discardDeferredCompaction(error: unknown): void { + this.emit({ + type: "compaction_end", + reason: "manual", + outcome: { + status: "failed", + reason: `Compaction failed: ${compactionErrorMessage(error, "Compaction failed")}`, + }, + }); + } + private async compactWithSessionWriteSettlement( customInstructions?: string, summaryOutputPolicy: SummaryOutputPolicy = "none", - ): Promise { + deferPersistence = false, + ): Promise> { this.disconnectFromAgent(); await this.abort(); const abortController = new AbortController(); @@ -80,6 +152,7 @@ export abstract class AgentSessionCompaction extends AgentSessionInspection { summaryOutputPolicy, settings, signal: abortController.signal, + deferPersistence, }); } catch (error) { const message = compactionErrorMessage(error, "Compaction failed"); @@ -103,17 +176,19 @@ export abstract class AgentSessionCompaction extends AgentSessionInspection { throw new Error("Compaction cancelled"); } - this.emit({ - type: "compaction_end", - reason: "manual", - outcome: { - status: "completed", - tokensBefore: outcome.result.tokensBefore, - tokensAfter: outcome.tokensAfter, - willRetry: false, - }, - }); - return outcome.result; + if (!deferPersistence) { + this.emit({ + type: "compaction_end", + reason: "manual", + outcome: { + status: "completed", + tokensBefore: outcome.result.tokensBefore, + tokensAfter: outcome.tokensAfter, + willRetry: false, + }, + }); + } + return outcome; } finally { if (this.compactionAbortController === abortController) { this.compactionAbortController = undefined; @@ -143,6 +218,7 @@ export abstract class AgentSessionCompaction extends AgentSessionInspection { customInstructions?: string; mode: "manual" | "auto"; summaryOutputPolicy: SummaryOutputPolicy; + deferPersistence?: boolean; }): Promise { const isManual = options.mode === "manual"; if (!this.model) { @@ -246,6 +322,21 @@ export abstract class AgentSessionCompaction extends AgentSessionInspection { summary: capCompactionSummary(compactionResult.summary), }; + if (options.deferPersistence) { + const entry = this.sessionManager.createCompactionEntry( + compactionResult.summary, + compactionResult.firstKeptEntryId, + compactionResult.tokensBefore, + compactionResult.details, + ); + return { + status: "completed", + result: compactionResult, + tokensAfter: estimateContextTokens(this.agent.state.messages).tokens, + deferred: Object.freeze({ entry, fromExtension, result: compactionResult }), + }; + } + this.sessionManager.appendCompaction( compactionResult.summary, compactionResult.firstKeptEntryId, diff --git a/src/agents/sessions/agent-session-loop-correctness.test.ts b/src/agents/sessions/agent-session-loop-correctness.test.ts index 6b4af6ae6957..f17b36754f2e 100644 --- a/src/agents/sessions/agent-session-loop-correctness.test.ts +++ b/src/agents/sessions/agent-session-loop-correctness.test.ts @@ -433,6 +433,28 @@ describe("AgentSession loop correctness", () => { } }); + it("produces a sealed compaction entry without publishing it", async () => { + const sessionManager = SessionManager.inMemory(); + appendHistory( + sessionManager, + createAssistant(testModel, [{ type: "text", text: "short answer" }]), + ); + const settingsManager = SettingsManager.inMemory({ + compaction: { enabled: true, reserveTokens: 0, keepRecentTokens: 10_000 }, + retry: { enabled: false }, + }); + const { session } = await createTestSession({ + sessionManager, + settingsManager, + resourceLoader: createResourceLoader(createCompactionHandlers()), + }); + + const deferred = await session.compactDeferred(); + + expect(deferred.entry).toMatchObject({ type: "compaction", summary: "condensed history" }); + expect(sessionManager.getBranch().some((entry) => entry.type === "compaction")).toBe(false); + }); + it("keeps a successful high-usage response and performs threshold maintenance without retry", async () => { const settingsManager = createAutoCompactionSettings(); const compactionEvents: AgentSessionEvent[] = []; diff --git a/src/agents/sessions/session-manager-entries.ts b/src/agents/sessions/session-manager-entries.ts index 943986c7f380..471750ab8eb3 100644 --- a/src/agents/sessions/session-manager-entries.ts +++ b/src/agents/sessions/session-manager-entries.ts @@ -1,4 +1,5 @@ import { + loadTranscriptEventsSync, readActiveTranscriptEntryAnchor, type TranscriptEntryAnchor, } from "../../config/sessions/session-accessor.js"; @@ -33,10 +34,13 @@ import type { ThinkingLevelChangeEntry, } from "./session-manager-types.js"; +/** Internal-only: callers cannot bypass persistence through public append options. */ +type AppendEntryOptions = AppendPersistenceOptions & Readonly<{ alreadyPersisted?: boolean }>; + export class SessionManagerEntries extends SessionManagerPersistence { protected appendEntry( entry: SessionEntry, - options?: AppendPersistenceOptions, + options?: AppendEntryOptions, ): TranscriptEntryAnchor | undefined { // oxlint-disable-next-line unicorn/prefer-structured-clone -- Match the persisted JSON/toJSON shape exactly. const canonicalEntry = JSON.parse(JSON.stringify(entry)) as SessionEntry; @@ -47,10 +51,15 @@ export class SessionManagerEntries extends SessionManagerPersistence { !this.pendingDeliberateAppend && this.appendMode !== "side" && !isSessionTranscriptSideAppendEntry(canonicalEntry); - const persistenceResult = this.persist(canonicalEntry, { - ...options, - ...(activeBranchAppend ? { appendIntent: "active-branch" } : {}), - }); + if (options?.alreadyPersisted && !this.persistenceTarget) { + throw new Error("A pre-persisted session entry requires a transcript target"); + } + const persistenceResult = options?.alreadyPersisted + ? undefined + : this.persist(canonicalEntry, { + ...options, + ...(activeBranchAppend ? { appendIntent: "active-branch" } : {}), + }); if (persistenceResult && typeof persistenceResult === "object") { if (persistenceResult.adoptedMessageId) { this.reloadPersistedTranscript(); @@ -195,14 +204,14 @@ export class SessionManagerEntries extends SessionManagerPersistence { return entry.id; } - appendCompaction( + createCompactionEntry( summary: string, firstKeptEntryId: string, tokensBefore: number, details?: unknown, fromHook?: boolean, - ): string { - const entry: CompactionEntry = { + ): CompactionEntry { + return Object.freeze({ type: "compaction", id: generateSessionEntryId(this.byId), parentId: this.appendParentId, @@ -212,7 +221,44 @@ export class SessionManagerEntries extends SessionManagerPersistence { tokensBefore, details, fromHook, - }; + }); + } + + /** Apply a compaction after the transcript owner committed the exact entry atomically. */ + applyPersistedCompaction(entry: CompactionEntry): string { + if (entry.parentId !== this.appendParentId || this.byId.has(entry.id)) { + throw new Error("Pre-persisted compaction no longer matches the active transcript leaf"); + } + const persisted = this.persistenceTarget + ? loadTranscriptEventsSync(this.persistenceTarget).some( + (candidate) => + isIndexedSessionEntry(candidate) && JSON.stringify(candidate) === JSON.stringify(entry), + ) + : false; + if (!persisted) { + throw new Error("Pre-persisted compaction was not committed to the transcript"); + } + this.appendEntry(entry, { + alreadyPersisted: true, + invalidateSerializedPrefixCache: entry.fromHook === true || entry.details !== undefined, + }); + return entry.id; + } + + appendCompaction( + summary: string, + firstKeptEntryId: string, + tokensBefore: number, + details?: unknown, + fromHook?: boolean, + ): string { + const entry = this.createCompactionEntry( + summary, + firstKeptEntryId, + tokensBefore, + details, + fromHook, + ); this.appendEntry(entry, { invalidateSerializedPrefixCache: fromHook === true || details !== undefined, }); diff --git a/src/agents/sessions/session-manager.test.ts b/src/agents/sessions/session-manager.test.ts index a6be3a68ff76..291d6e147790 100644 --- a/src/agents/sessions/session-manager.test.ts +++ b/src/agents/sessions/session-manager.test.ts @@ -8,6 +8,7 @@ import { parseSqliteSessionFileMarker, } from "../../config/sessions/legacy-sqlite-marker.js"; import { + appendTranscriptEvent, appendTranscriptMessage, loadSessionEntry, loadTranscriptEvents, @@ -34,6 +35,36 @@ function openMarker(marker: string, sessionKey: string, cwd: string): SessionMan } describe("SessionManager.open", () => { + it("applies a compaction entry only after its transaction owner persisted it", async () => { + const dir = tempDirs.make("openclaw-session-manager-"); + const storePath = path.join(dir, "sessions.json"); + const sessionId = "sealed-compaction-session"; + const sessionKey = "agent:main:dashboard:sealed-compaction"; + const scope = { agentId: "main", sessionId, sessionKey, storePath }; + await upsertSessionEntry(scope, { sessionId, updatedAt: 1 }); + await appendTranscriptMessage(scope, { + cwd: dir, + eventId: "source-message", + message: { role: "user", content: "retain this" }, + }); + const sessionManager = SessionManager.open(scope, dir); + const entry = sessionManager.createCompactionEntry("sealed summary", "source-message", 42); + + expect(() => sessionManager.applyPersistedCompaction(entry)).toThrow( + "Pre-persisted compaction was not committed to the transcript", + ); + await appendTranscriptEvent(scope, entry); + expect(sessionManager.applyPersistedCompaction(entry)).toBe(entry.id); + await expect(loadTranscriptEvents(scope)).resolves.toEqual( + expect.arrayContaining([ + expect.objectContaining({ id: entry.id, summary: "sealed summary", type: "compaction" }), + ]), + ); + expect(sessionManager.getEntries()).toEqual( + expect.arrayContaining([expect.objectContaining({ id: entry.id, type: "compaction" })]), + ); + }); + it("opens SQLite markers without creating marker-named files and persists assistant replies", async () => { const dir = tempDirs.make("openclaw-session-manager-"); const storePath = path.join(dir, "sessions.json"); diff --git a/src/agents/subagents/spawn/subagent-spawn-context.ts b/src/agents/subagents/spawn/subagent-spawn-context.ts index 987cf45bdeeb..75e2c47c3f2e 100644 --- a/src/agents/subagents/spawn/subagent-spawn-context.ts +++ b/src/agents/subagents/spawn/subagent-spawn-context.ts @@ -3,6 +3,7 @@ import { resolveThreadBindingSpawnPolicy } from "../../../channels/thread-bindin import type { SessionEntry } from "../../../config/sessions/types.js"; import type { OpenClawConfig } from "../../../config/types.openclaw.js"; import type { SubagentSpawnPreparation } from "../../../context-engine/types.js"; +import { resolveSubagentMemoryContextMode } from "../../memory-autonomous-run-policy.js"; import { summarizeSpawnError } from "../../spawn-pipeline.js"; import { getSubagentSpawnDeps } from "./subagent-spawn-deps.js"; import { resolveGatewaySessionStoreTarget } from "./subagent-spawn.runtime.js"; @@ -34,8 +35,24 @@ export async function prepareSubagentSessionContext(params: { requesterInternalKey: string; childSessionKey: string; }): Promise { - if (params.contextMode === "isolated") { - return { status: "ok", mode: "isolated" }; + const contextMode = resolveSubagentMemoryContextMode({ + requested: params.contextMode, + memoryIsolationActive: getSubagentSpawnDeps().isMemoryIsolationCutoverAgent( + params.requesterAgentId, + ), + }); + if (contextMode === "isolated") { + // A fork copies raw transcript rows before a child has an independently + // admitted memory view. Under cutover the empty intersection is the only + // safe default until an opaque delegation capability is issued. + return params.contextMode === "fork" + ? { + status: "ok", + mode: "isolated", + forkFallbackNote: + "context=\"fork\" is unavailable while memory isolation is active; starting with isolated context instead.", + } + : { status: "ok", mode: "isolated" }; } const childTarget = resolveGatewaySessionStoreTarget({ cfg: params.cfg, diff --git a/src/agents/subagents/spawn/subagent-spawn-deps.ts b/src/agents/subagents/spawn/subagent-spawn-deps.ts index e5e6b5c44d14..fbbaad6dbe8c 100644 --- a/src/agents/subagents/spawn/subagent-spawn-deps.ts +++ b/src/agents/subagents/spawn/subagent-spawn-deps.ts @@ -7,6 +7,7 @@ import { getGlobalHookRunner, getRuntimeConfig, hasInProcessGatewayContext, + isMemoryIsolationCutoverAgent, loadPreparedModelCatalog, resolveContextEngine, } from "./subagent-spawn.runtime.js"; @@ -18,6 +19,7 @@ type SubagentSpawnDeps = { getGlobalHookRunner: () => SubagentLifecycleHookRunner | null; getRuntimeConfig: typeof getRuntimeConfig; hasInProcessGatewayContext: typeof hasInProcessGatewayContext; + isMemoryIsolationCutoverAgent: typeof isMemoryIsolationCutoverAgent; ensureContextEnginesInitialized: typeof ensureContextEnginesInitialized; loadPreparedModelCatalog: typeof loadPreparedModelCatalog; resolveContextEngine: typeof resolveContextEngine; @@ -30,6 +32,7 @@ const defaultSubagentSpawnDeps: SubagentSpawnDeps = { getGlobalHookRunner, getRuntimeConfig, hasInProcessGatewayContext, + isMemoryIsolationCutoverAgent, ensureContextEnginesInitialized, loadPreparedModelCatalog, resolveContextEngine, diff --git a/src/agents/subagents/spawn/subagent-spawn.context.test.ts b/src/agents/subagents/spawn/subagent-spawn.context.test.ts index 03d00902baa2..ddfd26ec9894 100644 --- a/src/agents/subagents/spawn/subagent-spawn.context.test.ts +++ b/src/agents/subagents/spawn/subagent-spawn.context.test.ts @@ -17,6 +17,7 @@ describe("sessions_spawn context modes", () => { const forkSessionEntryFromParentMock = vi.fn(); const forkSessionFromParentMock = vi.fn(); const ensureContextEnginesInitializedMock = vi.fn(); + const isMemoryIsolationCutoverAgentMock = vi.fn(); const resolveContextEngineMock = vi.fn(); let spawnSubagentDirect: Awaited< ReturnType @@ -30,6 +31,7 @@ describe("sessions_spawn context modes", () => { forkSessionEntryFromParentMock, forkSessionFromParentMock, ensureContextEnginesInitializedMock, + isMemoryIsolationCutoverAgent: isMemoryIsolationCutoverAgentMock, resolveContextEngineMock, sessionStorePath: storePath, })); @@ -42,8 +44,10 @@ describe("sessions_spawn context modes", () => { forkSessionEntryFromParentMock.mockReset(); forkSessionFromParentMock.mockReset(); ensureContextEnginesInitializedMock.mockReset(); + isMemoryIsolationCutoverAgentMock.mockReset(); resolveContextEngineMock.mockReset(); setupAcceptedSubagentGatewayMock(callGatewayMock); + isMemoryIsolationCutoverAgentMock.mockReturnValue(false); resolveContextEngineMock.mockResolvedValue({}); }); @@ -224,6 +228,21 @@ describe("sessions_spawn context modes", () => { expect(prepareContext.childSessionFile).toBe("/tmp/forked-session.jsonl"); }); + it("does not fork raw parent context while memory isolation is active", async () => { + isMemoryIsolationCutoverAgentMock.mockReturnValue(true); + usePersistentStoreMock({ + main: { sessionId: "parent-session-id", updatedAt: 1 }, + }); + + const result = await spawnSubagentDirect( + { task: "inspect the current thread", context: "fork" }, + { agentSessionKey: "main" }, + ); + + expect(result.status).toBe("accepted"); + expect(forkSessionEntryFromParentMock).not.toHaveBeenCalled(); + }); + it("keeps the default spawn context isolated", async () => { const store: SessionStore = { main: { sessionId: "parent-session-id", updatedAt: 1 }, diff --git a/src/agents/subagents/spawn/subagent-spawn.runtime.ts b/src/agents/subagents/spawn/subagent-spawn.runtime.ts index 65b9059dc477..fcd630144fba 100644 --- a/src/agents/subagents/spawn/subagent-spawn.runtime.ts +++ b/src/agents/subagents/spawn/subagent-spawn.runtime.ts @@ -4,6 +4,7 @@ * entire gateway/channel stack. */ export { getRuntimeConfig } from "../../../config/config.js"; +export { isMemoryIsolationCutoverAgent } from "../../../plugins/memory-cutover.js"; export { loadSessionEntryReadOnly as loadSessionEntry, upsertSessionEntryCore, diff --git a/src/agents/subagents/spawn/subagent-spawn.test-helpers.ts b/src/agents/subagents/spawn/subagent-spawn.test-helpers.ts index 06e256f2b1cf..e858c34e6294 100644 --- a/src/agents/subagents/spawn/subagent-spawn.test-helpers.ts +++ b/src/agents/subagents/spawn/subagent-spawn.test-helpers.ts @@ -131,6 +131,7 @@ export async function loadSubagentSpawnModuleForTest(params: { callGatewayMock: MockFn; dispatchGatewayMethodInProcessMock?: MockFn; hasInProcessGatewayContextMock?: MockFn; + isMemoryIsolationCutoverAgent?: (agentId: string) => boolean; getRuntimeConfig?: () => Record; loadSessionStoreMock?: MockFn; loadPreparedModelCatalogMock?: MockFn; @@ -220,6 +221,8 @@ export async function loadSubagentSpawnModuleForTest(params: { dispatchGatewayMethodInProcess: (...args: unknown[]) => params.dispatchGatewayMethodInProcessMock?.(...args), hasInProcessGatewayContext: () => Boolean(params.hasInProcessGatewayContextMock?.()), + isMemoryIsolationCutoverAgent: (agentId: string) => + params.isMemoryIsolationCutoverAgent?.(agentId) ?? false, buildSubagentSystemPrompt: () => "system-prompt", forkSessionEntryFromParent: params.forkSessionEntryFromParentMock ?? diff --git a/src/auto-reply/dispatch.memory-egress.test.ts b/src/auto-reply/dispatch.memory-egress.test.ts index 6fbaca83f459..389836a10813 100644 --- a/src/auto-reply/dispatch.memory-egress.test.ts +++ b/src/auto-reply/dispatch.memory-egress.test.ts @@ -119,6 +119,14 @@ describe("memory egress at final dispatch", () => { expect(hoisted.prepareMemoryEgressAuthorization).toHaveBeenCalledWith( expect.objectContaining({ capabilityId: "reply.block" }), ); + // The finalized channel context has no authoritative session identity. The + // admission owner resolves sessionId/sessionKey from the registered run, + // so a stale inbound field cannot retarget memory egress. + expect( + hoisted.prepareMemoryEgressAuthorization.mock.calls.every(([params]) => + params.sessionId === undefined && params.sessionKey === undefined, + ), + ).toBe(true); expect(delivered).not.toHaveBeenCalled(); expect(dispatcher.getCancelledCounts?.()).toEqual({ tool: 0, block: 1, final: 1 }); }); diff --git a/src/auto-reply/dispatch.ts b/src/auto-reply/dispatch.ts index 778b4f0e08f1..050fcd54c96e 100644 --- a/src/auto-reply/dispatch.ts +++ b/src/auto-reply/dispatch.ts @@ -209,8 +209,6 @@ function installMemoryEgressAdmission( const resolveDeliveryFacts = () => resolveCurrentMemoryEgressDeliveryFacts(finalized); const memoryRunIdentity = { agentId: finalized.AgentId, - sessionId: finalized.SessionId, - sessionKey: finalized.SessionKey, }; if ( !appendReplyDispatcherPayloadPrepare(dispatcher, (payload, info) => { diff --git a/src/auto-reply/reply/agent-runner-memory.test.ts b/src/auto-reply/reply/agent-runner-memory.test.ts index a8ed9ac8d288..72fb59795f0f 100644 --- a/src/auto-reply/reply/agent-runner-memory.test.ts +++ b/src/auto-reply/reply/agent-runner-memory.test.ts @@ -55,8 +55,16 @@ const ensureSelectedAgentHarnessPluginMock = vi.fn(); const ensureMemoryFlushTargetFileMock = vi.fn(); const registerAgentRunContextMock = vi.fn(); const clearAgentRunContextMock = vi.fn(); +const memoryHostMocks = vi.hoisted(() => ({ + prepareAuthorizedTranscriptDerivationHost: vi.fn(), +})); const TEST_MAX_FLUSH_FAILURES = 3; +vi.mock("../../agents/memory-authorized-read-host.js", () => ({ + prepareAuthorizedTranscriptDerivationHost: + memoryHostMocks.prepareAuthorizedTranscriptDerivationHost, +})); + type MemoryFlushTestParams = Parameters[0] & { modelContextTokens?: number; }; @@ -241,6 +249,7 @@ type EmbeddedAgentParams = { prompt?: string; transcriptPrompt?: string; memoryFlushWritePath?: string; + authorizedMemoryWrite?: unknown; silentExpected?: boolean; extraSystemPrompt?: string; bootstrapPromptWarningSignaturesSeen?: string[]; @@ -451,6 +460,9 @@ describe("runMemoryFlushIfNeeded", () => { ensureSelectedAgentHarnessPluginMock.mockReset().mockResolvedValue(undefined); registerAgentRunContextMock.mockReset(); clearAgentRunContextMock.mockReset(); + memoryHostMocks.prepareAuthorizedTranscriptDerivationHost + .mockReset() + .mockResolvedValue(undefined); incrementCompactionCountMock.mockReset().mockImplementation(async (params) => { const sessionKey = String(params.sessionKey ?? ""); if (!sessionKey || !params.sessionStore?.[sessionKey]) { @@ -499,7 +511,7 @@ describe("runMemoryFlushIfNeeded", () => { await fs.rm(rootDir, { recursive: true, force: true }); }); - it("does not start or materialize a legacy memory flush for a cut-over agent", async () => { + it("does not let a cut-over flush model summarize transcript content without derivation lineage", async () => { markAgentCutOver("main"); const sessionKey = "agent:main:flush-policy"; const sessionEntry: SessionEntry = { @@ -538,6 +550,52 @@ describe("runMemoryFlushIfNeeded", () => { }); }); + it("routes a cut-over flush through its admitted transcript mutation without a workspace path", async () => { + markAgentCutOver("main"); + const authorizedMemoryWrite = { remember: vi.fn() }; + memoryHostMocks.prepareAuthorizedTranscriptDerivationHost.mockResolvedValue( + authorizedMemoryWrite, + ); + const sessionKey = "agent:main:authorized-flush"; + const sessionEntry: SessionEntry = { + sessionId: "session", + updatedAt: Date.now(), + totalTokens: 80_000, + totalTokensFresh: true, + totalTokensVersion: 1, + compactionCount: 1, + }; + const followupRun = createTestFollowupRun(); + followupRun.run.agentId = "main"; + followupRun.run.sessionKey = sessionKey; + followupRun.run.workspaceDir = rootDir; + + await runMemoryFlushIfNeeded({ + cfg: { agents: { defaults: { compaction: { memoryFlush: {} } } } }, + followupRun, + sessionCtx: { Provider: "whatsapp" } as unknown as TemplateContext, + defaultModel: "anthropic/claude-opus-4-6", + agentCfgContextTokens: 100_000, + resolvedVerboseLevel: "off", + sessionEntry, + sessionStore: { [sessionKey]: sessionEntry }, + sessionKey, + storePath: path.join(rootDir, "sessions.json"), + isHeartbeat: false, + replyOperation: createReplyOperation(), + }); + + expect(memoryHostMocks.prepareAuthorizedTranscriptDerivationHost).toHaveBeenCalledWith( + expect.objectContaining({ agentId: "main", sessionId: "session" }), + ); + const flushCall = requireEmbeddedAgentCall(); + expect(flushCall.authorizedMemoryWrite).toBe(authorizedMemoryWrite); + expect(flushCall.memoryFlushWritePath).toBeUndefined(); + expect(flushCall.prompt).toContain("memory_remember"); + expect(flushCall.extraSystemPrompt).toContain("Never use a workspace file"); + expect(ensureMemoryFlushTargetFileMock).not.toHaveBeenCalled(); + }); + it("runs exactly one auto-reply memory flush turn, rotates, and persists metadata", async () => { const followupRun = createTestFollowupRun({ authProfileId: "anthropic:work", diff --git a/src/auto-reply/reply/agent-runner-memory.ts b/src/auto-reply/reply/agent-runner-memory.ts index 17869024a4bc..b2efce3b96e7 100644 --- a/src/auto-reply/reply/agent-runner-memory.ts +++ b/src/auto-reply/reply/agent-runner-memory.ts @@ -15,6 +15,7 @@ import { estimateMessagesTokens } from "../../agents/compaction.js"; import { isBenignCompactionSkipResult } from "../../agents/embedded-agent-runner/compact-reasons.js"; import { runEmbeddedAgentEntry } from "../../agents/embedded-agent-runner/run-entry.js"; import { createToolResultPromptProjectionState } from "../../agents/embedded-agent-runner/session-prompt-state.js"; +import { prepareAuthorizedTranscriptDerivationHost } from "../../agents/memory-authorized-read-host.js"; import { isCliRuntimeAliasForProvider } from "../../agents/model-runtime-aliases.js"; import { isCliProvider } from "../../agents/model-selection.js"; import { resolveContextConfigProviderForRuntime } from "../../agents/openai-routing.js"; @@ -98,6 +99,10 @@ type UpdateSessionEntryParams = { const MAX_VISIBLE_MEMORY_FLUSH_ERROR_CHARS = 600; const MAX_FLUSH_FAILURES = 3; const MAX_FLUSH_ERROR_LENGTH = 200; +const AUTHORIZED_MEMORY_FLUSH_PROMPT = + "Pre-compaction memory flush. Use memory_remember to retain only durable facts for the current authorized subject. Do not write workspace files. If nothing should be retained, reply NO_REPLY."; +const AUTHORIZED_MEMORY_FLUSH_SYSTEM_PROMPT = + "Pre-compaction memory flush turn. The only durable mutation is memory_remember, which selects the authorized subject store and audience. Never use a workspace file as memory storage."; const embeddedAgentRuntimeLoader = createLazyImportLoader( () => import("../../agents/embedded-agent.js"), @@ -1105,9 +1110,6 @@ export async function runMemoryFlushIfNeeded(params: { params.followupRun.run.agentId ?? resolveDefaultAgentId(params.cfg), ) : (params.followupRun.run.agentId ?? resolveDefaultAgentId(params.cfg)); - if (isMemoryIsolationCutoverAgent(memoryFlushAgentId)) { - return { sessionEntry: params.sessionEntry, outcome: "skipped" }; - } const memoryFlushWritable = (() => { if (!params.sessionKey) { return true; @@ -1153,7 +1155,7 @@ export async function runMemoryFlushIfNeeded(params: { recordMemoryFlushFailure(error, params, activeSessionEntry); let memoryFlushPlan: MemoryFlushPlan | null; try { - memoryFlushPlan = resolveMemoryFlushPlan({ cfg: params.cfg }); + memoryFlushPlan = resolveMemoryFlushPlan({ cfg: params.cfg, agentId: memoryFlushAgentId }); } catch (error) { return await recordFailure(error); } @@ -1342,27 +1344,59 @@ export async function runMemoryFlushIfNeeded(params: { (params.sessionKey ? activeSessionStore?.[params.sessionKey]?.systemPromptReport : undefined), ); const prepareMemoryFlushAttempt = async () => { - const plan = resolveMemoryFlushPlan({ cfg: params.cfg, nowMs: memoryDeps.now() }); + const plan = resolveMemoryFlushPlan({ + cfg: params.cfg, + agentId: memoryFlushAgentId, + nowMs: memoryDeps.now(), + }); if (!plan) { return null; } const writePath = plan.relativePath; - await memoryDeps.ensureMemoryFlushTargetFile({ - workspaceDir: params.followupRun.run.workspaceDir, - relativePath: writePath, - }); - const absolutePath = path.join(params.followupRun.run.workspaceDir, writePath); - const readContent = () => - fs.promises.readFile(absolutePath, "utf8").catch((error: unknown) => { - if ((error as NodeJS.ErrnoException).code === "ENOENT") { - return ""; - } - throw error; + const usesAuthorizedMemoryStore = isMemoryIsolationCutoverAgent(memoryFlushAgentId); + const authorizedMemoryWrite = usesAuthorizedMemoryStore + ? await prepareAuthorizedTranscriptDerivationHost({ + agentId: memoryFlushAgentId, + sessionKey: + params.runtimePolicySessionKey ?? + params.followupRun.run.runtimePolicySessionKey ?? + params.sessionKey ?? + params.followupRun.run.sessionKey, + sessionId: activeSessionEntry?.sessionId ?? params.followupRun.run.sessionId, + runId: flushRunId, + messageChannel: params.followupRun.run.messageProvider, + agentAccountId: params.followupRun.run.agentAccountId, + }) + : undefined; + if (usesAuthorizedMemoryStore && !authorizedMemoryWrite) { + // A flush is a transcript derivation, not an ordinary append. Do not give a + // model raw history until the selected memory runtime can bind its output to + // the current transcript policy set and immutable source lineage. + return null; + } + let readContent: (() => Promise) | undefined; + let contentBefore: string | undefined; + if (!usesAuthorizedMemoryStore) { + await memoryDeps.ensureMemoryFlushTargetFile({ + workspaceDir: params.followupRun.run.workspaceDir, + relativePath: writePath, }); - // Capture one baseline before any write can start. Per-write snapshots can - // pair a failed later write with an earlier success and miss mixed content. - const contentBefore = await readContent(); - const systemPrompt = [params.followupRun.run.extraSystemPrompt, plan.systemPrompt] + const absolutePath = path.join(params.followupRun.run.workspaceDir, writePath); + readContent = () => + fs.promises.readFile(absolutePath, "utf8").catch((error: unknown) => { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return ""; + } + throw error; + }); + // Capture one baseline before any write can start. Per-write snapshots can + // pair a failed later write with an earlier success and miss mixed content. + contentBefore = await readContent(); + } + const systemPrompt = [ + params.followupRun.run.extraSystemPrompt, + usesAuthorizedMemoryStore ? AUTHORIZED_MEMORY_FLUSH_SYSTEM_PROMPT : plan.systemPrompt, + ] .filter(Boolean) .join("\n\n"); const selection = resolveMemoryFlushModelFallbackOptions( @@ -1384,6 +1418,8 @@ export async function runMemoryFlushIfNeeded(params: { systemPrompt, selection, preparedRunAdmission, + usesAuthorizedMemoryStore, + authorizedMemoryWrite, }; }; let preparedAttempt: Awaited>; @@ -1403,6 +1439,8 @@ export async function runMemoryFlushIfNeeded(params: { systemPrompt: flushSystemPrompt, selection, preparedRunAdmission, + usesAuthorizedMemoryStore, + authorizedMemoryWrite, } = preparedAttempt; let memoryCompactionCompleted = false; let memoryFlushWroteTarget = false; @@ -1500,8 +1538,11 @@ export async function runMemoryFlushIfNeeded(params: { allowGatewaySubagentBinding: true, silentExpected: true, trigger: "memory", - memoryFlushWritePath, - prompt: activeMemoryFlushPlan.prompt, + ...(usesAuthorizedMemoryStore ? {} : { memoryFlushWritePath }), + ...(authorizedMemoryWrite ? { authorizedMemoryWrite } : {}), + prompt: usesAuthorizedMemoryStore + ? AUTHORIZED_MEMORY_FLUSH_PROMPT + : activeMemoryFlushPlan.prompt, transcriptPrompt: "", extraSystemPrompt: flushSystemPrompt, isFinalFallbackAttempt: runOptions.isFinalFallbackAttempt, @@ -1513,7 +1554,10 @@ export async function runMemoryFlushIfNeeded(params: { contextEngineLogicalTurnLease: runOptions.contextEngineLogicalTurnLease, onContextEngineTurnCandidate: runOptions.onContextEngineTurnCandidate, onAgentEvent: (evt) => { - if (evt.stream === "tool" && evt.data.name === "write") { + if ( + evt.stream === "tool" && + evt.data.name === (usesAuthorizedMemoryStore ? "memory_remember" : "write") + ) { if (evt.data.phase === "result" && evt.data.isError !== true) { memoryFlushWroteTarget = true; } @@ -1536,11 +1580,16 @@ export async function runMemoryFlushIfNeeded(params: { return result; }, }); - if (activeMemoryFlushPlan.recordWriteProvenance && memoryFlushWroteTarget) { + if ( + !usesAuthorizedMemoryStore && + activeMemoryFlushPlan.recordWriteProvenance && + memoryFlushWroteTarget && + readMemoryFlushContent + ) { await activeMemoryFlushPlan.recordWriteProvenance({ workspaceDir: params.followupRun.run.workspaceDir, relativePath: memoryFlushWritePath, - contentBefore: memoryFlushContentBefore, + contentBefore: memoryFlushContentBefore ?? "", contentAfter: await readMemoryFlushContent(), originClass: params.followupRun.run.senderIsOwner && sessionLogSnapshot?.turnTainted !== true diff --git a/src/config/sessions/session-accessor.sqlite-scope.ts b/src/config/sessions/session-accessor.sqlite-scope.ts index 716df6ae78fb..bd6bcf1bae86 100644 --- a/src/config/sessions/session-accessor.sqlite-scope.ts +++ b/src/config/sessions/session-accessor.sqlite-scope.ts @@ -47,6 +47,7 @@ type SessionSqliteDatabase = Pick< | "transcript_rewrite_watermarks" | "trajectory_runtime_events" | "transcript_event_identities" + | "transcript_event_memory_policies" | "transcript_events" > & { sqlite_schema: { name: string | null; type: string }; diff --git a/src/config/sessions/session-accessor.sqlite-transcript-store.ts b/src/config/sessions/session-accessor.sqlite-transcript-store.ts index d1c305c59877..b8ca22fb78a0 100644 --- a/src/config/sessions/session-accessor.sqlite-transcript-store.ts +++ b/src/config/sessions/session-accessor.sqlite-transcript-store.ts @@ -43,17 +43,42 @@ import { startSessionTranscriptIndexReconcile } from "./session-transcript-recon import { createSessionTranscriptHeader } from "./transcript-header.js"; import { resolveVisibleTranscriptAppendParentId } from "./transcript-visible-events.js"; +type TranscriptEventAppendOptions = { + allowStoredAlias?: boolean; + dedupeByMessageIdempotency?: boolean; + onProjectionReconcileNeeded?: () => void; + scheduleProjectionReconcile?: boolean; + touchMutation?: boolean; +}; + +type InternalTranscriptEventAppendOptions = TranscriptEventAppendOptions & { + inheritedMemoryPolicy?: PreservedTranscriptMemoryPolicy; +}; + export function appendTranscriptEventInTransaction( database: OpenClawAgentDatabase, scope: ResolvedTranscriptScope, event: TranscriptEvent, - options: { - allowStoredAlias?: boolean; - dedupeByMessageIdempotency?: boolean; - onProjectionReconcileNeeded?: () => void; - scheduleProjectionReconcile?: boolean; - touchMutation?: boolean; - } = {}, + options: TranscriptEventAppendOptions = {}, +): boolean { + return appendTranscriptEventInternal(database, scope, event, options); +} + +/** Only sealed compaction may bind a newly appended output to its read source. */ +export function appendSealedCompactionTranscriptEventInTransaction( + database: OpenClawAgentDatabase, + scope: ResolvedTranscriptScope, + event: TranscriptEvent, + inheritedMemoryPolicy: PreservedTranscriptMemoryPolicy, +): boolean { + return appendTranscriptEventInternal(database, scope, event, { inheritedMemoryPolicy }); +} + +function appendTranscriptEventInternal( + database: OpenClawAgentDatabase, + scope: ResolvedTranscriptScope, + event: TranscriptEvent, + options: InternalTranscriptEventAppendOptions = {}, ): boolean { const persistedEvent = canonicalizeTranscriptEventMedia(event); const db = getSessionKysely(database.db); @@ -93,6 +118,7 @@ export function appendTranscriptEventInTransaction( sessionKey: scope.sessionKey, eventSeq: seq, createdAt, + ...(options.inheritedMemoryPolicy ? { inherited: options.inheritedMemoryPolicy } : {}), }); if (options.touchMutation !== false) { touchTranscriptMutationInTransaction(database, scope.sessionId); diff --git a/src/config/sessions/session-accessor.sqlite-transcript-write.ts b/src/config/sessions/session-accessor.sqlite-transcript-write.ts index e62166ad3456..8d21ea4d85cb 100644 --- a/src/config/sessions/session-accessor.sqlite-transcript-write.ts +++ b/src/config/sessions/session-accessor.sqlite-transcript-write.ts @@ -1,4 +1,5 @@ import { err, ok, type Result } from "@openclaw/normalization-core/result"; +import { executeSqliteQueryTakeFirstSync } from "../../infra/kysely-sync.js"; import { openOpenClawAgentDatabase, resolveOpenClawAgentSqlitePath, @@ -35,6 +36,8 @@ import { } from "./session-accessor.sqlite-read.js"; import { cloneSessionEntry, + getSessionKysely, + resolveSqliteTranscriptArchiveDirectory, resolveSqliteTranscriptScope, runExclusiveSqliteSessionWrite, toDatabaseOptions, @@ -46,14 +49,25 @@ import { readCommittedTranscriptMessageSequence, rememberCommittedTranscriptMessageSequencesInTransaction, } from "./session-accessor.sqlite-transcript-sequences.js"; -import { readTranscriptGenerationInTransaction } from "./session-accessor.sqlite-transcript-state.js"; +import { + readNextTranscriptSeq, + readTranscriptGenerationInTransaction, +} from "./session-accessor.sqlite-transcript-state.js"; import { appendTranscriptEventInTransaction, + appendSealedCompactionTranscriptEventInTransaction, replaceSqliteTranscriptEventsInTransaction, rewriteSqliteTranscriptEventRowsInTransaction, } from "./session-accessor.sqlite-transcript-store.js"; import type { SessionTranscriptWriteTransactionContext } from "./session-accessor.types.js"; -import { readAuthorizedTranscriptEventSeqs } from "./session-transcript-memory-policy.js"; +import { + persistSealedCompactionMemoryPolicyInTransaction, + readAuthorizedTranscriptDerivation, + readAuthorizedTranscriptEventSeqs, + readSealedCompactionOutputMemoryPolicyInTransaction, + type AuthorizedTranscriptDerivation, + type SealedCompactionMemoryPolicy, +} from "./session-transcript-memory-policy.js"; import type { SessionTranscriptTurnExpectedState, SessionTranscriptTurnLifecyclePatch, @@ -67,7 +81,7 @@ import { SessionTranscriptWriterClaimReboundError, withOwnedSessionTranscriptWriterFence, } from "./transcript-write-context.js"; -import type { InternalSessionEntry, SessionEntry } from "./types.js"; +import type { InternalSessionEntry, SessionCompactionCheckpoint, SessionEntry } from "./types.js"; import { mergeSessionEntry } from "./types.js"; // Transcript write owner. Queue coordination surrounds synchronous SQLite commit sections. @@ -655,6 +669,134 @@ export async function withTranscriptWriteTransaction( ); } +type SealedCompactionCommit = (params: { + database: OpenClawAgentDatabase; + compactionPolicy: SealedCompactionMemoryPolicy; + eventSeq: number; +}) => unknown; + +const MAX_SEALED_COMPACTION_CHECKPOINTS_PER_SESSION = 25; + +function persistSealedCompactionCheckpointInTransaction(params: { + database: OpenClawAgentDatabase; + resolved: ReturnType; + checkpoint: SessionCompactionCheckpoint; +}): void { + if (params.checkpoint.sessionId !== params.resolved.sessionId) { + throw new Error("sealed compaction checkpoint is unavailable"); + } + const current = readSessionEntryRow(params.database, params.resolved.sessionKey)?.entry; + if (!current || current.sessionId !== params.resolved.sessionId) { + throw new Error("sealed compaction checkpoint session is unavailable"); + } + const next = cloneSessionEntry(current); + // Cutover checkpoints are SQLite references, so all byte/file planning is + // complete before BEGIN. Retain the same bounded metadata history as the + // checkpoint owner without doing filesystem work in this transaction. + next.compactionCheckpoints = [...(current.compactionCheckpoints ?? []), params.checkpoint].slice( + -MAX_SEALED_COMPACTION_CHECKPOINTS_PER_SESSION, + ); + next.updatedAt = Math.max(current.updatedAt ?? 0, params.checkpoint.createdAt); + writeSessionEntry(params.database, params.resolved.sessionKey, next, { previousEntry: current }); +} + +function sameAuthorizedTranscriptDerivation( + left: AuthorizedTranscriptDerivation, + right: AuthorizedTranscriptDerivation, +): boolean { + return ( + left.sourcePolicySetId === right.sourcePolicySetId && + left.deliveryAudiencesJson === right.deliveryAudiencesJson && + left.eventSeqs.length === right.eventSeqs.length && + left.eventSeqs.every((eventSeq, index) => eventSeq === right.eventSeqs[index]) + ); +} + +/** + * Commits a compaction event and its derived-state owner in one SQLite transaction. + * The artifact owner must stage filesystem bytes before this call and keep its callback + * synchronous; a source-policy race or a partial callback rolls back every durable row. + */ +export async function commitSealedSqliteTranscriptCompaction(params: { + scope: SessionTranscriptWriteScope; + event: TranscriptEvent; + compactionPolicyId: string; + source: AuthorizedTranscriptDerivation; + checkpoint?: SessionCompactionCheckpoint; + commitDerivedState: SealedCompactionCommit; +}): Promise<{ compactionPolicy: SealedCompactionMemoryPolicy; eventSeq: number }> { + const resolved = resolveSqliteTranscriptScope(params.scope); + return await runExclusiveSqliteSessionWrite(resolved, async () => { + let result: { compactionPolicy: SealedCompactionMemoryPolicy; eventSeq: number } | undefined; + runOpenClawAgentWriteTransaction( + (database) => { + const currentSource = readAuthorizedTranscriptDerivation(database.db, resolved.sessionId); + if (!currentSource || !sameAuthorizedTranscriptDerivation(currentSource, params.source)) { + throw new Error("sealed compaction source policy is unavailable"); + } + const compactionPolicy = persistSealedCompactionMemoryPolicyInTransaction({ + db: database.db, + compactionPolicyId: params.compactionPolicyId, + sessionId: resolved.sessionId, + source: params.source, + }); + const outputPolicy = readSealedCompactionOutputMemoryPolicyInTransaction({ + database, + sessionId: resolved.sessionId, + source: params.source, + }); + if (!outputPolicy) { + throw new Error("sealed compaction output policy is unavailable"); + } + if ( + !appendSealedCompactionTranscriptEventInTransaction( + database, + resolved, + params.event, + outputPolicy, + ) + ) { + throw new Error("sealed compaction transcript event was not appended"); + } + const eventSeq = readNextTranscriptSeq(database, resolved.sessionId) - 1; + const eventPolicy = executeSqliteQueryTakeFirstSync( + database.db, + getSessionKysely(database.db) + .selectFrom("transcript_event_memory_policies") + .select(["authorization_status", "delivery_audiences_json", "source_policy_set_id"]) + .where("session_id", "=", resolved.sessionId) + .where("event_seq", "=", eventSeq), + ); + if ( + eventPolicy?.authorization_status !== "authorized" || + eventPolicy.source_policy_set_id !== params.source.sourcePolicySetId || + eventPolicy.delivery_audiences_json !== params.source.deliveryAudiencesJson + ) { + throw new Error("sealed compaction output policy is unavailable"); + } + if (params.checkpoint) { + persistSealedCompactionCheckpointInTransaction({ + database, + resolved, + checkpoint: params.checkpoint, + }); + } + const commitResult = params.commitDerivedState({ database, compactionPolicy, eventSeq }); + if (commitResult && typeof (commitResult as PromiseLike).then === "function") { + throw new Error("sealed compaction derived-state commit must be synchronous"); + } + result = { compactionPolicy, eventSeq }; + }, + toDatabaseOptions(resolved), + { operationLabel: "session.compaction.sealed" }, + ); + if (!result) { + throw new Error("sealed compaction transaction did not commit"); + } + return result; + }); +} + function isSqliteTranscriptSnapshotUnchanged( database: OpenClawAgentDatabase, sessionId: string, diff --git a/src/config/sessions/session-transcript-memory-policy.test.ts b/src/config/sessions/session-transcript-memory-policy.test.ts index 9199ef069f05..48f4eef7c4cf 100644 --- a/src/config/sessions/session-transcript-memory-policy.test.ts +++ b/src/config/sessions/session-transcript-memory-policy.test.ts @@ -22,7 +22,7 @@ import { } from "../../state/openclaw-agent-db.js"; import { readSessionTranscriptMessageEvents } from "./session-accessor.sqlite-active-events.js"; import { materializeSessionStateDeletePlans } from "./session-accessor.sqlite-archive.js"; -import { writeSessionEntry } from "./session-accessor.sqlite-entry-store.js"; +import { readSessionEntryRow, writeSessionEntry } from "./session-accessor.sqlite-entry-store.js"; import { planSessionStateDeleteIfUnreferenced } from "./session-accessor.sqlite-lifecycle-state.js"; import { loadLatestAssistantText, @@ -35,8 +35,11 @@ import { appendTranscriptMessage, replaceTranscriptEvents, trimTranscriptForManualCompact, + commitSealedSqliteTranscriptCompaction, } from "./session-accessor.sqlite-transcript-write.js"; import { + persistSealedCompactionMemoryPolicyInTransaction, + readAuthorizedTranscriptDerivation, preserveTranscriptMemoryPolicyTransitionInTransaction, readAuthorizedTranscriptEventSeqs, resetTranscriptMemoryPolicyForTest, @@ -272,6 +275,311 @@ afterEach(() => { }); describe("transcript memory policy companions", () => { + it("binds a sealed compaction policy to the exact current transcript source set", async () => { + const env = createEnv(); + const database = markCutOver(env); + persistExposure(database, { runId: "sealed-compaction-run" }); + await appendWithRun({ + env, + runId: "sealed-compaction-run", + text: "sealed compaction transcript source", + }); + const source = readAuthorizedTranscriptDerivation(database.db, SESSION_ID); + if (!source) { + throw new Error("fixture expected an authorized compaction source"); + } + expect(() => + persistSealedCompactionMemoryPolicyInTransaction({ + db: database.db, + compactionPolicyId: "compaction-policy-1", + sessionId: SESSION_ID, + source, + }), + ).toThrow("active transaction"); + + const persisted = runOpenClawAgentWriteTransaction( + (opened) => + persistSealedCompactionMemoryPolicyInTransaction({ + db: opened.db, + compactionPolicyId: "compaction-policy-1", + sessionId: SESSION_ID, + source, + createdAt: 123, + }), + { agentId: AGENT_ID, env }, + ); + expect(persisted).toMatchObject({ + compactionPolicyId: "compaction-policy-1", + sessionId: SESSION_ID, + sourcePolicySetId: source.sourcePolicySetId, + eventSeqs: source.eventSeqs, + createdAt: 123, + }); + expect( + database.db + .prepare( + `SELECT session_id, source_policy_set_id, retention_state, created_at + FROM memory_compaction_policies + WHERE compaction_policy_id = 'compaction-policy-1'`, + ) + .get(), + ).toEqual({ + session_id: SESSION_ID, + source_policy_set_id: source.sourcePolicySetId, + retention_state: "retained", + created_at: 123, + }); + + expect(() => + runOpenClawAgentWriteTransaction( + (opened) => + persistSealedCompactionMemoryPolicyInTransaction({ + db: opened.db, + compactionPolicyId: "compaction-policy-2", + sessionId: SESSION_ID, + source: { ...source, eventSeqs: [...source.eventSeqs, 999] }, + }), + { agentId: AGENT_ID, env }, + ), + ).toThrow("source policy is unavailable"); + expect( + database.db.prepare("SELECT count(*) AS count FROM memory_compaction_policies").get(), + ).toEqual({ count: 1 }); + }); + + it("atomically commits the authorized compaction event, policy, and derived-state callback", async () => { + const env = createEnv(); + const database = markCutOver(env); + writeSessionEntry(database, SESSION_KEY, { sessionId: SESSION_ID, updatedAt: 1 }); + persistExposure(database, { runId: "sealed-compaction-transaction" }); + await appendWithRun({ + env, + runId: "sealed-compaction-transaction", + text: "sealed transaction source", + }); + const source = readAuthorizedTranscriptDerivation(database.db, SESSION_ID); + if (!source) { + throw new Error("fixture expected an authorized compaction source"); + } + const sourceCompanion = database.db + .prepare( + `SELECT policy.run_id, detail.source_event_seq + FROM transcript_event_memory_policies AS policy + JOIN transcript_event_memory_policy_details AS detail + ON detail.session_id = policy.session_id AND detail.event_seq = policy.event_seq + WHERE policy.session_id = ? AND policy.event_seq = ?`, + ) + .get(SESSION_ID, source.eventSeqs[0]); + let committed: { eventSeq: number; policyId: string } | undefined; + const commit = async () => + await withOwnedSessionTranscriptWrites( + { + sessionTarget: { + agentId: AGENT_ID, + // The committing writer has no exposure of its own. The output can + // be authorized only by inheriting the transcript source it read. + expectedWriterRunId: "sealed-compaction-commit", + sessionId: SESSION_ID, + sessionKey: SESSION_KEY, + }, + withTranscriptWrite: async (run) => await run(), + }, + async () => + await commitSealedSqliteTranscriptCompaction({ + scope: scope(env), + event: { + type: "compaction", + id: "sealed-compaction-entry", + parentId: null, + timestamp: new Date(123).toISOString(), + summary: "sealed summary", + firstKeptEntryId: "source-message", + tokensBefore: 42, + }, + compactionPolicyId: "sealed-compaction-policy", + source, + checkpoint: { + checkpointId: "sealed-compaction-checkpoint", + sessionKey: SESSION_KEY, + sessionId: SESSION_ID, + createdAt: 123, + reason: "manual", + summary: "sealed summary", + firstKeptEntryId: "source-message", + preCompaction: { sessionId: SESSION_ID, leafId: "source-message" }, + postCompaction: { sessionId: SESSION_ID, entryId: "sealed-compaction-entry" }, + }, + commitDerivedState({ compactionPolicy, eventSeq }) { + committed = { eventSeq, policyId: compactionPolicy.compactionPolicyId }; + }, + }), + ); + + await expect(commit()).resolves.toMatchObject({ + compactionPolicy: { compactionPolicyId: "sealed-compaction-policy" }, + }); + expect(committed).toEqual({ + eventSeq: expect.any(Number), + policyId: "sealed-compaction-policy", + }); + expect( + readSessionEntryRow(database, SESSION_KEY)?.entry.compactionCheckpoints?.map( + (checkpoint) => checkpoint.checkpointId, + ), + ).toEqual(["sealed-compaction-checkpoint"]); + expect( + database.db.prepare("SELECT count(*) AS count FROM memory_compaction_policies").get(), + ).toEqual({ count: 1 }); + expect( + database.db + .prepare( + `SELECT source_session_id, source_event_seq, source_policy_set_id, delivery_audiences_json + FROM memory_compaction_policy_sources + WHERE compaction_policy_id = 'sealed-compaction-policy'`, + ) + .all(), + ).toEqual( + source.eventSeqs.map((sourceEventSeq) => ({ + source_session_id: SESSION_ID, + source_event_seq: sourceEventSeq, + source_policy_set_id: source.sourcePolicySetId, + delivery_audiences_json: source.deliveryAudiencesJson, + })), + ); + expect( + database.db + .prepare( + `SELECT policy.run_id, detail.source_event_seq + FROM transcript_event_memory_policies AS policy + JOIN transcript_event_memory_policy_details AS detail + ON detail.session_id = policy.session_id AND detail.event_seq = policy.event_seq + WHERE policy.session_id = ? AND policy.event_seq = ?`, + ) + .get(SESSION_ID, committed?.eventSeq), + ).toEqual(sourceCompanion); + expect( + loadTranscriptEventsSync(scope(env)).some( + (event) => + typeof event === "object" && + event !== null && + "id" in event && + event.id === "sealed-compaction-entry", + ), + ).toBe(true); + + const nextSource = readAuthorizedTranscriptDerivation(database.db, SESSION_ID); + if (!nextSource) { + throw new Error("fixture expected the committed compaction companion to remain authorized"); + } + await expect( + withOwnedSessionTranscriptWrites( + { + sessionTarget: { + agentId: AGENT_ID, + expectedWriterRunId: "sealed-compaction-commit", + sessionId: SESSION_ID, + sessionKey: SESSION_KEY, + }, + withTranscriptWrite: async (run) => await run(), + }, + async () => + await commitSealedSqliteTranscriptCompaction({ + scope: scope(env), + event: { + type: "compaction", + id: "rolled-back-compaction-entry", + parentId: null, + timestamp: new Date(124).toISOString(), + summary: "must roll back", + firstKeptEntryId: "source-message", + tokensBefore: 42, + }, + compactionPolicyId: "rolled-back-compaction-policy", + source: nextSource, + commitDerivedState() { + throw new Error("derived state failed"); + }, + }), + ), + ).rejects.toThrow("derived state failed"); + expect( + database.db.prepare("SELECT count(*) AS count FROM memory_compaction_policies").get(), + ).toEqual({ count: 1 }); + expect( + loadTranscriptEventsSync(scope(env)).some( + (event) => + typeof event === "object" && + event !== null && + "id" in event && + event.id === "rolled-back-compaction-entry", + ), + ).toBe(false); + + writeSessionEntry(database, SESSION_KEY, { + sessionId: SESSION_ID, + updatedAt: 125, + compactionCheckpoints: Array.from({ length: 25 }, (_, index) => ({ + checkpointId: `retained-checkpoint-${index}`, + sessionKey: SESSION_KEY, + sessionId: SESSION_ID, + createdAt: index, + reason: "manual" as const, + preCompaction: { sessionId: SESSION_ID, leafId: `pre-${index}` }, + postCompaction: { sessionId: SESSION_ID, entryId: `post-${index}` }, + })), + }); + await expect( + withOwnedSessionTranscriptWrites( + { + sessionTarget: { + agentId: AGENT_ID, + expectedWriterRunId: "sealed-compaction-commit", + sessionId: SESSION_ID, + sessionKey: SESSION_KEY, + }, + withTranscriptWrite: async (run) => await run(), + }, + async () => + await commitSealedSqliteTranscriptCompaction({ + scope: scope(env), + event: { + type: "compaction", + id: "checkpoint-cap-compaction-entry", + parentId: null, + timestamp: new Date(125).toISOString(), + summary: "bounded checkpoint summary", + firstKeptEntryId: "source-message", + tokensBefore: 42, + }, + compactionPolicyId: "checkpoint-cap-compaction-policy", + source: nextSource, + checkpoint: { + checkpointId: "checkpoint-cap-newest", + sessionKey: SESSION_KEY, + sessionId: SESSION_ID, + createdAt: 125, + reason: "manual", + preCompaction: { sessionId: SESSION_ID, leafId: "source-message" }, + postCompaction: { + sessionId: SESSION_ID, + entryId: "checkpoint-cap-compaction-entry", + }, + }, + commitDerivedState() {}, + }), + ), + ).resolves.toBeDefined(); + expect(readSessionEntryRow(database, SESSION_KEY)?.entry.compactionCheckpoints).toHaveLength( + 25, + ); + expect( + readSessionEntryRow(database, SESSION_KEY)?.entry.compactionCheckpoints?.[0], + ).toMatchObject({ checkpointId: "retained-checkpoint-1" }); + expect( + readSessionEntryRow(database, SESSION_KEY)?.entry.compactionCheckpoints?.at(-1), + ).toMatchObject({ checkpointId: "checkpoint-cap-newest" }); + }); + it("enforces Doctor shadow-read-only companion persistence for only its bound subject", async () => { const env = createEnv(); const options = { agentId: AGENT_ID, env }; @@ -812,6 +1120,9 @@ describe("transcript memory policy companions", () => { expect(companion.source_event_seq).toBeGreaterThanOrEqual(0); } expect(readAuthorizedTranscriptEventSeqs(database.db, SESSION_ID)?.size).toBe(7); + expect(readAuthorizedTranscriptDerivation(database.db, SESSION_ID)).toMatchObject({ + eventSeqs: [0, 1, 2, 3, 4, 5, 6], + }); }); it("uses the captured trusted actor and token-free delegation rather than reconstructing session facts", async () => { diff --git a/src/config/sessions/session-transcript-memory-policy.ts b/src/config/sessions/session-transcript-memory-policy.ts index b0c0e228ece2..53306ac7ce2f 100644 --- a/src/config/sessions/session-transcript-memory-policy.ts +++ b/src/config/sessions/session-transcript-memory-policy.ts @@ -18,6 +18,7 @@ type TranscriptMemoryPolicyDatabase = Pick< | "memory_policy_revisions" | "memory_policy_set_members" | "memory_policy_sets" + | "memory_compaction_policies" | "memory_resource_revisions" | "memory_run_exposure_resources" | "memory_run_exposures" @@ -72,6 +73,24 @@ export type TranscriptMemoryPolicyTransitionKind = | "switch" | "checkpoint"; +/** The complete, currently authorized transcript source set for one derivation. */ +export type AuthorizedTranscriptDerivation = Readonly<{ + eventSeqs: readonly number[]; + sourcePolicySetId: string; + deliveryAudiencesJson: string; +}>; + +/** Immutable policy evidence for one sealed compaction output. */ +export type SealedCompactionMemoryPolicy = Readonly<{ + compactionPolicyId: string; + sessionId: string; + sourcePolicySetId: string; + deliveryAudiencesJson: string; + eventSeqs: readonly number[]; + retentionState: "retained"; + createdAt: number; +}>; + const enforcementByDatabase = new WeakMap(); function policyDatabase(db: DatabaseSync) { @@ -1039,6 +1058,234 @@ export function readAuthorizedTranscriptEventSeqs( } } +/** + * Compaction is a derivation, so native session history cannot be treated as an implicit source. + * A mixed source set must be partitioned by its owner; this generic path denies it rather than + * letting a model phrase its way into a broader summary. + */ +export function readAuthorizedTranscriptDerivation( + db: DatabaseSync, + sessionId: string, +): AuthorizedTranscriptDerivation | undefined { + if (!isTranscriptMemoryPolicyEnforcedInDatabase(db)) { + return undefined; + } + try { + const policy = policyDatabase(db); + const eventRows = executeSqliteQuerySync( + db, + policy + .selectFrom("transcript_events") + .select("seq") + .where("session_id", "=", sessionId) + .orderBy("seq"), + ).rows; + if (eventRows.length === 0) { + return undefined; + } + const readable = readAuthorizedTranscriptEventSeqs(db, sessionId); + if ( + !readable || + readable.size !== eventRows.length || + eventRows.some((event) => !readable.has(event.seq)) + ) { + return undefined; + } + const rows = executeSqliteQuerySync( + db, + policy + .selectFrom("transcript_event_memory_policies as policy") + .innerJoin("transcript_event_memory_policy_details as detail", (join) => + join + .onRef("detail.session_id", "=", "policy.session_id") + .onRef("detail.event_seq", "=", "policy.event_seq"), + ) + .select([ + "policy.event_seq", + "policy.source_policy_set_id", + "policy.delivery_audiences_json", + ]) + .where("policy.session_id", "=", sessionId) + .where("policy.authorization_status", "=", "authorized") + .where("detail.retention_state", "=", "retained") + .orderBy("policy.event_seq"), + ).rows; + if ( + rows.length !== eventRows.length || + rows.some( + (row) => + !readable.has(row.event_seq) || + row.source_policy_set_id === null || + row.delivery_audiences_json === null, + ) + ) { + return undefined; + } + const sourcePolicySetIds = new Set(rows.map((row) => row.source_policy_set_id)); + const deliveryAudiences = new Set(rows.map((row) => row.delivery_audiences_json)); + if (sourcePolicySetIds.size !== 1 || deliveryAudiences.size !== 1) { + return undefined; + } + const sourcePolicySetId = rows[0]?.source_policy_set_id; + const deliveryAudiencesJson = rows[0]?.delivery_audiences_json; + if (!sourcePolicySetId || !deliveryAudiencesJson) { + return undefined; + } + return Object.freeze({ + eventSeqs: Object.freeze(eventRows.map((event) => event.seq)), + sourcePolicySetId, + deliveryAudiencesJson, + }); + } catch { + return undefined; + } +} + +/** + * Stores only a source set which is still the complete authorized transcript. + * The caller owns the surrounding transaction that also appends the summary, + * checkpoint, resource revision, and lineage; standalone policy rows would + * otherwise make an interrupted compaction look durable. + */ +export function persistSealedCompactionMemoryPolicyInTransaction(params: { + db: DatabaseSync; + compactionPolicyId: string; + sessionId: string; + source: AuthorizedTranscriptDerivation; + createdAt?: number; +}): SealedCompactionMemoryPolicy { + if (!params.db.isTransaction) { + throw new Error("sealed compaction policy requires an active transaction"); + } + const compactionPolicyId = params.compactionPolicyId.trim(); + const sessionId = params.sessionId.trim(); + if (!compactionPolicyId || !sessionId) { + throw new Error("sealed compaction policy is unavailable"); + } + const current = readAuthorizedTranscriptDerivation(params.db, sessionId); + const source = params.source; + if ( + !current || + current.sourcePolicySetId !== source.sourcePolicySetId || + current.deliveryAudiencesJson !== source.deliveryAudiencesJson || + current.eventSeqs.length !== source.eventSeqs.length || + current.eventSeqs.some((eventSeq, index) => eventSeq !== source.eventSeqs[index]) + ) { + throw new Error("sealed compaction source policy is unavailable"); + } + const createdAt = params.createdAt ?? Date.now(); + const db = policyDatabase(params.db); + const existing = executeSqliteQueryTakeFirstSync( + params.db, + db + .selectFrom("memory_compaction_policies") + .select(["session_id", "source_policy_set_id", "retention_state", "created_at"]) + .where("compaction_policy_id", "=", compactionPolicyId), + ); + if (existing) { + const persistedSources = executeSqliteQuerySync( + params.db, + db + .selectFrom("memory_compaction_policy_sources") + .select([ + "source_event_seq", + "source_policy_set_id", + "source_session_id", + "delivery_audiences_json", + ]) + .where("compaction_policy_id", "=", compactionPolicyId) + .orderBy("source_event_seq"), + ).rows; + if ( + existing.session_id !== sessionId || + existing.source_policy_set_id !== source.sourcePolicySetId || + existing.retention_state !== "retained" || + persistedSources.length !== source.eventSeqs.length || + persistedSources.some( + (persistedSource, index) => + persistedSource.source_session_id !== sessionId || + persistedSource.source_event_seq !== source.eventSeqs[index] || + persistedSource.source_policy_set_id !== source.sourcePolicySetId || + persistedSource.delivery_audiences_json !== source.deliveryAudiencesJson, + ) + ) { + throw new Error("sealed compaction policy idempotency conflict"); + } + return Object.freeze({ + compactionPolicyId, + sessionId, + sourcePolicySetId: source.sourcePolicySetId, + deliveryAudiencesJson: source.deliveryAudiencesJson, + eventSeqs: Object.freeze([...source.eventSeqs]), + retentionState: "retained", + createdAt: existing.created_at, + }); + } + executeSqliteQuerySync( + params.db, + db.insertInto("memory_compaction_policies").values({ + compaction_policy_id: compactionPolicyId, + session_id: sessionId, + source_policy_set_id: source.sourcePolicySetId, + retention_state: "retained", + created_at: createdAt, + }), + ); + for (const eventSeq of source.eventSeqs) { + executeSqliteQuerySync( + params.db, + db.insertInto("memory_compaction_policy_sources").values({ + compaction_policy_id: compactionPolicyId, + source_session_id: sessionId, + source_event_seq: eventSeq, + source_policy_set_id: source.sourcePolicySetId, + delivery_audiences_json: source.deliveryAudiencesJson, + created_at: createdAt, + }), + ); + } + return Object.freeze({ + compactionPolicyId, + sessionId, + sourcePolicySetId: source.sourcePolicySetId, + deliveryAudiencesJson: source.deliveryAudiencesJson, + eventSeqs: Object.freeze([...source.eventSeqs]), + retentionState: "retained", + createdAt, + }); +} + +/** + * The output event must inherit proof from an event the compactor actually + * read. A writer fence is only permission to commit; it is not source proof. + */ +export function readSealedCompactionOutputMemoryPolicyInTransaction(params: { + database: OpenClawAgentDatabase; + sessionId: string; + source: AuthorizedTranscriptDerivation; +}): PreservedTranscriptMemoryPolicy | undefined { + const current = readAuthorizedTranscriptDerivation(params.database.db, params.sessionId); + if ( + !current || + current.sourcePolicySetId !== params.source.sourcePolicySetId || + current.deliveryAudiencesJson !== params.source.deliveryAudiencesJson || + current.eventSeqs.length !== params.source.eventSeqs.length || + current.eventSeqs.some((eventSeq, index) => eventSeq !== params.source.eventSeqs[index]) + ) { + return undefined; + } + const firstEventSeq = params.source.eventSeqs[0]; + return [...readPreservedTranscriptMemoryPoliciesInTransaction(params.database, params.sessionId).values()] + .flat() + .find( + (policy) => + policy.eventSeq === firstEventSeq && + policy.sourcePolicySetId === params.source.sourcePolicySetId && + policy.deliveryAudiencesJson === params.source.deliveryAudiencesJson && + policy.retentionState === "retained", + ); +} + export function resetTranscriptMemoryPolicyForTest(db: DatabaseSync): void { enforcementByDatabase.delete(db); } diff --git a/src/cron/isolated-agent/run-prepare.ts b/src/cron/isolated-agent/run-prepare.ts index 1f50c5bd1788..0cca811627e1 100644 --- a/src/cron/isolated-agent/run-prepare.ts +++ b/src/cron/isolated-agent/run-prepare.ts @@ -1,6 +1,7 @@ /** Session identity and context preparation for isolated cron runs. */ import { isDeepStrictEqual } from "node:util"; import { hasAnyAuthProfileStoreSource } from "../../agents/auth-profiles/source-check.js"; +import { mayInjectAutonomousSourceTranscript } from "../../agents/memory-autonomous-run-policy.js"; import { findModelInCatalog } from "../../agents/model-catalog-lookup.js"; import { listOpenAIAuthProfileProvidersForAgentRuntime } from "../../agents/openai-routing.js"; import { loadAgentRuntimePluginRegistryHandle } from "../../agents/runtime-plugins.js"; @@ -11,6 +12,7 @@ import { resolveSessionWorkStartError } from "../../config/sessions/lifecycle.js import type { AgentDefaultsConfig } from "../../config/types.agent-defaults.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import type { SourceDeliveryPlan } from "../../infra/outbound/source-delivery-plan.js"; +import { isMemoryIsolationCutoverAgent } from "../../plugins/memory-cutover.js"; import type { PluginRegistry } from "../../plugins/registry-types.js"; import { isCronSessionKey, parseAgentSessionKey } from "../../routing/session-key.js"; import { @@ -522,6 +524,10 @@ export async function prepareCronRunContext(params: { // Current jobs stay detached; a bounded tail preserves context without transcript continuation. const currentConversationContext = input.job.sessionTarget === "current" && + mayInjectAutonomousSourceTranscript({ + sessionTarget: input.job.sessionTarget, + memoryIsolationActive: isMemoryIsolationCutoverAgent(agentId), + }) && agentPayload && sourceSessionKey && sourceSessionEntry diff --git a/src/infra/heartbeat-runner-execution.ts b/src/infra/heartbeat-runner-execution.ts index 0e3d868f47ff..0236ef16e6c2 100644 --- a/src/infra/heartbeat-runner-execution.ts +++ b/src/infra/heartbeat-runner-execution.ts @@ -32,6 +32,7 @@ import { loadExactSessionEntry, type SessionEntryLifecycleRemoval, } from "../config/sessions/session-accessor.js"; +import { buildSessionCreationStamp } from "../config/sessions/session-entry-provenance.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { hasActiveCronJobs, @@ -42,6 +43,7 @@ import { import { resolveCronSession } from "../cron/isolated-agent/session.js"; import { writeCronJobScratch } from "../cron/scratch-store.js"; import { resolveCronJobsStorePathFromConfig } from "../cron/store.js"; +import { isMemoryIsolationCutoverAgent } from "../plugins/memory-cutover.js"; import { getQueueSize, isCommandLaneTaskMarkerCurrent, @@ -70,6 +72,7 @@ import { shouldPreflightExecEventWake, } from "./heartbeat-runner-prompt.js"; import { + resolveMemoryIsolatedHeartbeatSessionKey, resolveHeartbeatSession, resolveIsolatedHeartbeatSessionKey, resolveStaleHeartbeatIsolatedSessionKey, @@ -403,7 +406,8 @@ export async function prepareHeartbeatRunStage(wake: ReadyHeartbeatWake) { // a new session ID (empty transcript) each run, avoiding the cost of // sending the full conversation history (~100K tokens) to the LLM. // Delivery routing still uses the main session entry (lastChannel, lastTo). - const useIsolatedSession = heartbeat?.isolatedSession === true; + const memoryIsolationActive = isMemoryIsolationCutoverAgent(agentId); + const useIsolatedSession = heartbeat?.isolatedSession === true || memoryIsolationActive; const delivery = await resolveHeartbeatDeliveryTargetWithSessionRoute({ cfg, agentId, @@ -488,22 +492,32 @@ export async function prepareHeartbeatRunStage(wake: ReadyHeartbeatWake) { let runSessionEntry = entry; let outboundPolicySessionKey: string | undefined; if (useIsolatedSession) { - const configuredSession = resolveHeartbeatSession(cfg, agentId, heartbeat); - // Collapse only the repeated `:heartbeat` suffixes introduced by wake-triggered - // re-entry for heartbeat-created isolated sessions. Real session keys that - // happen to end with `:heartbeat` still get a distinct isolated sibling. - const { isolatedSessionKey, isolatedBaseSessionKey } = resolveIsolatedHeartbeatSessionKey({ - agentId, - sessionKey, - configuredSessionKey: configuredSession.sessionKey, - sessionEntry: entry, - }); + // Under memory isolation, old session keys may still select a delivery + // target but cannot name the autonomous run or its policy context. + const isolatedSession = memoryIsolationActive + ? { + isolatedSessionKey: resolveMemoryIsolatedHeartbeatSessionKey(agentId), + isolatedBaseSessionKey: undefined, + } + : (() => { + const configuredSession = resolveHeartbeatSession(cfg, agentId, heartbeat); + return resolveIsolatedHeartbeatSessionKey({ + agentId, + sessionKey, + configuredSessionKey: configuredSession.sessionKey, + sessionEntry: entry, + }); + })(); + const { isolatedSessionKey } = isolatedSession; + const { isolatedBaseSessionKey } = isolatedSession; const isolatedStorePath = resolveSessionStorePathCore(cfg.session?.store, { agentId }); - const staleIsolatedSessionKey = resolveStaleHeartbeatIsolatedSessionKey({ - sessionKey, - isolatedSessionKey, - isolatedBaseSessionKey, - }); + const staleIsolatedSessionKey = isolatedBaseSessionKey + ? resolveStaleHeartbeatIsolatedSessionKey({ + sessionKey, + isolatedSessionKey, + isolatedBaseSessionKey, + }) + : undefined; if ( isReplyRunActive(isolatedSessionKey) || hasActiveRunForSession(isolatedSessionKey, listActiveEmbeddedRuns) @@ -551,7 +565,9 @@ export async function prepareHeartbeatRunStage(wake: ReadyHeartbeatWake) { }); const nextEntry = { ...cronSession.sessionEntry, - heartbeatIsolatedBaseSessionKey: isolatedBaseSessionKey, + ...(memoryIsolationActive + ? buildSessionCreationStamp({ via: "cron", actor: { type: "system" } }) + : { heartbeatIsolatedBaseSessionKey: isolatedBaseSessionKey }), }; runSessionEntry = nextEntry; return nextEntry; @@ -567,7 +583,7 @@ export async function prepareHeartbeatRunStage(wake: ReadyHeartbeatWake) { }); } runSessionKey = isolatedSessionKey; - outboundPolicySessionKey = isolatedBaseSessionKey; + outboundPolicySessionKey = memoryIsolationActive ? isolatedSessionKey : isolatedBaseSessionKey; const actualUseHeartbeatResponseToolPrompt = shouldUseHeartbeatResponseToolPrompt({ cfg, diff --git a/src/infra/heartbeat-runner-session.ts b/src/infra/heartbeat-runner-session.ts index 119b66f3224a..9a3b7c7b303b 100644 --- a/src/infra/heartbeat-runner-session.ts +++ b/src/infra/heartbeat-runner-session.ts @@ -196,6 +196,16 @@ export function resolveIsolatedHeartbeatSessionKey(params: { }; } +/** + * Memory-isolated heartbeats are autonomous service work, not a fork of the + * session that supplied their delivery route. Keep the identity stable per + * agent so repeated wakes retain service-only state without inheriting a + * human session's transcript or subject. + */ +export function resolveMemoryIsolatedHeartbeatSessionKey(agentId: string): string { + return toAgentStoreSessionKey({ agentId, requestKey: "service:heartbeat" }); +} + export function resolveStaleHeartbeatIsolatedSessionKey(params: { sessionKey: string; isolatedSessionKey: string; diff --git a/src/infra/heartbeat-runner.isolated-session-mirror.test.ts b/src/infra/heartbeat-runner.isolated-session-mirror.test.ts index 8e31d7902bdc..3a0aea9e1d4a 100644 --- a/src/infra/heartbeat-runner.isolated-session-mirror.test.ts +++ b/src/infra/heartbeat-runner.isolated-session-mirror.test.ts @@ -2,7 +2,13 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../config/config.js"; import { resolveMainSessionKey } from "../config/sessions.js"; +import { resetMemoryIsolationCutoverForTest } from "../plugins/memory-cutover.js"; +import { + closeOpenClawAgentDatabasesForTest, + openOpenClawAgentDatabase, +} from "../state/openclaw-agent-db.js"; import { runHeartbeatOnce } from "./heartbeat-runner.js"; +import { resolveMemoryIsolatedHeartbeatSessionKey } from "./heartbeat-runner-session.js"; import { installHeartbeatRunnerTestRuntime } from "./heartbeat-runner.test-harness.js"; import { readSessionStoreForTest, @@ -24,6 +30,8 @@ installHeartbeatRunnerTestRuntime(); afterEach(() => { deliverOutboundPayloadsInternal.mockClear(); + closeOpenClawAgentDatabasesForTest(); + resetMemoryIsolationCutoverForTest(); }); type DeliveryRequest = { @@ -43,7 +51,11 @@ function latestDeliveryRequest(): DeliveryRequest { return request as DeliveryRequest; } -function makeIsolatedLastTargetConfig(tmpDir: string, storePath: string): OpenClawConfig { +function makeIsolatedLastTargetConfig( + tmpDir: string, + storePath: string, + isolatedSession = true, +): OpenClawConfig { return { agents: { list: [{ id: "main", default: true }], @@ -52,7 +64,7 @@ function makeIsolatedLastTargetConfig(tmpDir: string, storePath: string): OpenCl heartbeat: { every: "5m", target: "last", - isolatedSession: true, + isolatedSession, }, }, }, @@ -160,4 +172,57 @@ describe("runHeartbeatOnce - isolated heartbeat outbound session mirror", () => }); }); }); + + it("uses a system service session under memory isolation instead of the routed user session", async () => { + await withTempHeartbeatSandbox(async ({ tmpDir, storePath, replySpy }) => { + const cfg = makeIsolatedLastTargetConfig(tmpDir, storePath, false); + const baseSessionKey = resolveMainSessionKey(cfg); + const serviceSessionKey = resolveMemoryIsolatedHeartbeatSessionKey("main"); + const nowMs = Date.now(); + const database = openOpenClawAgentDatabase({ agentId: "main" }); + database.db + .prepare( + `INSERT INTO memory_migrations + (migration_id, source_kind, source_hash, phase, classification_json, plan_hash, + verified_at, cutover_at, updated_at) + VALUES ('heartbeat-service-session', 'test', 'test-source', 'cutover', '{}', + 'test-plan', 1, 1, 1)`, + ) + .run(); + resetMemoryIsolationCutoverForTest(); + await seedHeartbeatScratchForTest({ content: "Check the scheduled service work." }); + await seedSessionStore(storePath, baseSessionKey, { + sessionId: "user-session", + updatedAt: nowMs - 1_000, + lastChannel: "whatsapp", + lastProvider: "whatsapp", + lastTo: "+15551234567", + }); + replySpy.mockResolvedValueOnce({ text: "Service work needs attention." }); + + const result = await runHeartbeatOnce({ + cfg, + deps: { + getReplyFromConfig: replySpy, + getQueueSize: () => 0, + nowMs: () => nowMs, + }, + }); + + expect(result.status).toBe("ran"); + expect(replySpy.mock.calls[0]?.[0]).toMatchObject({ SessionKey: serviceSessionKey }); + expect(latestDeliveryRequest()).toMatchObject({ + channel: "whatsapp", + to: "+15551234567", + session: { key: serviceSessionKey, policyKey: serviceSessionKey }, + }); + expect(readSessionStoreForTest(storePath)[serviceSessionKey]).toMatchObject({ + createdActor: { type: "system" }, + createdVia: "cron", + }); + expect(readSessionStoreForTest(storePath)[serviceSessionKey]).not.toHaveProperty( + "heartbeatIsolatedBaseSessionKey", + ); + }); + }); }); diff --git a/src/plugins/memory-authorization-runtime.ts b/src/plugins/memory-authorization-runtime.ts index 0a376ac1f960..264c327c5b66 100644 --- a/src/plugins/memory-authorization-runtime.ts +++ b/src/plugins/memory-authorization-runtime.ts @@ -297,6 +297,7 @@ export async function admitMemoryAuthorizationRuntime( return Object.freeze({ ok: false, reasonCode: "backend-nonconforming" }); } const source = runtime.value; + const stageSealedCompaction = readCallable(source, "stageSealedCompaction"); return Object.freeze({ ok: true, runtime: Object.freeze({ @@ -310,6 +311,13 @@ export async function admitMemoryAuthorizationRuntime( writeAuthorized: (methods.writeAuthorized as AuthorizedMemoryRuntime["writeAuthorized"]).bind( source, ), + ...(stageSealedCompaction + ? { + stageSealedCompaction: ( + stageSealedCompaction as NonNullable + ).bind(source), + } + : {}), importAuthorized: ( methods.importAuthorized as AuthorizedMemoryRuntime["importAuthorized"] ).bind(source), diff --git a/src/plugins/memory-invocation.test.ts b/src/plugins/memory-invocation.test.ts index 07ae72aff14e..fa90e30811b1 100644 --- a/src/plugins/memory-invocation.test.ts +++ b/src/plugins/memory-invocation.test.ts @@ -36,6 +36,7 @@ vi.mock("../logger.js", () => ({ const { MEMORY_INVOCATION_UNAVAILABLE, + createAuthorizedMemoryDeriveInvocation, createAuthorizedMemoryReadInvocation, createAuthorizedMemoryWriteInvocation, materializeAuthorizedMemoryVirtualView, @@ -760,6 +761,41 @@ describe("authorized memory read invocation", () => { expect(admittedVirtualView.readAuthorizedVirtualFile).not.toHaveBeenCalled(); }); + it("never materializes a derive invocation through the generic virtual filesystem", async () => { + const admittedVirtualView = { + materializeAuthorizedVirtualView: vi.fn(), + readAuthorizedVirtualFile: vi.fn(), + }; + const context = { + ...createContext(), + operation: "derive" as const, + }; + const plan = { + ...createPlan(), + operation: "derive" as const, + }; + mocks.materialize.mockReturnValue(context); + mocks.admit.mockResolvedValue({ + ok: true, + runtime: { + authorize: vi.fn().mockResolvedValue(plan), + searchAuthorized: vi.fn(), + readAuthorized: vi.fn(), + virtualView: admittedVirtualView, + }, + }); + + const invocation = await createAuthorizedMemoryDeriveInvocation({ context: {} as never }); + if (invocation === MEMORY_INVOCATION_UNAVAILABLE) { + throw new Error("fixture failed to admit derive invocation"); + } + + await expect(materializeAuthorizedMemoryVirtualView({ invocation })).resolves.toBe( + MEMORY_INVOCATION_UNAVAILABLE, + ); + expect(admittedVirtualView.materializeAuthorizedVirtualView).not.toHaveBeenCalled(); + }); + it("rejects malformed duplicate and case-colliding virtual views before any broker read", async () => { const admittedVirtualView = { materializeAuthorizedVirtualView: vi.fn(async () => ({ diff --git a/src/plugins/memory-invocation.ts b/src/plugins/memory-invocation.ts index 5555dbd83136..5397c90b0595 100644 --- a/src/plugins/memory-invocation.ts +++ b/src/plugins/memory-invocation.ts @@ -2,14 +2,18 @@ import { logWarn } from "../logger.js"; import type { AuthorizedMemoryVirtualView, AuthorizedMemoryMutation, + AuthorizedMemoryContentPlan, AuthorizedMemoryPlan, AuthorizedMemoryResultEnvelope, AuthorizedResourceHandle, AudienceRef, + MemoryContentAccessOperation, MemoryContentAccessContext, MemoryAccessContext, MemoryWriteResult, AuthorizedMemoryRuntime, + AuthorizedSealedCompactionArtifact, + AuthorizedTranscriptDerivationSource, } from "../memory-host-sdk/host/authorization.js"; import type { MemoryReadResult, @@ -67,8 +71,8 @@ export type AuthorizedMemoryWriteInvocation = Readonly<{ type InvocationState = Readonly<{ trustedContext: TrustedMemoryAccessContext; - context: MemoryContentAccessContext<"read">; - plan: AuthorizedMemoryPlan & Readonly<{ operation: "read" }>; + context: MemoryContentAccessContext; + plan: AuthorizedMemoryPlan & Readonly<{ operation: MemoryContentAccessOperation }>; authorizationStartedAtMs: number; runtime: AdmittedAuthorizedMemoryReadRuntime; /** Bound at admission; registry changes cannot replace this run's provider. */ @@ -83,6 +87,12 @@ type InvocationState = Readonly<{ runExposureRevisions: Set; }>; +type ReadInvocationState = Omit & + Readonly<{ + context: MemoryContentAccessContext<"read">; + plan: AuthorizedMemoryContentPlan<"read">; + }>; + const VIRTUAL_ROOT_PATTERN = /^[a-z][a-z0-9-]{0,63}$/u; const invocationStates = new WeakMap(); @@ -163,13 +173,14 @@ function readCurrentWriteContext(state: WriteInvocationState): MemoryAccessConte function readCurrentContext( state: InvocationState, -): MemoryContentAccessContext<"read"> | undefined { +): MemoryContentAccessContext | undefined { const current = materializeTrustedMemoryAccessContext(state.trustedContext); - if (!current || current.operation !== "read") { + if (!current || (current.operation !== "read" && current.operation !== "derive")) { return undefined; } - const readContext = current as MemoryContentAccessContext<"read">; + const readContext = current as MemoryContentAccessContext; if ( + readContext.operation !== state.context.operation || readContext.contextFingerprint !== state.context.contextFingerprint || readContext.runId !== state.context.runId || readContext.agentId !== state.context.agentId || @@ -185,9 +196,19 @@ function readCurrentContext( return readContext; } +function isReadContentContext( + context: MemoryContentAccessContext, +): context is MemoryContentAccessContext<"read"> { + return context.operation === "read"; +} + +function isReadInvocationState(state: InvocationState): state is ReadInvocationState { + return state.context.operation === "read" && state.plan.operation === "read"; +} + function validateEnvelope(params: { state: InvocationState; - context: MemoryContentAccessContext<"read">; + context: MemoryContentAccessContext; expectedRevisionHandles: readonly string[]; envelope: AuthorizedMemoryResultEnvelope; }): boolean { @@ -254,7 +275,7 @@ function mergeEnvelope( function readTranscriptExposure(params: { state: InvocationState; - context: MemoryContentAccessContext<"read">; + context: MemoryContentAccessContext; pendingEnvelope?: AuthorizedMemoryResultEnvelope; }) { const { state, context, pendingEnvelope } = params; @@ -298,7 +319,7 @@ function readTranscriptExposure(params: { */ function recordEnvelopeExposure(params: { state: InvocationState; - context: MemoryContentAccessContext<"read">; + context: MemoryContentAccessContext; envelope: AuthorizedMemoryResultEnvelope; }): void { if ( @@ -329,8 +350,8 @@ function readState(invocation: AuthorizedMemoryReadInvocation): InvocationState function canonicalizeAuthorizedVirtualView(params: { view: AuthorizedMemoryVirtualView; - context: MemoryContentAccessContext<"read">; - plan: AuthorizedMemoryPlan & Readonly<{ operation: "read" }>; + context: MemoryContentAccessContext; + plan: AuthorizedMemoryPlan & Readonly<{ operation: MemoryContentAccessOperation }>; }): AuthorizedMemoryVirtualView | undefined { const { view, context, plan } = params; const mountHandles = new Set(plan.mounts.map((mount) => mount.mountHandle)); @@ -409,16 +430,17 @@ function canonicalizeAuthorizedVirtualView(params: { * Creates a process-local, opaque read invocation. No caller can inject a serializable identity, * audience, plan, or continuation: all of those come from the trusted context and selected backend. */ -export async function createAuthorizedMemoryReadInvocation(params: { +async function createAuthorizedMemoryContentInvocation(params: { context: TrustedMemoryAccessContext; capability?: MemoryPluginCapability; + operation: MemoryContentAccessOperation; }): Promise { const materialized = materializeTrustedMemoryAccessContext(params.context); - if (!materialized || materialized.operation !== "read") { + if (!materialized || materialized.operation !== params.operation) { logMemoryInvocationDiagnostic("materialization-rejected"); return MEMORY_INVOCATION_UNAVAILABLE; } - const context = materialized as MemoryContentAccessContext<"read">; + const context = materialized as MemoryContentAccessContext; const capability = params.capability ?? resolveSelectedMemoryCapabilityRegistration(requireActivePluginRegistry())?.capability; @@ -430,7 +452,7 @@ export async function createAuthorizedMemoryReadInvocation(params: { try { const authorizationStartedAtMs = Date.now(); const plan = (await admission.runtime.authorize(context)) as AuthorizedMemoryPlan & - Readonly<{ operation: "read" }>; + Readonly<{ operation: MemoryContentAccessOperation }>; if (!isCurrentPlan({ context, plan, nowMs: Date.now() })) { logMemoryInvocationDiagnostic("invalid-plan"); return MEMORY_INVOCATION_UNAVAILABLE; @@ -461,6 +483,24 @@ export async function createAuthorizedMemoryReadInvocation(params: { } } +export async function createAuthorizedMemoryReadInvocation(params: { + context: TrustedMemoryAccessContext; + capability?: MemoryPluginCapability; +}): Promise { + return await createAuthorizedMemoryContentInvocation({ ...params, operation: "read" }); +} + +/** + * Creates an opaque content invocation for a derivation. Its content operations use the derive + * plan end to end, rather than checking derive once and then falling back to a read-only view. + */ +export async function createAuthorizedMemoryDeriveInvocation(params: { + context: TrustedMemoryAccessContext; + capability?: MemoryPluginCapability; +}): Promise { + return await createAuthorizedMemoryContentInvocation({ ...params, operation: "derive" }); +} + /** * Creates a process-local write invocation from host-minted facts. This is separate from read * exposure because a write is a resource lifecycle decision, never a continuation of a search hit. @@ -530,6 +570,40 @@ export async function writeAuthorizedMemoryForInvocation(params: { } } +/** + * Stages bytes before the caller opens its transaction. The returned closure is + * the only route that may insert the selected runtime's revision/catalog rows + * into the core-owned sealed compaction transaction. + */ +export async function stageAuthorizedMemorySealedCompactionForInvocation(params: { + invocation: AuthorizedMemoryWriteInvocation; + content: string; + transcriptSource: AuthorizedTranscriptDerivationSource; +}): Promise { + const state = writeInvocationStates.get(params.invocation); + const context = state ? readCurrentWriteContext(state) : undefined; + if ( + !state || + !context || + context.operation !== "derive" || + !state.runtime.stageSealedCompaction || + !isCurrentPlan({ context, plan: state.plan, nowMs: Date.now() }) + ) { + return MEMORY_INVOCATION_UNAVAILABLE; + } + try { + return await state.runtime.stageSealedCompaction({ + context, + plan: state.plan as AuthorizedMemoryPlan & Readonly<{ operation: "derive" }>, + content: params.content, + transcriptSource: params.transcriptSource, + }); + } catch { + logMemoryInvocationDiagnostic("authorization-failed"); + return MEMORY_INVOCATION_UNAVAILABLE; + } +} + /** * Obtains a selected-plugin projection for generic FS and sandbox consumers. * It is deliberately separate from search/read: no tool argument can turn a @@ -540,7 +614,15 @@ export async function materializeAuthorizedMemoryVirtualView(params: { }): Promise { const state = readState(params.invocation); const context = state ? readCurrentContext(state) : undefined; - if (!state || !context || !state.virtualView) { + // Derivations may expose source bytes only to their dedicated content path. A generic virtual + // filesystem is a read capability, so it must not become an alternate derive transport. + if ( + !state || + !context || + !isReadContentContext(context) || + !isReadInvocationState(state) || + !state.virtualView + ) { return MEMORY_INVOCATION_UNAVAILABLE; } try { @@ -576,6 +658,8 @@ export async function readAuthorizedMemoryVirtualFile(params: { if ( !state || !context || + !isReadContentContext(context) || + !isReadInvocationState(state) || !state.virtualView || !isCurrentPlan({ context, plan: state.plan, nowMs: Date.now() }) || state.virtualViews.get(params.view.viewId) !== params.view || @@ -632,7 +716,7 @@ export async function searchAuthorizedMemoryForInvocation(params: { ...(params.sources ? { sources: params.sources } : {}), limit: Math.max(1, Math.min(100, Math.trunc(params.limit ?? 10))), ...(params.signal ? { signal: params.signal } : {}), - }); + } as never); const revisionHandles = envelope.value.map((result) => result.resourceHandle.resourceRevision); if ( !validateEnvelope({ @@ -681,7 +765,7 @@ export async function readAuthorizedMemoryForInvocation(params: { handle, ...(params.from !== undefined ? { from: params.from } : {}), ...(params.lines !== undefined ? { lines: params.lines } : {}), - }); + } as never); if ( !validateEnvelope({ state, diff --git a/src/plugins/memory-state.test.ts b/src/plugins/memory-state.test.ts index db8d1a45b189..734e88d7b873 100644 --- a/src/plugins/memory-state.test.ts +++ b/src/plugins/memory-state.test.ts @@ -462,6 +462,17 @@ describe("memory plugin state", () => { expect(resolveMemoryFlushPlan({})?.relativePath).toBe("memory/same-owner.md"); }); + it("keeps the flush plan available for a cut-over agent", () => { + isMemoryIsolationCutoverAgentMock.mockReturnValue(true); + registerMemoryCapability("memory-core", { + flushPlanResolver: () => createMemoryFlushPlan("memory/authorized.md"), + }); + + expect(resolveMemoryFlushPlan({ agentId: "main" })?.relativePath).toBe( + "memory/authorized.md", + ); + }); + it("passes citations mode through to the prompt builder", () => { registerTestMemoryPromptBuilder(({ citationsMode }) => [ `citations: ${citationsMode ?? "default"}`, diff --git a/src/plugins/memory-state.ts b/src/plugins/memory-state.ts index 4c098c9374d4..cb9ddc6bd37e 100644 --- a/src/plugins/memory-state.ts +++ b/src/plugins/memory-state.ts @@ -317,9 +317,6 @@ export function resolveMemoryFlushPlan(params: { agentId?: string; nowMs?: number; }): MemoryFlushPlan | null { - if (params.agentId && isMemoryIsolationCutoverAgent(params.agentId)) { - return null; - } return getMemoryCapability()?.capability.flushPlanResolver?.(params) ?? null; } export function getMemoryRuntime(): MemoryPluginRuntime | undefined { diff --git a/src/plugins/registry-contribution-types.ts b/src/plugins/registry-contribution-types.ts index 81619b34e2dc..cb23699206e1 100644 --- a/src/plugins/registry-contribution-types.ts +++ b/src/plugins/registry-contribution-types.ts @@ -284,6 +284,7 @@ export type MemoryPluginRuntime = { searchAuthorized?: AuthorizedMemoryRuntime["searchAuthorized"]; readAuthorized?: AuthorizedMemoryRuntime["readAuthorized"]; writeAuthorized?: AuthorizedMemoryRuntime["writeAuthorized"]; + stageSealedCompaction?: AuthorizedMemoryRuntime["stageSealedCompaction"]; importAuthorized?: AuthorizedMemoryRuntime["importAuthorized"]; syncAuthorized?: AuthorizedMemoryRuntime["syncAuthorized"]; exportAuthorized?: AuthorizedMemoryRuntime["exportAuthorized"]; diff --git a/src/state/openclaw-agent-db.generated.d.ts b/src/state/openclaw-agent-db.generated.d.ts index 8a9e9a1172de..8af5e6d7fec3 100644 --- a/src/state/openclaw-agent-db.generated.d.ts +++ b/src/state/openclaw-agent-db.generated.d.ts @@ -165,6 +165,15 @@ export interface MemoryCompactionPolicies { source_policy_set_id: string; } +export interface MemoryCompactionPolicySources { + compaction_policy_id: string; + created_at: number; + delivery_audiences_json: string; + source_event_seq: number; + source_policy_set_id: string; + source_session_id: string; +} + export interface MemoryEmbeddingCache { dims: number | null; embedding: string; @@ -301,6 +310,23 @@ export interface MemoryPolicySets { policy_set_id: string; } +export interface MemoryRevisionPolicyRequirements { + created_at: number; + expected_revision_id: string; + expected_revocation_epoch: number; + policy_id: string; + requirement_kind: string; + revision_id: string; +} + +export interface MemoryLineageEdges { + child_revision_id: string; + created_at: number; + parent_id: string; + parent_kind: string; + relation_kind: string; +} + export interface MemoryPreoutputExposureLedger { agent_id: string; context_fingerprint: string; @@ -849,6 +875,7 @@ export interface DB { heartbeat_outcomes: HeartbeatOutcomes; memory_audit_outbox: MemoryAuditOutbox; memory_compaction_policies: MemoryCompactionPolicies; + memory_compaction_policy_sources: MemoryCompactionPolicySources; memory_embedding_cache: MemoryEmbeddingCache; memory_index_chunk_provenance: MemoryIndexChunkProvenance; memory_index_chunk_recall_metadata: MemoryIndexChunkRecallMetadata; @@ -863,6 +890,8 @@ export interface DB { memory_policy_revisions: MemoryPolicyRevisions; memory_policy_set_members: MemoryPolicySetMembers; memory_policy_sets: MemoryPolicySets; + memory_revision_policy_requirements: MemoryRevisionPolicyRequirements; + memory_lineage_edges: MemoryLineageEdges; memory_preoutput_exposure_authorization_facts: MemoryPreoutputExposureAuthorizationFacts; memory_preoutput_exposure_ledger: MemoryPreoutputExposureLedger; memory_resource_revisions: MemoryResourceRevisions; diff --git a/src/state/openclaw-agent-schema.sql b/src/state/openclaw-agent-schema.sql index 31d041a038eb..c93590798fa8 100644 --- a/src/state/openclaw-agent-schema.sql +++ b/src/state/openclaw-agent-schema.sql @@ -790,6 +790,63 @@ BEGIN SELECT RAISE(ABORT, 'tombstoned memory resource revisions cannot be reactivated'); END; +-- A derived revision records every stable policy that must still be current before it can be +-- exposed. Missing requirements are a durable deny, never a reason to infer access from content. +CREATE TABLE IF NOT EXISTS memory_revision_policy_requirements ( + revision_id TEXT NOT NULL, + policy_id TEXT NOT NULL, + expected_revision_id TEXT NOT NULL, + expected_revocation_epoch INTEGER NOT NULL CHECK (expected_revocation_epoch >= 0), + requirement_kind TEXT NOT NULL CHECK (requirement_kind IN ('output-policy', 'source-policy')), + created_at INTEGER NOT NULL, + PRIMARY KEY (revision_id, policy_id), + FOREIGN KEY (revision_id) REFERENCES memory_resource_revisions(revision_id) ON DELETE RESTRICT, + FOREIGN KEY (policy_id) REFERENCES memory_policies(policy_id) ON DELETE RESTRICT, + FOREIGN KEY (expected_revision_id) REFERENCES memory_policy_revisions(revision_id) ON DELETE RESTRICT +) STRICT; + +CREATE INDEX IF NOT EXISTS idx_memory_revision_policy_requirements_policy + ON memory_revision_policy_requirements(policy_id, expected_revision_id, expected_revocation_epoch); + +CREATE TRIGGER IF NOT EXISTS memory_revision_policy_requirements_no_update +BEFORE UPDATE ON memory_revision_policy_requirements +BEGIN + SELECT RAISE(ABORT, 'memory revision policy requirements are immutable'); +END; + +CREATE TRIGGER IF NOT EXISTS memory_revision_policy_requirements_no_delete +BEFORE DELETE ON memory_revision_policy_requirements +BEGIN + SELECT RAISE(ABORT, 'memory revision policy requirements cannot be deleted'); +END; + +-- Parent revisions are immutable derivation facts. Readers traverse resource parents rather than +-- maintaining a mutable descendant cache, so an ancestor tombstone takes effect immediately. +CREATE TABLE IF NOT EXISTS memory_lineage_edges ( + child_revision_id TEXT NOT NULL, + parent_kind TEXT NOT NULL CHECK (parent_kind IN ('resource-revision', 'transcript-policy-set', 'compaction-policy', 'checkpoint', 'export', 'child-artifact')), + parent_id TEXT NOT NULL, + relation_kind TEXT NOT NULL CHECK (relation_kind IN ('derived-from', 'compacted-from', 'flushed-from', 'dreamed-from', 'promoted-from', 'exported-from', 'child-produced')), + created_at INTEGER NOT NULL, + PRIMARY KEY (child_revision_id, parent_kind, parent_id, relation_kind), + FOREIGN KEY (child_revision_id) REFERENCES memory_resource_revisions(revision_id) ON DELETE RESTRICT +) STRICT; + +CREATE INDEX IF NOT EXISTS idx_memory_lineage_edges_parent + ON memory_lineage_edges(parent_kind, parent_id, child_revision_id); + +CREATE TRIGGER IF NOT EXISTS memory_lineage_edges_no_update +BEFORE UPDATE ON memory_lineage_edges +BEGIN + SELECT RAISE(ABORT, 'memory lineage edges are immutable'); +END; + +CREATE TRIGGER IF NOT EXISTS memory_lineage_edges_no_delete +BEFORE DELETE ON memory_lineage_edges +BEGIN + SELECT RAISE(ABORT, 'memory lineage edges cannot be deleted'); +END; + CREATE TABLE IF NOT EXISTS memory_resource_subjects ( revision_id TEXT NOT NULL, subject_kind TEXT NOT NULL CHECK (subject_kind IN ('person', 'project', 'conversation', 'topic')), @@ -1268,6 +1325,36 @@ BEGIN SELECT RAISE(ABORT, 'memory compaction policies cannot be deleted'); END; +-- A compaction policy names the complete transcript source set, not merely its +-- common policy set. Transcript rows may later be reset or archived, so this +-- immutable provenance deliberately has no foreign key to mutable event rows. +CREATE TABLE IF NOT EXISTS memory_compaction_policy_sources ( + compaction_policy_id TEXT NOT NULL, + source_session_id TEXT NOT NULL, + source_event_seq INTEGER NOT NULL CHECK (source_event_seq >= 0), + source_policy_set_id TEXT NOT NULL, + delivery_audiences_json TEXT NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (compaction_policy_id, source_session_id, source_event_seq), + FOREIGN KEY (compaction_policy_id) REFERENCES memory_compaction_policies(compaction_policy_id) ON DELETE RESTRICT, + FOREIGN KEY (source_policy_set_id) REFERENCES memory_policy_sets(policy_set_id) ON DELETE RESTRICT +) STRICT; + +CREATE INDEX IF NOT EXISTS idx_memory_compaction_policy_sources_source + ON memory_compaction_policy_sources(source_session_id, source_event_seq, compaction_policy_id); + +CREATE TRIGGER IF NOT EXISTS memory_compaction_policy_sources_no_update +BEFORE UPDATE ON memory_compaction_policy_sources +BEGIN + SELECT RAISE(ABORT, 'memory compaction policy sources are immutable'); +END; + +CREATE TRIGGER IF NOT EXISTS memory_compaction_policy_sources_no_delete +BEFORE DELETE ON memory_compaction_policy_sources +BEGIN + SELECT RAISE(ABORT, 'memory compaction policy sources cannot be deleted'); +END; + CREATE TABLE IF NOT EXISTS standing_intents ( intent_key INTEGER PRIMARY KEY, id TEXT NOT NULL UNIQUE, diff --git a/src/state/openclaw-agent-scoped-memory-schema.ts b/src/state/openclaw-agent-scoped-memory-schema.ts index b9d654be1e2c..7c045109bed9 100644 --- a/src/state/openclaw-agent-scoped-memory-schema.ts +++ b/src/state/openclaw-agent-scoped-memory-schema.ts @@ -10,6 +10,8 @@ export const AGENT_SCOPED_MEMORY_TABLES = [ "memory_policy_entries", "memory_resources", "memory_resource_revisions", + "memory_revision_policy_requirements", + "memory_lineage_edges", "memory_resource_subjects", "memory_scoped_chunks", "memory_scoped_chunk_vectors", @@ -26,6 +28,7 @@ export const AGENT_SCOPED_MEMORY_TABLES = [ "transcript_event_memory_policy_details", "transcript_event_memory_policy_transitions", "memory_compaction_policies", + "memory_compaction_policy_sources", ] as const; export const AGENT_SCOPED_MEMORY_FTS_TABLE = "memory_scoped_chunks_fts";