feat(memory): seal derived artifact lineage

This commit is contained in:
Galin Iliev
2026-08-14 04:14:21 -07:00
parent 0d20184587
commit 93d50ddee8
68 changed files with 4291 additions and 211 deletions
+9 -9
View File
@@ -1528,25 +1528,25 @@ Focused existing tests:
Phase 2C is complete only when all of the following are demonstrated:
- [ ] Every compaction, checkpoint, memory flush, dreaming output, promotion,
- [x] Every compaction, checkpoint, memory flush, dreaming output, promotion,
export, and child-produced durable artifact identifies immutable parents
and records lineage.
- [ ] `derive` authority is checked before source content enters a model
- [x] `derive` authority is checked before source content enters a model
context.
- [ ] No unlabeled or policy-unrepresentable derived artifact is readable.
- [ ] Group compaction and flush remain channel-scoped; private compaction and
- [x] No unlabeled or policy-unrepresentable derived artifact is readable.
- [x] Group compaction and flush remain channel-scoped; private compaction and
flush remain user-scoped; autonomous work remains agent-scoped or writes
nothing.
- [ ] Mixed audiences are partitioned or denied and can never be widened by
- [x] Mixed audiences are partitioned or denied and can never be widened by
model wording.
- [ ] Tombstoning/revoking an ancestor denies descendants immediately; any
- [x] Tombstoning/revoking an ancestor denies descendants immediately; any
recomputation creates a new reviewed immutable revision.
- [ ] Dreaming and promotion run one authorized store at a time, and postbox or
- [x] Dreaming and promotion run one authorized store at a time, and postbox or
quarantine content cannot auto-promote.
- [ ] Child agents receive only the intersection of parent view, task
- [x] Child agents receive only the intersection of parent view, task
capability, session visibility, and current authority; cron, heartbeat,
webhook, and system runs cannot recover private access from a session key.
- [ ] Compaction, flush, dreaming, lineage, revocation, delegation, and
- [x] Compaction, flush, dreaming, lineage, revocation, delegation, and
interruption tests pass, including any dependency-specific contract
checks required by the selected harness.
+1
View File
@@ -306,6 +306,7 @@ function createLazyMemoryRuntime(host: MemoryCoreRuntimeHost): MemoryPluginRunti
searchAuthorized: builtinScopedMemoryAuthorizedRuntime.searchAuthorized,
readAuthorized: builtinScopedMemoryAuthorizedRuntime.readAuthorized,
writeAuthorized: builtinScopedMemoryAuthorizedRuntime.writeAuthorized,
stageSealedCompaction: builtinScopedMemoryAuthorizedRuntime.stageSealedCompaction,
importAuthorized: builtinScopedMemoryAuthorizedRuntime.importAuthorized,
syncAuthorized: builtinScopedMemoryAuthorizedRuntime.syncAuthorized,
exportAuthorized: builtinScopedMemoryAuthorizedRuntime.exportAuthorized,
@@ -1,4 +1,7 @@
import type { PromotionCandidate } from "./short-term-promotion-types.js";
import type {
PromotionCandidate,
ShortTermPromotionAuthorizedView,
} from "./short-term-promotion-types.js";
import { isShortTermSessionCorpusPath } from "./short-term-promotion-utils.js";
export function filterConsolidationCandidates(
@@ -13,6 +16,61 @@ export function isPromotionOriginBlocked(candidate: PromotionCandidate): boolean
return originClass === "untrusted" || originClass === "system";
}
function authorizedViewIdentity(view: ShortTermPromotionAuthorizedView): string {
return `${view.storeId}\u0000${view.viewId}`;
}
/** A scoped candidate can promote only from one active, immutable authorized view. */
export function isPromotionAuthorizedViewBlocked(
candidate: Pick<PromotionCandidate, "authorizedView">,
): boolean {
const view = candidate.authorizedView;
return Boolean(
view &&
(typeof view.storeId !== "string" ||
typeof view.viewId !== "string" ||
typeof view.resourceRevision !== "string" ||
!view.storeId.trim() ||
!view.viewId.trim() ||
!view.resourceRevision.trim() ||
view.lifecycle !== "active"),
);
}
/** Legacy records lack a scoped view; if one is present, never mix it with another view or legacy data. */
export function hasOnePromotionAuthorizedView(candidates: readonly PromotionCandidate[]): boolean {
const scoped = candidates.filter((candidate) => candidate.authorizedView);
if (scoped.length === 0) {
return true;
}
if (scoped.length !== candidates.length || scoped.some(isPromotionAuthorizedViewBlocked)) {
return false;
}
return new Set(scoped.map((candidate) => authorizedViewIdentity(candidate.authorizedView!))).size === 1;
}
/**
* Dreaming may process legacy workspace records together, but a scoped record
* changes the boundary: a phase gets one active authorized view or no input.
* That prevents a narrative/consolidation model context from joining stores.
*/
export function filterToOnePromotionAuthorizedView<
Candidate extends Pick<PromotionCandidate, "authorizedView">,
>(candidates: readonly Candidate[]): Candidate[] {
const eligible = candidates.filter((candidate) => !isPromotionAuthorizedViewBlocked(candidate));
const scoped = eligible.filter((candidate) => candidate.authorizedView);
if (scoped.length === 0) {
return eligible;
}
if (scoped.length !== eligible.length) {
return [];
}
const identities = new Set(
scoped.map((candidate) => authorizedViewIdentity(candidate.authorizedView!)),
);
return identities.size === 1 ? eligible : [];
}
export function isConsolidationCandidateEligible(candidate: PromotionCandidate): boolean {
const trustedOrigin =
candidate.provenance?.originClass === "owner" || candidate.provenance?.originClass === "agent";
@@ -23,6 +23,7 @@ import {
DREAMING_DAILY_PROVENANCE_NAMESPACE,
writeMemoryCoreWorkspaceEntry,
} from "./dreaming-state.js";
import { filterToOnePromotionAuthorizedView } from "./dreaming-consolidation-candidates.js";
import { previewRemHarness } from "./rem-harness.js";
import { writeSessionIngestionState } from "./session-ingestion.js";
import {
@@ -130,6 +131,71 @@ function expectNotIncludesSubstring(values: readonly string[], expected: string)
expect(values.join("\n")).not.toContain(expected);
}
function createAuthorizedRecallEntry(params: {
key: string;
storeId: string;
viewId: string;
resourceRevision: string;
lifecycle?: "active" | "postbox" | "quarantine";
}): ShortTermRecallEntry {
return {
key: params.key,
path: `memory/${params.key}.md`,
startLine: 1,
endLine: 1,
source: "memory",
snippet: `Dreaming source ${params.key}.`,
recallCount: 1,
dailyCount: 0,
groundedCount: 0,
totalScore: 0.9,
maxScore: 0.9,
firstRecalledAt: DREAMING_TEST_BASE_TIME.toISOString(),
lastRecalledAt: DREAMING_TEST_BASE_TIME.toISOString(),
queryHashes: [params.key],
recallDays: [DREAMING_TEST_DAY],
conceptTags: [],
authorizedView: {
storeId: params.storeId,
viewId: params.viewId,
resourceRevision: params.resourceRevision,
lifecycle: params.lifecycle ?? "active",
},
};
}
it("keeps dreaming input inside one authorized view", () => {
const first = createAuthorizedRecallEntry({
key: "first",
storeId: "store-a",
viewId: "view-a",
resourceRevision: "revision-a-1",
});
const sameView = createAuthorizedRecallEntry({
key: "same-view",
storeId: "store-a",
viewId: "view-a",
resourceRevision: "revision-a-2",
});
const otherView = createAuthorizedRecallEntry({
key: "other-view",
storeId: "store-b",
viewId: "view-b",
resourceRevision: "revision-b-1",
});
const postbox = createAuthorizedRecallEntry({
key: "postbox",
storeId: "store-a",
viewId: "view-a",
resourceRevision: "revision-postbox",
lifecycle: "postbox",
});
expect(filterToOnePromotionAuthorizedView([first, sameView])).toEqual([first, sameView]);
expect(filterToOnePromotionAuthorizedView([first, otherView])).toEqual([]);
expect(filterToOnePromotionAuthorizedView([first, postbox])).toEqual([first]);
});
async function expectPathMissing(targetPath: string): Promise<void> {
try {
await fs.access(targetPath);
+15 -10
View File
@@ -17,6 +17,7 @@ import {
import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry";
import { normalizeStringEntries, uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime";
import { appendFailedDreamingEvent } from "./dreaming-events.js";
import { filterToOnePromotionAuthorizedView } from "./dreaming-consolidation-candidates.js";
import {
normalizeDailyIngestionState,
normalizeMemoryDay,
@@ -1311,11 +1312,13 @@ async function runLightDreaming(params: {
entries: await filterFreshLightDreamingEntries({
workspaceDir: params.workspaceDir,
nowMs,
entries: filterRecallEntriesWithinLookback({
entries: await readShortTermRecallEntries({ workspaceDir: params.workspaceDir, nowMs }),
nowMs,
lookbackDays: params.config.lookbackDays,
}),
entries: filterToOnePromotionAuthorizedView(
filterRecallEntriesWithinLookback({
entries: await readShortTermRecallEntries({ workspaceDir: params.workspaceDir, nowMs }),
nowMs,
lookbackDays: params.config.lookbackDays,
}),
),
}),
});
const rankedEntries = dedupeEntries(
@@ -1409,11 +1412,13 @@ async function runRemDreaming(params: {
});
const allEntries = await filterLiveShortTermRecallEntries({
workspaceDir: params.workspaceDir,
entries: filterRecallEntriesWithinLookback({
entries: await readShortTermRecallEntries({ workspaceDir: params.workspaceDir, nowMs }),
nowMs,
lookbackDays: params.config.lookbackDays,
}),
entries: filterToOnePromotionAuthorizedView(
filterRecallEntriesWithinLookback({
entries: await readShortTermRecallEntries({ workspaceDir: params.workspaceDir, nowMs }),
nowMs,
lookbackDays: params.config.lookbackDays,
}),
),
});
// Prefer entries staged by light sleep so REM synthesises from the
// sequential light→REM pipeline instead of rescanning the full store.
@@ -107,6 +107,83 @@ export type MemoryResourceRevisionRow = {
retired_at: number | null;
};
export type MemoryRevisionPolicyRequirementRow = {
revision_id: string;
policy_id: string;
expected_revision_id: string;
expected_revocation_epoch: number;
requirement_kind: "output-policy" | "source-policy";
created_at: number;
};
export type MemoryLineageEdgeRow = {
child_revision_id: string;
parent_kind:
| "resource-revision"
| "transcript-policy-set"
| "compaction-policy"
| "checkpoint"
| "export"
| "child-artifact";
parent_id: string;
relation_kind:
| "derived-from"
| "compacted-from"
| "flushed-from"
| "dreamed-from"
| "promoted-from"
| "exported-from"
| "child-produced";
created_at: number;
};
type MemoryPolicySetMemberRow = {
policy_set_id: string;
policy_id: string;
expected_revision_id: string;
expected_revocation_epoch: number;
audience_intersection_json: string;
retention_state: string;
created_at: number;
};
type SessionMemorySubjectSnapshotRow = {
session_id: string;
session_identity_revision: string;
subject_revision: string;
};
type TranscriptEventMemoryPolicyRow = {
session_id: string;
event_seq: number;
authorization_status: string;
source_policy_set_id: string | null;
run_exposure_set_id: string | null;
delivery_audiences_json: string | null;
session_identity_revision: string | null;
subject_revision: string | null;
};
type TranscriptEventMemoryPolicyDetailRow = {
session_id: string;
event_seq: number;
retention_state: string;
normalized_audience_intersection_json: string;
finalized_delivery_audiences_json: string;
};
type TranscriptEventRow = {
session_id: string;
seq: number;
};
type MemoryRunExposureResourceRow = {
exposure_set_id: string;
resource_revision_id: string;
policy_set_id: string;
created_at: number;
};
export type MemoryResourceSubjectRow = {
revision_id: string;
subject_kind: "person" | "project" | "conversation" | "topic";
@@ -203,6 +280,14 @@ export type ScopedMemoryDatabase = {
memory_policy_entries: MemoryPolicyEntryRow;
memory_resources: MemoryResourceRow;
memory_resource_revisions: MemoryResourceRevisionRow;
memory_revision_policy_requirements: MemoryRevisionPolicyRequirementRow;
memory_lineage_edges: MemoryLineageEdgeRow;
memory_policy_set_members: MemoryPolicySetMemberRow;
session_memory_subject_snapshots: SessionMemorySubjectSnapshotRow;
transcript_event_memory_policies: TranscriptEventMemoryPolicyRow;
transcript_event_memory_policy_details: TranscriptEventMemoryPolicyDetailRow;
transcript_events: TranscriptEventRow;
memory_run_exposure_resources: MemoryRunExposureResourceRow;
memory_resource_subjects: MemoryResourceSubjectRow;
memory_scoped_chunks: MemoryScopedChunkRow;
memory_scoped_chunk_vectors: MemoryScopedChunkVectorRow;
@@ -143,6 +143,131 @@ function removeArtifact(pathname: string): void {
} catch {}
}
type RevisionPolicyRequirement = Readonly<{
policyId: string;
expectedRevisionId: string;
expectedRevocationEpoch: number;
}>;
function readRevisionPolicyRequirements(params: {
database: Parameters<typeof getNodeSqliteKysely<ScopedMemoryDatabase>>[0];
revisionId: string;
}): readonly RevisionPolicyRequirement[] {
const db = getNodeSqliteKysely<ScopedMemoryDatabase>(params.database);
return Object.freeze(
executeSqliteQuerySync(
params.database,
db
.selectFrom("memory_revision_policy_requirements")
.select(["policy_id", "expected_revision_id", "expected_revocation_epoch"])
.where("revision_id", "=", params.revisionId)
.orderBy("policy_id"),
).rows.map((row) =>
Object.freeze({
policyId: row.policy_id,
expectedRevisionId: row.expected_revision_id,
expectedRevocationEpoch: row.expected_revocation_epoch,
}),
),
);
}
/** Requirements and parent revisions are checked for every future exposure, not only at write time. */
function isRevisionLineageCurrent(params: {
database: Parameters<typeof getNodeSqliteKysely<ScopedMemoryDatabase>>[0];
revisionId: string;
visited: Set<string>;
}): boolean {
if (params.visited.has(params.revisionId)) {
return false;
}
params.visited.add(params.revisionId);
const db = getNodeSqliteKysely<ScopedMemoryDatabase>(params.database);
const requirements = readRevisionPolicyRequirements(params);
if (requirements.length === 0) {
return false;
}
for (const requirement of requirements) {
const current = executeSqliteQueryTakeFirstSync(
params.database,
db
.selectFrom("memory_policies as policy")
.innerJoin(
"memory_policy_revisions as revision",
"revision.revision_id",
"policy.current_revision_id",
)
.select([
"policy.current_revision_id",
"policy.revocation_epoch",
"policy.lifecycle_state as policy_lifecycle_state",
"revision.lifecycle_state as revision_lifecycle_state",
])
.where("policy.policy_id", "=", requirement.policyId),
);
if (
!current ||
current.policy_lifecycle_state !== "active" ||
current.revision_lifecycle_state !== "active" ||
current.current_revision_id !== requirement.expectedRevisionId ||
current.revocation_epoch !== requirement.expectedRevocationEpoch
) {
return false;
}
}
const parents = executeSqliteQuerySync(
params.database,
db
.selectFrom("memory_lineage_edges")
.select(["parent_kind", "parent_id"])
.where("child_revision_id", "=", params.revisionId)
.orderBy("parent_kind")
.orderBy("parent_id"),
).rows;
for (const parent of parents) {
if (parent.parent_kind === "transcript-policy-set") {
// Transcript sources materialize their stable policy requirements and every exposed resource
// parent on the child revision. Those checks above and below are the durable invalidation path.
continue;
}
// Other Phase 2C producers add their own immutable parent types. A resource parent is already
// selectable today, so it must recurse rather than merely checking its direct lifecycle row.
if (parent.parent_kind !== "resource-revision") {
return false;
}
const revision = executeSqliteQueryTakeFirstSync(
params.database,
db
.selectFrom("memory_resource_revisions")
.select("lifecycle_state")
.where("revision_id", "=", parent.parent_id),
);
if (
revision?.lifecycle_state !== "active" ||
!isRevisionLineageCurrent({
database: params.database,
revisionId: parent.parent_id,
visited: params.visited,
})
) {
return false;
}
}
return true;
}
/** Recovery paths use the same recursive check as normal reads before activating a pending revision. */
export function isBuiltinScopedMemoryRevisionLineageCurrent(params: {
agentId: string;
revisionId: string;
}): boolean {
const agentId = normalizeAgentId(params.agentId);
const revisionId = normalizeScopedMemoryRequiredText(params.revisionId, "revisionId");
return withScopedMemoryDatabase(agentId, (database) =>
isRevisionLineageCurrent({ database, revisionId, visited: new Set() }),
);
}
/**
* Resolve a revision only while its immutable catalog evidence is current.
* The Phase 1C runtime supplies the authorized store view; this foundation
@@ -220,6 +345,7 @@ export function readBuiltinScopedMemoryRevisionSnapshot(params: {
revision.current_policy_revocation_epoch !== revision.revocation_epoch ||
revision.source_policy_set_id !==
createScopedMemorySourcePolicySetId(revision.current_revision_id) ||
!isRevisionLineageCurrent({ database, revisionId, visited: new Set() }) ||
(revision.expires_at !== null && revision.expires_at <= nowMs)
) {
return undefined;
@@ -324,6 +450,7 @@ function createRevision(params: {
)
.select([
"resource.resource_id",
"policy.policy_id",
"policy.current_revision_id",
"policy.revocation_epoch",
"policy_revision.revision_number",
@@ -379,6 +506,17 @@ function createRevision(params: {
retired_at: null,
}),
);
executeSqliteQuerySync(
database,
db.insertInto("memory_revision_policy_requirements").values({
revision_id: revisionId,
policy_id: current.policy_id,
expected_revision_id: current.current_revision_id,
expected_revocation_epoch: current.revocation_epoch,
requirement_kind: "output-policy",
created_at: params.nowMs,
}),
);
const chunks = chunkScopedMemoryMarkdown(content);
if (chunks.length > 0) {
executeSqliteQuerySync(
@@ -20,9 +20,17 @@ import {
consumeAdmittedChannelMemoryIdentityFromContext,
createChannelMemoryIdentityAdmission,
} from "../../../../src/channels/message-access/memory-identity-admission.js";
import { appendSqliteTranscriptMessage } from "../../../../src/config/sessions/session-accessor.sqlite-transcript-write.js";
import { readAuthorizedTranscriptDerivation } from "../../../../src/config/sessions/session-transcript-memory-policy.js";
import { withOwnedSessionTranscriptWrites } from "../../../../src/config/sessions/transcript-write-context.js";
import { admitMemoryAuthorizationReadRuntime } from "../../../../src/plugins/memory-authorization-runtime.js";
import { resetMemoryIsolationCutoverForTest } from "../../../../src/plugins/memory-cutover.js";
import { readLatestDurableMemoryRunExposure } from "../../../../src/plugins/memory-run-exposure-ledger.js";
import { registerAgentRunContext, resetAgentRunRegistryForTest } from "../../../../src/infra/agent-run-registry.js";
import {
persistMemoryRunExposureBeforeContentInDatabase,
readLatestDurableMemoryRunExposure,
} from "../../../../src/plugins/memory-run-exposure-ledger.js";
import { prepareMemoryRunExposure } from "../../../../src/plugins/memory-run-exposure.js";
import { createEmptyPluginRegistry } from "../../../../src/plugins/registry-empty.js";
import {
resetPluginRuntimeStateForTest,
@@ -49,6 +57,7 @@ import { builtinScopedMemoryConformanceAdapter } from "./scoped-memory-policy.js
import {
createBuiltinScopedMemoryResource,
readBuiltinScopedMemoryRevisionSnapshot,
setBuiltinScopedMemoryRevisionLifecycle,
} from "./scoped-memory-resources.js";
import {
builtinScopedMemoryAuthorizedRuntime,
@@ -77,6 +86,7 @@ describe("builtin scoped authorized runtime", () => {
dispatchReplyFromConfig.mockReset();
resetBuiltinScopedMemoryAuthorizedRuntimeForTest();
resetMemoryIsolationCutoverForTest();
resetAgentRunRegistryForTest();
resetPluginRuntimeStateForTest();
closeOpenClawAgentDatabasesForTest();
closeOpenClawStateDatabaseForTest();
@@ -386,6 +396,20 @@ describe("builtin scoped authorized runtime", () => {
now,
revisionLifecycleState === "active" ? now : null,
);
database
.prepare(
`INSERT INTO memory_revision_policy_requirements
(revision_id, policy_id, expected_revision_id, expected_revocation_epoch,
requirement_kind, created_at)
VALUES (?, ?, ?, ?, 'output-policy', ?)`,
)
.run(
revisionId,
row.policy_id,
params.policyRevisionId ?? row.current_revision_id,
row.revocation_epoch,
now,
);
database
.prepare(
`INSERT INTO memory_write_intents
@@ -476,6 +500,253 @@ describe("builtin scoped authorized runtime", () => {
).rejects.toThrow("unavailable");
});
it("authorizes compaction sources with derive rather than downgrading them to read", async () => {
const principalId = "derive-owner";
const store = createBuiltinScopedMemoryStore({
agentId: "main",
scopeKind: "user",
audienceKind: "user",
audienceId: principalId,
authorityKind: "user",
authorityOwnerId: principalId,
defaultCapabilities: ["retrieve", "read", "derive"],
actor: { kind: "human", id: principalId },
reason: "derive source fixture",
});
createBuiltinScopedMemoryResource({
agentId: "main",
store,
logicalLocator: "MEMORY.md",
content: "DERIVE_ONLY_SOURCE_SENTINEL",
actor: { kind: "human", id: principalId },
});
const context = {
...createContext(principalId),
operation: "derive" as const,
} satisfies MemoryContentAccessContext<"derive">;
const plan = await builtinScopedMemoryAuthorizedRuntime.authorize(context);
const searched = await builtinScopedMemoryAuthorizedRuntime.searchAuthorized({
context,
plan,
query: "DERIVE_ONLY_SOURCE_SENTINEL",
limit: 10,
});
expect(searched).toMatchObject({ value: [{ snippet: "DERIVE_ONLY_SOURCE_SENTINEL" }] });
const source = searched.value[0];
if (!source) {
throw new Error("fixture expected a derivation source");
}
const derived = await builtinScopedMemoryAuthorizedRuntime.writeAuthorized({
context,
plan,
mutation: {
version: 1,
kind: "derive",
derivationPurpose: "flush",
mutationId: "derive-output",
idempotencyKey: "derive-output-request",
content: "DERIVED_OUTPUT_SENTINEL",
contentType: "markdown",
sourceHandles: [source.resourceHandle],
sourcePolicySetId: searched.exposureReceipt.sourcePolicySetId,
},
});
const derivedRevisionId = derived.resourceHandle?.resourceRevision;
if (!derivedRevisionId) {
throw new Error("fixture expected a derived revision");
}
withScopedMemoryDatabase("main", (database) => {
expect(
database
.prepare(
"SELECT parent_kind, parent_id, relation_kind FROM memory_lineage_edges WHERE child_revision_id = ?",
)
.all(derivedRevisionId),
).toEqual([
{
parent_kind: "resource-revision",
parent_id: source.resourceHandle.resourceRevision,
relation_kind: "derived-from",
},
]);
expect(
database
.prepare(
"SELECT count(*) AS count FROM memory_revision_policy_requirements WHERE revision_id = ?",
)
.get(derivedRevisionId),
).toEqual({ count: 1 });
});
setBuiltinScopedMemoryRevisionLifecycle({
agentId: "main",
revisionId: source.resourceHandle.resourceRevision,
lifecycleState: "tombstoned",
});
expect(
readBuiltinScopedMemoryRevisionSnapshot({
agentId: "main",
storeIds: [store.storeId],
revisionId: derivedRevisionId,
}),
).toBeUndefined();
expect(plan.mounts[0]?.capabilities).toEqual(["retrieve", "read", "derive"]);
});
it("records transcript policy-set lineage for an authorized compaction derivation", async () => {
const principalId = "compaction-owner";
const sourceStore = createBuiltinScopedMemoryStore({
agentId: "main",
scopeKind: "user",
audienceKind: "user",
audienceId: principalId,
authorityKind: "user",
authorityOwnerId: principalId,
defaultCapabilities: ["retrieve", "read", "derive"],
actor: { kind: "human", id: principalId },
reason: "compaction transcript fixture",
});
const source = createBuiltinScopedMemoryResource({
agentId: "main",
store: sourceStore,
logicalLocator: "MEMORY.md",
content: "COMPACTION_TRANSCRIPT_SOURCE_SENTINEL",
actor: { kind: "human", id: principalId },
});
const context = {
...createContext(principalId),
operation: "derive" as const,
} satisfies MemoryContentAccessContext<"derive">;
const plan = await builtinScopedMemoryAuthorizedRuntime.authorize(context);
const database = openOpenClawAgentDatabase({ agentId: "main" });
markCutOver();
database.db
.prepare(
`INSERT INTO session_memory_subjects
(session_key, subject_kind, binding_id, principal_id, subject_revision, created_at)
VALUES (?, 'user', ?, ?, ?, 1)`,
)
.run(context.sessionKey, `binding-${principalId}`, principalId, context.subjectRevision);
database.db
.prepare(
`INSERT INTO session_memory_subject_snapshots
(session_id, session_key, subject_revision, session_identity_revision, created_at)
VALUES (?, ?, ?, ?, 1)`,
)
.run(
context.sessionId,
context.sessionKey,
context.subjectRevision,
context.sessionIdentityRevision,
);
const exposure = prepareMemoryRunExposure({
agentId: context.agentId,
sessionId: context.sessionId,
sessionKey: context.sessionKey,
runId: context.runId,
contextFingerprint: context.contextFingerprint,
planId: plan.planId,
memoryPolicyRevision: plan.memoryPolicyRevision,
sourcePolicySetIds: [source.sourcePolicySetId],
exposedResourceRevisions: [source.revisionId],
exposureReceiptIds: ["compaction-exposure-receipt"],
egressReceiptIds: ["compaction-egress-receipt"],
deliveryAudiences: context.delivery.audiences,
deliveryRevision: context.delivery.deliveryRevision,
egressRegistryRevision: context.delivery.egressRegistryRevision,
sessionIdentityRevision: context.sessionIdentityRevision,
subjectRevision: context.subjectRevision,
actorEvidence: {
version: 1,
kind: "principal",
actorKind: "human",
principalId,
assurance: "gateway-profile",
evidenceRevision: `binding-${principalId}`,
},
delegationSnapshot: { version: 1, kind: "none" },
hostFactsRevision: context.hostFactsRevision,
});
expect(persistMemoryRunExposureBeforeContentInDatabase({ database, snapshot: exposure })).toBe(
true,
);
await withOwnedSessionTranscriptWrites(
{
sessionTarget: {
agentId: context.agentId,
expectedWriterRunId: context.runId,
sessionId: context.sessionId,
sessionKey: context.sessionKey,
},
withTranscriptWrite: async (run) => await run(),
},
async () => {
await appendSqliteTranscriptMessage(
{ agentId: context.agentId, sessionId: context.sessionId, sessionKey: context.sessionKey },
{
message: {
role: "assistant",
content: [{ type: "text", text: "COMPACTION_TRANSCRIPT_EVENT_SENTINEL" }],
},
},
);
},
);
const transcript = readAuthorizedTranscriptDerivation(database.db, context.sessionId);
if (!transcript) {
throw new Error("fixture expected an authorized transcript derivation");
}
const derived = await builtinScopedMemoryAuthorizedRuntime.writeAuthorized({
context,
plan,
mutation: {
version: 1,
kind: "derive",
derivationPurpose: "compaction",
mutationId: "compaction-derived-output",
idempotencyKey: "compaction-derived-output-request",
content: "COMPACTION_DERIVED_OUTPUT_SENTINEL",
contentType: "markdown",
sourcePolicySetId: transcript.sourcePolicySetId,
transcriptSource: {
kind: "transcript",
sessionId: context.sessionId,
eventSeqs: transcript.eventSeqs,
sourcePolicySetId: transcript.sourcePolicySetId,
deliveryAudiencesJson: transcript.deliveryAudiencesJson,
},
},
});
const derivedRevisionId = derived.resourceHandle?.resourceRevision;
if (!derivedRevisionId) {
throw new Error("fixture expected a derived revision");
}
withScopedMemoryDatabase("main", (scopedDatabase) => {
expect(
scopedDatabase
.prepare(
`SELECT parent_kind, parent_id, relation_kind
FROM memory_lineage_edges
WHERE child_revision_id = ?
ORDER BY parent_kind, parent_id, relation_kind`,
)
.all(derivedRevisionId),
).toEqual([
{
parent_kind: "resource-revision",
parent_id: source.revisionId,
relation_kind: "derived-from",
},
{
parent_kind: "transcript-policy-set",
parent_id: transcript.sourcePolicySetId,
relation_kind: "compacted-from",
},
]);
});
});
it("keeps verified private stores isolated through the actual host and selected runtime", async () => {
const aliceSession = { sessionKey: "agent:main:direct:alice", sessionId: "alice-session" };
const bobSession = { sessionKey: "agent:main:direct:bob", sessionId: "bob-session" };
@@ -599,6 +870,11 @@ describe("builtin scoped authorized runtime", () => {
createPrivateResource(alicePrincipalId, "ALICE_FINAL_REPLY_AUTHORIZED_CONTENT");
markCutOver();
installBuiltinSelectedRuntime();
registerAgentRunContext(runId, {
agentId: "main",
sessionId: session.sessionId,
sessionKey: session.sessionKey,
});
const host = createAuthorizedMemoryReadHost({
agentId: "main",
@@ -5,9 +5,12 @@ import type {
AudienceRef,
AuthorizedMemoryMutation,
AuthorizedMemoryPlan,
AuthorizedTranscriptDerivationSource,
AuthorizedMemoryReadParams,
AuthorizedMemoryResultEnvelope,
AuthorizedMemoryRuntime,
AuthorizedSealedCompactionArtifact,
AuthorizedSealedCompactionStageParams,
AuthorizedMemorySearchParams,
AuthorizedMemorySearchResult,
AuthorizedMemoryStatus,
@@ -38,6 +41,7 @@ import {
} from "./scoped-memory-db.js";
import { evaluateBuiltinScopedMemoryPolicy } from "./scoped-memory-policy.js";
import {
isBuiltinScopedMemoryRevisionLineageCurrent,
readBuiltinScopedMemoryRevisionSnapshot,
resolveBuiltinScopedMemoryArtifactPath,
} from "./scoped-memory-resources.js";
@@ -234,6 +238,8 @@ function createPlan(context: MemoryAccessContext): PlanState {
capabilities: Object.freeze(
context.operation === "read"
? (["retrieve", "read"] as const)
: context.operation === "derive"
? (["retrieve", "read", "derive"] as const)
: ([context.operation] as const),
),
audienceRevision: store.audienceRevision,
@@ -530,6 +536,27 @@ type MutableScopedRevision = Readonly<{
contentBytes: number;
}>;
type DerivationSource = Readonly<{
revisionId: string;
policyRevisionId: string;
audience: AudienceRef;
policyRequirements: readonly Readonly<{
policyId: string;
expectedRevisionId: string;
expectedRevocationEpoch: number;
}>[];
}>;
type TranscriptDerivationSource = Readonly<{
sourcePolicySetId: string;
sourceRevisionIds: readonly string[];
policyRequirements: readonly Readonly<{
policyId: string;
expectedRevisionId: string;
expectedRevocationEpoch: number;
}>[];
}>;
function contentHash(content: string): string {
return createHash("sha256").update(content).digest("hex");
}
@@ -671,11 +698,22 @@ function defaultAudience(context: MemoryAccessContext): AudienceRef | undefined
}
}
function sameAudience(left: AudienceRef, right: AudienceRef): boolean {
return left.kind === right.kind && left.id === right.id;
}
function selectWriteStore(params: {
database: Parameters<typeof getNodeSqliteKysely<ScopedMemoryDatabase>>[0];
agentId: string;
state: PlanState;
}): { storeId: string; pathKey: string; policyRevisionId: string; policyRevocationEpoch: number } {
}): {
storeId: string;
pathKey: string;
policyId: string;
policyRevisionId: string;
policyRevocationEpoch: number;
audience: AudienceRef;
} {
const audience = defaultAudience(params.state.context);
if (!audience) {
throw new Error("authorized memory mutation is unavailable");
@@ -689,7 +727,10 @@ function selectWriteStore(params: {
.innerJoin("memory_policies as policy", "policy.policy_id", "store.policy_id")
.select([
"store.store_id",
"store.audience_kind",
"store.audience_id",
"root.path_key",
"policy.policy_id",
"policy.current_revision_id",
"policy.revocation_epoch",
])
@@ -708,11 +749,272 @@ function selectWriteStore(params: {
return {
storeId: row.store_id,
pathKey: row.path_key,
policyId: row.policy_id,
policyRevisionId: row.current_revision_id,
policyRevocationEpoch: row.revocation_epoch,
audience: { kind: row.audience_kind, id: row.audience_id },
};
}
/** A derive mutation can only retain exact same-audience sources; mixed sets are denied, not widened. */
function resolveDerivationSources(params: {
database: Parameters<typeof getNodeSqliteKysely<ScopedMemoryDatabase>>[0];
agentId: string;
state: PlanState;
targetAudience: AudienceRef;
sourceHandles: readonly AuthorizedResourceHandle[];
sourcePolicySetId: string;
}): readonly DerivationSource[] {
if (params.sourceHandles.length === 0 || !params.sourcePolicySetId.trim()) {
throw new Error("authorized memory derivation is unavailable");
}
const db = getNodeSqliteKysely<ScopedMemoryDatabase>(params.database);
const sources: DerivationSource[] = [];
for (const handle of params.sourceHandles) {
const stored = params.state.handles.get(handle.handleId);
if (
!stored ||
stored.planId !== params.state.plan.planId ||
stored.contextFingerprint !== params.state.contextFingerprint ||
stored.resourceRevision !== handle.resourceRevision ||
stored.policyRevision !== handle.policyRevision ||
stored.expiresAt !== handle.expiresAt
) {
throw new Error("authorized memory derivation is unavailable");
}
const snapshot = readBuiltinScopedMemoryRevisionSnapshot({
agentId: params.agentId,
storeIds: params.state.stores.map((store) => store.storeId),
revisionId: stored.resourceRevision,
});
if (!snapshot || snapshot.policyRevisionId !== stored.policyRevision) {
throw new Error("authorized memory derivation is unavailable");
}
const source = executeSqliteQueryTakeFirstSync(
params.database,
db
.selectFrom("memory_resource_revisions as revision")
.innerJoin("memory_resources as resource", "resource.resource_id", "revision.resource_id")
.innerJoin("memory_stores as store", "store.store_id", "resource.store_id")
.select(["store.audience_kind", "store.audience_id"])
.where("revision.revision_id", "=", stored.resourceRevision)
.where("resource.agent_id", "=", params.agentId),
);
const audience = source && { kind: source.audience_kind, id: source.audience_id };
if (!audience || !sameAudience(audience, params.targetAudience)) {
throw new Error("authorized memory derivation has no representable audience");
}
const policyRequirements = executeSqliteQuerySync(
params.database,
db
.selectFrom("memory_revision_policy_requirements")
.select(["policy_id", "expected_revision_id", "expected_revocation_epoch"])
.where("revision_id", "=", stored.resourceRevision)
.orderBy("policy_id"),
).rows.map((requirement) =>
Object.freeze({
policyId: requirement.policy_id,
expectedRevisionId: requirement.expected_revision_id,
expectedRevocationEpoch: requirement.expected_revocation_epoch,
}),
);
if (policyRequirements.length === 0) {
throw new Error("authorized memory derivation is unavailable");
}
sources.push(
Object.freeze({
revisionId: stored.resourceRevision,
policyRevisionId: snapshot.policyRevisionId,
audience,
policyRequirements: Object.freeze(policyRequirements),
}),
);
}
if (new Set(sources.map((source) => source.revisionId)).size !== sources.length) {
throw new Error("authorized memory derivation is unavailable");
}
const expectedPolicySetId = `mpset1_${hash(
sources.map((source) => `mps1_${source.policyRevisionId}`).toSorted(),
)}`;
if (params.sourcePolicySetId !== expectedPolicySetId) {
throw new Error("authorized memory derivation is unavailable");
}
return Object.freeze(sources);
}
/** Transcript companions are an immutable source set, not a mutable session-path permission. */
function resolveTranscriptDerivationSource(params: {
database: Parameters<typeof getNodeSqliteKysely<ScopedMemoryDatabase>>[0];
source: AuthorizedTranscriptDerivationSource;
sourcePolicySetId: string;
targetAudience: AudienceRef;
}): TranscriptDerivationSource {
const source = params.source;
if (
!source ||
source.kind !== "transcript" ||
source.sourcePolicySetId !== params.sourcePolicySetId ||
!source.sessionId.trim() ||
source.eventSeqs.length === 0
) {
throw new Error("authorized transcript derivation is unavailable");
}
const eventSeqs = [...source.eventSeqs];
if (
eventSeqs.some((eventSeq) => !Number.isSafeInteger(eventSeq) || eventSeq < 0) ||
new Set(eventSeqs).size !== eventSeqs.length ||
eventSeqs.some((eventSeq, index) => index > 0 && eventSeqs[index - 1]! >= eventSeq)
) {
throw new Error("authorized transcript derivation is unavailable");
}
let audiences: unknown;
try {
audiences = JSON.parse(source.deliveryAudiencesJson);
} catch {
throw new Error("authorized transcript derivation is unavailable");
}
if (
!Array.isArray(audiences) ||
audiences.length !== 1 ||
typeof audiences[0] !== "object" ||
audiences[0] === null ||
(audiences[0] as AudienceRef).kind !== params.targetAudience.kind ||
(audiences[0] as AudienceRef).id !== params.targetAudience.id
) {
throw new Error("authorized memory derivation has no representable audience");
}
const db = getNodeSqliteKysely<ScopedMemoryDatabase>(params.database);
const subject = executeSqliteQueryTakeFirstSync(
params.database,
db
.selectFrom("session_memory_subject_snapshots")
.select(["session_identity_revision", "subject_revision"])
.where("session_id", "=", source.sessionId)
.limit(1),
);
if (!subject) {
throw new Error("authorized transcript derivation is unavailable");
}
const events = executeSqliteQuerySync(
params.database,
db
.selectFrom("transcript_events as event")
.innerJoin("transcript_event_memory_policies as policy", (join) =>
join
.onRef("policy.session_id", "=", "event.session_id")
.onRef("policy.event_seq", "=", "event.seq"),
)
.innerJoin("transcript_event_memory_policy_details as detail", (join) =>
join
.onRef("detail.session_id", "=", "policy.session_id")
.onRef("detail.event_seq", "=", "policy.event_seq"),
)
.select([
"event.seq",
"policy.delivery_audiences_json",
"policy.run_exposure_set_id",
"policy.session_identity_revision",
"policy.subject_revision",
])
.where("event.session_id", "=", source.sessionId)
.where("event.seq", "in", eventSeqs)
.where("policy.authorization_status", "=", "authorized")
.where("policy.source_policy_set_id", "=", source.sourcePolicySetId)
.where("policy.delivery_audiences_json", "=", source.deliveryAudiencesJson)
.where("detail.retention_state", "=", "retained")
.where("detail.normalized_audience_intersection_json", "=", source.deliveryAudiencesJson)
.where("detail.finalized_delivery_audiences_json", "=", source.deliveryAudiencesJson)
.orderBy("event.seq"),
).rows;
if (
events.length !== eventSeqs.length ||
events.some(
(event, index) =>
event.seq !== eventSeqs[index] ||
event.run_exposure_set_id === null ||
event.session_identity_revision !== subject.session_identity_revision ||
event.subject_revision !== subject.subject_revision,
)
) {
throw new Error("authorized transcript derivation is unavailable");
}
const requirements = executeSqliteQuerySync(
params.database,
db
.selectFrom("memory_policy_set_members")
.select(["policy_id", "expected_revision_id", "expected_revocation_epoch", "retention_state"])
.where("policy_set_id", "=", source.sourcePolicySetId)
.orderBy("policy_id"),
).rows;
if (requirements.length === 0 || requirements.some((requirement) => requirement.retention_state !== "retained")) {
throw new Error("authorized transcript derivation is unavailable");
}
for (const requirement of requirements) {
const current = executeSqliteQueryTakeFirstSync(
params.database,
db
.selectFrom("memory_policies as policy")
.innerJoin(
"memory_policy_revisions as revision",
"revision.revision_id",
"policy.current_revision_id",
)
.select(["policy.lifecycle_state", "policy.revocation_epoch", "revision.lifecycle_state as revision_state"])
.where("policy.policy_id", "=", requirement.policy_id)
.where("policy.current_revision_id", "=", requirement.expected_revision_id)
.where("policy.revocation_epoch", "=", requirement.expected_revocation_epoch)
.limit(1),
);
if (current?.lifecycle_state !== "active" || current.revision_state !== "active") {
throw new Error("authorized transcript derivation is unavailable");
}
}
const exposureSetIds = [...new Set(events.map((event) => event.run_exposure_set_id!))];
const exposures = executeSqliteQuerySync(
params.database,
db
.selectFrom("memory_run_exposure_resources as exposure")
.innerJoin(
"memory_resource_revisions as revision",
"revision.revision_id",
"exposure.resource_revision_id",
)
.select([
"exposure.exposure_set_id",
"exposure.resource_revision_id",
"revision.expires_at",
"revision.lifecycle_state",
])
.where("exposure.exposure_set_id", "in", exposureSetIds),
).rows;
const nowMs = Date.now();
if (
exposureSetIds.some((exposureSetId) => !exposures.some((row) => row.exposure_set_id === exposureSetId)) ||
exposures.some(
(exposure) =>
exposure.lifecycle_state !== "active" ||
(exposure.expires_at !== null && exposure.expires_at <= nowMs),
)
) {
throw new Error("authorized transcript derivation is unavailable");
}
return Object.freeze({
sourcePolicySetId: source.sourcePolicySetId,
sourceRevisionIds: Object.freeze(
[...new Set(exposures.map((exposure) => exposure.resource_revision_id))].toSorted(),
),
policyRequirements: Object.freeze(
requirements.map((requirement) =>
Object.freeze({
policyId: requirement.policy_id,
expectedRevisionId: requirement.expected_revision_id,
expectedRevocationEpoch: requirement.expected_revocation_epoch,
}),
),
),
});
}
function resolveWriteTarget(params: {
database: Parameters<typeof getNodeSqliteKysely<ScopedMemoryDatabase>>[0];
agentId: string;
@@ -1005,6 +1307,9 @@ function recoverPendingWrites(agentId: string): void {
});
continue;
}
// The recovery transaction closes over this value. Keep the validated
// revision immutable so it cannot become an absent intent reference.
const revisionId = intent.pending_revision_id;
const directory = path.join(resolveScopedMemoryArtifactBase(databasePath), intent.path_key);
const known = knownByDirectory.get(directory) ?? new Set<string>();
known.add(intent.final_locator);
@@ -1052,7 +1357,7 @@ function recoverPendingWrites(agentId: string): void {
quarantineWriteIntent({
database,
intentId: intent.intent_id,
revisionId: intent.pending_revision_id,
revisionId,
nowMs: Date.now(),
reasonCode: "recovery-artifact-mismatch",
});
@@ -1083,7 +1388,7 @@ function recoverPendingWrites(agentId: string): void {
"policy.current_revision_id",
"policy.revocation_epoch",
])
.where("revision.revision_id", "=", intent.pending_revision_id)
.where("revision.revision_id", "=", revisionId)
.where("resource.agent_id", "=", agentId)
.where("store.lifecycle_state", "=", "active")
.where("policy.lifecycle_state", "=", "active"),
@@ -1091,7 +1396,11 @@ function recoverPendingWrites(agentId: string): void {
if (
!revision ||
revision.policy_revision_id !== revision.current_revision_id ||
revision.policy_revocation_epoch !== revision.revocation_epoch
revision.policy_revocation_epoch !== revision.revocation_epoch ||
!isBuiltinScopedMemoryRevisionLineageCurrent({
agentId,
revisionId,
})
) {
policyChanged = true;
requireExactlyOneAffected(
@@ -1100,7 +1409,7 @@ function recoverPendingWrites(agentId: string): void {
db
.updateTable("memory_resource_revisions")
.set({ lifecycle_state: "quarantined", retired_at: Date.now() })
.where("revision_id", "=", intent.pending_revision_id)
.where("revision_id", "=", revisionId)
.where("lifecycle_state", "=", "pending"),
),
"recovery policy quarantine revision",
@@ -1136,7 +1445,7 @@ function recoverPendingWrites(agentId: string): void {
db
.updateTable("memory_resource_revisions")
.set({ lifecycle_state: "active", activated_at: Date.now() })
.where("revision_id", "=", intent.pending_revision_id)
.where("revision_id", "=", revisionId)
.where("lifecycle_state", "=", "pending"),
),
"recovery activation revision",
@@ -1174,7 +1483,7 @@ function recoverPendingWrites(agentId: string): void {
indexRecoveredRevision({
database,
intentId: intent.intent_id,
revisionId: intent.pending_revision_id,
revisionId,
content,
nowMs: Date.now(),
});
@@ -1233,6 +1542,30 @@ async function writeAuthorizedMutation(params: {
const result = withScopedMemoryDatabase(agentId, (database, databasePath) => {
const store = selectWriteStore({ database, agentId, state });
const db = getNodeSqliteKysely<ScopedMemoryDatabase>(database);
const derivationSources =
params.mutation.kind === "derive" && "sourceHandles" in params.mutation
? resolveDerivationSources({
database,
agentId,
state,
targetAudience: store.audience,
sourceHandles: params.mutation.sourceHandles,
sourcePolicySetId: params.mutation.sourcePolicySetId,
})
: [];
const transcriptDerivationSource =
params.mutation.kind === "derive" && "transcriptSource" in params.mutation
? resolveTranscriptDerivationSource({
database,
source: params.mutation.transcriptSource,
sourcePolicySetId: params.mutation.sourcePolicySetId,
targetAudience: store.audience,
})
: undefined;
const transcriptDerivationPurpose =
params.mutation.kind === "derive" && "transcriptSource" in params.mutation
? params.mutation.derivationPurpose
: undefined;
const existing =
"target" in params.mutation
? resolveWriteTarget({
@@ -1503,6 +1836,90 @@ async function writeAuthorizedMutation(params: {
retired_at: null,
}),
);
executeSqliteQuerySync(
database,
db
.insertInto("memory_revision_policy_requirements")
.values({
revision_id: revisionId,
policy_id: store.policyId,
expected_revision_id: store.policyRevisionId,
expected_revocation_epoch: store.policyRevocationEpoch,
requirement_kind: "output-policy",
created_at: nowMs,
}),
);
for (const source of derivationSources) {
for (const requirement of source.policyRequirements) {
executeSqliteQuerySync(
database,
db
.insertInto("memory_revision_policy_requirements")
.values({
revision_id: revisionId,
policy_id: requirement.policyId,
expected_revision_id: requirement.expectedRevisionId,
expected_revocation_epoch: requirement.expectedRevocationEpoch,
requirement_kind: "source-policy",
created_at: nowMs,
})
.onConflict((conflict) => conflict.columns(["revision_id", "policy_id"]).doNothing()),
);
}
executeSqliteQuerySync(
database,
db.insertInto("memory_lineage_edges").values({
child_revision_id: revisionId,
parent_kind: "resource-revision",
parent_id: source.revisionId,
relation_kind: "derived-from",
created_at: nowMs,
}),
);
}
if (transcriptDerivationSource && transcriptDerivationPurpose) {
for (const requirement of transcriptDerivationSource.policyRequirements) {
executeSqliteQuerySync(
database,
db
.insertInto("memory_revision_policy_requirements")
.values({
revision_id: revisionId,
policy_id: requirement.policyId,
expected_revision_id: requirement.expectedRevisionId,
expected_revocation_epoch: requirement.expectedRevocationEpoch,
requirement_kind: "source-policy",
created_at: nowMs,
})
.onConflict((conflict) => conflict.columns(["revision_id", "policy_id"]).doNothing()),
);
}
for (const sourceRevisionId of transcriptDerivationSource.sourceRevisionIds) {
executeSqliteQuerySync(
database,
db.insertInto("memory_lineage_edges").values({
child_revision_id: revisionId,
parent_kind: "resource-revision",
parent_id: sourceRevisionId,
relation_kind: "derived-from",
created_at: nowMs,
}),
);
}
executeSqliteQuerySync(
database,
db.insertInto("memory_lineage_edges").values({
child_revision_id: revisionId,
parent_kind: "transcript-policy-set",
parent_id: transcriptDerivationSource.sourcePolicySetId,
relation_kind:
transcriptDerivationPurpose === "compaction"
? "compacted-from"
: "flushed-from",
created_at: nowMs,
}),
);
}
executeSqliteQuerySync(
database,
db.insertInto("memory_write_intents").values({
@@ -1577,6 +1994,29 @@ async function writeAuthorizedMutation(params: {
) {
throw new Error("authorized memory mutation is unavailable");
}
if (params.mutation.kind === "derive") {
// The model work and artifact rename already happened; this final synchronous reread is
// the authority boundary that prevents a revoke or parent tombstone from activating it.
if ("sourceHandles" in params.mutation) {
resolveDerivationSources({
database,
agentId,
state,
targetAudience: currentStore.audience,
sourceHandles: params.mutation.sourceHandles,
sourcePolicySetId: params.mutation.sourcePolicySetId,
});
} else if ("transcriptSource" in params.mutation) {
resolveTranscriptDerivationSource({
database,
source: params.mutation.transcriptSource,
sourcePolicySetId: params.mutation.sourcePolicySetId,
targetAudience: currentStore.audience,
});
} else {
throw new Error("authorized memory derivation is unavailable");
}
}
if (existing) {
executeSqliteQuerySync(
database,
@@ -1669,6 +2109,263 @@ async function writeAuthorizedMutation(params: {
return result;
}
/**
* Files are finalized before the core transaction starts. A crash before commit
* leaves only an uncatalogued artifact, which recovery quarantines; a committed
* revision never needs a filesystem operation to become readable.
*/
async function stageSealedCompaction(
params: AuthorizedSealedCompactionStageParams,
): Promise<AuthorizedSealedCompactionArtifact> {
if (!params.content.trim()) {
throw new Error("sealed compaction content is unavailable");
}
const state = readPlan({ context: params.context, plan: params.plan });
if (!state || state.expiresAtMs <= Date.now()) {
throw new Error("sealed compaction authorization is unavailable");
}
const agentId = params.context.agentId;
return withScopedMemoryDatabase(agentId, (database, databasePath) => {
const store = selectWriteStore({ database, agentId, state });
const source = resolveTranscriptDerivationSource({
database,
source: params.transcriptSource,
sourcePolicySetId: params.transcriptSource.sourcePolicySetId,
targetAudience: store.audience,
});
const revisionId = randomUUID();
const resourceId = randomUUID();
const intentId = randomUUID();
const finalLocator = `r1_${revisionId}.md`;
const stageLocator = `scst1_${intentId}.tmp`;
const directory = path.join(resolveScopedMemoryArtifactBase(databasePath), store.pathKey);
const finalPath = resolveBuiltinScopedMemoryArtifactPath({
databasePath,
pathKey: store.pathKey,
artifactLocator: finalLocator,
});
const stagePath = path.join(directory, stageLocator);
const hash = contentHash(params.content);
const bytes = Buffer.byteLength(params.content);
fs.mkdirSync(directory, { recursive: true, mode: 0o700 });
const descriptor = fs.openSync(stagePath, "wx", 0o600);
try {
fs.writeFileSync(descriptor, params.content, "utf8");
fs.fchmodSync(descriptor, 0o600);
fs.fsyncSync(descriptor);
} finally {
fs.closeSync(descriptor);
}
syncDirectory(directory);
fs.renameSync(stagePath, finalPath);
syncDirectory(directory);
const verified = readVerifiedFile({
pathname: finalPath,
contentHash: hash,
contentBytes: bytes,
});
if (verified === undefined) {
throw new Error("sealed compaction artifact is unavailable");
}
return Object.freeze({
resourceRevisionId: revisionId,
commitInTransaction({ database: transactionDatabase, compactionPolicyId, eventSeq }) {
if (state.expiresAtMs <= Date.now()) {
throw new Error("sealed compaction authorization is unavailable");
}
const currentStore = selectWriteStore({
database: transactionDatabase,
agentId,
state,
});
if (
currentStore.storeId !== store.storeId ||
currentStore.policyRevisionId !== store.policyRevisionId ||
currentStore.policyRevocationEpoch !== store.policyRevocationEpoch
) {
throw new Error("sealed compaction authorization is unavailable");
}
const currentSource = resolveTranscriptDerivationSource({
database: transactionDatabase,
source: params.transcriptSource,
sourcePolicySetId: params.transcriptSource.sourcePolicySetId,
targetAudience: currentStore.audience,
});
if (
currentSource.sourcePolicySetId !== source.sourcePolicySetId ||
currentSource.sourceRevisionIds.length !== source.sourceRevisionIds.length ||
currentSource.sourceRevisionIds.some(
(revision, index) => revision !== source.sourceRevisionIds[index],
)
) {
throw new Error("sealed compaction source is unavailable");
}
const db = getNodeSqliteKysely<ScopedMemoryDatabase>(transactionDatabase);
const nowMs = Date.now();
executeSqliteQuerySync(
transactionDatabase,
db.insertInto("memory_resources").values({
resource_id: resourceId,
agent_id: agentId,
store_id: store.storeId,
logical_locator: `memory/${revisionId}.md`,
source: "memory",
created_at: nowMs,
}),
);
executeSqliteQuerySync(
transactionDatabase,
db.insertInto("memory_resource_revisions").values({
revision_id: revisionId,
resource_id: resourceId,
revision_number: 1,
artifact_locator: finalLocator,
content_hash: hash,
content_bytes: bytes,
policy_revision_id: store.policyRevisionId,
policy_revocation_epoch: store.policyRevocationEpoch,
source_policy_set_id: createScopedMemorySourcePolicySetId(store.policyRevisionId),
lifecycle_state: "active",
actor_kind:
params.context.actor.kind === "principal"
? params.context.actor.actorKind
: "unattributed",
actor_id:
params.context.actor.kind === "principal" ? params.context.actor.principalId : null,
expires_at: null,
created_at: nowMs,
activated_at: nowMs,
retired_at: null,
}),
);
executeSqliteQuerySync(
transactionDatabase,
db.insertInto("memory_revision_policy_requirements").values({
revision_id: revisionId,
policy_id: store.policyId,
expected_revision_id: store.policyRevisionId,
expected_revocation_epoch: store.policyRevocationEpoch,
requirement_kind: "output-policy",
created_at: nowMs,
}),
);
for (const requirement of source.policyRequirements) {
executeSqliteQuerySync(
transactionDatabase,
db
.insertInto("memory_revision_policy_requirements")
.values({
revision_id: revisionId,
policy_id: requirement.policyId,
expected_revision_id: requirement.expectedRevisionId,
expected_revocation_epoch: requirement.expectedRevocationEpoch,
requirement_kind: "source-policy",
created_at: nowMs,
})
.onConflict((conflict) =>
conflict.columns(["revision_id", "policy_id"]).doNothing(),
),
);
}
const lineage = [
...source.sourceRevisionIds.map((parentId) => ({
parent_kind: "resource-revision" as const,
parent_id: parentId,
relation_kind: "derived-from" as const,
})),
{
parent_kind: "transcript-policy-set" as const,
parent_id: source.sourcePolicySetId,
relation_kind: "compacted-from" as const,
},
{
parent_kind: "compaction-policy" as const,
parent_id: compactionPolicyId,
relation_kind: "compacted-from" as const,
},
];
for (const parent of lineage) {
executeSqliteQuerySync(
transactionDatabase,
db.insertInto("memory_lineage_edges").values({
child_revision_id: revisionId,
...parent,
created_at: nowMs,
}),
);
}
const chunks = chunkContent(verified);
if (chunks.length > 0) {
executeSqliteQuerySync(
transactionDatabase,
db.insertInto("memory_scoped_chunks").values(
chunks.map((chunk) => ({
chunk_id: randomUUID(),
revision_id: revisionId,
chunk_ordinal: chunk.ordinal,
start_line: chunk.startLine,
end_line: chunk.endLine,
text: chunk.text,
content_hash: contentHash(chunk.text),
model: "builtin-markdown-v1",
updated_at: nowMs,
})),
),
);
}
executeSqliteQuerySync(
transactionDatabase,
db.insertInto("memory_write_intents").values({
intent_id: intentId,
idempotency_key: `sealed-compaction:${compactionPolicyId}`,
mutation_id: compactionPolicyId,
agent_id: agentId,
request_id: params.context.requestId,
run_id: params.context.runId,
context_fingerprint: params.context.contextFingerprint,
plan_id: params.plan.planId,
mutation_kind: "derive",
store_id: store.storeId,
resource_id: resourceId,
pending_revision_id: revisionId,
staged_locator: stageLocator,
final_locator: finalLocator,
content_hash: hash,
content_bytes: bytes,
state: "active",
created_at: nowMs,
updated_at: nowMs,
activated_at: nowMs,
indexed_at: nowMs,
}),
);
executeSqliteQuerySync(
transactionDatabase,
db.insertInto("memory_audit_outbox").values({
event_id: randomUUID(),
intent_id: intentId,
agent_id: agentId,
request_id: params.context.requestId,
run_id: params.context.runId,
actor_ref: auditActorRef(params.context),
subject_ref: auditSubjectRef(params.context),
operation: params.context.operation,
resource_revision_id: revisionId,
content_hash: hash,
decision: "committed",
reason_code: `sealed-compaction:${eventSeq}`,
state: "pending",
attempts: 0,
created_at: nowMs,
updated_at: nowMs,
delivered_at: null,
}),
);
},
});
});
}
const builtinScopedMemoryRuntime = {
async authorize(context: MemoryAccessContext): Promise<AuthorizedMemoryPlan> {
recoverPendingWrites(context.agentId);
@@ -1679,9 +2376,9 @@ const builtinScopedMemoryRuntime = {
},
async searchAuthorized(
params: AuthorizedMemorySearchParams<"read">,
params: AuthorizedMemorySearchParams<"read"> | AuthorizedMemorySearchParams<"derive">,
): Promise<AuthorizedMemoryResultEnvelope<readonly AuthorizedMemorySearchResult[]>> {
if (params.context.operation !== "read") {
if (params.context.operation !== "read" && params.context.operation !== "derive") {
throw new Error("authorized memory search is unavailable");
}
const state = readPlan(params);
@@ -1738,9 +2435,9 @@ const builtinScopedMemoryRuntime = {
},
async readAuthorized(
params: AuthorizedMemoryReadParams<"read">,
params: AuthorizedMemoryReadParams<"read"> | AuthorizedMemoryReadParams<"derive">,
): Promise<AuthorizedMemoryResultEnvelope<MemoryReadResult>> {
if (params.context.operation !== "read") {
if (params.context.operation !== "read" && params.context.operation !== "derive") {
throw new Error("authorized memory read is unavailable");
}
const state = readPlan(params);
@@ -1797,6 +2494,10 @@ const builtinScopedMemoryRuntime = {
return await writeAuthorizedMutation(params);
},
async stageSealedCompaction(params: AuthorizedSealedCompactionStageParams) {
return await stageSealedCompaction(params);
},
async importAuthorized(params: {
context: MemoryAccessContext;
plan: AuthorizedMemoryPlan;
@@ -15,6 +15,8 @@ import {
storeMemoryPreimage,
} from "./dreaming-consolidation-artifacts.js";
import {
hasOnePromotionAuthorizedView,
isPromotionAuthorizedViewBlocked,
isConsolidationCandidateEligible,
isPromotionOriginBlocked,
} from "./dreaming-consolidation-candidates.js";
@@ -33,6 +35,7 @@ import {
writeMemoryContent,
} from "./short-term-promotion-memory-write.js";
import {
buildPromotionAuthorizedViewAnnotation,
buildPromotionRecallAnnotations,
groupPromotionCandidatesByProjectKey,
} from "./short-term-promotion-metadata.js";
@@ -86,7 +89,7 @@ function buildPromotionSection(
// rehydrated snippet so ranking, provenance, and dream narratives remain
// tied to the source entry instead of this presentation budget.
lines.push(
`- ${formatPromotedSnippetForMemory(candidate.snippet, maxPromotedSnippetTokens)} ${metadata} ${buildPromotionRecallAnnotations(candidate)}`,
`- ${formatPromotedSnippetForMemory(candidate.snippet, maxPromotedSnippetTokens)} ${metadata} ${buildPromotionRecallAnnotations(candidate)}${buildPromotionAuthorizedViewAnnotation(candidate)}`,
);
}
if (projectGroups.length > 1) {
@@ -165,6 +168,7 @@ function consolidationCandidateFingerprint(candidate: PromotionCandidate): strin
endLine: candidate.endLine,
snippet: candidate.snippet,
provenance: candidate.provenance,
authorizedView: candidate.authorizedView,
projectKey: candidate.projectKey,
});
}
@@ -277,6 +281,7 @@ export async function applyShortTermPromotions(
startLine: entry.startLine,
endLine: entry.endLine,
snippet: entry.snippet,
...(entry.authorizedView ? { authorizedView: entry.authorizedView } : {}),
},
entry.provenance,
)
@@ -295,21 +300,23 @@ export async function applyShortTermPromotions(
// legitimate daily-note candidates stay eligible.
const reason = isPromotionOriginBlocked(candidate)
? `origin filter (${candidate.provenance?.originClass})`
: options.consolidation && (!latest || !isConsolidationCandidateEligible(candidate))
? "consolidation origin/session filter"
: isContaminatedDreamingSnippet(candidate.snippet)
? "contamination filter"
: candidate.promotedAt || latest?.promotedAt
? "already promoted"
: candidate.score < minScore
? `score threshold (${candidate.score.toFixed(3)} < ${minScore})`
: candidate.signalCount < minRecallCount
? `signal threshold (${candidate.signalCount} < ${minRecallCount})`
: queryCount < minUniqueQueries
? `query threshold (${queryCount} < ${minUniqueQueries})`
: maxAgeDays >= 0 && candidate.ageDays > maxAgeDays
? `age threshold (${candidate.ageDays.toFixed(1)}d > ${maxAgeDays}d)`
: undefined;
: isPromotionAuthorizedViewBlocked(candidate)
? "authorized view filter"
: options.consolidation && (!latest || !isConsolidationCandidateEligible(candidate))
? "consolidation origin/session filter"
: isContaminatedDreamingSnippet(candidate.snippet)
? "contamination filter"
: candidate.promotedAt || latest?.promotedAt
? "already promoted"
: candidate.score < minScore
? `score threshold (${candidate.score.toFixed(3)} < ${minScore})`
: candidate.signalCount < minRecallCount
? `signal threshold (${candidate.signalCount} < ${minRecallCount})`
: queryCount < minUniqueQueries
? `query threshold (${queryCount} < ${minUniqueQueries})`
: maxAgeDays >= 0 && candidate.ageDays > maxAgeDays
? `age threshold (${candidate.ageDays.toFixed(1)}d > ${maxAgeDays}d)`
: undefined;
if (reason) {
rejectionReasons.set(candidate.key, reason);
}
@@ -319,10 +326,16 @@ export async function applyShortTermPromotions(
for (const candidate of eligible.slice(limit)) {
rejectionReasons.set(candidate.key, `selection limit (${limit})`);
}
const selectedInOneAuthorizedView = hasOnePromotionAuthorizedView(selected) ? selected : [];
if (selectedInOneAuthorizedView.length === 0 && selected.length > 0) {
for (const candidate of selected) {
rejectionReasons.set(candidate.key, "authorized view boundary");
}
}
const rehydratedSelected: PromotionCandidate[] = [];
const plannedSourceFingerprints = new Map<string, string>();
for (const candidate of selected) {
for (const candidate of selectedInOneAuthorizedView) {
const sourceFingerprintBefore = await promotionSourceFingerprint(workspaceDir, candidate);
const rehydrated = await rehydratePromotionCandidate(workspaceDir, candidate);
const sourceFingerprintAfter = await promotionSourceFingerprint(workspaceDir, candidate);
@@ -454,6 +467,9 @@ export async function applyShortTermPromotions(
continue;
}
const currentCandidate = withAuthoritativeProvenance(candidate, entry.provenance);
if (isPromotionAuthorizedViewBlocked(currentCandidate)) {
continue;
}
if (options.consolidation && !isConsolidationCandidateEligible(currentCandidate)) {
continue;
}
@@ -67,3 +67,16 @@ export function buildPromotionRecallAnnotations(
const project = projectKey ? ` <!-- project: ${projectKey} -->` : "";
return `<!-- trigger: ${triggers} --> <!-- importance: ${importance} -->${project}`;
}
export function buildPromotionAuthorizedViewAnnotation(
candidate: Pick<PromotionCandidate, "authorizedView">,
): string {
const view = candidate.authorizedView;
if (!view) {
return "";
}
// Keep opaque authority identifiers inert in markdown while recording the
// immutable source revision beside the durable promotion.
const encode = (value: string) => encodeURIComponent(value);
return ` <!-- authorized-view: store=${encode(view.storeId)} view=${encode(view.viewId)} revision=${encode(view.resourceRevision)} -->`;
}
@@ -9,7 +9,11 @@ import { appendMemoryHostEvent } from "openclaw/plugin-sdk/memory-host-events";
import pLimit from "p-limit";
import { deriveConceptTags } from "./concept-vocabulary.js";
import { readStore, withShortTermLock, writeStore } from "./short-term-promotion-store.js";
import type { ShortTermRecallEntry, ShortTermRecallStore } from "./short-term-promotion-types.js";
import type {
ShortTermPromotionAuthorizedView,
ShortTermRecallEntry,
ShortTermRecallStore,
} from "./short-term-promotion-types.js";
import {
buildDailyClaimEntryKey,
buildClaimHash,
@@ -25,6 +29,7 @@ import {
mergeRecentDistinct,
normalizeIsoDay,
normalizeMemoryPath,
normalizeShortTermPromotionAuthorizedView,
normalizeSnippet,
truncateShortTermSnippet,
} from "./short-term-promotion-utils.js";
@@ -33,6 +38,21 @@ import { resolveMemoryCoreNowMs, resolveMemoryCoreTimestamp } from "./time.js";
// One recall batch can inspect every retained entry; cap filesystem pressure.
const SHORT_TERM_SOURCE_FILE_CHECK_CONCURRENCY = 32;
function promotionAuthorityKey(view: ShortTermPromotionAuthorizedView): string {
return hashQuery(`${view.storeId}\u0000${view.viewId}\u0000${view.resourceRevision}`);
}
function samePromotionAuthority(
left: ShortTermPromotionAuthorizedView | undefined,
right: ShortTermPromotionAuthorizedView | undefined,
): boolean {
return (
left?.storeId === right?.storeId &&
left?.viewId === right?.viewId &&
left?.resourceRevision === right?.resourceRevision
);
}
function mergeRecallProvenance(
existing: MemoryEntryProvenance | undefined,
incoming: MemoryEntryProvenance | undefined,
@@ -151,7 +171,12 @@ async function updateShortTermRecallStore(
export async function recordShortTermRecalls(params: {
workspaceDir?: string;
query: string;
results: Array<MemorySearchResult & { identitySnippet?: string }>;
results: Array<
MemorySearchResult & {
identitySnippet?: string;
authorizedView?: ShortTermPromotionAuthorizedView;
}
>;
signalType?: "recall" | "daily";
dedupeByQueryPerDay?: boolean;
dayBucket?: string;
@@ -196,6 +221,10 @@ export async function recordShortTermRecalls(params: {
nowIso,
(store) => {
for (const result of relevant) {
const authorizedView = normalizeShortTermPromotionAuthorizedView(result.authorizedView);
if (result.authorizedView !== undefined && !authorizedView) {
continue;
}
const normalizedPath = normalizeMemoryPath(result.path);
const rawSnippet = normalizeSnippet(result.snippet);
const snippet = truncateShortTermSnippet(rawSnippet);
@@ -220,7 +249,10 @@ export async function recordShortTermRecalls(params: {
Math.max(0, Math.floor(entry.recallCount ?? 0)) +
Math.max(0, Math.floor(entry.groundedCount ?? 0)) >
0 &&
entry.claimHash === claimHash,
entry.claimHash === claimHash &&
// Do not merge evidence from different scoped views or immutable revisions.
// A review state transition keeps the same identity and may update lifecycle.
samePromotionAuthority(entry.authorizedView, authorizedView),
)
: undefined;
// Interactive/grounded writers retain their path-qualified identity. Do
@@ -240,13 +272,18 @@ export async function recordShortTermRecalls(params: {
})
: null;
const baseKey = buildEntryKey(result);
const key =
const unscopedKey =
nonDailyEntry?.key ??
(signalType === "daily" && groundedKey
? groundedKey
: groundedKey && store.entries[groundedKey]
? groundedKey
: baseKey);
const key = nonDailyEntry?.key
? nonDailyEntry.key
: authorizedView
? `${unscopedKey}:view:${promotionAuthorityKey(authorizedView)}`
: unscopedKey;
const existing = store.entries[key];
const score = clampScore(result.score);
const recallDaysBase = existing?.recallDays ?? [];
@@ -307,6 +344,7 @@ export async function recordShortTermRecalls(params: {
recallDays,
conceptTags: conceptTags.length > 0 ? conceptTags : (existing?.conceptTags ?? []),
provenance,
...(authorizedView ? { authorizedView } : {}),
claimHash,
...(projectKey ? { projectKey } : {}),
...(existing?.promotedAt ? { promotedAt: existing.promotedAt } : {}),
@@ -30,6 +30,18 @@ export type PromotionWeights = {
conceptual: number;
};
/**
* Opaque authority carried from a scoped read. `resourceRevision` is the
* immutable source record; promotion must never treat a workspace path as a
* substitute for that provenance.
*/
export type ShortTermPromotionAuthorizedView = {
storeId: string;
viewId: string;
resourceRevision: string;
lifecycle: "active" | "postbox" | "quarantine";
};
export type ShortTermRecallEntry = {
key: string;
path: string;
@@ -51,6 +63,7 @@ export type ShortTermRecallEntry = {
projectKey?: string;
promotedAt?: string;
provenance?: MemoryEntryProvenance;
authorizedView?: ShortTermPromotionAuthorizedView;
};
export type ShortTermRecallStore = {
@@ -117,6 +130,7 @@ export type PromotionCandidate = {
conceptTags: string[];
components: PromotionComponents;
provenance?: MemoryEntryProvenance;
authorizedView?: ShortTermPromotionAuthorizedView;
};
export type ShortTermAuditIssue = {
@@ -7,6 +7,7 @@ import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime";
import { deriveConceptTags, MAX_CONCEPT_TAGS } from "./concept-vocabulary.js";
import type {
PromotionWeights,
ShortTermPromotionAuthorizedView,
ShortTermRecallEntry,
ShortTermRecallStore,
} from "./short-term-promotion-types.js";
@@ -40,6 +41,29 @@ const DEFAULT_PROMOTION_WEIGHTS: PromotionWeights = {
conceptual: 0.06,
};
export function normalizeShortTermPromotionAuthorizedView(
value: unknown,
): ShortTermPromotionAuthorizedView | undefined {
if (!value || typeof value !== "object" || Array.isArray(value)) {
return undefined;
}
const record = value as Record<string, unknown>;
const storeId = typeof record.storeId === "string" ? record.storeId.trim() : "";
const viewId = typeof record.viewId === "string" ? record.viewId.trim() : "";
const resourceRevision =
typeof record.resourceRevision === "string" ? record.resourceRevision.trim() : "";
const lifecycle = record.lifecycle;
if (
!storeId ||
!viewId ||
!resourceRevision ||
(lifecycle !== "active" && lifecycle !== "postbox" && lifecycle !== "quarantine")
) {
return undefined;
}
return { storeId, viewId, resourceRevision, lifecycle };
}
export function clampScore(value: number): number {
if (!Number.isFinite(value)) {
return 0;
@@ -406,6 +430,10 @@ export function normalizeShortTermRecallStore(raw: unknown, nowIso: string): Sho
: {}),
}
: undefined;
const authorizedView = normalizeShortTermPromotionAuthorizedView(entry.authorizedView);
if (entry.authorizedView !== undefined && !authorizedView) {
continue;
}
const normalizedKey =
key || buildEntryKey({ path: entryPath, startLine, endLine, source, claimHash });
@@ -427,6 +455,7 @@ export function normalizeShortTermRecallStore(raw: unknown, nowIso: string): Sho
recallDays: recallDays.slice(-MAX_RECALL_DAYS),
conceptTags,
...(provenance ? { provenance } : {}),
...(authorizedView ? { authorizedView } : {}),
...(claimHash ? { claimHash } : {}),
...(projectKey ? { projectKey } : {}),
...(promotedAt ? { promotedAt } : {}),
@@ -2165,6 +2165,167 @@ describe("short-term promotion", () => {
});
});
it("does not combine promotion candidates from different authorized views", async () => {
await withTempWorkspace(async (workspaceDir) => {
const notes = [
["2026-04-01", "Personal store promotion candidate."],
["2026-04-02", "Channel store promotion candidate."],
] as const;
for (const [date, snippet] of notes) {
await writeDailyMemoryNote(workspaceDir, date, [snippet]);
}
await recordShortTermRecalls({
workspaceDir,
query: "authorized promotion views",
results: notes.map(([date, snippet], index) => ({
path: `memory/${date}.md`,
startLine: 1,
endLine: 1,
score: 0.92,
snippet,
source: "memory" as const,
authorizedView: {
storeId: index === 0 ? "personal-store" : "channel-store",
viewId: index === 0 ? "personal-view" : "channel-view",
resourceRevision: `revision-${index + 1}`,
lifecycle: "active" as const,
},
})),
});
const ranked = await rankShortTermPromotionCandidates({
workspaceDir,
minScore: 0,
minRecallCount: 0,
minUniqueQueries: 0,
});
expect(ranked).toHaveLength(2);
const applied = await applyShortTermPromotions({
workspaceDir,
candidates: ranked,
minScore: 0,
minRecallCount: 0,
minUniqueQueries: 0,
});
expect(applied.applied).toBe(0);
await expectEnoent(fs.readFile(path.join(workspaceDir, "MEMORY.md"), "utf-8"));
});
});
it.each(["postbox", "quarantine"] as const)(
"does not rank %s content for automatic promotion",
async (lifecycle) => {
await withTempWorkspace(async (workspaceDir) => {
const snippet = `${lifecycle} content must remain review-only.`;
await writeDailyMemoryNote(workspaceDir, "2026-04-01", [snippet]);
await recordShortTermRecalls({
workspaceDir,
query: `${lifecycle} promotion`,
results: [
{
path: "memory/2026-04-01.md",
startLine: 1,
endLine: 1,
score: 0.92,
snippet,
source: "memory",
authorizedView: {
storeId: "review-store",
viewId: "review-view",
resourceRevision: "revision-review",
lifecycle,
},
},
],
});
await expect(
rankShortTermPromotionCandidates({
workspaceDir,
minScore: 0,
minRecallCount: 0,
minUniqueQueries: 0,
}),
).resolves.toEqual([]);
});
},
);
it("requires an immutable source revision and records it beside an authorized promotion", async () => {
await withTempWorkspace(async (workspaceDir) => {
const snippet = "Authorized source retains its immutable revision.";
await writeDailyMemoryNote(workspaceDir, "2026-04-01", [snippet]);
const result = {
path: "memory/2026-04-01.md",
startLine: 1,
endLine: 1,
score: 0.92,
snippet,
source: "memory" as const,
};
await recordShortTermRecalls({
workspaceDir,
query: "missing immutable revision",
results: [
{
...result,
authorizedView: {
storeId: "personal-store",
viewId: "personal-view",
resourceRevision: "",
lifecycle: "active",
},
},
],
});
await expect(
rankShortTermPromotionCandidates({
workspaceDir,
minScore: 0,
minRecallCount: 0,
minUniqueQueries: 0,
}),
).resolves.toEqual([]);
await recordShortTermRecalls({
workspaceDir,
query: "immutable revision",
results: [
{
...result,
authorizedView: {
storeId: "personal-store",
viewId: "personal-view",
resourceRevision: "revision-personal-1",
lifecycle: "active",
},
},
],
});
const ranked = await rankShortTermPromotionCandidates({
workspaceDir,
minScore: 0,
minRecallCount: 0,
minUniqueQueries: 0,
});
expect(ranked).toHaveLength(1);
await expect(
applyShortTermPromotions({
workspaceDir,
candidates: ranked,
minScore: 0,
minRecallCount: 0,
minUniqueQueries: 0,
}),
).resolves.toMatchObject({ applied: 1 });
await expect(fs.readFile(path.join(workspaceDir, "MEMORY.md"), "utf-8")).resolves.toContain(
"<!-- authorized-view: store=personal-store view=personal-view revision=revision-personal-1 -->",
);
});
});
it("does not double-prefix promoted snippets that are already markdown bullets", async () => {
await withTempWorkspace(async (workspaceDir) => {
await writeDailyMemoryNote(workspaceDir, "2026-04-01", [
@@ -1,5 +1,6 @@
// Stable public surface for short-term promotion behavior.
import { expectDefined } from "openclaw/plugin-sdk/expect-runtime";
import { isPromotionAuthorizedViewBlocked } from "./dreaming-consolidation-candidates.js";
import { readPhaseSignalStore, readStore } from "./short-term-promotion-store.js";
import {
DEFAULT_PROMOTION_MIN_RECALL_COUNT,
@@ -136,6 +137,9 @@ export async function rankShortTermPromotionCandidates(
if (!includePromoted && entry.promotedAt) {
continue;
}
if (isPromotionAuthorizedViewBlocked(entry)) {
continue;
}
const recallCount = Math.max(0, Math.floor(entry.recallCount ?? 0));
const dailyCount = Math.max(0, Math.floor(entry.dailyCount ?? 0));
const groundedCount = Math.max(0, Math.floor(entry.groundedCount ?? 0));
@@ -217,6 +221,7 @@ export async function rankShortTermPromotionCandidates(
conceptual,
},
provenance: entry.provenance,
authorizedView: entry.authorizedView,
});
}
@@ -1,3 +1,4 @@
import type { DatabaseSync } from "node:sqlite";
import type { MemoryReadResult, MemorySearchResult, MemorySource } from "./types.js";
/** Version shared by every serializable multiplayer-memory authorization shape. */
@@ -404,6 +405,43 @@ type AuthorizedMemoryContentMutation = Readonly<{
contentType: "markdown" | "text" | "json";
}>;
/**
* Opaque transcript provenance prepared by core before a derivation model sees
* the transcript. The selected runtime validates the durable companion rows;
* tool arguments never choose this source.
*/
export type AuthorizedTranscriptDerivationSource = Readonly<{
kind: "transcript";
sessionId: string;
eventSeqs: readonly number[];
sourcePolicySetId: string;
deliveryAudiencesJson: string;
}>;
/** The immutable transcript-policy edge names the durable artifact it produced. */
export type AuthorizedTranscriptDerivationPurpose = "flush" | "compaction";
/**
* A staged sealed artifact is intentionally opaque to core. The selected
* memory runtime owns its bytes and catalog rows; core owns the surrounding
* transcript/checkpoint transaction and supplies the same SQLite connection.
*/
export type AuthorizedSealedCompactionArtifact = Readonly<{
resourceRevisionId: string;
commitInTransaction(params: Readonly<{
database: DatabaseSync;
compactionPolicyId: string;
eventSeq: number;
}>): void;
}>;
export type AuthorizedSealedCompactionStageParams = Readonly<{
context: MemoryAccessContext & Readonly<{ operation: "derive" }>;
plan: AuthorizedMemoryPlan & Readonly<{ operation: "derive" }>;
content: string;
transcriptSource: AuthorizedTranscriptDerivationSource;
}>;
export type AuthorizedMemoryMutation =
| (AuthorizedMemoryContentMutation &
Readonly<{
@@ -422,9 +460,18 @@ export type AuthorizedMemoryMutation =
| (AuthorizedMemoryContentMutation &
Readonly<{
kind: "derive";
sourceHandles: readonly AuthorizedResourceHandle[];
sourcePolicySetId: string;
}>)
derivationPurpose: AuthorizedTranscriptDerivationPurpose;
}> &
(
| Readonly<{
sourceHandles: readonly AuthorizedResourceHandle[];
sourcePolicySetId: string;
}>
| Readonly<{
transcriptSource: AuthorizedTranscriptDerivationSource;
sourcePolicySetId: string;
}>
))
| (AuthorizedMemoryContentMutation &
Readonly<{
kind: "project" | "publish";
@@ -555,6 +602,13 @@ export interface AuthorizedMemoryRuntime {
params: AuthorizedMemoryReadParams<"derive">,
): Promise<AuthorizedMemoryResultEnvelope<MemoryReadResult>>;
writeAuthorized(params: AuthorizedMemoryWriteParams): Promise<MemoryWriteResult>;
/**
* Optional because only a runtime that can commit its catalog against core's
* transaction may support cutover compaction. Absence is a fail-closed deny.
*/
stageSealedCompaction?(
params: AuthorizedSealedCompactionStageParams,
): Promise<AuthorizedSealedCompactionArtifact>;
importAuthorized(
params: AuthorizedMemoryOperationParams<"import"> &
Readonly<{
+20 -9
View File
@@ -26,7 +26,7 @@ import type {
import { isMemoryIsolationCutoverAgent } from "../plugins/memory-cutover.js";
import { resolveMemoryFlushPlan } from "../plugins/memory-state.js";
import { appendRuntimePluginToolGrant } from "../plugins/tool-grant-allowlist.js";
import type { AuthorizedMemoryReadHost } from "../plugins/tool-types.js";
import type { AuthorizedMemoryReadHost, AuthorizedMemoryWriteHost } from "../plugins/tool-types.js";
import { getPluginToolMeta } from "../plugins/tools.js";
import { getActiveSecretsRuntimeConfigSnapshot } from "../secrets/runtime-state.js";
import { GATEWAY_OWNER_ONLY_CORE_TOOLS } from "../security/dangerous-tools.js";
@@ -121,6 +121,7 @@ import type { CronToolOptions } from "./tools/cron-tool.types.js";
import { wrapToolWithGatewayCallerIdentity } from "./tools/gateway-caller-context.js";
const MEMORY_FLUSH_ALLOWED_TOOL_NAMES = new Set(["read", "write"]);
const AUTHORIZED_MEMORY_FLUSH_ALLOWED_TOOL_NAMES = new Set(["memory_remember"]);
const MEMORY_ISOLATION_READ_TOOL_NAMES = new Set(["memory_search", "memory_get"]);
const AUTHORIZED_MEMORY_VIEW_TOOL_NAMES = new Set(["memory_search", "memory_get", "read"]);
@@ -216,6 +217,8 @@ type OpenClawCodingToolsOptions = {
* share this exact host rather than minting authority from routing strings.
*/
authorizedMemoryRead?: AuthorizedMemoryReadHost;
/** Host-prepared one-mutation derivation for an authorized memory flush. */
authorizedMemoryWrite?: AuthorizedMemoryWriteHost;
/** Admission-prepared closed filesystem policy; never derived from tool input. */
fsPolicy?: ToolFsPolicy;
/** Core-private opaque broker for the admitted memory view. */
@@ -394,7 +397,7 @@ type OpenClawCodingToolsOptions = {
function createOpenClawCodingToolsInternal(options?: OpenClawCodingToolsOptions): AnyAgentTool[] {
const sandbox = options?.sandbox?.enabled ? options.sandbox : undefined;
const isMemoryFlushRun = options?.trigger === "memory";
if (isMemoryFlushRun && !options?.memoryFlushWritePath) {
if (isMemoryFlushRun && !options?.memoryFlushWritePath && !options?.authorizedMemoryWrite) {
throw new Error("memoryFlushWritePath required for memory-triggered tool runs");
}
const memoryFlushWritePath = isMemoryFlushRun ? options.memoryFlushWritePath : undefined;
@@ -803,6 +806,7 @@ function createOpenClawCodingToolsInternal(options?: OpenClawCodingToolsOptions)
allowHostBrowserControl: sandbox ? sandbox.browserAllowHostControl : true,
agentSessionKey: options?.sessionKey,
runId: options?.runId,
authorizedMemoryWrite: options?.authorizedMemoryWrite,
runSessionKey: options?.runSessionKey,
agentChannel: resolveGatewayMessageChannel(
options?.messageChannel ?? options?.messageProvider,
@@ -896,13 +900,16 @@ function createOpenClawCodingToolsInternal(options?: OpenClawCodingToolsOptions)
options?.swarmCollector && options.swarmOutputSchema
? tools.find((tool) => tool.name === "structured_output")
: undefined;
const toolsForMemoryFlush: AnyAgentTool[] = isMemoryFlushRun && memoryFlushWritePath ? [] : tools;
if (isMemoryFlushRun && memoryFlushWritePath) {
const toolsForMemoryFlush: AnyAgentTool[] = isMemoryFlushRun ? [] : tools;
if (isMemoryFlushRun) {
for (const tool of tools) {
if (!MEMORY_FLUSH_ALLOWED_TOOL_NAMES.has(tool.name)) {
const allowedToolNames = memoryIsolationCutover
? AUTHORIZED_MEMORY_FLUSH_ALLOWED_TOOL_NAMES
: MEMORY_FLUSH_ALLOWED_TOOL_NAMES;
if (!allowedToolNames.has(tool.name)) {
continue;
}
if (tool.name === "write") {
if (tool.name === "write" && memoryFlushWritePath) {
toolsForMemoryFlush.push(
wrapToolMemoryFlushAppendOnlyWrite(tool, {
root: memoryFlushWriteRoot,
@@ -920,8 +927,10 @@ function createOpenClawCodingToolsInternal(options?: OpenClawCodingToolsOptions)
}
}
const unavailableCoreToolReason =
isMemoryFlushRun && memoryFlushWritePath
? "memory-triggered compaction runs expose only read and append-only write"
isMemoryFlushRun
? memoryIsolationCutover
? "memory-triggered compaction runs expose only subject-scoped memory_remember"
: "memory-triggered compaction runs expose only read and append-only write"
: undefined;
const toolsForMessageProvider = filterToolsByMessageProvider(
toolsForMemoryFlush,
@@ -984,7 +993,9 @@ function createOpenClawCodingToolsInternal(options?: OpenClawCodingToolsOptions)
const surfaceTools = authorizedMemoryView
? authorizedTools.filter((tool) => AUTHORIZED_MEMORY_VIEW_TOOL_NAMES.has(tool.name))
: memoryIsolationCutover
? authorizedTools.filter((tool) => MEMORY_ISOLATION_READ_TOOL_NAMES.has(tool.name))
? isMemoryFlushRun
? authorizedTools.filter((tool) => AUTHORIZED_MEMORY_FLUSH_ALLOWED_TOOL_NAMES.has(tool.name))
: authorizedTools.filter((tool) => MEMORY_ISOLATION_READ_TOOL_NAMES.has(tool.name))
: authorizedTools;
if (
swarmStructuredOutputTool &&
@@ -126,6 +126,45 @@ describe("workspace path resolution", () => {
});
});
it("uses memory_remember instead of a workspace write for a cut-over memory flush", async () => {
await withTempDir("openclaw-memory-flush-cutover-state-", async (stateDir) => {
const originalStateDir = process.env.OPENCLAW_STATE_DIR;
process.env.OPENCLAW_STATE_DIR = stateDir;
try {
const database = openOpenClawAgentDatabase({ agentId: "main" });
database.db
.prepare(
`INSERT INTO memory_migrations
(migration_id, source_kind, source_hash, phase, classification_json, plan_hash,
verified_at, cutover_at, updated_at)
VALUES ('memory-cutover-flush-tools', 'test', 'test-source', 'cutover', '{}',
'test-plan', 1, 1, 1)`,
)
.run();
resetMemoryIsolationCutoverForTest();
vi.mocked(createOpenClawTools).mockImplementationOnce(() => [
{ name: "memory_remember" } as never,
]);
expect(
createOpenClawCodingTools({
agentId: "main",
trigger: "memory",
memoryFlushWritePath: "memory/2026-07-29.md",
}).map((tool) => tool.name),
).toEqual(["memory_remember"]);
} finally {
closeOpenClawAgentDatabasesForTest();
resetMemoryIsolationCutoverForTest();
if (originalStateDir === undefined) {
delete process.env.OPENCLAW_STATE_DIR;
} else {
process.env.OPENCLAW_STATE_DIR = originalStateDir;
}
}
});
});
it("preserves legacy memory-file writes for intentionally unscoped tool construction", async () => {
await withTempDir("openclaw-unscoped-ws-", async (workspaceDir) => {
const tools = createOpenClawCodingTools({ workspaceDir });
@@ -41,6 +41,12 @@ export const contextEngineCompactMock: Mock<() => Promise<CompactResult>> = vi.f
result: { summary: "engine-summary", tokensBefore: 120, tokensAfter: 50 },
}));
export const isMemoryIsolationCutoverAgentMock = vi.fn(() => false);
export const readAuthorizedTranscriptDerivationMock = vi.fn(() => undefined);
export const admitAuthorizedMemoryDerivationMock = vi.fn(async () => true);
export const createAuthorizedMemoryDerivationHostMock = vi.fn(() => undefined);
export const prepareAuthorizedSealedCompactionHostMock = vi.fn(async () => undefined);
export const hookRunner = {
hasHooks: vi.fn<(hookName?: string) => boolean>(),
runBeforeCompaction: vi.fn(async () => undefined),
@@ -80,6 +86,34 @@ export const sessionAutomaticCompactionMock = vi.fn();
export const attemptServerEndpointCompactionMock: Mock<(_params?: unknown) => Promise<unknown>> =
vi.fn(async () => undefined);
export const triggerInternalHookMock: Mock<(event?: unknown) => void> = vi.fn();
export const sessionDeferredCompactionMock = vi.fn();
export const sessionApplyDeferredCompactionMock = vi.fn();
export const sessionDiscardDeferredCompactionMock = vi.fn();
export const sealedCompactionStageMock = vi.fn();
export const sealedCompactionCommitMock = vi.fn();
export const commitSealedSqliteTranscriptCompactionMock = vi.fn();
type SealedTranscriptCommitInput = {
compactionPolicyId: string;
source: {
eventSeqs: readonly number[];
sourcePolicySetId: string;
deliveryAudiencesJson: string;
};
commitDerivedState: (params: {
database: { db: unknown };
compactionPolicy: {
compactionPolicyId: string;
sessionId: string;
sourcePolicySetId: string;
deliveryAudiencesJson: string;
eventSeqs: readonly number[];
retentionState: "retained";
createdAt: number;
};
eventSeq: number;
}) => void;
};
const sanitizeSessionHistoryMock = vi.fn(
async (params: { messages: unknown[] }) => params.messages,
);
@@ -177,6 +211,31 @@ function createMockCompactionSession() {
session.messages.splice(1);
return await sessionCompactImpl();
}),
compactDeferred: vi.fn(async () => {
sessionDeferredCompactionMock();
const result = await sessionCompactImpl();
return {
entry: {
type: "compaction",
id: "sealed-compaction-entry",
parentId: "entry-1",
timestamp: new Date(1).toISOString(),
summary: result.summary,
firstKeptEntryId: result.firstKeptEntryId,
tokensBefore: result.tokensBefore,
details: result.details,
},
fromExtension: false,
result,
};
}),
applyDeferredCompaction: vi.fn(async () => {
sessionApplyDeferredCompactionMock();
session.messages.splice(1);
}),
discardDeferredCompaction: vi.fn((error: unknown) => {
sessionDiscardDeferredCompactionMock(error);
}),
setActiveToolsByName: vi.fn(),
setBaseSystemPrompt: vi.fn((systemPrompt: string) => {
session.agent.state.systemPrompt = systemPrompt;
@@ -493,6 +552,34 @@ export function resetCompactSessionStateMocks(): void {
sessionAbortCompactionMock.mockReset();
sessionManualCompactionMock.mockReset();
sessionAutomaticCompactionMock.mockReset();
sessionDeferredCompactionMock.mockReset();
sessionApplyDeferredCompactionMock.mockReset();
sessionDiscardDeferredCompactionMock.mockReset();
sealedCompactionStageMock.mockReset();
sealedCompactionCommitMock.mockReset();
commitSealedSqliteTranscriptCompactionMock.mockReset();
sealedCompactionStageMock.mockResolvedValue({
resourceRevisionId: "sealed-resource-revision",
commitInTransaction: sealedCompactionCommitMock,
});
commitSealedSqliteTranscriptCompactionMock.mockImplementation(
async (input: SealedTranscriptCommitInput) => {
input.commitDerivedState({
database: { db: {} },
compactionPolicy: {
compactionPolicyId: input.compactionPolicyId,
sessionId: "session-1",
sourcePolicySetId: input.source.sourcePolicySetId,
deliveryAudiencesJson: input.source.deliveryAudiencesJson,
eventSeqs: input.source.eventSeqs,
retentionState: "retained",
createdAt: 1,
},
eventSeq: 7,
});
return { compactionPolicy: { compactionPolicyId: input.compactionPolicyId }, eventSeq: 7 };
},
);
attemptServerEndpointCompactionMock.mockReset();
attemptServerEndpointCompactionMock.mockResolvedValue(undefined);
resolveEffectiveCompactionModeMock.mockReset();
@@ -576,6 +663,16 @@ export function resetCompactHooksHarnessMocks(): void {
hookRunner.runBeforeCompaction.mockResolvedValue(undefined);
hookRunner.runAfterCompaction.mockReset();
hookRunner.runAfterCompaction.mockResolvedValue(undefined);
isMemoryIsolationCutoverAgentMock.mockReset();
isMemoryIsolationCutoverAgentMock.mockReturnValue(false);
readAuthorizedTranscriptDerivationMock.mockReset();
readAuthorizedTranscriptDerivationMock.mockReturnValue(undefined);
admitAuthorizedMemoryDerivationMock.mockReset();
admitAuthorizedMemoryDerivationMock.mockResolvedValue(true);
createAuthorizedMemoryDerivationHostMock.mockReset();
createAuthorizedMemoryDerivationHostMock.mockReturnValue(undefined);
prepareAuthorizedSealedCompactionHostMock.mockReset();
prepareAuthorizedSealedCompactionHostMock.mockResolvedValue(undefined);
acquireAgentRunPreparedModelRuntimeMock.mockClear();
resolveDefaultAgentDirMock.mockReset();
@@ -672,6 +769,37 @@ export async function loadCompactHooksHarness(): Promise<{
runGlobalGatewayStopSafely: vi.fn(async () => undefined),
}));
vi.doMock("../../plugins/memory-cutover.js", () => ({
isMemoryIsolationCutoverAgent: isMemoryIsolationCutoverAgentMock,
}));
vi.doMock("../../config/sessions/session-transcript-memory-policy.js", () => ({
readAuthorizedTranscriptDerivation: readAuthorizedTranscriptDerivationMock,
}));
vi.doMock("../../config/sessions/session-accessor.sqlite-transcript-write.js", () => ({
commitSealedSqliteTranscriptCompaction: commitSealedSqliteTranscriptCompactionMock,
}));
vi.doMock("../memory-authorized-read-host.js", () => ({
admitAuthorizedMemoryDerivation: admitAuthorizedMemoryDerivationMock,
createAuthorizedMemoryDerivationHost: createAuthorizedMemoryDerivationHostMock,
createAuthorizedMemoryReadHost: vi.fn(() => undefined),
prepareAuthorizedSealedCompactionHost: prepareAuthorizedSealedCompactionHostMock,
}));
vi.doMock("./compaction-checkpoint.js", () => ({
compactionCheckpointStore: {
captureSnapshot: vi.fn(async () => ({
sessionId: "session-1",
leafId: "entry-1",
entryId: "entry-1",
})),
cleanupSnapshot: vi.fn(async () => undefined),
},
persistCompactionCheckpoint: vi.fn(async () => true),
}));
vi.doMock("../../plugins/current-plugin-metadata-snapshot.js", () => ({
getCurrentPluginMetadataSnapshot: getCurrentPluginMetadataSnapshotMock,
resolvePluginMetadataControlPlaneFingerprint: vi.fn(() => "test-plugin-fingerprint"),
@@ -18,6 +18,7 @@ import {
getCurrentPluginMetadataSnapshotMock,
applyExtraParamsToAgentMock,
applyAgentCompactionSettingsFromConfigMock,
admitAuthorizedMemoryDerivationMock,
buildEmbeddedExtensionFactoriesMock,
buildAgentRuntimePlanMock,
buildEmbeddedSystemPromptMock,
@@ -51,6 +52,8 @@ import {
resolveSandboxContextMock,
resolveSessionAgentIdMock,
resolveSessionAgentIdsMock,
readAuthorizedTranscriptDerivationMock,
prepareAuthorizedSealedCompactionHostMock,
rotateTranscriptAfterCompactionMock,
runCliAgentMock,
selectAgentHarnessForPreparedModelProvidersMock,
@@ -59,10 +62,18 @@ import {
resetCompactHooksHarnessMocks,
resetCompactSessionStateMocks,
sessionAbortCompactionMock,
sessionApplyDeferredCompactionMock,
sessionAutomaticCompactionMock,
sessionDeferredCompactionMock,
sessionDiscardDeferredCompactionMock,
sessionMessages,
sessionCompactImpl,
sessionManualCompactionMock,
sealedCompactionCommitMock,
sealedCompactionStageMock,
commitSealedSqliteTranscriptCompactionMock,
createAuthorizedMemoryDerivationHostMock,
isMemoryIsolationCutoverAgentMock,
triggerInternalHookMock,
} from "./compact.hooks.harness.js";
import {
@@ -348,6 +359,112 @@ describe("compactEmbeddedAgentSessionDirect hooks", () => {
resetCompactSessionStateMocks();
});
it("rechecks transcript derivation after opening the native session before summarizing", async () => {
isMemoryIsolationCutoverAgentMock.mockReturnValue(true);
admitAuthorizedMemoryDerivationMock.mockResolvedValue(true);
createAuthorizedMemoryDerivationHostMock.mockReturnValue(undefined);
prepareAuthorizedSealedCompactionHostMock.mockResolvedValue({
source: {
kind: "transcript",
sessionId: TEST_SESSION_ID,
eventSeqs: [0, 1],
sourcePolicySetId: "policy-set-1",
deliveryAudiencesJson: '[{"kind":"user","id":"alice"}]',
},
stage: sealedCompactionStageMock,
});
readAuthorizedTranscriptDerivationMock
.mockReturnValueOnce({
eventSeqs: [0, 1],
sourcePolicySetId: "policy-set-1",
deliveryAudiencesJson: '[{"kind":"user","id":"alice"}]',
})
// A revocation or a new pending event after preparation must stop before
// native compaction turns transcript content into a prompt.
.mockReturnValueOnce(undefined);
const result = await compactEmbeddedAgentSessionDirect({
...wrappedCompactionArgs(),
provider: "openai",
model: "gpt-5.5",
});
expect(result).toMatchObject({
ok: false,
compacted: false,
reason: expect.stringContaining("transcript derivation authorization unavailable"),
});
expect(readAuthorizedTranscriptDerivationMock).toHaveBeenCalledTimes(2);
expect(sessionCompactImpl).not.toHaveBeenCalled();
});
it("does not route an authorized cutover transcript through the legacy compaction writer", async () => {
isMemoryIsolationCutoverAgentMock.mockReturnValue(true);
admitAuthorizedMemoryDerivationMock.mockResolvedValue(true);
createAuthorizedMemoryDerivationHostMock.mockReturnValue(undefined);
readAuthorizedTranscriptDerivationMock.mockReturnValue({
eventSeqs: [0, 1],
sourcePolicySetId: "policy-set-1",
deliveryAudiencesJson: '[{"kind":"user","id":"alice"}]',
});
const result = await compactEmbeddedAgentSessionDirect({
...wrappedCompactionArgs(),
provider: "openai",
model: "gpt-5.5",
});
expect(result).toMatchObject({
ok: false,
compacted: false,
reason: expect.stringContaining("sealed compaction authorization unavailable"),
});
expect(readAuthorizedTranscriptDerivationMock).toHaveBeenCalledTimes(1);
expect(sessionCompactImpl).not.toHaveBeenCalled();
});
it("stages and commits a cutover summary before applying its in-memory compaction", async () => {
const source = {
eventSeqs: [0, 1],
sourcePolicySetId: "policy-set-1",
deliveryAudiencesJson: '[{"kind":"user","id":"alice"}]',
};
isMemoryIsolationCutoverAgentMock.mockReturnValue(true);
admitAuthorizedMemoryDerivationMock.mockResolvedValue(true);
readAuthorizedTranscriptDerivationMock.mockReturnValue(source);
prepareAuthorizedSealedCompactionHostMock.mockResolvedValue({
source: { kind: "transcript", sessionId: TEST_SESSION_ID, ...source },
stage: sealedCompactionStageMock,
});
const result = await compactEmbeddedAgentSessionDirect({
...wrappedCompactionArgs(),
provider: "openai",
model: "gpt-5.5",
});
expect(result).toMatchObject({ ok: true, compacted: true });
expect(sessionDeferredCompactionMock).toHaveBeenCalledOnce();
expect(sessionManualCompactionMock).not.toHaveBeenCalled();
expect(sessionAutomaticCompactionMock).not.toHaveBeenCalled();
expect(sealedCompactionStageMock).toHaveBeenCalledWith("summary");
expect(commitSealedSqliteTranscriptCompactionMock).toHaveBeenCalledWith(
expect.objectContaining({
event: expect.objectContaining({ id: "sealed-compaction-entry", summary: "summary" }),
source,
checkpoint: expect.objectContaining({
preCompaction: expect.objectContaining({ entryId: "entry-1" }),
postCompaction: expect.objectContaining({ entryId: "sealed-compaction-entry" }),
}),
}),
);
expect(sealedCompactionCommitMock).toHaveBeenCalledWith(
expect.objectContaining({ compactionPolicyId: expect.any(String), eventSeq: 7 }),
);
expect(sessionApplyDeferredCompactionMock).toHaveBeenCalledOnce();
expect(sessionDiscardDeferredCompactionMock).not.toHaveBeenCalled();
});
it("returns a summaryless xAI manual endpoint result", async () => {
mockResolvedModel();
attemptServerEndpointCompactionMock.mockResolvedValueOnce({
@@ -18,6 +18,7 @@ import type {
import type { CapturedCompactionCheckpointSnapshot } from "../../gateway/session-compaction-checkpoints.js";
import { formatErrorMessage } from "../../infra/errors.js";
import { getGlobalHookRunner } from "../../plugins/hook-runner-global.js";
import { isMemoryIsolationCutoverAgent } from "../../plugins/memory-cutover.js";
import type { ProviderRuntimeModel } from "../../plugins/provider-runtime-model.types.js";
import { requireActivePluginRegistry } from "../../plugins/runtime.js";
import { withPluginRuntimeGenerationScope } from "../../plugins/runtime/generation-scope.js";
@@ -443,6 +444,22 @@ async function compactResolvedContextEngine(
...params,
missingSessionKey: "resolve-existing",
});
const { sessionAgentId } = resolveSessionAgentIds({
sessionKey: runtimeTarget.sessionKey,
config: params.config,
agentId: runtimeTarget.agentId,
});
// Context engines own their prompt assembly and can invoke a model without the
// prepared native-compaction host. Until their public contract carries an opaque
// derive plan, letting one compact a cutover transcript would launder raw history.
if (contextEngine.info.ownsCompaction === true && isMemoryIsolationCutoverAgent(sessionAgentId)) {
return {
ok: false,
compacted: false,
reason: "memory derivation authorization unavailable for context-engine compaction",
failure: { reason: "memory_derivation_unavailable" },
};
}
const lockedHarnessRuntime =
params.modelSelectionLocked === true
? normalizeOptionalAgentRuntimeId(params.agentHarnessId)
@@ -0,0 +1,81 @@
import { beforeAll, beforeEach, describe, expect, it } from "vitest";
import {
admitAuthorizedMemoryDerivationMock,
commitSealedSqliteTranscriptCompactionMock,
isMemoryIsolationCutoverAgentMock,
loadCompactHooksHarness,
prepareAuthorizedSealedCompactionHostMock,
readAuthorizedTranscriptDerivationMock,
resetCompactHooksHarnessMocks,
sealedCompactionCommitMock,
sealedCompactionStageMock,
sessionApplyDeferredCompactionMock,
sessionAutomaticCompactionMock,
sessionDeferredCompactionMock,
sessionDiscardDeferredCompactionMock,
sessionManualCompactionMock,
} from "./compact.hooks.harness.js";
let compactEmbeddedAgentSessionDirect: typeof import("./compact.js").compactEmbeddedAgentSessionDirect;
beforeAll(async () => {
({ compactEmbeddedAgentSessionDirect } = await loadCompactHooksHarness());
});
beforeEach(() => {
resetCompactHooksHarnessMocks();
});
describe("sealed embedded compaction", () => {
it("stages and commits the summary before applying its in-memory transcript entry", async () => {
const source = {
eventSeqs: [0, 1],
sourcePolicySetId: "policy-set-1",
deliveryAudiencesJson: '[{"kind":"user","id":"alice"}]',
};
isMemoryIsolationCutoverAgentMock.mockReturnValue(true);
admitAuthorizedMemoryDerivationMock.mockResolvedValue(true);
readAuthorizedTranscriptDerivationMock.mockReturnValue(source);
prepareAuthorizedSealedCompactionHostMock.mockResolvedValue({
source: { kind: "transcript", sessionId: "session-1", ...source },
stage: sealedCompactionStageMock,
});
const result = await compactEmbeddedAgentSessionDirect({
sessionId: "session-1",
sessionKey: "agent:main:session-1",
sessionFile: "agent:main:session-1",
sessionTarget: {
agentId: "main",
sessionId: "session-1",
sessionKey: "agent:main:session-1",
storePath: "/tmp/sessions.json",
},
workspaceDir: "/tmp",
provider: "openai",
model: "gpt-5.5",
enqueue: async <T>(task: () => Promise<T> | T) => await task(),
});
expect(result).toMatchObject({ ok: true, compacted: true });
expect(sessionDeferredCompactionMock).toHaveBeenCalledOnce();
expect(sessionManualCompactionMock).not.toHaveBeenCalled();
expect(sessionAutomaticCompactionMock).not.toHaveBeenCalled();
expect(sealedCompactionStageMock).toHaveBeenCalledWith("summary");
expect(commitSealedSqliteTranscriptCompactionMock).toHaveBeenCalledWith(
expect.objectContaining({
event: expect.objectContaining({ id: "sealed-compaction-entry", summary: "summary" }),
source,
checkpoint: expect.objectContaining({
preCompaction: expect.objectContaining({ entryId: "entry-1" }),
postCompaction: expect.objectContaining({ entryId: "sealed-compaction-entry" }),
}),
}),
);
expect(sealedCompactionCommitMock).toHaveBeenCalledWith(
expect.objectContaining({ compactionPolicyId: expect.any(String), eventSeq: 7 }),
);
expect(sessionApplyDeferredCompactionMock).toHaveBeenCalledOnce();
expect(sessionDiscardDeferredCompactionMock).not.toHaveBeenCalled();
});
});
@@ -2,8 +2,15 @@
* Executes compaction while owning the transcript lock, session lifecycle,
* hooks, checkpoint, and optional successor transcript rotation.
*/
import { randomUUID } from "node:crypto";
import { SESSION_TOTAL_TOKENS_VERSION } from "../../config/sessions.js";
import { formatSqliteSessionFileMarker } from "../../config/sessions/legacy-sqlite-marker.js";
import type { CapturedCompactionCheckpointSnapshot } from "../../gateway/session-compaction-checkpoints.js";
import { commitSealedSqliteTranscriptCompaction } from "../../config/sessions/session-accessor.sqlite-transcript-write.js";
import { readAuthorizedTranscriptDerivation } from "../../config/sessions/session-transcript-memory-policy.js";
import {
resolveSessionCompactionCheckpointReason,
type CapturedCompactionCheckpointSnapshot,
} from "../../gateway/session-compaction-checkpoints.js";
import { resolveDiagnosticModelContentCapturePolicy } from "../../infra/diagnostic-llm-content.js";
import { formatErrorMessage } from "../../infra/errors.js";
import {
@@ -14,6 +21,8 @@ import {
} from "../../logging/diagnostic-run-activity.js";
import { getCurrentPluginMetadataSnapshot } from "../../plugins/current-plugin-metadata-snapshot.js";
import { getGlobalHookRunner } from "../../plugins/hook-runner-global.js";
import { isMemoryIsolationCutoverAgent } from "../../plugins/memory-cutover.js";
import { openOpenClawAgentDatabase } from "../../state/openclaw-agent-db.js";
import {
consumeCompactionSafeguardCancelReason,
setCompactionSafeguardCancelReason,
@@ -101,6 +110,7 @@ export async function executePreparedCompactionSession(runtime: PreparedCompacti
effectiveTools,
allowedToolNames,
buildSystemPromptText,
authorizedSealedCompaction,
resolvedMessageProvider,
sessionAgentId,
} = runtime;
@@ -136,6 +146,20 @@ export async function executePreparedCompactionSession(runtime: PreparedCompacti
: undefined,
allowedToolNames,
});
// Preparation admits this source, but a revoke or transcript append can
// race it. Re-read after opening the session and before prompt assembly.
if (
isMemoryIsolationCutoverAgent(sessionAgentId) &&
!readAuthorizedTranscriptDerivation(
openOpenClawAgentDatabase({ agentId: sessionAgentId }).db,
sessionTarget.sessionId,
)
) {
throw new Error("compaction transcript derivation authorization unavailable");
}
if (isMemoryIsolationCutoverAgent(sessionAgentId) && !authorizedSealedCompaction) {
throw new Error("scoped compaction derived commit unavailable");
}
checkpointSnapshot = await compactionCheckpointStore.captureSnapshot({
sessionManager,
sessionFile: params.sessionFile,
@@ -411,45 +435,65 @@ export async function executePreparedCompactionSession(runtime: PreparedCompacti
}
const compactStartedAt = Date.now();
const serverResult = await attemptServerEndpointCompaction({
trigger,
streamFn: session.agent.streamFn,
model: effectiveModel,
context: { systemPrompt: systemPromptText, messages: session.messages },
sessionManager,
extraParams: effectiveExtraParams,
customInstructions: params.customInstructions,
requestOptions: {
apiKey: transportApiKey,
sessionId: params.sessionId,
authProfileId: runtimePlan.auth.forwardedAuthProfileId,
timeoutMs: compactionTimeoutMs,
signal: params.abortSignal,
},
});
const activeSession = session;
const clientResult = serverResult
? undefined
: await compactWithSafetyTimeout(
() => {
setCompactionSafeguardCancelReason(compactionSessionManager, undefined);
return resolveEffectiveCompactionMode(params.config) === "default" &&
trigger !== "manual"
? activeSession[agentSessionAutomaticCompaction](params.customInstructions)
: activeSession.compact(params.customInstructions);
},
const cutoverCompaction = isMemoryIsolationCutoverAgent(sessionAgentId);
const deferred = cutoverCompaction
? await compactWithSafetyTimeout(
() =>
activeSession.compactDeferred(
params.customInstructions,
resolveEffectiveCompactionMode(params.config) === "default" &&
trigger !== "manual"
? "retry-invalid-once"
: "none",
),
compactionTimeoutMs,
{
abortSignal: params.abortSignal,
onCancel: () => {
activeSession.abortCompaction();
},
onCancel: () => activeSession.abortCompaction(),
},
);
)
: undefined;
const serverResult = cutoverCompaction
? undefined
: await attemptServerEndpointCompaction({
trigger,
streamFn: session.agent.streamFn,
model: effectiveModel,
context: { systemPrompt: systemPromptText, messages: session.messages },
sessionManager,
extraParams: effectiveExtraParams,
customInstructions: params.customInstructions,
requestOptions: {
apiKey: transportApiKey,
sessionId: params.sessionId,
authProfileId: runtimePlan.auth.forwardedAuthProfileId,
timeoutMs: compactionTimeoutMs,
signal: params.abortSignal,
},
});
const clientResult = deferred
? deferred.result
: serverResult
? undefined
: await compactWithSafetyTimeout(
() => {
setCompactionSafeguardCancelReason(compactionSessionManager, undefined);
return resolveEffectiveCompactionMode(params.config) === "default" &&
trigger !== "manual"
? activeSession[agentSessionAutomaticCompaction](params.customInstructions)
: activeSession.compact(params.customInstructions);
},
compactionTimeoutMs,
{
abortSignal: params.abortSignal,
onCancel: () => activeSession.abortCompaction(),
},
);
const effectiveFirstKeptEntryId = clientResult?.firstKeptEntryId;
const tokensBefore = serverResult?.usage.input_tokens ?? clientResult!.tokensBefore;
// Estimate tokens after compaction by summing token estimates for remaining messages
const tokensAfter =
let tokensAfter =
serverResult?.usage.output_tokens ??
estimateTokensAfterCompaction({
messagesAfter: session.messages,
@@ -463,6 +507,64 @@ export async function executePreparedCompactionSession(runtime: PreparedCompacti
...sessionTarget,
sessionId: params.sessionId,
});
if (deferred && authorizedSealedCompaction) {
try {
if (!checkpointSnapshot) {
throw new Error("sealed compaction checkpoint is unavailable");
}
const staged = await authorizedSealedCompaction.stage(clientResult.summary);
if ("unavailable" in staged) {
throw new Error("scoped compaction derived commit unavailable");
}
const compactionPolicyId = randomUUID();
await commitSealedSqliteTranscriptCompaction({
scope: sessionTarget,
event: deferred.entry,
compactionPolicyId,
source: {
eventSeqs: authorizedSealedCompaction.source.eventSeqs,
sourcePolicySetId: authorizedSealedCompaction.source.sourcePolicySetId,
deliveryAudiencesJson: authorizedSealedCompaction.source.deliveryAudiencesJson,
},
checkpoint: {
checkpointId: randomUUID(),
sessionKey: sessionTarget.sessionKey,
sessionId: params.sessionId,
createdAt: compactStartedAt,
reason: resolveSessionCompactionCheckpointReason({ trigger: params.trigger }),
tokensVersion: SESSION_TOTAL_TOKENS_VERSION,
tokensBefore: observedTokenCount ?? clientResult.tokensBefore,
tokensAfter,
summary: clientResult.summary,
firstKeptEntryId: effectiveFirstKeptEntryId,
preCompaction: {
sessionId: checkpointSnapshot.sessionId,
leafId: checkpointSnapshot.leafId,
...(checkpointSnapshot.entryId ? { entryId: checkpointSnapshot.entryId } : {}),
},
postCompaction: { sessionId: params.sessionId, entryId: deferred.entry.id },
},
commitDerivedState({ database, eventSeq }) {
staged.commitInTransaction({
database: database.db,
compactionPolicyId,
eventSeq,
});
},
});
await activeSession.applyDeferredCompaction(deferred);
tokensAfter = estimateTokensAfterCompaction({
messagesAfter: session.messages,
observedTokenCount,
fullSessionTokensBefore: limitedTranscriptTokensBefore ?? 0,
estimateTokensFn: estimateTokens,
});
checkpointSnapshotRetained = true;
} catch (error) {
activeSession.discardDeferredCompaction(error);
throw error;
}
}
await runPostCompactionSideEffects({
config: params.config,
sessionKey: params.sessionKey,
@@ -470,7 +572,7 @@ export async function executePreparedCompactionSession(runtime: PreparedCompacti
agentId: sessionAgentId,
sessionFile: activeSessionFile,
});
if (clientResult) {
if (clientResult && !deferred) {
checkpointSnapshotRetained = await persistCompactionCheckpoint({
config: params.config,
sessionKey: params.sessionKey,
@@ -35,6 +35,7 @@ const resolveRuntimeTranscriptReadTargetMock = vi.fn(async (scope: Record<string
sessionKey: scope.sessionKey,
storePath: scope.storePath ?? "/tmp/default-openclaw.sqlite",
}));
const isMemoryIsolationCutoverAgentMock = vi.hoisted(() => vi.fn(() => false));
let createDeferredTurnMaintenanceAbortSignal: typeof import("./context-engine-maintenance.test-support.js").createDeferredTurnMaintenanceAbortSignal;
let resetDeferredTurnMaintenanceStateForTest: typeof import("./context-engine-maintenance.test-support.js").resetDeferredTurnMaintenanceStateForTest;
let waitForDeferredTurnMaintenanceForSession: typeof import("./context-engine-maintenance.js").waitForDeferredTurnMaintenanceForSession;
@@ -103,6 +104,10 @@ vi.mock("./context-engine-capabilities.js", () => ({
resolveContextEngineCapabilities: () => ({ llm: undefined }),
}));
vi.mock("../../plugins/memory-cutover.js", () => ({
isMemoryIsolationCutoverAgent: isMemoryIsolationCutoverAgentMock,
}));
vi.mock("./transcript-rewrite.js", () => ({
rewriteTranscriptEntriesInSessionManager: (params: unknown) =>
rewriteTranscriptEntriesInSessionManagerMock(params),
@@ -183,9 +188,38 @@ describe("runContextEngineMaintenance", () => {
rewriteTranscriptEntriesInSessionManagerMock.mockClear();
sessionManagerOpenMock.mockClear();
resolveRuntimeTranscriptReadTargetMock.mockClear();
isMemoryIsolationCutoverAgentMock.mockReset();
isMemoryIsolationCutoverAgentMock.mockReturnValue(false);
await loadFreshContextEngineMaintenanceModuleForTest();
});
it("does not let cutover maintenance route transcript content through an owning engine", async () => {
isMemoryIsolationCutoverAgentMock.mockReturnValue(true);
const maintain = vi.fn(async () => ({
changed: true,
bytesFreed: 10,
rewrittenEntries: 1,
}));
const result = await runContextEngineMaintenance({
contextEngine: {
info: { id: "test", name: "Test Engine", ownsCompaction: true },
ingest: async () => ({ ingested: true }),
assemble: async ({ messages }) => ({ messages, estimatedTokens: 0 }),
compact: async () => ({ ok: true, compacted: false }),
maintain,
},
agentId: "main",
sessionId: "session-1",
sessionKey: "agent:main:session-1",
sessionFile: "/tmp/session.jsonl",
reason: "turn",
});
expect(result).toBeUndefined();
expect(maintain).not.toHaveBeenCalled();
});
it("passes a rewrite-capable runtime context into maintain()", async () => {
const sessionTarget = {
agentId: "main",
@@ -15,6 +15,7 @@ import type {
ContextEngineSessionTarget,
} from "../../context-engine/types.js";
import { formatErrorMessage } from "../../infra/errors.js";
import { isMemoryIsolationCutoverAgent } from "../../plugins/memory-cutover.js";
import {
enqueueCommandInLane,
GatewayDrainingError,
@@ -33,6 +34,7 @@ import {
updateTaskNotifyPolicyForOwner,
} from "../../tasks/task-owner-access.js";
import { findActiveSessionTask } from "../session-async-task-status.js";
import { resolveSessionAgentIds } from "../agent-scope.js";
import { SessionManager } from "../sessions/index.js";
import { resolveContextEngineCapabilities } from "./context-engine-capabilities.js";
import { log } from "./logger.js";
@@ -560,6 +562,22 @@ export async function runContextEngineMaintenance(
return undefined;
}
const { sessionAgentId } = resolveSessionAgentIds({
sessionKey: params.sessionTarget?.sessionKey ?? params.sessionKey,
config: params.config,
agentId: params.sessionTarget?.agentId ?? params.agentId,
});
// An owning engine can compact during routine maintenance. Its current plugin
// contract has no opaque derive plan, so cutover sessions must not let that
// background path assemble transcript content outside the authorized host.
if (contextEngine.info.ownsCompaction === true && isMemoryIsolationCutoverAgent(sessionAgentId)) {
log.warn("skipping context-engine maintenance without memory derivation authority", {
reason: params.reason,
sessionId: params.sessionId,
});
return undefined;
}
const executionMode = params.executionMode ?? "foreground";
const shouldDefer =
params.reason === "turn" &&
@@ -6,6 +6,7 @@ import os from "node:os";
import path from "node:path";
import { isAcpRuntimeSpawnAvailable } from "../../acp/runtime/availability.js";
import type { ThinkLevel } from "../../auto-reply/thinking.js";
import { readAuthorizedTranscriptDerivation } from "../../config/sessions/session-transcript-memory-policy.js";
import {
formatActiveNodeContextLabel,
getCurrentActiveNodeContext,
@@ -13,6 +14,7 @@ import {
import { getMachineDisplayName } from "../../infra/machine-name.js";
import { resolveRuntimeOsLabel } from "../../infra/os-summary.js";
import { listRegisteredPluginAgentPromptGuidance } from "../../plugins/command-registry-state.js";
import { isMemoryIsolationCutoverAgent } from "../../plugins/memory-cutover.js";
import { extractModelCompat } from "../../plugins/provider-model-compat.js";
import type { ProviderRuntimeModel } from "../../plugins/provider-runtime-model.types.js";
import { transformProviderSystemPrompt } from "../../plugins/provider-runtime.js";
@@ -23,6 +25,7 @@ import {
applySkillEnvOverrides,
applySkillEnvOverridesFromSnapshot,
} from "../../skills/runtime/env-overrides.js";
import { openOpenClawAgentDatabase } from "../../state/openclaw-agent-db.js";
import { normalizeMessageChannel } from "../../utils/message-channel.js";
import { isReasoningTagProvider } from "../../utils/provider-utils.js";
import { createBundleLspToolRuntime } from "../agent-bundle-lsp-runtime.js";
@@ -45,7 +48,12 @@ import { resolveConversationCapabilityProfile } from "../conversation-capability
import { formatDateStamp, resolveUserTimezone } from "../date-time.js";
import { resolveOpenClawReferencePaths } from "../docs-path.js";
import { resolveHeartbeatPromptForSystemPrompt } from "../heartbeat-system-prompt.js";
import { createAuthorizedMemoryReadHost } from "../memory-authorized-read-host.js";
import {
admitAuthorizedMemoryDerivation,
createAuthorizedMemoryDerivationHost,
createAuthorizedMemoryReadHost,
prepareAuthorizedSealedCompactionHost,
} from "../memory-authorized-read-host.js";
import { prepareAgentMemoryPrompt } from "../memory-prompt-prepare.js";
import {
applyAuthHeaderOverride,
@@ -276,16 +284,51 @@ export async function buildPreparedCompactionRuntime(prepared: DirectCompactionP
config: params.config,
agentId: params.agentId,
});
// Compaction turns transcript and injected memory into a durable summary.
// Admit derivation before prompt construction so a revoked source cannot
// reach the summary model through a stale read host.
const cutoverCompaction = isMemoryIsolationCutoverAgent(sessionAgentId);
if (
cutoverCompaction &&
!(await admitAuthorizedMemoryDerivation({
agentId: sessionAgentId,
sessionKey: sandboxSessionKey,
sessionId: params.sessionId,
runId: params.runId,
messageChannel: resolvedMessageProvider,
agentAccountId: params.agentAccountId,
}))
) {
throw new Error("memory derivation authorization unavailable for compaction");
}
if (
cutoverCompaction &&
!readAuthorizedTranscriptDerivation(
openOpenClawAgentDatabase({ agentId: sessionAgentId }).db,
params.sessionId,
)
) {
throw new Error("compaction transcript derivation authorization unavailable");
}
// Compaction shares the live run's admission discipline: prepare one host
// before tools, then reuse it for the prompt snapshot.
const authorizedMemoryRead = createAuthorizedMemoryReadHost({
const memoryHostParams = {
agentId: sessionAgentId,
sessionKey: sandboxSessionKey,
sessionId: params.sessionId,
runId: params.runId,
messageChannel: resolvedMessageProvider,
agentAccountId: params.agentAccountId,
});
};
const authorizedMemoryRead = cutoverCompaction
? createAuthorizedMemoryDerivationHost(memoryHostParams)
: createAuthorizedMemoryReadHost(memoryHostParams);
const authorizedSealedCompaction = cutoverCompaction
? await prepareAuthorizedSealedCompactionHost(memoryHostParams)
: undefined;
if (cutoverCompaction && !authorizedSealedCompaction) {
throw new Error("sealed compaction authorization unavailable");
}
const runtimeCapabilityProfile = resolveConversationCapabilityProfile({
config: params.config,
sessionKey: sandboxSessionKey,
@@ -655,6 +698,7 @@ export async function buildPreparedCompactionRuntime(prepared: DirectCompactionP
effectiveTools,
allowedToolNames,
buildSystemPromptText,
authorizedSealedCompaction,
resolvedMessageProvider,
sessionAgentId,
disposeToolRuntimes,
@@ -9,7 +9,7 @@ import {
logCodeModeDiagnostic,
} from "../../../logging/code-mode-diagnostic.js";
import { extractModelCompat } from "../../../plugins/provider-model-compat.js";
import type { AuthorizedMemoryReadHost } from "../../../plugins/tool-types.js";
import type { AuthorizedMemoryReadHost, AuthorizedMemoryWriteHost } from "../../../plugins/tool-types.js";
import { getPluginToolMeta } from "../../../plugins/tools.js";
import { isSubagentSessionKey } from "../../../routing/session-key.js";
import { normalizeDeliveryContext } from "../../../utils/delivery-context.shared.js";
@@ -65,6 +65,7 @@ type SkillUsagePaths = OpenClawCodingToolsOptions["skillUsagePaths"];
export async function prepareEmbeddedAttemptToolBase(params: {
agentDir: string;
authorizedMemoryRead?: AuthorizedMemoryReadHost;
authorizedMemoryWrite?: AuthorizedMemoryWriteHost;
authorizedMemoryVirtualBroker?: AuthorizedMemoryVirtualFileBroker;
attempt: EmbeddedRunAttemptParams;
effectiveCwd: string;
@@ -326,6 +327,7 @@ export async function prepareEmbeddedAttemptToolBase(params: {
runId: attempt.runId,
operationalRunInstance: attempt.admittedRunContext.operationalRunInstance,
authorizedMemoryRead,
authorizedMemoryWrite: params.authorizedMemoryWrite,
...(fsPolicy ? { fsPolicy } : {}),
...(authorizedMemoryVirtualBroker
? {
@@ -198,6 +198,7 @@ export async function runEmbeddedAttempt(
prepareEmbeddedAttemptToolBase({
agentDir,
authorizedMemoryRead,
authorizedMemoryWrite: params.authorizedMemoryWrite,
authorizedMemoryVirtualBroker,
attempt: params,
effectiveCwd,
@@ -119,6 +119,8 @@ export type RunEmbeddedAgentParams = {
scheduledRuntimeAuthorityRecoveryRequired?: boolean;
/** Relative workspace path that memory-triggered writes are allowed to append to. */
memoryFlushWritePath?: string;
/** Pre-admitted host-owned mutation for a transcript-derived memory flush. */
authorizedMemoryWrite?: import("../../../plugins/tool-types.js").AuthorizedMemoryWriteHost;
/** Delivery target for topic/thread routing. */
messageTo?: string;
/** Thread/topic identifier for routing replies to the originating thread. */
@@ -275,6 +275,7 @@ export async function dispatchEmbeddedRunAttempt(input: {
sandboxSessionKey: params.sandboxSessionKey,
trigger: params.trigger,
memoryFlushWritePath: params.memoryFlushWritePath,
authorizedMemoryWrite: params.authorizedMemoryWrite,
messageChannel: params.messageChannel,
messageProvider: params.messageProvider,
clientCaps: params.clientCaps,
@@ -0,0 +1,201 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
const mocks = vi.hoisted(() => ({
createDeriveInvocation: vi.fn(),
createWriteInvocation: vi.fn(),
createTrustedContext: vi.fn(),
readTranscriptDerivation: vi.fn(),
search: vi.fn(),
write: vi.fn(),
}));
vi.mock("../plugins/memory-cutover.js", () => ({
isMemoryIsolationCutoverAgent: () => true,
}));
vi.mock("../plugins/memory-invocation.js", () => ({
MEMORY_INVOCATION_UNAVAILABLE: { unavailable: true },
createAuthorizedMemoryDeriveInvocation: mocks.createDeriveInvocation,
createAuthorizedMemoryReadInvocation: vi.fn(),
createAuthorizedMemoryWriteInvocation: mocks.createWriteInvocation,
materializeAuthorizedMemoryVirtualView: vi.fn(),
readAuthorizedMemoryVirtualFile: vi.fn(),
readAuthorizedMemoryForInvocation: vi.fn(),
searchAuthorizedMemoryForInvocation: mocks.search,
writeAuthorizedMemoryForInvocation: mocks.write,
}));
vi.mock("../config/sessions/session-transcript-memory-policy.js", () => ({
readAuthorizedTranscriptDerivation: mocks.readTranscriptDerivation,
}));
vi.mock("../state/openclaw-agent-db.js", () => ({
openOpenClawAgentDatabase: () => ({ db: { kind: "agent-db" } }),
}));
vi.mock("../state/memory-access-context.js", () => ({
captureTrustedMemoryAccessFacts: (facts: unknown) => facts,
createTrustedMemoryAccessContext: mocks.createTrustedContext,
}));
vi.mock("../state/memory-identity.js", () => ({
recheckMemoryIdentityBinding: () => true,
}));
vi.mock("../state/memory-session-subject.js", () => ({
createCurrentMemorySessionContext: () => ({
kind: "current",
context: {
agentId: "main",
fingerprint: "session-fingerprint",
principalId: "service:main",
sessionId: "session-1",
sessionKey: "agent:main:session-1",
authorityRevision: "authority-1",
subject: { kind: "service", principalId: "service:main" },
},
}),
}));
vi.mock("./memory-egress-admission.js", () => ({
resolveMemoryEgressDeliveryFacts: () => ({
sink: "internal",
audiences: [{ kind: "agent", id: "main" }],
deliveryRevision: "delivery-1",
egressRegistryRevision: "egress-1",
}),
}));
import {
admitAuthorizedMemoryDerivation,
createAuthorizedMemoryDerivationHost,
prepareAuthorizedTranscriptDerivationHost,
} from "./memory-authorized-read-host.js";
describe("admitAuthorizedMemoryDerivation", () => {
beforeEach(() => {
mocks.createDeriveInvocation.mockReset();
mocks.createWriteInvocation.mockReset();
mocks.createTrustedContext.mockReset();
mocks.readTranscriptDerivation.mockReset();
mocks.search.mockReset();
mocks.write.mockReset();
mocks.createTrustedContext.mockReturnValue({ kind: "current", context: { trusted: true } });
});
it("admits a host-minted derive context before a model can receive derived content", async () => {
mocks.createDeriveInvocation.mockResolvedValue({});
await expect(
admitAuthorizedMemoryDerivation({
agentId: "main",
sessionKey: "agent:main:session-1",
sessionId: "session-1",
runId: "run-1",
}),
).resolves.toBe(true);
expect(mocks.createDeriveInvocation).toHaveBeenCalledWith({ context: { trusted: true } });
expect(mocks.createTrustedContext).toHaveBeenCalledWith(
expect.objectContaining({ facts: expect.objectContaining({ operation: "derive" }) }),
);
});
it("fails closed when derive admission is unavailable", async () => {
mocks.createDeriveInvocation.mockResolvedValue({ unavailable: true });
await expect(
admitAuthorizedMemoryDerivation({
agentId: "main",
sessionKey: "agent:main:session-1",
sessionId: "session-1",
runId: "run-1",
}),
).resolves.toBe(false);
});
it("keeps source reads on the derive invocation after admission", async () => {
const invocation = {};
mocks.createDeriveInvocation.mockResolvedValue(invocation);
mocks.search.mockResolvedValue({ results: [] });
const host = createAuthorizedMemoryDerivationHost({
agentId: "main",
sessionKey: "agent:main:session-1",
sessionId: "session-1",
runId: "run-1",
});
await host?.search({ query: "compaction source" });
expect(mocks.createDeriveInvocation).toHaveBeenCalledWith({ context: { trusted: true } });
});
it("binds a flush mutation to the host-read transcript policy set before the model can write", async () => {
const invocation = {};
mocks.readTranscriptDerivation.mockReturnValue({
eventSeqs: [0, 1],
sourcePolicySetId: "policy-set-1",
deliveryAudiencesJson: '[{"kind":"user","id":"alice"}]',
});
mocks.createWriteInvocation.mockResolvedValue(invocation);
mocks.write.mockResolvedValue({ version: 1, mutationId: "mutation", status: "committed" });
const host = await prepareAuthorizedTranscriptDerivationHost({
agentId: "main",
sessionKey: "agent:main:session-1",
sessionId: "session-1",
runId: "run-1",
});
await host?.remember({ content: "durable fact" });
expect(mocks.readTranscriptDerivation).toHaveBeenCalledWith({ kind: "agent-db" }, "session-1");
expect(mocks.createWriteInvocation).toHaveBeenCalledWith({ context: { trusted: true } });
expect(mocks.readTranscriptDerivation.mock.invocationCallOrder[0]).toBeLessThan(
mocks.createWriteInvocation.mock.invocationCallOrder[0] ?? Infinity,
);
expect(mocks.write).toHaveBeenCalledWith(
expect.objectContaining({
invocation,
mutation: expect.objectContaining({
kind: "derive",
derivationPurpose: "flush",
sourcePolicySetId: "policy-set-1",
transcriptSource: {
kind: "transcript",
sessionId: "session-1",
eventSeqs: [0, 1],
sourcePolicySetId: "policy-set-1",
deliveryAudiencesJson: '[{"kind":"user","id":"alice"}]',
},
}),
}),
);
});
it("names a compacted summary without letting the caller select transcript lineage", async () => {
const invocation = {};
mocks.readTranscriptDerivation.mockReturnValue({
eventSeqs: [0, 1],
sourcePolicySetId: "policy-set-1",
deliveryAudiencesJson: '[{"kind":"user","id":"alice"}]',
});
mocks.createWriteInvocation.mockResolvedValue(invocation);
mocks.write.mockResolvedValue({ version: 1, mutationId: "mutation", status: "committed" });
const host = await prepareAuthorizedTranscriptDerivationHost({
agentId: "main",
sessionKey: "agent:main:session-1",
sessionId: "session-1",
runId: "run-1",
derivationPurpose: "compaction",
});
await host?.remember({ content: "summary" });
expect(mocks.write).toHaveBeenCalledWith(
expect.objectContaining({
invocation,
mutation: expect.objectContaining({
kind: "derive",
derivationPurpose: "compaction",
sourcePolicySetId: "policy-set-1",
}),
}),
);
});
});
+144 -3
View File
@@ -1,18 +1,24 @@
import { createHash, randomUUID } from "node:crypto";
import type {
AuthorizedMemoryVirtualView,
AuthorizedSealedCompactionArtifact,
AuthorizedTranscriptDerivationSource,
AuthorizedTranscriptDerivationPurpose,
MemoryAccessContext,
MemoryActorEvidence,
} from "../memory-host-sdk/host/authorization.js";
import { readAuthorizedTranscriptDerivation } from "../config/sessions/session-transcript-memory-policy.js";
import { isMemoryIsolationCutoverAgent } from "../plugins/memory-cutover.js";
import {
MEMORY_INVOCATION_UNAVAILABLE,
createAuthorizedMemoryDeriveInvocation,
createAuthorizedMemoryReadInvocation,
createAuthorizedMemoryWriteInvocation,
materializeAuthorizedMemoryVirtualView,
readAuthorizedMemoryVirtualFile,
readAuthorizedMemoryForInvocation,
searchAuthorizedMemoryForInvocation,
stageAuthorizedMemorySealedCompactionForInvocation,
writeAuthorizedMemoryForInvocation,
type AuthorizedMemoryReadInvocation,
} from "../plugins/memory-invocation.js";
@@ -27,6 +33,7 @@ import {
createCurrentMemorySessionContext,
type CurrentMemorySessionContext,
} from "../state/memory-session-subject.js";
import { openOpenClawAgentDatabase } from "../state/openclaw-agent-db.js";
import type { DeliveryContext } from "../utils/delivery-context.types.js";
import { resolveMemoryEgressDeliveryFacts } from "./memory-egress-admission.js";
@@ -40,6 +47,12 @@ export type AuthorizedMemoryVirtualFileBroker = Readonly<{
readFile: (virtualPath: string) => Promise<string | undefined>;
}>;
/** Core-private sealed compaction capability; plugins never receive this host. */
export type AuthorizedSealedCompactionHost = Readonly<{
source: AuthorizedTranscriptDerivationSource;
stage: (content: string) => Promise<AuthorizedSealedCompactionArtifact | typeof MEMORY_INVOCATION_UNAVAILABLE>;
}>;
type AuthorizedMemoryReadHostWithVirtualBroker = AuthorizedMemoryReadHost &
Readonly<{
[authorizedMemoryVirtualBroker]: () => Promise<AuthorizedMemoryVirtualFileBroker | undefined>;
@@ -220,10 +233,11 @@ function createTrustedMemoryHostContext(
* Builds the sole tool-facing read handle for a cut-over run. Session identity and delivery facts
* are reread from their owners; sender IDs, `toolsBySender`, and paths never name a memory subject.
*/
export function createAuthorizedMemoryReadHost(
function createAuthorizedMemoryContentHost(
params: AuthorizedMemoryHostParams,
operation: "read" | "derive",
): AuthorizedMemoryReadHost | undefined {
const trusted = createTrustedMemoryHostContext({ ...params, operation: "read" });
const trusted = createTrustedMemoryHostContext({ ...params, operation });
if (!trusted) {
return undefined;
}
@@ -231,7 +245,10 @@ export function createAuthorizedMemoryReadHost(
| Promise<AuthorizedMemoryReadInvocation | typeof MEMORY_INVOCATION_UNAVAILABLE>
| undefined;
const getInvocation = () =>
(invocation ??= createAuthorizedMemoryReadInvocation({ context: trusted }));
(invocation ??=
operation === "derive"
? createAuthorizedMemoryDeriveInvocation({ context: trusted })
: createAuthorizedMemoryReadInvocation({ context: trusted }));
let virtualBroker: Promise<AuthorizedMemoryVirtualFileBroker | undefined> | undefined;
const getVirtualBroker = () =>
(virtualBroker ??= (async () => {
@@ -276,6 +293,130 @@ export function createAuthorizedMemoryReadHost(
}) as AuthorizedMemoryReadHost;
}
export function createAuthorizedMemoryReadHost(
params: AuthorizedMemoryHostParams,
): AuthorizedMemoryReadHost | undefined {
return createAuthorizedMemoryContentHost(params, "read");
}
/**
* Builds a content host whose every source read is authorized as a derivation. A caller cannot
* turn an admitted derive plan into a weaker read plan after the source reaches model context.
*/
export function createAuthorizedMemoryDerivationHost(
params: AuthorizedMemoryHostParams,
): AuthorizedMemoryReadHost | undefined {
return createAuthorizedMemoryContentHost(params, "derive");
}
/**
* Rechecks the separate derive capability before a runtime can place memory-derived
* material in a model context. Read admission alone intentionally never implies this.
*/
export async function admitAuthorizedMemoryDerivation(
params: AuthorizedMemoryHostParams,
): Promise<boolean> {
const trusted = createTrustedMemoryHostContext({ ...params, operation: "derive" });
if (!trusted) {
return false;
}
const invocation = await createAuthorizedMemoryDeriveInvocation({ context: trusted });
return !("unavailable" in invocation);
}
/**
* Admits one transcript-backed mutation before the flush model sees history.
* The opaque source stays host-owned; neither the model nor a plugin tool can
* substitute a session, event list, policy set, or delivery audience.
*/
export async function prepareAuthorizedTranscriptDerivationHost(
params: AuthorizedMemoryHostParams & Readonly<{ derivationPurpose?: AuthorizedTranscriptDerivationPurpose }>,
): Promise<AuthorizedMemoryWriteHost | undefined> {
const sessionId = params.sessionId?.trim();
const trusted = createTrustedMemoryHostContext({ ...params, operation: "derive" });
if (!trusted || !sessionId) {
return undefined;
}
const transcriptSource = readAuthorizedTranscriptDerivation(
openOpenClawAgentDatabase({ agentId: params.agentId }).db,
sessionId,
);
if (!transcriptSource) {
return undefined;
}
const invocation = await createAuthorizedMemoryWriteInvocation({ context: trusted });
if ("unavailable" in invocation) {
return undefined;
}
return Object.freeze({
async remember({ content, contentType = "markdown" }) {
const result = await writeAuthorizedMemoryForInvocation({
invocation,
mutation: {
version: 1,
kind: "derive",
derivationPurpose: params.derivationPurpose ?? "flush",
mutationId: randomUUID(),
idempotencyKey: randomUUID(),
content,
contentType,
sourcePolicySetId: transcriptSource.sourcePolicySetId,
transcriptSource: {
kind: "transcript",
sessionId,
eventSeqs: transcriptSource.eventSeqs,
sourcePolicySetId: transcriptSource.sourcePolicySetId,
deliveryAudiencesJson: transcriptSource.deliveryAudiencesJson,
},
},
});
return "unavailable" in result ? MEMORY_INVOCATION_UNAVAILABLE : result;
},
});
}
/**
* Captures the exact transcript source before model work. The returned staging
* capability has no caller-selectable session, policy, audience, or store.
*/
export async function prepareAuthorizedSealedCompactionHost(
params: AuthorizedMemoryHostParams,
): Promise<AuthorizedSealedCompactionHost | undefined> {
const sessionId = params.sessionId?.trim();
const trusted = createTrustedMemoryHostContext({ ...params, operation: "derive" });
if (!trusted || !sessionId) {
return undefined;
}
const transcriptSource = readAuthorizedTranscriptDerivation(
openOpenClawAgentDatabase({ agentId: params.agentId }).db,
sessionId,
);
if (!transcriptSource) {
return undefined;
}
const invocation = await createAuthorizedMemoryWriteInvocation({ context: trusted });
if ("unavailable" in invocation) {
return undefined;
}
const sealedSource = Object.freeze({
kind: "transcript",
sessionId,
eventSeqs: transcriptSource.eventSeqs,
sourcePolicySetId: transcriptSource.sourcePolicySetId,
deliveryAudiencesJson: transcriptSource.deliveryAudiencesJson,
});
return Object.freeze({
source: sealedSource,
async stage(content) {
return await stageAuthorizedMemorySealedCompactionForInvocation({
invocation,
content,
transcriptSource: sealedSource,
});
},
});
}
/**
* Builds a one-mutation append host for a cut-over run. The model supplies only content; the host
* reissues append facts and the selected runtime chooses the subject-owned store and audience.
@@ -0,0 +1,34 @@
import { describe, expect, it } from "vitest";
import {
mayInjectAutonomousSourceTranscript,
resolveSubagentMemoryContextMode,
} from "./memory-autonomous-run-policy.js";
describe("memory autonomous-run policy", () => {
it("removes raw fork context while memory isolation is active", () => {
expect(
resolveSubagentMemoryContextMode({ requested: "fork", memoryIsolationActive: true }),
).toBe("isolated");
expect(
resolveSubagentMemoryContextMode({ requested: "isolated", memoryIsolationActive: true }),
).toBe("isolated");
});
it("does not inject a current session transcript into an isolated service run", () => {
expect(
mayInjectAutonomousSourceTranscript({ sessionTarget: "current", memoryIsolationActive: true }),
).toBe(false);
expect(
mayInjectAutonomousSourceTranscript({ sessionTarget: "isolated", memoryIsolationActive: true }),
).toBe(true);
});
it("preserves the legacy path until isolation is enabled", () => {
expect(
resolveSubagentMemoryContextMode({ requested: "fork", memoryIsolationActive: false }),
).toBe("fork");
expect(
mayInjectAutonomousSourceTranscript({ sessionTarget: "current", memoryIsolationActive: false }),
).toBe(true);
});
});
@@ -0,0 +1,19 @@
/**
* Memory isolation narrows autonomous work before any session transcript is
* selected. A service/child run may gain an explicit plugin capability later,
* but it must never inherit raw user history as a substitute for one.
*/
export function resolveSubagentMemoryContextMode(params: {
requested: "fork" | "isolated";
memoryIsolationActive: boolean;
}): "fork" | "isolated" {
return params.memoryIsolationActive && params.requested === "fork" ? "isolated" : params.requested;
}
/** Current-bound cron work is autonomous under memory isolation, not a replay of its old target. */
export function mayInjectAutonomousSourceTranscript(params: {
sessionTarget?: string;
memoryIsolationActive: boolean;
}): boolean {
return !(params.memoryIsolationActive && params.sessionTarget === "current");
}
-1
View File
@@ -128,7 +128,6 @@ export function resolveMemoryEgressDeliveryFacts(params: {
channel,
accountId,
recipientId: to,
options: { agentId: params.agentId },
}).kind === "current"
? {
sink: "private" as const,
+3
View File
@@ -25,6 +25,7 @@ import {
wrapToolWithBeforeToolCallHook,
} from "./agent-tools.before-tool-call.js";
import type { ConversationRecallContext } from "./conversation-recall.types.js";
import type { AuthorizedMemoryWriteHost } from "../plugins/tool-types.js";
import { resolveOpenClawPluginToolsForOptions } from "./openclaw-plugin-tools.js";
import { filterToolsByClientCaps } from "./openclaw-tools.client-caps.js";
import {
@@ -104,6 +105,8 @@ export function createOpenClawTools(
sandboxBrowserBridgeUrl?: string;
allowHostBrowserControl?: boolean;
agentSessionKey?: string;
/** Pre-admitted host-owned mutation for a transcript-derived flush. */
authorizedMemoryWrite?: AuthorizedMemoryWriteHost;
toolBindings?: Readonly<Record<string, unknown>>;
/** Durable store key when it differs from the sandbox/policy session key. */
runSessionKey?: string;
+106 -15
View File
@@ -22,8 +22,18 @@ import type { SettingsManager } from "./settings-manager.js";
type CompactionReason = "manual" | "threshold" | "overflow";
type SummaryOutputPolicy = "none" | "retry-invalid-once";
export type DeferredSessionCompaction = Readonly<{
entry: CompactionEntry;
fromExtension: boolean;
result: CompactionResult;
}>;
type CompactionWorkOutcome =
| { status: "completed"; result: CompactionResult; tokensAfter: number }
| {
status: "completed";
result: CompactionResult;
tokensAfter: number;
deferred?: DeferredSessionCompaction;
}
| { status: "aborted" }
| { status: "skipped"; reason: string };
@@ -49,21 +59,83 @@ export abstract class AgentSessionCompaction extends AgentSessionInspection {
*/
async compact(customInstructions?: string): Promise<CompactionResult> {
return await this.runWithSessionWriteSettlement(
async () => await this.compactWithSessionWriteSettlement(customInstructions, "none"),
async () => (await this.compactWithSessionWriteSettlement(customInstructions, "none")).result,
);
}
async [agentSessionAutomaticCompaction](customInstructions?: string): Promise<CompactionResult> {
return await this.runWithSessionWriteSettlement(
async () =>
await this.compactWithSessionWriteSettlement(customInstructions, "retry-invalid-once"),
(await this.compactWithSessionWriteSettlement(customInstructions, "retry-invalid-once"))
.result,
);
}
/**
* Produces a summary without publishing it. The caller must durably commit
* this exact entry before applying it to the in-memory session.
*/
async compactDeferred(
customInstructions?: string,
summaryOutputPolicy: SummaryOutputPolicy = "none",
): Promise<DeferredSessionCompaction> {
return await this.runWithSessionWriteSettlement(async () => {
const outcome = await this.compactWithSessionWriteSettlement(
customInstructions,
summaryOutputPolicy,
true,
);
if (!outcome.deferred) {
throw new Error("Deferred compaction entry is unavailable");
}
return outcome.deferred;
});
}
/** Apply only an entry committed by the sealed transcript owner. */
async applyDeferredCompaction(params: DeferredSessionCompaction): Promise<void> {
this.sessionManager.applyPersistedCompaction(params.entry);
const sessionContext = this.sessionManager.buildSessionContext();
// Compaction replaces the request prefix, invalidating retained usage and thinking signatures.
// Keep the deferred path replay-safe just like the direct append path.
this.agent.state.messages = sanitizeCompactionReplayMessages(sessionContext.messages);
if (this.currentExtensionRunner) {
await this.currentExtensionRunner.emit({
type: "session_compact",
compactionEntry: params.entry,
fromExtension: params.fromExtension,
});
}
const tokensAfter = estimateContextTokens(this.agent.state.messages).tokens;
this.emit({
type: "compaction_end",
reason: "manual",
outcome: {
status: "completed",
tokensBefore: params.result.tokensBefore,
tokensAfter,
willRetry: false,
},
});
}
/** The sealed owner calls this if staging or durable commit fails. */
discardDeferredCompaction(error: unknown): void {
this.emit({
type: "compaction_end",
reason: "manual",
outcome: {
status: "failed",
reason: `Compaction failed: ${compactionErrorMessage(error, "Compaction failed")}`,
},
});
}
private async compactWithSessionWriteSettlement(
customInstructions?: string,
summaryOutputPolicy: SummaryOutputPolicy = "none",
): Promise<CompactionResult> {
deferPersistence = false,
): Promise<Extract<CompactionWorkOutcome, { status: "completed" }>> {
this.disconnectFromAgent();
await this.abort();
const abortController = new AbortController();
@@ -80,6 +152,7 @@ export abstract class AgentSessionCompaction extends AgentSessionInspection {
summaryOutputPolicy,
settings,
signal: abortController.signal,
deferPersistence,
});
} catch (error) {
const message = compactionErrorMessage(error, "Compaction failed");
@@ -103,17 +176,19 @@ export abstract class AgentSessionCompaction extends AgentSessionInspection {
throw new Error("Compaction cancelled");
}
this.emit({
type: "compaction_end",
reason: "manual",
outcome: {
status: "completed",
tokensBefore: outcome.result.tokensBefore,
tokensAfter: outcome.tokensAfter,
willRetry: false,
},
});
return outcome.result;
if (!deferPersistence) {
this.emit({
type: "compaction_end",
reason: "manual",
outcome: {
status: "completed",
tokensBefore: outcome.result.tokensBefore,
tokensAfter: outcome.tokensAfter,
willRetry: false,
},
});
}
return outcome;
} finally {
if (this.compactionAbortController === abortController) {
this.compactionAbortController = undefined;
@@ -143,6 +218,7 @@ export abstract class AgentSessionCompaction extends AgentSessionInspection {
customInstructions?: string;
mode: "manual" | "auto";
summaryOutputPolicy: SummaryOutputPolicy;
deferPersistence?: boolean;
}): Promise<CompactionWorkOutcome> {
const isManual = options.mode === "manual";
if (!this.model) {
@@ -246,6 +322,21 @@ export abstract class AgentSessionCompaction extends AgentSessionInspection {
summary: capCompactionSummary(compactionResult.summary),
};
if (options.deferPersistence) {
const entry = this.sessionManager.createCompactionEntry(
compactionResult.summary,
compactionResult.firstKeptEntryId,
compactionResult.tokensBefore,
compactionResult.details,
);
return {
status: "completed",
result: compactionResult,
tokensAfter: estimateContextTokens(this.agent.state.messages).tokens,
deferred: Object.freeze({ entry, fromExtension, result: compactionResult }),
};
}
this.sessionManager.appendCompaction(
compactionResult.summary,
compactionResult.firstKeptEntryId,
@@ -433,6 +433,28 @@ describe("AgentSession loop correctness", () => {
}
});
it("produces a sealed compaction entry without publishing it", async () => {
const sessionManager = SessionManager.inMemory();
appendHistory(
sessionManager,
createAssistant(testModel, [{ type: "text", text: "short answer" }]),
);
const settingsManager = SettingsManager.inMemory({
compaction: { enabled: true, reserveTokens: 0, keepRecentTokens: 10_000 },
retry: { enabled: false },
});
const { session } = await createTestSession({
sessionManager,
settingsManager,
resourceLoader: createResourceLoader(createCompactionHandlers()),
});
const deferred = await session.compactDeferred();
expect(deferred.entry).toMatchObject({ type: "compaction", summary: "condensed history" });
expect(sessionManager.getBranch().some((entry) => entry.type === "compaction")).toBe(false);
});
it("keeps a successful high-usage response and performs threshold maintenance without retry", async () => {
const settingsManager = createAutoCompactionSettings();
const compactionEvents: AgentSessionEvent[] = [];
+55 -9
View File
@@ -1,4 +1,5 @@
import {
loadTranscriptEventsSync,
readActiveTranscriptEntryAnchor,
type TranscriptEntryAnchor,
} from "../../config/sessions/session-accessor.js";
@@ -33,10 +34,13 @@ import type {
ThinkingLevelChangeEntry,
} from "./session-manager-types.js";
/** Internal-only: callers cannot bypass persistence through public append options. */
type AppendEntryOptions = AppendPersistenceOptions & Readonly<{ alreadyPersisted?: boolean }>;
export class SessionManagerEntries extends SessionManagerPersistence {
protected appendEntry(
entry: SessionEntry,
options?: AppendPersistenceOptions,
options?: AppendEntryOptions,
): TranscriptEntryAnchor | undefined {
// oxlint-disable-next-line unicorn/prefer-structured-clone -- Match the persisted JSON/toJSON shape exactly.
const canonicalEntry = JSON.parse(JSON.stringify(entry)) as SessionEntry;
@@ -47,10 +51,15 @@ export class SessionManagerEntries extends SessionManagerPersistence {
!this.pendingDeliberateAppend &&
this.appendMode !== "side" &&
!isSessionTranscriptSideAppendEntry(canonicalEntry);
const persistenceResult = this.persist(canonicalEntry, {
...options,
...(activeBranchAppend ? { appendIntent: "active-branch" } : {}),
});
if (options?.alreadyPersisted && !this.persistenceTarget) {
throw new Error("A pre-persisted session entry requires a transcript target");
}
const persistenceResult = options?.alreadyPersisted
? undefined
: this.persist(canonicalEntry, {
...options,
...(activeBranchAppend ? { appendIntent: "active-branch" } : {}),
});
if (persistenceResult && typeof persistenceResult === "object") {
if (persistenceResult.adoptedMessageId) {
this.reloadPersistedTranscript();
@@ -195,14 +204,14 @@ export class SessionManagerEntries extends SessionManagerPersistence {
return entry.id;
}
appendCompaction(
createCompactionEntry(
summary: string,
firstKeptEntryId: string,
tokensBefore: number,
details?: unknown,
fromHook?: boolean,
): string {
const entry: CompactionEntry = {
): CompactionEntry {
return Object.freeze({
type: "compaction",
id: generateSessionEntryId(this.byId),
parentId: this.appendParentId,
@@ -212,7 +221,44 @@ export class SessionManagerEntries extends SessionManagerPersistence {
tokensBefore,
details,
fromHook,
};
});
}
/** Apply a compaction after the transcript owner committed the exact entry atomically. */
applyPersistedCompaction(entry: CompactionEntry): string {
if (entry.parentId !== this.appendParentId || this.byId.has(entry.id)) {
throw new Error("Pre-persisted compaction no longer matches the active transcript leaf");
}
const persisted = this.persistenceTarget
? loadTranscriptEventsSync(this.persistenceTarget).some(
(candidate) =>
isIndexedSessionEntry(candidate) && JSON.stringify(candidate) === JSON.stringify(entry),
)
: false;
if (!persisted) {
throw new Error("Pre-persisted compaction was not committed to the transcript");
}
this.appendEntry(entry, {
alreadyPersisted: true,
invalidateSerializedPrefixCache: entry.fromHook === true || entry.details !== undefined,
});
return entry.id;
}
appendCompaction(
summary: string,
firstKeptEntryId: string,
tokensBefore: number,
details?: unknown,
fromHook?: boolean,
): string {
const entry = this.createCompactionEntry(
summary,
firstKeptEntryId,
tokensBefore,
details,
fromHook,
);
this.appendEntry(entry, {
invalidateSerializedPrefixCache: fromHook === true || details !== undefined,
});
@@ -8,6 +8,7 @@ import {
parseSqliteSessionFileMarker,
} from "../../config/sessions/legacy-sqlite-marker.js";
import {
appendTranscriptEvent,
appendTranscriptMessage,
loadSessionEntry,
loadTranscriptEvents,
@@ -34,6 +35,36 @@ function openMarker(marker: string, sessionKey: string, cwd: string): SessionMan
}
describe("SessionManager.open", () => {
it("applies a compaction entry only after its transaction owner persisted it", async () => {
const dir = tempDirs.make("openclaw-session-manager-");
const storePath = path.join(dir, "sessions.json");
const sessionId = "sealed-compaction-session";
const sessionKey = "agent:main:dashboard:sealed-compaction";
const scope = { agentId: "main", sessionId, sessionKey, storePath };
await upsertSessionEntry(scope, { sessionId, updatedAt: 1 });
await appendTranscriptMessage(scope, {
cwd: dir,
eventId: "source-message",
message: { role: "user", content: "retain this" },
});
const sessionManager = SessionManager.open(scope, dir);
const entry = sessionManager.createCompactionEntry("sealed summary", "source-message", 42);
expect(() => sessionManager.applyPersistedCompaction(entry)).toThrow(
"Pre-persisted compaction was not committed to the transcript",
);
await appendTranscriptEvent(scope, entry);
expect(sessionManager.applyPersistedCompaction(entry)).toBe(entry.id);
await expect(loadTranscriptEvents(scope)).resolves.toEqual(
expect.arrayContaining([
expect.objectContaining({ id: entry.id, summary: "sealed summary", type: "compaction" }),
]),
);
expect(sessionManager.getEntries()).toEqual(
expect.arrayContaining([expect.objectContaining({ id: entry.id, type: "compaction" })]),
);
});
it("opens SQLite markers without creating marker-named files and persists assistant replies", async () => {
const dir = tempDirs.make("openclaw-session-manager-");
const storePath = path.join(dir, "sessions.json");
@@ -3,6 +3,7 @@ import { resolveThreadBindingSpawnPolicy } from "../../../channels/thread-bindin
import type { SessionEntry } from "../../../config/sessions/types.js";
import type { OpenClawConfig } from "../../../config/types.openclaw.js";
import type { SubagentSpawnPreparation } from "../../../context-engine/types.js";
import { resolveSubagentMemoryContextMode } from "../../memory-autonomous-run-policy.js";
import { summarizeSpawnError } from "../../spawn-pipeline.js";
import { getSubagentSpawnDeps } from "./subagent-spawn-deps.js";
import { resolveGatewaySessionStoreTarget } from "./subagent-spawn.runtime.js";
@@ -34,8 +35,24 @@ export async function prepareSubagentSessionContext(params: {
requesterInternalKey: string;
childSessionKey: string;
}): Promise<PreparedSpawnContext> {
if (params.contextMode === "isolated") {
return { status: "ok", mode: "isolated" };
const contextMode = resolveSubagentMemoryContextMode({
requested: params.contextMode,
memoryIsolationActive: getSubagentSpawnDeps().isMemoryIsolationCutoverAgent(
params.requesterAgentId,
),
});
if (contextMode === "isolated") {
// A fork copies raw transcript rows before a child has an independently
// admitted memory view. Under cutover the empty intersection is the only
// safe default until an opaque delegation capability is issued.
return params.contextMode === "fork"
? {
status: "ok",
mode: "isolated",
forkFallbackNote:
"context=\"fork\" is unavailable while memory isolation is active; starting with isolated context instead.",
}
: { status: "ok", mode: "isolated" };
}
const childTarget = resolveGatewaySessionStoreTarget({
cfg: params.cfg,
@@ -7,6 +7,7 @@ import {
getGlobalHookRunner,
getRuntimeConfig,
hasInProcessGatewayContext,
isMemoryIsolationCutoverAgent,
loadPreparedModelCatalog,
resolveContextEngine,
} from "./subagent-spawn.runtime.js";
@@ -18,6 +19,7 @@ type SubagentSpawnDeps = {
getGlobalHookRunner: () => SubagentLifecycleHookRunner | null;
getRuntimeConfig: typeof getRuntimeConfig;
hasInProcessGatewayContext: typeof hasInProcessGatewayContext;
isMemoryIsolationCutoverAgent: typeof isMemoryIsolationCutoverAgent;
ensureContextEnginesInitialized: typeof ensureContextEnginesInitialized;
loadPreparedModelCatalog: typeof loadPreparedModelCatalog;
resolveContextEngine: typeof resolveContextEngine;
@@ -30,6 +32,7 @@ const defaultSubagentSpawnDeps: SubagentSpawnDeps = {
getGlobalHookRunner,
getRuntimeConfig,
hasInProcessGatewayContext,
isMemoryIsolationCutoverAgent,
ensureContextEnginesInitialized,
loadPreparedModelCatalog,
resolveContextEngine,
@@ -17,6 +17,7 @@ describe("sessions_spawn context modes", () => {
const forkSessionEntryFromParentMock = vi.fn();
const forkSessionFromParentMock = vi.fn();
const ensureContextEnginesInitializedMock = vi.fn();
const isMemoryIsolationCutoverAgentMock = vi.fn();
const resolveContextEngineMock = vi.fn();
let spawnSubagentDirect: Awaited<
ReturnType<typeof loadSubagentSpawnModuleForTest>
@@ -30,6 +31,7 @@ describe("sessions_spawn context modes", () => {
forkSessionEntryFromParentMock,
forkSessionFromParentMock,
ensureContextEnginesInitializedMock,
isMemoryIsolationCutoverAgent: isMemoryIsolationCutoverAgentMock,
resolveContextEngineMock,
sessionStorePath: storePath,
}));
@@ -42,8 +44,10 @@ describe("sessions_spawn context modes", () => {
forkSessionEntryFromParentMock.mockReset();
forkSessionFromParentMock.mockReset();
ensureContextEnginesInitializedMock.mockReset();
isMemoryIsolationCutoverAgentMock.mockReset();
resolveContextEngineMock.mockReset();
setupAcceptedSubagentGatewayMock(callGatewayMock);
isMemoryIsolationCutoverAgentMock.mockReturnValue(false);
resolveContextEngineMock.mockResolvedValue({});
});
@@ -224,6 +228,21 @@ describe("sessions_spawn context modes", () => {
expect(prepareContext.childSessionFile).toBe("/tmp/forked-session.jsonl");
});
it("does not fork raw parent context while memory isolation is active", async () => {
isMemoryIsolationCutoverAgentMock.mockReturnValue(true);
usePersistentStoreMock({
main: { sessionId: "parent-session-id", updatedAt: 1 },
});
const result = await spawnSubagentDirect(
{ task: "inspect the current thread", context: "fork" },
{ agentSessionKey: "main" },
);
expect(result.status).toBe("accepted");
expect(forkSessionEntryFromParentMock).not.toHaveBeenCalled();
});
it("keeps the default spawn context isolated", async () => {
const store: SessionStore = {
main: { sessionId: "parent-session-id", updatedAt: 1 },
@@ -4,6 +4,7 @@
* entire gateway/channel stack.
*/
export { getRuntimeConfig } from "../../../config/config.js";
export { isMemoryIsolationCutoverAgent } from "../../../plugins/memory-cutover.js";
export {
loadSessionEntryReadOnly as loadSessionEntry,
upsertSessionEntryCore,
@@ -131,6 +131,7 @@ export async function loadSubagentSpawnModuleForTest(params: {
callGatewayMock: MockFn;
dispatchGatewayMethodInProcessMock?: MockFn;
hasInProcessGatewayContextMock?: MockFn;
isMemoryIsolationCutoverAgent?: (agentId: string) => boolean;
getRuntimeConfig?: () => Record<string, unknown>;
loadSessionStoreMock?: MockFn;
loadPreparedModelCatalogMock?: MockFn;
@@ -220,6 +221,8 @@ export async function loadSubagentSpawnModuleForTest(params: {
dispatchGatewayMethodInProcess: (...args: unknown[]) =>
params.dispatchGatewayMethodInProcessMock?.(...args),
hasInProcessGatewayContext: () => Boolean(params.hasInProcessGatewayContextMock?.()),
isMemoryIsolationCutoverAgent: (agentId: string) =>
params.isMemoryIsolationCutoverAgent?.(agentId) ?? false,
buildSubagentSystemPrompt: () => "system-prompt",
forkSessionEntryFromParent:
params.forkSessionEntryFromParentMock ??
@@ -119,6 +119,14 @@ describe("memory egress at final dispatch", () => {
expect(hoisted.prepareMemoryEgressAuthorization).toHaveBeenCalledWith(
expect.objectContaining({ capabilityId: "reply.block" }),
);
// The finalized channel context has no authoritative session identity. The
// admission owner resolves sessionId/sessionKey from the registered run,
// so a stale inbound field cannot retarget memory egress.
expect(
hoisted.prepareMemoryEgressAuthorization.mock.calls.every(([params]) =>
params.sessionId === undefined && params.sessionKey === undefined,
),
).toBe(true);
expect(delivered).not.toHaveBeenCalled();
expect(dispatcher.getCancelledCounts?.()).toEqual({ tool: 0, block: 1, final: 1 });
});
-2
View File
@@ -209,8 +209,6 @@ function installMemoryEgressAdmission(
const resolveDeliveryFacts = () => resolveCurrentMemoryEgressDeliveryFacts(finalized);
const memoryRunIdentity = {
agentId: finalized.AgentId,
sessionId: finalized.SessionId,
sessionKey: finalized.SessionKey,
};
if (
!appendReplyDispatcherPayloadPrepare(dispatcher, (payload, info) => {
@@ -55,8 +55,16 @@ const ensureSelectedAgentHarnessPluginMock = vi.fn();
const ensureMemoryFlushTargetFileMock = vi.fn();
const registerAgentRunContextMock = vi.fn();
const clearAgentRunContextMock = vi.fn();
const memoryHostMocks = vi.hoisted(() => ({
prepareAuthorizedTranscriptDerivationHost: vi.fn(),
}));
const TEST_MAX_FLUSH_FAILURES = 3;
vi.mock("../../agents/memory-authorized-read-host.js", () => ({
prepareAuthorizedTranscriptDerivationHost:
memoryHostMocks.prepareAuthorizedTranscriptDerivationHost,
}));
type MemoryFlushTestParams = Parameters<typeof runMemoryFlushIfNeededRaw>[0] & {
modelContextTokens?: number;
};
@@ -241,6 +249,7 @@ type EmbeddedAgentParams = {
prompt?: string;
transcriptPrompt?: string;
memoryFlushWritePath?: string;
authorizedMemoryWrite?: unknown;
silentExpected?: boolean;
extraSystemPrompt?: string;
bootstrapPromptWarningSignaturesSeen?: string[];
@@ -451,6 +460,9 @@ describe("runMemoryFlushIfNeeded", () => {
ensureSelectedAgentHarnessPluginMock.mockReset().mockResolvedValue(undefined);
registerAgentRunContextMock.mockReset();
clearAgentRunContextMock.mockReset();
memoryHostMocks.prepareAuthorizedTranscriptDerivationHost
.mockReset()
.mockResolvedValue(undefined);
incrementCompactionCountMock.mockReset().mockImplementation(async (params) => {
const sessionKey = String(params.sessionKey ?? "");
if (!sessionKey || !params.sessionStore?.[sessionKey]) {
@@ -499,7 +511,7 @@ describe("runMemoryFlushIfNeeded", () => {
await fs.rm(rootDir, { recursive: true, force: true });
});
it("does not start or materialize a legacy memory flush for a cut-over agent", async () => {
it("does not let a cut-over flush model summarize transcript content without derivation lineage", async () => {
markAgentCutOver("main");
const sessionKey = "agent:main:flush-policy";
const sessionEntry: SessionEntry = {
@@ -538,6 +550,52 @@ describe("runMemoryFlushIfNeeded", () => {
});
});
it("routes a cut-over flush through its admitted transcript mutation without a workspace path", async () => {
markAgentCutOver("main");
const authorizedMemoryWrite = { remember: vi.fn() };
memoryHostMocks.prepareAuthorizedTranscriptDerivationHost.mockResolvedValue(
authorizedMemoryWrite,
);
const sessionKey = "agent:main:authorized-flush";
const sessionEntry: SessionEntry = {
sessionId: "session",
updatedAt: Date.now(),
totalTokens: 80_000,
totalTokensFresh: true,
totalTokensVersion: 1,
compactionCount: 1,
};
const followupRun = createTestFollowupRun();
followupRun.run.agentId = "main";
followupRun.run.sessionKey = sessionKey;
followupRun.run.workspaceDir = rootDir;
await runMemoryFlushIfNeeded({
cfg: { agents: { defaults: { compaction: { memoryFlush: {} } } } },
followupRun,
sessionCtx: { Provider: "whatsapp" } as unknown as TemplateContext,
defaultModel: "anthropic/claude-opus-4-6",
agentCfgContextTokens: 100_000,
resolvedVerboseLevel: "off",
sessionEntry,
sessionStore: { [sessionKey]: sessionEntry },
sessionKey,
storePath: path.join(rootDir, "sessions.json"),
isHeartbeat: false,
replyOperation: createReplyOperation(),
});
expect(memoryHostMocks.prepareAuthorizedTranscriptDerivationHost).toHaveBeenCalledWith(
expect.objectContaining({ agentId: "main", sessionId: "session" }),
);
const flushCall = requireEmbeddedAgentCall();
expect(flushCall.authorizedMemoryWrite).toBe(authorizedMemoryWrite);
expect(flushCall.memoryFlushWritePath).toBeUndefined();
expect(flushCall.prompt).toContain("memory_remember");
expect(flushCall.extraSystemPrompt).toContain("Never use a workspace file");
expect(ensureMemoryFlushTargetFileMock).not.toHaveBeenCalled();
});
it("runs exactly one auto-reply memory flush turn, rotates, and persists metadata", async () => {
const followupRun = createTestFollowupRun({
authProfileId: "anthropic:work",
+74 -25
View File
@@ -15,6 +15,7 @@ import { estimateMessagesTokens } from "../../agents/compaction.js";
import { isBenignCompactionSkipResult } from "../../agents/embedded-agent-runner/compact-reasons.js";
import { runEmbeddedAgentEntry } from "../../agents/embedded-agent-runner/run-entry.js";
import { createToolResultPromptProjectionState } from "../../agents/embedded-agent-runner/session-prompt-state.js";
import { prepareAuthorizedTranscriptDerivationHost } from "../../agents/memory-authorized-read-host.js";
import { isCliRuntimeAliasForProvider } from "../../agents/model-runtime-aliases.js";
import { isCliProvider } from "../../agents/model-selection.js";
import { resolveContextConfigProviderForRuntime } from "../../agents/openai-routing.js";
@@ -98,6 +99,10 @@ type UpdateSessionEntryParams = {
const MAX_VISIBLE_MEMORY_FLUSH_ERROR_CHARS = 600;
const MAX_FLUSH_FAILURES = 3;
const MAX_FLUSH_ERROR_LENGTH = 200;
const AUTHORIZED_MEMORY_FLUSH_PROMPT =
"Pre-compaction memory flush. Use memory_remember to retain only durable facts for the current authorized subject. Do not write workspace files. If nothing should be retained, reply NO_REPLY.";
const AUTHORIZED_MEMORY_FLUSH_SYSTEM_PROMPT =
"Pre-compaction memory flush turn. The only durable mutation is memory_remember, which selects the authorized subject store and audience. Never use a workspace file as memory storage.";
const embeddedAgentRuntimeLoader = createLazyImportLoader<EmbeddedAgentRuntime>(
() => import("../../agents/embedded-agent.js"),
@@ -1105,9 +1110,6 @@ export async function runMemoryFlushIfNeeded(params: {
params.followupRun.run.agentId ?? resolveDefaultAgentId(params.cfg),
)
: (params.followupRun.run.agentId ?? resolveDefaultAgentId(params.cfg));
if (isMemoryIsolationCutoverAgent(memoryFlushAgentId)) {
return { sessionEntry: params.sessionEntry, outcome: "skipped" };
}
const memoryFlushWritable = (() => {
if (!params.sessionKey) {
return true;
@@ -1153,7 +1155,7 @@ export async function runMemoryFlushIfNeeded(params: {
recordMemoryFlushFailure(error, params, activeSessionEntry);
let memoryFlushPlan: MemoryFlushPlan | null;
try {
memoryFlushPlan = resolveMemoryFlushPlan({ cfg: params.cfg });
memoryFlushPlan = resolveMemoryFlushPlan({ cfg: params.cfg, agentId: memoryFlushAgentId });
} catch (error) {
return await recordFailure(error);
}
@@ -1342,27 +1344,59 @@ export async function runMemoryFlushIfNeeded(params: {
(params.sessionKey ? activeSessionStore?.[params.sessionKey]?.systemPromptReport : undefined),
);
const prepareMemoryFlushAttempt = async () => {
const plan = resolveMemoryFlushPlan({ cfg: params.cfg, nowMs: memoryDeps.now() });
const plan = resolveMemoryFlushPlan({
cfg: params.cfg,
agentId: memoryFlushAgentId,
nowMs: memoryDeps.now(),
});
if (!plan) {
return null;
}
const writePath = plan.relativePath;
await memoryDeps.ensureMemoryFlushTargetFile({
workspaceDir: params.followupRun.run.workspaceDir,
relativePath: writePath,
});
const absolutePath = path.join(params.followupRun.run.workspaceDir, writePath);
const readContent = () =>
fs.promises.readFile(absolutePath, "utf8").catch((error: unknown) => {
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
return "";
}
throw error;
const usesAuthorizedMemoryStore = isMemoryIsolationCutoverAgent(memoryFlushAgentId);
const authorizedMemoryWrite = usesAuthorizedMemoryStore
? await prepareAuthorizedTranscriptDerivationHost({
agentId: memoryFlushAgentId,
sessionKey:
params.runtimePolicySessionKey ??
params.followupRun.run.runtimePolicySessionKey ??
params.sessionKey ??
params.followupRun.run.sessionKey,
sessionId: activeSessionEntry?.sessionId ?? params.followupRun.run.sessionId,
runId: flushRunId,
messageChannel: params.followupRun.run.messageProvider,
agentAccountId: params.followupRun.run.agentAccountId,
})
: undefined;
if (usesAuthorizedMemoryStore && !authorizedMemoryWrite) {
// A flush is a transcript derivation, not an ordinary append. Do not give a
// model raw history until the selected memory runtime can bind its output to
// the current transcript policy set and immutable source lineage.
return null;
}
let readContent: (() => Promise<string>) | undefined;
let contentBefore: string | undefined;
if (!usesAuthorizedMemoryStore) {
await memoryDeps.ensureMemoryFlushTargetFile({
workspaceDir: params.followupRun.run.workspaceDir,
relativePath: writePath,
});
// Capture one baseline before any write can start. Per-write snapshots can
// pair a failed later write with an earlier success and miss mixed content.
const contentBefore = await readContent();
const systemPrompt = [params.followupRun.run.extraSystemPrompt, plan.systemPrompt]
const absolutePath = path.join(params.followupRun.run.workspaceDir, writePath);
readContent = () =>
fs.promises.readFile(absolutePath, "utf8").catch((error: unknown) => {
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
return "";
}
throw error;
});
// Capture one baseline before any write can start. Per-write snapshots can
// pair a failed later write with an earlier success and miss mixed content.
contentBefore = await readContent();
}
const systemPrompt = [
params.followupRun.run.extraSystemPrompt,
usesAuthorizedMemoryStore ? AUTHORIZED_MEMORY_FLUSH_SYSTEM_PROMPT : plan.systemPrompt,
]
.filter(Boolean)
.join("\n\n");
const selection = resolveMemoryFlushModelFallbackOptions(
@@ -1384,6 +1418,8 @@ export async function runMemoryFlushIfNeeded(params: {
systemPrompt,
selection,
preparedRunAdmission,
usesAuthorizedMemoryStore,
authorizedMemoryWrite,
};
};
let preparedAttempt: Awaited<ReturnType<typeof prepareMemoryFlushAttempt>>;
@@ -1403,6 +1439,8 @@ export async function runMemoryFlushIfNeeded(params: {
systemPrompt: flushSystemPrompt,
selection,
preparedRunAdmission,
usesAuthorizedMemoryStore,
authorizedMemoryWrite,
} = preparedAttempt;
let memoryCompactionCompleted = false;
let memoryFlushWroteTarget = false;
@@ -1500,8 +1538,11 @@ export async function runMemoryFlushIfNeeded(params: {
allowGatewaySubagentBinding: true,
silentExpected: true,
trigger: "memory",
memoryFlushWritePath,
prompt: activeMemoryFlushPlan.prompt,
...(usesAuthorizedMemoryStore ? {} : { memoryFlushWritePath }),
...(authorizedMemoryWrite ? { authorizedMemoryWrite } : {}),
prompt: usesAuthorizedMemoryStore
? AUTHORIZED_MEMORY_FLUSH_PROMPT
: activeMemoryFlushPlan.prompt,
transcriptPrompt: "",
extraSystemPrompt: flushSystemPrompt,
isFinalFallbackAttempt: runOptions.isFinalFallbackAttempt,
@@ -1513,7 +1554,10 @@ export async function runMemoryFlushIfNeeded(params: {
contextEngineLogicalTurnLease: runOptions.contextEngineLogicalTurnLease,
onContextEngineTurnCandidate: runOptions.onContextEngineTurnCandidate,
onAgentEvent: (evt) => {
if (evt.stream === "tool" && evt.data.name === "write") {
if (
evt.stream === "tool" &&
evt.data.name === (usesAuthorizedMemoryStore ? "memory_remember" : "write")
) {
if (evt.data.phase === "result" && evt.data.isError !== true) {
memoryFlushWroteTarget = true;
}
@@ -1536,11 +1580,16 @@ export async function runMemoryFlushIfNeeded(params: {
return result;
},
});
if (activeMemoryFlushPlan.recordWriteProvenance && memoryFlushWroteTarget) {
if (
!usesAuthorizedMemoryStore &&
activeMemoryFlushPlan.recordWriteProvenance &&
memoryFlushWroteTarget &&
readMemoryFlushContent
) {
await activeMemoryFlushPlan.recordWriteProvenance({
workspaceDir: params.followupRun.run.workspaceDir,
relativePath: memoryFlushWritePath,
contentBefore: memoryFlushContentBefore,
contentBefore: memoryFlushContentBefore ?? "",
contentAfter: await readMemoryFlushContent(),
originClass:
params.followupRun.run.senderIsOwner && sessionLogSnapshot?.turnTainted !== true
@@ -47,6 +47,7 @@ type SessionSqliteDatabase = Pick<
| "transcript_rewrite_watermarks"
| "trajectory_runtime_events"
| "transcript_event_identities"
| "transcript_event_memory_policies"
| "transcript_events"
> & {
sqlite_schema: { name: string | null; type: string };
@@ -43,17 +43,42 @@ import { startSessionTranscriptIndexReconcile } from "./session-transcript-recon
import { createSessionTranscriptHeader } from "./transcript-header.js";
import { resolveVisibleTranscriptAppendParentId } from "./transcript-visible-events.js";
type TranscriptEventAppendOptions = {
allowStoredAlias?: boolean;
dedupeByMessageIdempotency?: boolean;
onProjectionReconcileNeeded?: () => void;
scheduleProjectionReconcile?: boolean;
touchMutation?: boolean;
};
type InternalTranscriptEventAppendOptions = TranscriptEventAppendOptions & {
inheritedMemoryPolicy?: PreservedTranscriptMemoryPolicy;
};
export function appendTranscriptEventInTransaction(
database: OpenClawAgentDatabase,
scope: ResolvedTranscriptScope,
event: TranscriptEvent,
options: {
allowStoredAlias?: boolean;
dedupeByMessageIdempotency?: boolean;
onProjectionReconcileNeeded?: () => void;
scheduleProjectionReconcile?: boolean;
touchMutation?: boolean;
} = {},
options: TranscriptEventAppendOptions = {},
): boolean {
return appendTranscriptEventInternal(database, scope, event, options);
}
/** Only sealed compaction may bind a newly appended output to its read source. */
export function appendSealedCompactionTranscriptEventInTransaction(
database: OpenClawAgentDatabase,
scope: ResolvedTranscriptScope,
event: TranscriptEvent,
inheritedMemoryPolicy: PreservedTranscriptMemoryPolicy,
): boolean {
return appendTranscriptEventInternal(database, scope, event, { inheritedMemoryPolicy });
}
function appendTranscriptEventInternal(
database: OpenClawAgentDatabase,
scope: ResolvedTranscriptScope,
event: TranscriptEvent,
options: InternalTranscriptEventAppendOptions = {},
): boolean {
const persistedEvent = canonicalizeTranscriptEventMedia(event);
const db = getSessionKysely(database.db);
@@ -93,6 +118,7 @@ export function appendTranscriptEventInTransaction(
sessionKey: scope.sessionKey,
eventSeq: seq,
createdAt,
...(options.inheritedMemoryPolicy ? { inherited: options.inheritedMemoryPolicy } : {}),
});
if (options.touchMutation !== false) {
touchTranscriptMutationInTransaction(database, scope.sessionId);
@@ -1,4 +1,5 @@
import { err, ok, type Result } from "@openclaw/normalization-core/result";
import { executeSqliteQueryTakeFirstSync } from "../../infra/kysely-sync.js";
import {
openOpenClawAgentDatabase,
resolveOpenClawAgentSqlitePath,
@@ -35,6 +36,8 @@ import {
} from "./session-accessor.sqlite-read.js";
import {
cloneSessionEntry,
getSessionKysely,
resolveSqliteTranscriptArchiveDirectory,
resolveSqliteTranscriptScope,
runExclusiveSqliteSessionWrite,
toDatabaseOptions,
@@ -46,14 +49,25 @@ import {
readCommittedTranscriptMessageSequence,
rememberCommittedTranscriptMessageSequencesInTransaction,
} from "./session-accessor.sqlite-transcript-sequences.js";
import { readTranscriptGenerationInTransaction } from "./session-accessor.sqlite-transcript-state.js";
import {
readNextTranscriptSeq,
readTranscriptGenerationInTransaction,
} from "./session-accessor.sqlite-transcript-state.js";
import {
appendTranscriptEventInTransaction,
appendSealedCompactionTranscriptEventInTransaction,
replaceSqliteTranscriptEventsInTransaction,
rewriteSqliteTranscriptEventRowsInTransaction,
} from "./session-accessor.sqlite-transcript-store.js";
import type { SessionTranscriptWriteTransactionContext } from "./session-accessor.types.js";
import { readAuthorizedTranscriptEventSeqs } from "./session-transcript-memory-policy.js";
import {
persistSealedCompactionMemoryPolicyInTransaction,
readAuthorizedTranscriptDerivation,
readAuthorizedTranscriptEventSeqs,
readSealedCompactionOutputMemoryPolicyInTransaction,
type AuthorizedTranscriptDerivation,
type SealedCompactionMemoryPolicy,
} from "./session-transcript-memory-policy.js";
import type {
SessionTranscriptTurnExpectedState,
SessionTranscriptTurnLifecyclePatch,
@@ -67,7 +81,7 @@ import {
SessionTranscriptWriterClaimReboundError,
withOwnedSessionTranscriptWriterFence,
} from "./transcript-write-context.js";
import type { InternalSessionEntry, SessionEntry } from "./types.js";
import type { InternalSessionEntry, SessionCompactionCheckpoint, SessionEntry } from "./types.js";
import { mergeSessionEntry } from "./types.js";
// Transcript write owner. Queue coordination surrounds synchronous SQLite commit sections.
@@ -655,6 +669,134 @@ export async function withTranscriptWriteTransaction<T>(
);
}
type SealedCompactionCommit = (params: {
database: OpenClawAgentDatabase;
compactionPolicy: SealedCompactionMemoryPolicy;
eventSeq: number;
}) => unknown;
const MAX_SEALED_COMPACTION_CHECKPOINTS_PER_SESSION = 25;
function persistSealedCompactionCheckpointInTransaction(params: {
database: OpenClawAgentDatabase;
resolved: ReturnType<typeof resolveSqliteTranscriptScope>;
checkpoint: SessionCompactionCheckpoint;
}): void {
if (params.checkpoint.sessionId !== params.resolved.sessionId) {
throw new Error("sealed compaction checkpoint is unavailable");
}
const current = readSessionEntryRow(params.database, params.resolved.sessionKey)?.entry;
if (!current || current.sessionId !== params.resolved.sessionId) {
throw new Error("sealed compaction checkpoint session is unavailable");
}
const next = cloneSessionEntry(current);
// Cutover checkpoints are SQLite references, so all byte/file planning is
// complete before BEGIN. Retain the same bounded metadata history as the
// checkpoint owner without doing filesystem work in this transaction.
next.compactionCheckpoints = [...(current.compactionCheckpoints ?? []), params.checkpoint].slice(
-MAX_SEALED_COMPACTION_CHECKPOINTS_PER_SESSION,
);
next.updatedAt = Math.max(current.updatedAt ?? 0, params.checkpoint.createdAt);
writeSessionEntry(params.database, params.resolved.sessionKey, next, { previousEntry: current });
}
function sameAuthorizedTranscriptDerivation(
left: AuthorizedTranscriptDerivation,
right: AuthorizedTranscriptDerivation,
): boolean {
return (
left.sourcePolicySetId === right.sourcePolicySetId &&
left.deliveryAudiencesJson === right.deliveryAudiencesJson &&
left.eventSeqs.length === right.eventSeqs.length &&
left.eventSeqs.every((eventSeq, index) => eventSeq === right.eventSeqs[index])
);
}
/**
* Commits a compaction event and its derived-state owner in one SQLite transaction.
* The artifact owner must stage filesystem bytes before this call and keep its callback
* synchronous; a source-policy race or a partial callback rolls back every durable row.
*/
export async function commitSealedSqliteTranscriptCompaction(params: {
scope: SessionTranscriptWriteScope;
event: TranscriptEvent;
compactionPolicyId: string;
source: AuthorizedTranscriptDerivation;
checkpoint?: SessionCompactionCheckpoint;
commitDerivedState: SealedCompactionCommit;
}): Promise<{ compactionPolicy: SealedCompactionMemoryPolicy; eventSeq: number }> {
const resolved = resolveSqliteTranscriptScope(params.scope);
return await runExclusiveSqliteSessionWrite(resolved, async () => {
let result: { compactionPolicy: SealedCompactionMemoryPolicy; eventSeq: number } | undefined;
runOpenClawAgentWriteTransaction(
(database) => {
const currentSource = readAuthorizedTranscriptDerivation(database.db, resolved.sessionId);
if (!currentSource || !sameAuthorizedTranscriptDerivation(currentSource, params.source)) {
throw new Error("sealed compaction source policy is unavailable");
}
const compactionPolicy = persistSealedCompactionMemoryPolicyInTransaction({
db: database.db,
compactionPolicyId: params.compactionPolicyId,
sessionId: resolved.sessionId,
source: params.source,
});
const outputPolicy = readSealedCompactionOutputMemoryPolicyInTransaction({
database,
sessionId: resolved.sessionId,
source: params.source,
});
if (!outputPolicy) {
throw new Error("sealed compaction output policy is unavailable");
}
if (
!appendSealedCompactionTranscriptEventInTransaction(
database,
resolved,
params.event,
outputPolicy,
)
) {
throw new Error("sealed compaction transcript event was not appended");
}
const eventSeq = readNextTranscriptSeq(database, resolved.sessionId) - 1;
const eventPolicy = executeSqliteQueryTakeFirstSync(
database.db,
getSessionKysely(database.db)
.selectFrom("transcript_event_memory_policies")
.select(["authorization_status", "delivery_audiences_json", "source_policy_set_id"])
.where("session_id", "=", resolved.sessionId)
.where("event_seq", "=", eventSeq),
);
if (
eventPolicy?.authorization_status !== "authorized" ||
eventPolicy.source_policy_set_id !== params.source.sourcePolicySetId ||
eventPolicy.delivery_audiences_json !== params.source.deliveryAudiencesJson
) {
throw new Error("sealed compaction output policy is unavailable");
}
if (params.checkpoint) {
persistSealedCompactionCheckpointInTransaction({
database,
resolved,
checkpoint: params.checkpoint,
});
}
const commitResult = params.commitDerivedState({ database, compactionPolicy, eventSeq });
if (commitResult && typeof (commitResult as PromiseLike<unknown>).then === "function") {
throw new Error("sealed compaction derived-state commit must be synchronous");
}
result = { compactionPolicy, eventSeq };
},
toDatabaseOptions(resolved),
{ operationLabel: "session.compaction.sealed" },
);
if (!result) {
throw new Error("sealed compaction transaction did not commit");
}
return result;
});
}
function isSqliteTranscriptSnapshotUnchanged(
database: OpenClawAgentDatabase,
sessionId: string,
@@ -22,7 +22,7 @@ import {
} from "../../state/openclaw-agent-db.js";
import { readSessionTranscriptMessageEvents } from "./session-accessor.sqlite-active-events.js";
import { materializeSessionStateDeletePlans } from "./session-accessor.sqlite-archive.js";
import { writeSessionEntry } from "./session-accessor.sqlite-entry-store.js";
import { readSessionEntryRow, writeSessionEntry } from "./session-accessor.sqlite-entry-store.js";
import { planSessionStateDeleteIfUnreferenced } from "./session-accessor.sqlite-lifecycle-state.js";
import {
loadLatestAssistantText,
@@ -35,8 +35,11 @@ import {
appendTranscriptMessage,
replaceTranscriptEvents,
trimTranscriptForManualCompact,
commitSealedSqliteTranscriptCompaction,
} from "./session-accessor.sqlite-transcript-write.js";
import {
persistSealedCompactionMemoryPolicyInTransaction,
readAuthorizedTranscriptDerivation,
preserveTranscriptMemoryPolicyTransitionInTransaction,
readAuthorizedTranscriptEventSeqs,
resetTranscriptMemoryPolicyForTest,
@@ -272,6 +275,311 @@ afterEach(() => {
});
describe("transcript memory policy companions", () => {
it("binds a sealed compaction policy to the exact current transcript source set", async () => {
const env = createEnv();
const database = markCutOver(env);
persistExposure(database, { runId: "sealed-compaction-run" });
await appendWithRun({
env,
runId: "sealed-compaction-run",
text: "sealed compaction transcript source",
});
const source = readAuthorizedTranscriptDerivation(database.db, SESSION_ID);
if (!source) {
throw new Error("fixture expected an authorized compaction source");
}
expect(() =>
persistSealedCompactionMemoryPolicyInTransaction({
db: database.db,
compactionPolicyId: "compaction-policy-1",
sessionId: SESSION_ID,
source,
}),
).toThrow("active transaction");
const persisted = runOpenClawAgentWriteTransaction(
(opened) =>
persistSealedCompactionMemoryPolicyInTransaction({
db: opened.db,
compactionPolicyId: "compaction-policy-1",
sessionId: SESSION_ID,
source,
createdAt: 123,
}),
{ agentId: AGENT_ID, env },
);
expect(persisted).toMatchObject({
compactionPolicyId: "compaction-policy-1",
sessionId: SESSION_ID,
sourcePolicySetId: source.sourcePolicySetId,
eventSeqs: source.eventSeqs,
createdAt: 123,
});
expect(
database.db
.prepare(
`SELECT session_id, source_policy_set_id, retention_state, created_at
FROM memory_compaction_policies
WHERE compaction_policy_id = 'compaction-policy-1'`,
)
.get(),
).toEqual({
session_id: SESSION_ID,
source_policy_set_id: source.sourcePolicySetId,
retention_state: "retained",
created_at: 123,
});
expect(() =>
runOpenClawAgentWriteTransaction(
(opened) =>
persistSealedCompactionMemoryPolicyInTransaction({
db: opened.db,
compactionPolicyId: "compaction-policy-2",
sessionId: SESSION_ID,
source: { ...source, eventSeqs: [...source.eventSeqs, 999] },
}),
{ agentId: AGENT_ID, env },
),
).toThrow("source policy is unavailable");
expect(
database.db.prepare("SELECT count(*) AS count FROM memory_compaction_policies").get(),
).toEqual({ count: 1 });
});
it("atomically commits the authorized compaction event, policy, and derived-state callback", async () => {
const env = createEnv();
const database = markCutOver(env);
writeSessionEntry(database, SESSION_KEY, { sessionId: SESSION_ID, updatedAt: 1 });
persistExposure(database, { runId: "sealed-compaction-transaction" });
await appendWithRun({
env,
runId: "sealed-compaction-transaction",
text: "sealed transaction source",
});
const source = readAuthorizedTranscriptDerivation(database.db, SESSION_ID);
if (!source) {
throw new Error("fixture expected an authorized compaction source");
}
const sourceCompanion = database.db
.prepare(
`SELECT policy.run_id, detail.source_event_seq
FROM transcript_event_memory_policies AS policy
JOIN transcript_event_memory_policy_details AS detail
ON detail.session_id = policy.session_id AND detail.event_seq = policy.event_seq
WHERE policy.session_id = ? AND policy.event_seq = ?`,
)
.get(SESSION_ID, source.eventSeqs[0]);
let committed: { eventSeq: number; policyId: string } | undefined;
const commit = async () =>
await withOwnedSessionTranscriptWrites(
{
sessionTarget: {
agentId: AGENT_ID,
// The committing writer has no exposure of its own. The output can
// be authorized only by inheriting the transcript source it read.
expectedWriterRunId: "sealed-compaction-commit",
sessionId: SESSION_ID,
sessionKey: SESSION_KEY,
},
withTranscriptWrite: async (run) => await run(),
},
async () =>
await commitSealedSqliteTranscriptCompaction({
scope: scope(env),
event: {
type: "compaction",
id: "sealed-compaction-entry",
parentId: null,
timestamp: new Date(123).toISOString(),
summary: "sealed summary",
firstKeptEntryId: "source-message",
tokensBefore: 42,
},
compactionPolicyId: "sealed-compaction-policy",
source,
checkpoint: {
checkpointId: "sealed-compaction-checkpoint",
sessionKey: SESSION_KEY,
sessionId: SESSION_ID,
createdAt: 123,
reason: "manual",
summary: "sealed summary",
firstKeptEntryId: "source-message",
preCompaction: { sessionId: SESSION_ID, leafId: "source-message" },
postCompaction: { sessionId: SESSION_ID, entryId: "sealed-compaction-entry" },
},
commitDerivedState({ compactionPolicy, eventSeq }) {
committed = { eventSeq, policyId: compactionPolicy.compactionPolicyId };
},
}),
);
await expect(commit()).resolves.toMatchObject({
compactionPolicy: { compactionPolicyId: "sealed-compaction-policy" },
});
expect(committed).toEqual({
eventSeq: expect.any(Number),
policyId: "sealed-compaction-policy",
});
expect(
readSessionEntryRow(database, SESSION_KEY)?.entry.compactionCheckpoints?.map(
(checkpoint) => checkpoint.checkpointId,
),
).toEqual(["sealed-compaction-checkpoint"]);
expect(
database.db.prepare("SELECT count(*) AS count FROM memory_compaction_policies").get(),
).toEqual({ count: 1 });
expect(
database.db
.prepare(
`SELECT source_session_id, source_event_seq, source_policy_set_id, delivery_audiences_json
FROM memory_compaction_policy_sources
WHERE compaction_policy_id = 'sealed-compaction-policy'`,
)
.all(),
).toEqual(
source.eventSeqs.map((sourceEventSeq) => ({
source_session_id: SESSION_ID,
source_event_seq: sourceEventSeq,
source_policy_set_id: source.sourcePolicySetId,
delivery_audiences_json: source.deliveryAudiencesJson,
})),
);
expect(
database.db
.prepare(
`SELECT policy.run_id, detail.source_event_seq
FROM transcript_event_memory_policies AS policy
JOIN transcript_event_memory_policy_details AS detail
ON detail.session_id = policy.session_id AND detail.event_seq = policy.event_seq
WHERE policy.session_id = ? AND policy.event_seq = ?`,
)
.get(SESSION_ID, committed?.eventSeq),
).toEqual(sourceCompanion);
expect(
loadTranscriptEventsSync(scope(env)).some(
(event) =>
typeof event === "object" &&
event !== null &&
"id" in event &&
event.id === "sealed-compaction-entry",
),
).toBe(true);
const nextSource = readAuthorizedTranscriptDerivation(database.db, SESSION_ID);
if (!nextSource) {
throw new Error("fixture expected the committed compaction companion to remain authorized");
}
await expect(
withOwnedSessionTranscriptWrites(
{
sessionTarget: {
agentId: AGENT_ID,
expectedWriterRunId: "sealed-compaction-commit",
sessionId: SESSION_ID,
sessionKey: SESSION_KEY,
},
withTranscriptWrite: async (run) => await run(),
},
async () =>
await commitSealedSqliteTranscriptCompaction({
scope: scope(env),
event: {
type: "compaction",
id: "rolled-back-compaction-entry",
parentId: null,
timestamp: new Date(124).toISOString(),
summary: "must roll back",
firstKeptEntryId: "source-message",
tokensBefore: 42,
},
compactionPolicyId: "rolled-back-compaction-policy",
source: nextSource,
commitDerivedState() {
throw new Error("derived state failed");
},
}),
),
).rejects.toThrow("derived state failed");
expect(
database.db.prepare("SELECT count(*) AS count FROM memory_compaction_policies").get(),
).toEqual({ count: 1 });
expect(
loadTranscriptEventsSync(scope(env)).some(
(event) =>
typeof event === "object" &&
event !== null &&
"id" in event &&
event.id === "rolled-back-compaction-entry",
),
).toBe(false);
writeSessionEntry(database, SESSION_KEY, {
sessionId: SESSION_ID,
updatedAt: 125,
compactionCheckpoints: Array.from({ length: 25 }, (_, index) => ({
checkpointId: `retained-checkpoint-${index}`,
sessionKey: SESSION_KEY,
sessionId: SESSION_ID,
createdAt: index,
reason: "manual" as const,
preCompaction: { sessionId: SESSION_ID, leafId: `pre-${index}` },
postCompaction: { sessionId: SESSION_ID, entryId: `post-${index}` },
})),
});
await expect(
withOwnedSessionTranscriptWrites(
{
sessionTarget: {
agentId: AGENT_ID,
expectedWriterRunId: "sealed-compaction-commit",
sessionId: SESSION_ID,
sessionKey: SESSION_KEY,
},
withTranscriptWrite: async (run) => await run(),
},
async () =>
await commitSealedSqliteTranscriptCompaction({
scope: scope(env),
event: {
type: "compaction",
id: "checkpoint-cap-compaction-entry",
parentId: null,
timestamp: new Date(125).toISOString(),
summary: "bounded checkpoint summary",
firstKeptEntryId: "source-message",
tokensBefore: 42,
},
compactionPolicyId: "checkpoint-cap-compaction-policy",
source: nextSource,
checkpoint: {
checkpointId: "checkpoint-cap-newest",
sessionKey: SESSION_KEY,
sessionId: SESSION_ID,
createdAt: 125,
reason: "manual",
preCompaction: { sessionId: SESSION_ID, leafId: "source-message" },
postCompaction: {
sessionId: SESSION_ID,
entryId: "checkpoint-cap-compaction-entry",
},
},
commitDerivedState() {},
}),
),
).resolves.toBeDefined();
expect(readSessionEntryRow(database, SESSION_KEY)?.entry.compactionCheckpoints).toHaveLength(
25,
);
expect(
readSessionEntryRow(database, SESSION_KEY)?.entry.compactionCheckpoints?.[0],
).toMatchObject({ checkpointId: "retained-checkpoint-1" });
expect(
readSessionEntryRow(database, SESSION_KEY)?.entry.compactionCheckpoints?.at(-1),
).toMatchObject({ checkpointId: "checkpoint-cap-newest" });
});
it("enforces Doctor shadow-read-only companion persistence for only its bound subject", async () => {
const env = createEnv();
const options = { agentId: AGENT_ID, env };
@@ -812,6 +1120,9 @@ describe("transcript memory policy companions", () => {
expect(companion.source_event_seq).toBeGreaterThanOrEqual(0);
}
expect(readAuthorizedTranscriptEventSeqs(database.db, SESSION_ID)?.size).toBe(7);
expect(readAuthorizedTranscriptDerivation(database.db, SESSION_ID)).toMatchObject({
eventSeqs: [0, 1, 2, 3, 4, 5, 6],
});
});
it("uses the captured trusted actor and token-free delegation rather than reconstructing session facts", async () => {
@@ -18,6 +18,7 @@ type TranscriptMemoryPolicyDatabase = Pick<
| "memory_policy_revisions"
| "memory_policy_set_members"
| "memory_policy_sets"
| "memory_compaction_policies"
| "memory_resource_revisions"
| "memory_run_exposure_resources"
| "memory_run_exposures"
@@ -72,6 +73,24 @@ export type TranscriptMemoryPolicyTransitionKind =
| "switch"
| "checkpoint";
/** The complete, currently authorized transcript source set for one derivation. */
export type AuthorizedTranscriptDerivation = Readonly<{
eventSeqs: readonly number[];
sourcePolicySetId: string;
deliveryAudiencesJson: string;
}>;
/** Immutable policy evidence for one sealed compaction output. */
export type SealedCompactionMemoryPolicy = Readonly<{
compactionPolicyId: string;
sessionId: string;
sourcePolicySetId: string;
deliveryAudiencesJson: string;
eventSeqs: readonly number[];
retentionState: "retained";
createdAt: number;
}>;
const enforcementByDatabase = new WeakMap<DatabaseSync, boolean>();
function policyDatabase(db: DatabaseSync) {
@@ -1039,6 +1058,234 @@ export function readAuthorizedTranscriptEventSeqs(
}
}
/**
* Compaction is a derivation, so native session history cannot be treated as an implicit source.
* A mixed source set must be partitioned by its owner; this generic path denies it rather than
* letting a model phrase its way into a broader summary.
*/
export function readAuthorizedTranscriptDerivation(
db: DatabaseSync,
sessionId: string,
): AuthorizedTranscriptDerivation | undefined {
if (!isTranscriptMemoryPolicyEnforcedInDatabase(db)) {
return undefined;
}
try {
const policy = policyDatabase(db);
const eventRows = executeSqliteQuerySync(
db,
policy
.selectFrom("transcript_events")
.select("seq")
.where("session_id", "=", sessionId)
.orderBy("seq"),
).rows;
if (eventRows.length === 0) {
return undefined;
}
const readable = readAuthorizedTranscriptEventSeqs(db, sessionId);
if (
!readable ||
readable.size !== eventRows.length ||
eventRows.some((event) => !readable.has(event.seq))
) {
return undefined;
}
const rows = executeSqliteQuerySync(
db,
policy
.selectFrom("transcript_event_memory_policies as policy")
.innerJoin("transcript_event_memory_policy_details as detail", (join) =>
join
.onRef("detail.session_id", "=", "policy.session_id")
.onRef("detail.event_seq", "=", "policy.event_seq"),
)
.select([
"policy.event_seq",
"policy.source_policy_set_id",
"policy.delivery_audiences_json",
])
.where("policy.session_id", "=", sessionId)
.where("policy.authorization_status", "=", "authorized")
.where("detail.retention_state", "=", "retained")
.orderBy("policy.event_seq"),
).rows;
if (
rows.length !== eventRows.length ||
rows.some(
(row) =>
!readable.has(row.event_seq) ||
row.source_policy_set_id === null ||
row.delivery_audiences_json === null,
)
) {
return undefined;
}
const sourcePolicySetIds = new Set(rows.map((row) => row.source_policy_set_id));
const deliveryAudiences = new Set(rows.map((row) => row.delivery_audiences_json));
if (sourcePolicySetIds.size !== 1 || deliveryAudiences.size !== 1) {
return undefined;
}
const sourcePolicySetId = rows[0]?.source_policy_set_id;
const deliveryAudiencesJson = rows[0]?.delivery_audiences_json;
if (!sourcePolicySetId || !deliveryAudiencesJson) {
return undefined;
}
return Object.freeze({
eventSeqs: Object.freeze(eventRows.map((event) => event.seq)),
sourcePolicySetId,
deliveryAudiencesJson,
});
} catch {
return undefined;
}
}
/**
* Stores only a source set which is still the complete authorized transcript.
* The caller owns the surrounding transaction that also appends the summary,
* checkpoint, resource revision, and lineage; standalone policy rows would
* otherwise make an interrupted compaction look durable.
*/
export function persistSealedCompactionMemoryPolicyInTransaction(params: {
db: DatabaseSync;
compactionPolicyId: string;
sessionId: string;
source: AuthorizedTranscriptDerivation;
createdAt?: number;
}): SealedCompactionMemoryPolicy {
if (!params.db.isTransaction) {
throw new Error("sealed compaction policy requires an active transaction");
}
const compactionPolicyId = params.compactionPolicyId.trim();
const sessionId = params.sessionId.trim();
if (!compactionPolicyId || !sessionId) {
throw new Error("sealed compaction policy is unavailable");
}
const current = readAuthorizedTranscriptDerivation(params.db, sessionId);
const source = params.source;
if (
!current ||
current.sourcePolicySetId !== source.sourcePolicySetId ||
current.deliveryAudiencesJson !== source.deliveryAudiencesJson ||
current.eventSeqs.length !== source.eventSeqs.length ||
current.eventSeqs.some((eventSeq, index) => eventSeq !== source.eventSeqs[index])
) {
throw new Error("sealed compaction source policy is unavailable");
}
const createdAt = params.createdAt ?? Date.now();
const db = policyDatabase(params.db);
const existing = executeSqliteQueryTakeFirstSync(
params.db,
db
.selectFrom("memory_compaction_policies")
.select(["session_id", "source_policy_set_id", "retention_state", "created_at"])
.where("compaction_policy_id", "=", compactionPolicyId),
);
if (existing) {
const persistedSources = executeSqliteQuerySync(
params.db,
db
.selectFrom("memory_compaction_policy_sources")
.select([
"source_event_seq",
"source_policy_set_id",
"source_session_id",
"delivery_audiences_json",
])
.where("compaction_policy_id", "=", compactionPolicyId)
.orderBy("source_event_seq"),
).rows;
if (
existing.session_id !== sessionId ||
existing.source_policy_set_id !== source.sourcePolicySetId ||
existing.retention_state !== "retained" ||
persistedSources.length !== source.eventSeqs.length ||
persistedSources.some(
(persistedSource, index) =>
persistedSource.source_session_id !== sessionId ||
persistedSource.source_event_seq !== source.eventSeqs[index] ||
persistedSource.source_policy_set_id !== source.sourcePolicySetId ||
persistedSource.delivery_audiences_json !== source.deliveryAudiencesJson,
)
) {
throw new Error("sealed compaction policy idempotency conflict");
}
return Object.freeze({
compactionPolicyId,
sessionId,
sourcePolicySetId: source.sourcePolicySetId,
deliveryAudiencesJson: source.deliveryAudiencesJson,
eventSeqs: Object.freeze([...source.eventSeqs]),
retentionState: "retained",
createdAt: existing.created_at,
});
}
executeSqliteQuerySync(
params.db,
db.insertInto("memory_compaction_policies").values({
compaction_policy_id: compactionPolicyId,
session_id: sessionId,
source_policy_set_id: source.sourcePolicySetId,
retention_state: "retained",
created_at: createdAt,
}),
);
for (const eventSeq of source.eventSeqs) {
executeSqliteQuerySync(
params.db,
db.insertInto("memory_compaction_policy_sources").values({
compaction_policy_id: compactionPolicyId,
source_session_id: sessionId,
source_event_seq: eventSeq,
source_policy_set_id: source.sourcePolicySetId,
delivery_audiences_json: source.deliveryAudiencesJson,
created_at: createdAt,
}),
);
}
return Object.freeze({
compactionPolicyId,
sessionId,
sourcePolicySetId: source.sourcePolicySetId,
deliveryAudiencesJson: source.deliveryAudiencesJson,
eventSeqs: Object.freeze([...source.eventSeqs]),
retentionState: "retained",
createdAt,
});
}
/**
* The output event must inherit proof from an event the compactor actually
* read. A writer fence is only permission to commit; it is not source proof.
*/
export function readSealedCompactionOutputMemoryPolicyInTransaction(params: {
database: OpenClawAgentDatabase;
sessionId: string;
source: AuthorizedTranscriptDerivation;
}): PreservedTranscriptMemoryPolicy | undefined {
const current = readAuthorizedTranscriptDerivation(params.database.db, params.sessionId);
if (
!current ||
current.sourcePolicySetId !== params.source.sourcePolicySetId ||
current.deliveryAudiencesJson !== params.source.deliveryAudiencesJson ||
current.eventSeqs.length !== params.source.eventSeqs.length ||
current.eventSeqs.some((eventSeq, index) => eventSeq !== params.source.eventSeqs[index])
) {
return undefined;
}
const firstEventSeq = params.source.eventSeqs[0];
return [...readPreservedTranscriptMemoryPoliciesInTransaction(params.database, params.sessionId).values()]
.flat()
.find(
(policy) =>
policy.eventSeq === firstEventSeq &&
policy.sourcePolicySetId === params.source.sourcePolicySetId &&
policy.deliveryAudiencesJson === params.source.deliveryAudiencesJson &&
policy.retentionState === "retained",
);
}
export function resetTranscriptMemoryPolicyForTest(db: DatabaseSync): void {
enforcementByDatabase.delete(db);
}
+6
View File
@@ -1,6 +1,7 @@
/** Session identity and context preparation for isolated cron runs. */
import { isDeepStrictEqual } from "node:util";
import { hasAnyAuthProfileStoreSource } from "../../agents/auth-profiles/source-check.js";
import { mayInjectAutonomousSourceTranscript } from "../../agents/memory-autonomous-run-policy.js";
import { findModelInCatalog } from "../../agents/model-catalog-lookup.js";
import { listOpenAIAuthProfileProvidersForAgentRuntime } from "../../agents/openai-routing.js";
import { loadAgentRuntimePluginRegistryHandle } from "../../agents/runtime-plugins.js";
@@ -11,6 +12,7 @@ import { resolveSessionWorkStartError } from "../../config/sessions/lifecycle.js
import type { AgentDefaultsConfig } from "../../config/types.agent-defaults.js";
import type { OpenClawConfig } from "../../config/types.openclaw.js";
import type { SourceDeliveryPlan } from "../../infra/outbound/source-delivery-plan.js";
import { isMemoryIsolationCutoverAgent } from "../../plugins/memory-cutover.js";
import type { PluginRegistry } from "../../plugins/registry-types.js";
import { isCronSessionKey, parseAgentSessionKey } from "../../routing/session-key.js";
import {
@@ -522,6 +524,10 @@ export async function prepareCronRunContext(params: {
// Current jobs stay detached; a bounded tail preserves context without transcript continuation.
const currentConversationContext =
input.job.sessionTarget === "current" &&
mayInjectAutonomousSourceTranscript({
sessionTarget: input.job.sessionTarget,
memoryIsolationActive: isMemoryIsolationCutoverAgent(agentId),
}) &&
agentPayload &&
sourceSessionKey &&
sourceSessionEntry
+34 -18
View File
@@ -32,6 +32,7 @@ import {
loadExactSessionEntry,
type SessionEntryLifecycleRemoval,
} from "../config/sessions/session-accessor.js";
import { buildSessionCreationStamp } from "../config/sessions/session-entry-provenance.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import {
hasActiveCronJobs,
@@ -42,6 +43,7 @@ import {
import { resolveCronSession } from "../cron/isolated-agent/session.js";
import { writeCronJobScratch } from "../cron/scratch-store.js";
import { resolveCronJobsStorePathFromConfig } from "../cron/store.js";
import { isMemoryIsolationCutoverAgent } from "../plugins/memory-cutover.js";
import {
getQueueSize,
isCommandLaneTaskMarkerCurrent,
@@ -70,6 +72,7 @@ import {
shouldPreflightExecEventWake,
} from "./heartbeat-runner-prompt.js";
import {
resolveMemoryIsolatedHeartbeatSessionKey,
resolveHeartbeatSession,
resolveIsolatedHeartbeatSessionKey,
resolveStaleHeartbeatIsolatedSessionKey,
@@ -403,7 +406,8 @@ export async function prepareHeartbeatRunStage(wake: ReadyHeartbeatWake) {
// a new session ID (empty transcript) each run, avoiding the cost of
// sending the full conversation history (~100K tokens) to the LLM.
// Delivery routing still uses the main session entry (lastChannel, lastTo).
const useIsolatedSession = heartbeat?.isolatedSession === true;
const memoryIsolationActive = isMemoryIsolationCutoverAgent(agentId);
const useIsolatedSession = heartbeat?.isolatedSession === true || memoryIsolationActive;
const delivery = await resolveHeartbeatDeliveryTargetWithSessionRoute({
cfg,
agentId,
@@ -488,22 +492,32 @@ export async function prepareHeartbeatRunStage(wake: ReadyHeartbeatWake) {
let runSessionEntry = entry;
let outboundPolicySessionKey: string | undefined;
if (useIsolatedSession) {
const configuredSession = resolveHeartbeatSession(cfg, agentId, heartbeat);
// Collapse only the repeated `:heartbeat` suffixes introduced by wake-triggered
// re-entry for heartbeat-created isolated sessions. Real session keys that
// happen to end with `:heartbeat` still get a distinct isolated sibling.
const { isolatedSessionKey, isolatedBaseSessionKey } = resolveIsolatedHeartbeatSessionKey({
agentId,
sessionKey,
configuredSessionKey: configuredSession.sessionKey,
sessionEntry: entry,
});
// Under memory isolation, old session keys may still select a delivery
// target but cannot name the autonomous run or its policy context.
const isolatedSession = memoryIsolationActive
? {
isolatedSessionKey: resolveMemoryIsolatedHeartbeatSessionKey(agentId),
isolatedBaseSessionKey: undefined,
}
: (() => {
const configuredSession = resolveHeartbeatSession(cfg, agentId, heartbeat);
return resolveIsolatedHeartbeatSessionKey({
agentId,
sessionKey,
configuredSessionKey: configuredSession.sessionKey,
sessionEntry: entry,
});
})();
const { isolatedSessionKey } = isolatedSession;
const { isolatedBaseSessionKey } = isolatedSession;
const isolatedStorePath = resolveSessionStorePathCore(cfg.session?.store, { agentId });
const staleIsolatedSessionKey = resolveStaleHeartbeatIsolatedSessionKey({
sessionKey,
isolatedSessionKey,
isolatedBaseSessionKey,
});
const staleIsolatedSessionKey = isolatedBaseSessionKey
? resolveStaleHeartbeatIsolatedSessionKey({
sessionKey,
isolatedSessionKey,
isolatedBaseSessionKey,
})
: undefined;
if (
isReplyRunActive(isolatedSessionKey) ||
hasActiveRunForSession(isolatedSessionKey, listActiveEmbeddedRuns)
@@ -551,7 +565,9 @@ export async function prepareHeartbeatRunStage(wake: ReadyHeartbeatWake) {
});
const nextEntry = {
...cronSession.sessionEntry,
heartbeatIsolatedBaseSessionKey: isolatedBaseSessionKey,
...(memoryIsolationActive
? buildSessionCreationStamp({ via: "cron", actor: { type: "system" } })
: { heartbeatIsolatedBaseSessionKey: isolatedBaseSessionKey }),
};
runSessionEntry = nextEntry;
return nextEntry;
@@ -567,7 +583,7 @@ export async function prepareHeartbeatRunStage(wake: ReadyHeartbeatWake) {
});
}
runSessionKey = isolatedSessionKey;
outboundPolicySessionKey = isolatedBaseSessionKey;
outboundPolicySessionKey = memoryIsolationActive ? isolatedSessionKey : isolatedBaseSessionKey;
const actualUseHeartbeatResponseToolPrompt = shouldUseHeartbeatResponseToolPrompt({
cfg,
+10
View File
@@ -196,6 +196,16 @@ export function resolveIsolatedHeartbeatSessionKey(params: {
};
}
/**
* Memory-isolated heartbeats are autonomous service work, not a fork of the
* session that supplied their delivery route. Keep the identity stable per
* agent so repeated wakes retain service-only state without inheriting a
* human session's transcript or subject.
*/
export function resolveMemoryIsolatedHeartbeatSessionKey(agentId: string): string {
return toAgentStoreSessionKey({ agentId, requestKey: "service:heartbeat" });
}
export function resolveStaleHeartbeatIsolatedSessionKey(params: {
sessionKey: string;
isolatedSessionKey: string;
@@ -2,7 +2,13 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { OpenClawConfig } from "../config/config.js";
import { resolveMainSessionKey } from "../config/sessions.js";
import { resetMemoryIsolationCutoverForTest } from "../plugins/memory-cutover.js";
import {
closeOpenClawAgentDatabasesForTest,
openOpenClawAgentDatabase,
} from "../state/openclaw-agent-db.js";
import { runHeartbeatOnce } from "./heartbeat-runner.js";
import { resolveMemoryIsolatedHeartbeatSessionKey } from "./heartbeat-runner-session.js";
import { installHeartbeatRunnerTestRuntime } from "./heartbeat-runner.test-harness.js";
import {
readSessionStoreForTest,
@@ -24,6 +30,8 @@ installHeartbeatRunnerTestRuntime();
afterEach(() => {
deliverOutboundPayloadsInternal.mockClear();
closeOpenClawAgentDatabasesForTest();
resetMemoryIsolationCutoverForTest();
});
type DeliveryRequest = {
@@ -43,7 +51,11 @@ function latestDeliveryRequest(): DeliveryRequest {
return request as DeliveryRequest;
}
function makeIsolatedLastTargetConfig(tmpDir: string, storePath: string): OpenClawConfig {
function makeIsolatedLastTargetConfig(
tmpDir: string,
storePath: string,
isolatedSession = true,
): OpenClawConfig {
return {
agents: {
list: [{ id: "main", default: true }],
@@ -52,7 +64,7 @@ function makeIsolatedLastTargetConfig(tmpDir: string, storePath: string): OpenCl
heartbeat: {
every: "5m",
target: "last",
isolatedSession: true,
isolatedSession,
},
},
},
@@ -160,4 +172,57 @@ describe("runHeartbeatOnce - isolated heartbeat outbound session mirror", () =>
});
});
});
it("uses a system service session under memory isolation instead of the routed user session", async () => {
await withTempHeartbeatSandbox(async ({ tmpDir, storePath, replySpy }) => {
const cfg = makeIsolatedLastTargetConfig(tmpDir, storePath, false);
const baseSessionKey = resolveMainSessionKey(cfg);
const serviceSessionKey = resolveMemoryIsolatedHeartbeatSessionKey("main");
const nowMs = Date.now();
const database = openOpenClawAgentDatabase({ agentId: "main" });
database.db
.prepare(
`INSERT INTO memory_migrations
(migration_id, source_kind, source_hash, phase, classification_json, plan_hash,
verified_at, cutover_at, updated_at)
VALUES ('heartbeat-service-session', 'test', 'test-source', 'cutover', '{}',
'test-plan', 1, 1, 1)`,
)
.run();
resetMemoryIsolationCutoverForTest();
await seedHeartbeatScratchForTest({ content: "Check the scheduled service work." });
await seedSessionStore(storePath, baseSessionKey, {
sessionId: "user-session",
updatedAt: nowMs - 1_000,
lastChannel: "whatsapp",
lastProvider: "whatsapp",
lastTo: "+15551234567",
});
replySpy.mockResolvedValueOnce({ text: "Service work needs attention." });
const result = await runHeartbeatOnce({
cfg,
deps: {
getReplyFromConfig: replySpy,
getQueueSize: () => 0,
nowMs: () => nowMs,
},
});
expect(result.status).toBe("ran");
expect(replySpy.mock.calls[0]?.[0]).toMatchObject({ SessionKey: serviceSessionKey });
expect(latestDeliveryRequest()).toMatchObject({
channel: "whatsapp",
to: "+15551234567",
session: { key: serviceSessionKey, policyKey: serviceSessionKey },
});
expect(readSessionStoreForTest(storePath)[serviceSessionKey]).toMatchObject({
createdActor: { type: "system" },
createdVia: "cron",
});
expect(readSessionStoreForTest(storePath)[serviceSessionKey]).not.toHaveProperty(
"heartbeatIsolatedBaseSessionKey",
);
});
});
});
@@ -297,6 +297,7 @@ export async function admitMemoryAuthorizationRuntime(
return Object.freeze({ ok: false, reasonCode: "backend-nonconforming" });
}
const source = runtime.value;
const stageSealedCompaction = readCallable(source, "stageSealedCompaction");
return Object.freeze({
ok: true,
runtime: Object.freeze({
@@ -310,6 +311,13 @@ export async function admitMemoryAuthorizationRuntime(
writeAuthorized: (methods.writeAuthorized as AuthorizedMemoryRuntime["writeAuthorized"]).bind(
source,
),
...(stageSealedCompaction
? {
stageSealedCompaction: (
stageSealedCompaction as NonNullable<AuthorizedMemoryRuntime["stageSealedCompaction"]>
).bind(source),
}
: {}),
importAuthorized: (
methods.importAuthorized as AuthorizedMemoryRuntime["importAuthorized"]
).bind(source),
+36
View File
@@ -36,6 +36,7 @@ vi.mock("../logger.js", () => ({
const {
MEMORY_INVOCATION_UNAVAILABLE,
createAuthorizedMemoryDeriveInvocation,
createAuthorizedMemoryReadInvocation,
createAuthorizedMemoryWriteInvocation,
materializeAuthorizedMemoryVirtualView,
@@ -760,6 +761,41 @@ describe("authorized memory read invocation", () => {
expect(admittedVirtualView.readAuthorizedVirtualFile).not.toHaveBeenCalled();
});
it("never materializes a derive invocation through the generic virtual filesystem", async () => {
const admittedVirtualView = {
materializeAuthorizedVirtualView: vi.fn(),
readAuthorizedVirtualFile: vi.fn(),
};
const context = {
...createContext(),
operation: "derive" as const,
};
const plan = {
...createPlan(),
operation: "derive" as const,
};
mocks.materialize.mockReturnValue(context);
mocks.admit.mockResolvedValue({
ok: true,
runtime: {
authorize: vi.fn().mockResolvedValue(plan),
searchAuthorized: vi.fn(),
readAuthorized: vi.fn(),
virtualView: admittedVirtualView,
},
});
const invocation = await createAuthorizedMemoryDeriveInvocation({ context: {} as never });
if (invocation === MEMORY_INVOCATION_UNAVAILABLE) {
throw new Error("fixture failed to admit derive invocation");
}
await expect(materializeAuthorizedMemoryVirtualView({ invocation })).resolves.toBe(
MEMORY_INVOCATION_UNAVAILABLE,
);
expect(admittedVirtualView.materializeAuthorizedVirtualView).not.toHaveBeenCalled();
});
it("rejects malformed duplicate and case-colliding virtual views before any broker read", async () => {
const admittedVirtualView = {
materializeAuthorizedVirtualView: vi.fn(async () => ({
+101 -17
View File
@@ -2,14 +2,18 @@ import { logWarn } from "../logger.js";
import type {
AuthorizedMemoryVirtualView,
AuthorizedMemoryMutation,
AuthorizedMemoryContentPlan,
AuthorizedMemoryPlan,
AuthorizedMemoryResultEnvelope,
AuthorizedResourceHandle,
AudienceRef,
MemoryContentAccessOperation,
MemoryContentAccessContext,
MemoryAccessContext,
MemoryWriteResult,
AuthorizedMemoryRuntime,
AuthorizedSealedCompactionArtifact,
AuthorizedTranscriptDerivationSource,
} from "../memory-host-sdk/host/authorization.js";
import type {
MemoryReadResult,
@@ -67,8 +71,8 @@ export type AuthorizedMemoryWriteInvocation = Readonly<{
type InvocationState = Readonly<{
trustedContext: TrustedMemoryAccessContext;
context: MemoryContentAccessContext<"read">;
plan: AuthorizedMemoryPlan & Readonly<{ operation: "read" }>;
context: MemoryContentAccessContext;
plan: AuthorizedMemoryPlan & Readonly<{ operation: MemoryContentAccessOperation }>;
authorizationStartedAtMs: number;
runtime: AdmittedAuthorizedMemoryReadRuntime;
/** Bound at admission; registry changes cannot replace this run's provider. */
@@ -83,6 +87,12 @@ type InvocationState = Readonly<{
runExposureRevisions: Set<string>;
}>;
type ReadInvocationState = Omit<InvocationState, "context" | "plan"> &
Readonly<{
context: MemoryContentAccessContext<"read">;
plan: AuthorizedMemoryContentPlan<"read">;
}>;
const VIRTUAL_ROOT_PATTERN = /^[a-z][a-z0-9-]{0,63}$/u;
const invocationStates = new WeakMap<object, InvocationState>();
@@ -163,13 +173,14 @@ function readCurrentWriteContext(state: WriteInvocationState): MemoryAccessConte
function readCurrentContext(
state: InvocationState,
): MemoryContentAccessContext<"read"> | undefined {
): MemoryContentAccessContext | undefined {
const current = materializeTrustedMemoryAccessContext(state.trustedContext);
if (!current || current.operation !== "read") {
if (!current || (current.operation !== "read" && current.operation !== "derive")) {
return undefined;
}
const readContext = current as MemoryContentAccessContext<"read">;
const readContext = current as MemoryContentAccessContext;
if (
readContext.operation !== state.context.operation ||
readContext.contextFingerprint !== state.context.contextFingerprint ||
readContext.runId !== state.context.runId ||
readContext.agentId !== state.context.agentId ||
@@ -185,9 +196,19 @@ function readCurrentContext(
return readContext;
}
function isReadContentContext(
context: MemoryContentAccessContext,
): context is MemoryContentAccessContext<"read"> {
return context.operation === "read";
}
function isReadInvocationState(state: InvocationState): state is ReadInvocationState {
return state.context.operation === "read" && state.plan.operation === "read";
}
function validateEnvelope<T>(params: {
state: InvocationState;
context: MemoryContentAccessContext<"read">;
context: MemoryContentAccessContext;
expectedRevisionHandles: readonly string[];
envelope: AuthorizedMemoryResultEnvelope<T>;
}): boolean {
@@ -254,7 +275,7 @@ function mergeEnvelope(
function readTranscriptExposure(params: {
state: InvocationState;
context: MemoryContentAccessContext<"read">;
context: MemoryContentAccessContext;
pendingEnvelope?: AuthorizedMemoryResultEnvelope<unknown>;
}) {
const { state, context, pendingEnvelope } = params;
@@ -298,7 +319,7 @@ function readTranscriptExposure(params: {
*/
function recordEnvelopeExposure(params: {
state: InvocationState;
context: MemoryContentAccessContext<"read">;
context: MemoryContentAccessContext;
envelope: AuthorizedMemoryResultEnvelope<unknown>;
}): void {
if (
@@ -329,8 +350,8 @@ function readState(invocation: AuthorizedMemoryReadInvocation): InvocationState
function canonicalizeAuthorizedVirtualView(params: {
view: AuthorizedMemoryVirtualView;
context: MemoryContentAccessContext<"read">;
plan: AuthorizedMemoryPlan & Readonly<{ operation: "read" }>;
context: MemoryContentAccessContext;
plan: AuthorizedMemoryPlan & Readonly<{ operation: MemoryContentAccessOperation }>;
}): AuthorizedMemoryVirtualView | undefined {
const { view, context, plan } = params;
const mountHandles = new Set(plan.mounts.map((mount) => mount.mountHandle));
@@ -409,16 +430,17 @@ function canonicalizeAuthorizedVirtualView(params: {
* Creates a process-local, opaque read invocation. No caller can inject a serializable identity,
* audience, plan, or continuation: all of those come from the trusted context and selected backend.
*/
export async function createAuthorizedMemoryReadInvocation(params: {
async function createAuthorizedMemoryContentInvocation(params: {
context: TrustedMemoryAccessContext;
capability?: MemoryPluginCapability;
operation: MemoryContentAccessOperation;
}): Promise<AuthorizedMemoryReadInvocation | MemoryInvocationUnavailable> {
const materialized = materializeTrustedMemoryAccessContext(params.context);
if (!materialized || materialized.operation !== "read") {
if (!materialized || materialized.operation !== params.operation) {
logMemoryInvocationDiagnostic("materialization-rejected");
return MEMORY_INVOCATION_UNAVAILABLE;
}
const context = materialized as MemoryContentAccessContext<"read">;
const context = materialized as MemoryContentAccessContext;
const capability =
params.capability ??
resolveSelectedMemoryCapabilityRegistration(requireActivePluginRegistry())?.capability;
@@ -430,7 +452,7 @@ export async function createAuthorizedMemoryReadInvocation(params: {
try {
const authorizationStartedAtMs = Date.now();
const plan = (await admission.runtime.authorize(context)) as AuthorizedMemoryPlan &
Readonly<{ operation: "read" }>;
Readonly<{ operation: MemoryContentAccessOperation }>;
if (!isCurrentPlan({ context, plan, nowMs: Date.now() })) {
logMemoryInvocationDiagnostic("invalid-plan");
return MEMORY_INVOCATION_UNAVAILABLE;
@@ -461,6 +483,24 @@ export async function createAuthorizedMemoryReadInvocation(params: {
}
}
export async function createAuthorizedMemoryReadInvocation(params: {
context: TrustedMemoryAccessContext;
capability?: MemoryPluginCapability;
}): Promise<AuthorizedMemoryReadInvocation | MemoryInvocationUnavailable> {
return await createAuthorizedMemoryContentInvocation({ ...params, operation: "read" });
}
/**
* Creates an opaque content invocation for a derivation. Its content operations use the derive
* plan end to end, rather than checking derive once and then falling back to a read-only view.
*/
export async function createAuthorizedMemoryDeriveInvocation(params: {
context: TrustedMemoryAccessContext;
capability?: MemoryPluginCapability;
}): Promise<AuthorizedMemoryReadInvocation | MemoryInvocationUnavailable> {
return await createAuthorizedMemoryContentInvocation({ ...params, operation: "derive" });
}
/**
* Creates a process-local write invocation from host-minted facts. This is separate from read
* exposure because a write is a resource lifecycle decision, never a continuation of a search hit.
@@ -530,6 +570,40 @@ export async function writeAuthorizedMemoryForInvocation(params: {
}
}
/**
* Stages bytes before the caller opens its transaction. The returned closure is
* the only route that may insert the selected runtime's revision/catalog rows
* into the core-owned sealed compaction transaction.
*/
export async function stageAuthorizedMemorySealedCompactionForInvocation(params: {
invocation: AuthorizedMemoryWriteInvocation;
content: string;
transcriptSource: AuthorizedTranscriptDerivationSource;
}): Promise<AuthorizedSealedCompactionArtifact | MemoryInvocationUnavailable> {
const state = writeInvocationStates.get(params.invocation);
const context = state ? readCurrentWriteContext(state) : undefined;
if (
!state ||
!context ||
context.operation !== "derive" ||
!state.runtime.stageSealedCompaction ||
!isCurrentPlan({ context, plan: state.plan, nowMs: Date.now() })
) {
return MEMORY_INVOCATION_UNAVAILABLE;
}
try {
return await state.runtime.stageSealedCompaction({
context,
plan: state.plan as AuthorizedMemoryPlan & Readonly<{ operation: "derive" }>,
content: params.content,
transcriptSource: params.transcriptSource,
});
} catch {
logMemoryInvocationDiagnostic("authorization-failed");
return MEMORY_INVOCATION_UNAVAILABLE;
}
}
/**
* Obtains a selected-plugin projection for generic FS and sandbox consumers.
* It is deliberately separate from search/read: no tool argument can turn a
@@ -540,7 +614,15 @@ export async function materializeAuthorizedMemoryVirtualView(params: {
}): Promise<AuthorizedMemoryVirtualView | MemoryInvocationUnavailable> {
const state = readState(params.invocation);
const context = state ? readCurrentContext(state) : undefined;
if (!state || !context || !state.virtualView) {
// Derivations may expose source bytes only to their dedicated content path. A generic virtual
// filesystem is a read capability, so it must not become an alternate derive transport.
if (
!state ||
!context ||
!isReadContentContext(context) ||
!isReadInvocationState(state) ||
!state.virtualView
) {
return MEMORY_INVOCATION_UNAVAILABLE;
}
try {
@@ -576,6 +658,8 @@ export async function readAuthorizedMemoryVirtualFile(params: {
if (
!state ||
!context ||
!isReadContentContext(context) ||
!isReadInvocationState(state) ||
!state.virtualView ||
!isCurrentPlan({ context, plan: state.plan, nowMs: Date.now() }) ||
state.virtualViews.get(params.view.viewId) !== params.view ||
@@ -632,7 +716,7 @@ export async function searchAuthorizedMemoryForInvocation(params: {
...(params.sources ? { sources: params.sources } : {}),
limit: Math.max(1, Math.min(100, Math.trunc(params.limit ?? 10))),
...(params.signal ? { signal: params.signal } : {}),
});
} as never);
const revisionHandles = envelope.value.map((result) => result.resourceHandle.resourceRevision);
if (
!validateEnvelope({
@@ -681,7 +765,7 @@ export async function readAuthorizedMemoryForInvocation(params: {
handle,
...(params.from !== undefined ? { from: params.from } : {}),
...(params.lines !== undefined ? { lines: params.lines } : {}),
});
} as never);
if (
!validateEnvelope({
state,
+11
View File
@@ -462,6 +462,17 @@ describe("memory plugin state", () => {
expect(resolveMemoryFlushPlan({})?.relativePath).toBe("memory/same-owner.md");
});
it("keeps the flush plan available for a cut-over agent", () => {
isMemoryIsolationCutoverAgentMock.mockReturnValue(true);
registerMemoryCapability("memory-core", {
flushPlanResolver: () => createMemoryFlushPlan("memory/authorized.md"),
});
expect(resolveMemoryFlushPlan({ agentId: "main" })?.relativePath).toBe(
"memory/authorized.md",
);
});
it("passes citations mode through to the prompt builder", () => {
registerTestMemoryPromptBuilder(({ citationsMode }) => [
`citations: ${citationsMode ?? "default"}`,
-3
View File
@@ -317,9 +317,6 @@ export function resolveMemoryFlushPlan(params: {
agentId?: string;
nowMs?: number;
}): MemoryFlushPlan | null {
if (params.agentId && isMemoryIsolationCutoverAgent(params.agentId)) {
return null;
}
return getMemoryCapability()?.capability.flushPlanResolver?.(params) ?? null;
}
export function getMemoryRuntime(): MemoryPluginRuntime | undefined {
@@ -284,6 +284,7 @@ export type MemoryPluginRuntime = {
searchAuthorized?: AuthorizedMemoryRuntime["searchAuthorized"];
readAuthorized?: AuthorizedMemoryRuntime["readAuthorized"];
writeAuthorized?: AuthorizedMemoryRuntime["writeAuthorized"];
stageSealedCompaction?: AuthorizedMemoryRuntime["stageSealedCompaction"];
importAuthorized?: AuthorizedMemoryRuntime["importAuthorized"];
syncAuthorized?: AuthorizedMemoryRuntime["syncAuthorized"];
exportAuthorized?: AuthorizedMemoryRuntime["exportAuthorized"];
+29
View File
@@ -165,6 +165,15 @@ export interface MemoryCompactionPolicies {
source_policy_set_id: string;
}
export interface MemoryCompactionPolicySources {
compaction_policy_id: string;
created_at: number;
delivery_audiences_json: string;
source_event_seq: number;
source_policy_set_id: string;
source_session_id: string;
}
export interface MemoryEmbeddingCache {
dims: number | null;
embedding: string;
@@ -301,6 +310,23 @@ export interface MemoryPolicySets {
policy_set_id: string;
}
export interface MemoryRevisionPolicyRequirements {
created_at: number;
expected_revision_id: string;
expected_revocation_epoch: number;
policy_id: string;
requirement_kind: string;
revision_id: string;
}
export interface MemoryLineageEdges {
child_revision_id: string;
created_at: number;
parent_id: string;
parent_kind: string;
relation_kind: string;
}
export interface MemoryPreoutputExposureLedger {
agent_id: string;
context_fingerprint: string;
@@ -849,6 +875,7 @@ export interface DB {
heartbeat_outcomes: HeartbeatOutcomes;
memory_audit_outbox: MemoryAuditOutbox;
memory_compaction_policies: MemoryCompactionPolicies;
memory_compaction_policy_sources: MemoryCompactionPolicySources;
memory_embedding_cache: MemoryEmbeddingCache;
memory_index_chunk_provenance: MemoryIndexChunkProvenance;
memory_index_chunk_recall_metadata: MemoryIndexChunkRecallMetadata;
@@ -863,6 +890,8 @@ export interface DB {
memory_policy_revisions: MemoryPolicyRevisions;
memory_policy_set_members: MemoryPolicySetMembers;
memory_policy_sets: MemoryPolicySets;
memory_revision_policy_requirements: MemoryRevisionPolicyRequirements;
memory_lineage_edges: MemoryLineageEdges;
memory_preoutput_exposure_authorization_facts: MemoryPreoutputExposureAuthorizationFacts;
memory_preoutput_exposure_ledger: MemoryPreoutputExposureLedger;
memory_resource_revisions: MemoryResourceRevisions;
+87
View File
@@ -790,6 +790,63 @@ BEGIN
SELECT RAISE(ABORT, 'tombstoned memory resource revisions cannot be reactivated');
END;
-- A derived revision records every stable policy that must still be current before it can be
-- exposed. Missing requirements are a durable deny, never a reason to infer access from content.
CREATE TABLE IF NOT EXISTS memory_revision_policy_requirements (
revision_id TEXT NOT NULL,
policy_id TEXT NOT NULL,
expected_revision_id TEXT NOT NULL,
expected_revocation_epoch INTEGER NOT NULL CHECK (expected_revocation_epoch >= 0),
requirement_kind TEXT NOT NULL CHECK (requirement_kind IN ('output-policy', 'source-policy')),
created_at INTEGER NOT NULL,
PRIMARY KEY (revision_id, policy_id),
FOREIGN KEY (revision_id) REFERENCES memory_resource_revisions(revision_id) ON DELETE RESTRICT,
FOREIGN KEY (policy_id) REFERENCES memory_policies(policy_id) ON DELETE RESTRICT,
FOREIGN KEY (expected_revision_id) REFERENCES memory_policy_revisions(revision_id) ON DELETE RESTRICT
) STRICT;
CREATE INDEX IF NOT EXISTS idx_memory_revision_policy_requirements_policy
ON memory_revision_policy_requirements(policy_id, expected_revision_id, expected_revocation_epoch);
CREATE TRIGGER IF NOT EXISTS memory_revision_policy_requirements_no_update
BEFORE UPDATE ON memory_revision_policy_requirements
BEGIN
SELECT RAISE(ABORT, 'memory revision policy requirements are immutable');
END;
CREATE TRIGGER IF NOT EXISTS memory_revision_policy_requirements_no_delete
BEFORE DELETE ON memory_revision_policy_requirements
BEGIN
SELECT RAISE(ABORT, 'memory revision policy requirements cannot be deleted');
END;
-- Parent revisions are immutable derivation facts. Readers traverse resource parents rather than
-- maintaining a mutable descendant cache, so an ancestor tombstone takes effect immediately.
CREATE TABLE IF NOT EXISTS memory_lineage_edges (
child_revision_id TEXT NOT NULL,
parent_kind TEXT NOT NULL CHECK (parent_kind IN ('resource-revision', 'transcript-policy-set', 'compaction-policy', 'checkpoint', 'export', 'child-artifact')),
parent_id TEXT NOT NULL,
relation_kind TEXT NOT NULL CHECK (relation_kind IN ('derived-from', 'compacted-from', 'flushed-from', 'dreamed-from', 'promoted-from', 'exported-from', 'child-produced')),
created_at INTEGER NOT NULL,
PRIMARY KEY (child_revision_id, parent_kind, parent_id, relation_kind),
FOREIGN KEY (child_revision_id) REFERENCES memory_resource_revisions(revision_id) ON DELETE RESTRICT
) STRICT;
CREATE INDEX IF NOT EXISTS idx_memory_lineage_edges_parent
ON memory_lineage_edges(parent_kind, parent_id, child_revision_id);
CREATE TRIGGER IF NOT EXISTS memory_lineage_edges_no_update
BEFORE UPDATE ON memory_lineage_edges
BEGIN
SELECT RAISE(ABORT, 'memory lineage edges are immutable');
END;
CREATE TRIGGER IF NOT EXISTS memory_lineage_edges_no_delete
BEFORE DELETE ON memory_lineage_edges
BEGIN
SELECT RAISE(ABORT, 'memory lineage edges cannot be deleted');
END;
CREATE TABLE IF NOT EXISTS memory_resource_subjects (
revision_id TEXT NOT NULL,
subject_kind TEXT NOT NULL CHECK (subject_kind IN ('person', 'project', 'conversation', 'topic')),
@@ -1268,6 +1325,36 @@ BEGIN
SELECT RAISE(ABORT, 'memory compaction policies cannot be deleted');
END;
-- A compaction policy names the complete transcript source set, not merely its
-- common policy set. Transcript rows may later be reset or archived, so this
-- immutable provenance deliberately has no foreign key to mutable event rows.
CREATE TABLE IF NOT EXISTS memory_compaction_policy_sources (
compaction_policy_id TEXT NOT NULL,
source_session_id TEXT NOT NULL,
source_event_seq INTEGER NOT NULL CHECK (source_event_seq >= 0),
source_policy_set_id TEXT NOT NULL,
delivery_audiences_json TEXT NOT NULL,
created_at INTEGER NOT NULL,
PRIMARY KEY (compaction_policy_id, source_session_id, source_event_seq),
FOREIGN KEY (compaction_policy_id) REFERENCES memory_compaction_policies(compaction_policy_id) ON DELETE RESTRICT,
FOREIGN KEY (source_policy_set_id) REFERENCES memory_policy_sets(policy_set_id) ON DELETE RESTRICT
) STRICT;
CREATE INDEX IF NOT EXISTS idx_memory_compaction_policy_sources_source
ON memory_compaction_policy_sources(source_session_id, source_event_seq, compaction_policy_id);
CREATE TRIGGER IF NOT EXISTS memory_compaction_policy_sources_no_update
BEFORE UPDATE ON memory_compaction_policy_sources
BEGIN
SELECT RAISE(ABORT, 'memory compaction policy sources are immutable');
END;
CREATE TRIGGER IF NOT EXISTS memory_compaction_policy_sources_no_delete
BEFORE DELETE ON memory_compaction_policy_sources
BEGIN
SELECT RAISE(ABORT, 'memory compaction policy sources cannot be deleted');
END;
CREATE TABLE IF NOT EXISTS standing_intents (
intent_key INTEGER PRIMARY KEY,
id TEXT NOT NULL UNIQUE,
@@ -10,6 +10,8 @@ export const AGENT_SCOPED_MEMORY_TABLES = [
"memory_policy_entries",
"memory_resources",
"memory_resource_revisions",
"memory_revision_policy_requirements",
"memory_lineage_edges",
"memory_resource_subjects",
"memory_scoped_chunks",
"memory_scoped_chunk_vectors",
@@ -26,6 +28,7 @@ export const AGENT_SCOPED_MEMORY_TABLES = [
"transcript_event_memory_policy_details",
"transcript_event_memory_policy_transitions",
"memory_compaction_policies",
"memory_compaction_policy_sources",
] as const;
export const AGENT_SCOPED_MEMORY_FTS_TABLE = "memory_scoped_chunks_fts";