From 0d201845875705463a3a7561ab2be08fefbada90 Mon Sep 17 00:00:00 2001 From: Galin Iliev Date: Fri, 14 Aug 2026 00:26:12 -0700 Subject: [PATCH] feat(memory): retain transcript policy lineage --- docs/concepts/2026-07-29-memory-impl-plan.md | 31 +- .../memory-core/src/session-ingestion.test.ts | 33 +- .../memory-core/src/session-ingestion.ts | 12 + ...ion-accessor.sqlite-archive.worker.test.ts | 342 ++++++++ .../session-accessor.sqlite-archive.worker.ts | 218 ++++- .../session-accessor.sqlite-checkpoint.ts | 62 +- .../session-accessor.sqlite-import.ts | 155 ++++ .../session-accessor.sqlite-message-cut.ts | 24 +- .../session-accessor.sqlite-parent-session.ts | 113 ++- ...ession-accessor.sqlite-transcript-store.ts | 22 +- .../session-transcript-memory-policy.test.ts | 538 ++++++++++++- .../session-transcript-memory-policy.ts | 761 +++++++++++++++++- .../session-transcript-policy-archive.test.ts | 64 ++ .../session-transcript-policy-archive.ts | 383 +++++++++ .../sessions/session-transcript-search.ts | 20 +- src/plugins/memory-invocation.ts | 7 +- .../memory-run-exposure-ledger.test.ts | 122 +++ src/plugins/memory-run-exposure-ledger.ts | 225 +++++- src/plugins/memory-run-exposure.ts | 137 +++- src/state/openclaw-agent-db.generated.d.ts | 67 ++ src/state/openclaw-agent-schema.sql | 173 ++++ .../openclaw-agent-scoped-memory-schema.ts | 6 + 22 files changed, 3367 insertions(+), 148 deletions(-) create mode 100644 src/config/sessions/session-transcript-policy-archive.test.ts create mode 100644 src/config/sessions/session-transcript-policy-archive.ts diff --git a/docs/concepts/2026-07-29-memory-impl-plan.md b/docs/concepts/2026-07-29-memory-impl-plan.md index 49db9d31d1e8..92b29f5000d9 100644 --- a/docs/concepts/2026-07-29-memory-impl-plan.md +++ b/docs/concepts/2026-07-29-memory-impl-plan.md @@ -1359,23 +1359,38 @@ Add: Phase 2B is complete only when all of the following are demonstrated: -- [ ] Every readable user, assistant, tool-result, summary, checkpoint, and +- [x] Every readable user, assistant, tool-result, summary, checkpoint, and system event has an atomic, evaluable policy companion row. -- [ ] Every scoped exposure can be mapped to the durable events, policy-set +- [x] Every scoped exposure can be mapped to the durable events, policy-set revision, delivery audience, and run exposure revision it influenced. -- [ ] Stable policy IDs are revalidated against current active revisions and +- [x] Stable policy IDs are revalidated against current active revisions and revocation epochs; captured historical allows are not permanent grants. -- [ ] Reset, rollover, fork, rewind, checkpoint restore, archive, export, and +- [x] Reset, rollover, fork, rewind, checkpoint restore, archive, export, and confirmed import preserve subject and policy lineage exactly. -- [ ] Missing, invalid, stale, or authorization-pending labels exclude events +- [x] Missing, invalid, stale, or authorization-pending labels exclude events from replay, search, compaction, export, and derivation. -- [ ] Authorization is never reconstructed from session-key shape, transcript +- [x] Authorization is never reconstructed from session-key shape, transcript JSON, rendered prompt text, or `InputProvenance`. -- [ ] No plugin call or async work occurs inside the transcript commit +- [x] No plugin call or async work occurs inside the transcript commit transaction. -- [ ] Atomic-write, transition, policy-revision, revoke-race, session-rebound, +- [x] Atomic-write, transition, policy-revision, revoke-race, session-rebound, and legacy-unlabeled transcript tests pass. +Proof (2026-08-14): policy companion/evidence and current-policy tests in +`src/config/sessions/session-transcript-memory-policy.test.ts`; archive/export +and confirmed-import rollback tests in +`src/config/sessions/session-accessor.sqlite-archive.worker.test.ts`; canonical +archive parsing in `src/config/sessions/session-transcript-policy-archive.test.ts`; +checkpoint and parent-fork lifecycle coverage in +`src/config/sessions/session-accessor.conformance.test.ts` and +`src/config/sessions/session-accessor.parent-fork.test.ts`; durable actor and +delegation ledger coverage in `src/plugins/memory-run-exposure-ledger.test.ts`; +and enforced archive-ingestion quarantine in +`extensions/memory-core/src/session-ingestion.test.ts`. Blacksmith Testbox +`tbx_01kzzhjsz3x1fsrrrm035egekp` passed 104 runtime-config and 10 plugin-ledger +tests for these files. `src/config/sessions/session-transcript-memory-policy.ts` +keeps policy evaluation and all plugin work outside the synchronous transaction. + ### Phase 2B rollback Disable transcript recall, compaction, flush, dreaming, and derivation for the diff --git a/extensions/memory-core/src/session-ingestion.test.ts b/extensions/memory-core/src/session-ingestion.test.ts index af63b27cfa07..4310a2f6e946 100644 --- a/extensions/memory-core/src/session-ingestion.test.ts +++ b/extensions/memory-core/src/session-ingestion.test.ts @@ -1,7 +1,14 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +const isLegacyMemorySurfaceDisabled = vi.hoisted(() => vi.fn(() => false)); + +vi.mock("openclaw/plugin-sdk/memory-core-host-runtime-core", async (importOriginal) => ({ + ...(await importOriginal()), + isLegacyMemorySurfaceDisabled, +})); import { foreignSessionIngestionSource, scanSessionIngestionSource, @@ -11,6 +18,8 @@ import { const tempDirs: string[] = []; afterEach(async () => { + isLegacyMemorySurfaceDisabled.mockReset(); + isLegacyMemorySurfaceDisabled.mockReturnValue(false); await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true }))); }); @@ -27,6 +36,28 @@ describe("session ingestion", () => { expect(source?.scope).toBe("main:foo.jsonl"); }); + it("excludes unconfirmed archive artifacts from cut-over derivation", () => { + isLegacyMemorySurfaceDisabled.mockReturnValue(true); + + expect( + sessionIngestionSourceFromCorpus({ + agentId: "main", + artifactKind: "archive-artifact", + sessionFile: path.join(os.tmpdir(), "archived.jsonl.deleted.2026-08-14"), + sessionId: "archived", + sessionKind: "interactive", + }), + ).toBeNull(); + }); + + it("rejects caller-supplied archive files after cut-over with a visible transfer outcome", () => { + isLegacyMemorySurfaceDisabled.mockReturnValue(true); + + expect(() => foreignSessionIngestionSource("main", "/tmp/archived.jsonl")).toThrow( + "confirmed transcript import is available", + ); + }); + it("verifies backfill content despite an unchanged size and mtime", async () => { const dir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-session-ingestion-")); tempDirs.push(dir); diff --git a/extensions/memory-core/src/session-ingestion.ts b/extensions/memory-core/src/session-ingestion.ts index f0fd1c3ac414..76f5947bd83c 100644 --- a/extensions/memory-core/src/session-ingestion.ts +++ b/extensions/memory-core/src/session-ingestion.ts @@ -9,6 +9,7 @@ import { statSessionEntrySync, type SessionTranscriptCorpusEntry, } from "openclaw/plugin-sdk/memory-core-host-engine-sessions"; +import { isLegacyMemorySurfaceDisabled } from "openclaw/plugin-sdk/memory-core-host-runtime-core"; import type { MemorySearchResult } from "openclaw/plugin-sdk/memory-core-host-runtime-files"; import { formatMemoryDreamingDay } from "openclaw/plugin-sdk/memory-core-host-status"; import { appendRegularFile } from "openclaw/plugin-sdk/security-runtime"; @@ -91,6 +92,12 @@ function sessionPathFromCorpus(entry: SessionTranscriptCorpusEntry): string { export function sessionIngestionSourceFromCorpus( entry: SessionTranscriptCorpusEntry, ): SessionIngestionSource | null { + // Archive JSONL has no confirmed transfer package yet. In cut-over mode it + // cannot carry a currently revalidated transcript policy, so deny it before + // any parser turns the bytes into derivable memory. + if (entry.artifactKind === "archive-artifact" && isLegacyMemorySurfaceDisabled(entry.agentId)) { + return null; + } const sessionPath = sessionPathFromCorpus(entry); if (entry.sessionKind !== "interactive") { return null; @@ -133,6 +140,11 @@ export function foreignSessionIngestionSource( agentId: string, archiveFile: string, ): SessionIngestionSource { + if (isLegacyMemorySurfaceDisabled(agentId)) { + throw new Error( + "Archive-file session backfill is unavailable after scoped-memory cutover; retain the archive until a confirmed transcript import is available.", + ); + } const absolutePath = path.resolve(archiveFile); const normalizedPath = absolutePath.replaceAll("\\", "/"); return { diff --git a/src/config/sessions/session-accessor.sqlite-archive.worker.test.ts b/src/config/sessions/session-accessor.sqlite-archive.worker.test.ts index f80f4554da4d..4a967e15d496 100644 --- a/src/config/sessions/session-accessor.sqlite-archive.worker.test.ts +++ b/src/config/sessions/session-accessor.sqlite-archive.worker.test.ts @@ -6,6 +6,7 @@ import path from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { recordAcpParentStreamEvents } from "../../agents/subagents/spawn/acp-parent-stream-store.sqlite.js"; import { executeSqliteQuerySync, getNodeSqliteKysely } from "../../infra/kysely-sync.js"; +import { resetMemoryIsolationCutoverForTest } from "../../plugins/memory-cutover.js"; import type { DB as OpenClawAgentKyselyDatabase } from "../../state/openclaw-agent-db.generated.js"; import { closeOpenClawAgentDatabasesForTest, @@ -23,6 +24,8 @@ import { replaceSessionEntry, } from "./session-accessor.js"; import { materializeSessionStateDeletePlans } from "./session-accessor.sqlite-archive.js"; +import { materializeTranscriptArchiveInWorker } from "./session-accessor.sqlite-archive.worker.js"; +import { importConfirmedSqliteTranscriptPolicyArchive } from "./session-accessor.sqlite-import.js"; import { deleteMaterializedSessionStatePlans, planSessionStateDeleteIfUnreferenced, @@ -30,6 +33,14 @@ import { import { touchTranscriptMutationInTransaction } from "./session-accessor.sqlite-transcript-state.js"; import { replaceTranscriptEvents } from "./session-accessor.sqlite-transcript-write.js"; import { resolveSqliteTargetFromSessionStorePath } from "./session-sqlite-target.js"; +import { + readAuthorizedTranscriptEventSeqs, + resetTranscriptMemoryPolicyForTest, +} from "./session-transcript-memory-policy.js"; +import { + parseTranscriptPolicyArchive, + restoreConfirmedTranscriptPolicyArchiveInTransaction, +} from "./session-transcript-policy-archive.js"; type TestTranscriptEvent = { id: string; @@ -46,6 +57,7 @@ describe("SQLite transcript archive worker", () => { }); afterEach(() => { + resetMemoryIsolationCutoverForTest(); closeOpenClawAgentDatabasesForTest(); fs.rmSync(tempDir, { recursive: true, force: true }); }); @@ -221,6 +233,213 @@ describe("SQLite transcript archive worker", () => { ).toMatchObject({ published_at: expect.any(Number) }); }); + it("keeps legacy archives raw JSONL and exports enforced events with immutable companions", async () => { + const legacySessionId = "legacy-policy-archive-session"; + const legacySessionKey = "agent:main:legacy-policy-archive"; + const legacyEvent = createTranscriptEvent(legacySessionId, "legacy archive bytes"); + await replaceSessionEntry( + { sessionKey: legacySessionKey, storePath }, + { sessionId: legacySessionId, updatedAt: Date.now() }, + ); + await replaceTranscriptEvents( + { sessionKey: legacySessionKey, sessionId: legacySessionId, storePath }, + [legacyEvent], + ); + const database = openLifecycleTestDatabase(storePath); + const legacyArchive = materializeTranscriptArchiveInWorker( + planArchiveWorker(database, path.dirname(storePath), legacySessionId), + ); + expect(readMaterializedArchiveLines(legacyArchive.archive)).toEqual([ + JSON.stringify(legacyEvent), + ]); + + const sessionId = "enforced-policy-archive-session"; + const sessionKey = "agent:main:enforced-policy-archive"; + const event = createTranscriptEvent(sessionId, "audited archive bytes"); + await replaceSessionEntry({ sessionKey, storePath }, { sessionId, updatedAt: Date.now() }); + await replaceTranscriptEvents({ sessionKey, sessionId, storePath }, [event]); + const { eventSeq } = seedEnforcedArchivePolicy(database, { sessionId }); + + const archive = materializeTranscriptArchiveInWorker( + planArchiveWorker(database, path.dirname(storePath), sessionId), + ); + const lines = readMaterializedArchiveLines(archive.archive); + expect(lines).toHaveLength(2); + expect(lines[0]).toBe(JSON.stringify(event)); + expect(JSON.parse(lines[1] ?? "")).toMatchObject({ + type: "openclaw.memory-policy-archive-v1", + version: 1, + agentId: "main", + sessionId, + eventSeq, + subject: expect.objectContaining({ + sessionKey, + sessionIdentityRevision: expect.any(String), + subjectRevision: expect.any(String), + }), + policy: expect.objectContaining({ + runExposureSetId: "archive-exposure-set", + sourcePolicySetId: "archive-policy-set", + }), + detail: expect.objectContaining({ + sourceEventSeq: eventSeq, + sourceSessionId: sessionId, + }), + }); + + const parsed = parseTranscriptPolicyArchive(readMaterializedArchiveContent(archive.archive)); + expect(parsed).toBeDefined(); + if (!parsed) { + throw new Error("expected a confirmed archive envelope"); + } + database.db.exec(/* sqlite-allow-raw: fixture resets a just-restored pending companion. */ ` + DELETE FROM transcript_event_memory_policy_details + WHERE session_id = '${sessionId}'; + UPDATE transcript_event_memory_policies + SET authorization_status = 'pending', + source_policy_set_id = NULL, + run_exposure_set_id = NULL, + run_exposure_revision = NULL, + delivery_audiences_json = NULL, + session_identity_revision = NULL, + subject_revision = NULL, + run_id = NULL, + context_fingerprint = NULL + WHERE session_id = '${sessionId}'; + `); + runOpenClawAgentWriteTransaction( + (transactionDatabase) => + restoreConfirmedTranscriptPolicyArchiveInTransaction({ + archive: parsed, + database: transactionDatabase, + sessionId, + sessionKey, + }), + { agentId: database.agentId, path: database.path }, + ); + expect(readAuthorizedTranscriptEventSeqs(database.db, sessionId)).toEqual(new Set([eventSeq])); + expect(readMaterializedArchiveLines(archive.archive)).toHaveLength(2); + }); + + it("restores a confirmed archive only through its original immutable session subject", async () => { + const sessionId = "confirmed-policy-archive-session"; + const sessionKey = "agent:main:confirmed-policy-archive"; + const event = createTranscriptEvent(sessionId, "confirmed archive restore bytes"); + await replaceSessionEntry({ sessionKey, storePath }, { sessionId, updatedAt: Date.now() }); + await replaceTranscriptEvents({ sessionKey, sessionId, storePath }, [event]); + const database = openLifecycleTestDatabase(storePath); + const { eventSeq } = seedEnforcedArchivePolicy(database, { sessionId }); + const archive = materializeTranscriptArchiveInWorker( + planArchiveWorker(database, path.dirname(storePath), sessionId), + ); + const archiveContent = readMaterializedArchiveContent(archive.archive); + const sourceSnapshot = database.db + .prepare( + `SELECT session_identity_revision, subject_revision + FROM session_memory_subject_snapshots + WHERE session_id = ?`, + ) + .get(sessionId); + + // Lifecycle deletion removes rows owned by the old session window while the + // immutable subject snapshot remains available for a confirmed same-agent restore. + database.db.exec(/* sqlite-allow-raw: fixture simulates post-archive lifecycle reclamation. */ ` + DELETE FROM session_nodes WHERE session_key = '${sessionKey}'; + DELETE FROM session_windows WHERE session_id = '${sessionId}'; + `); + expect( + database.db + .prepare("SELECT COUNT(*) AS count FROM session_memory_subjects WHERE session_key = ?") + .get(sessionKey), + ).toEqual({ count: 1 }); + + await expect( + importConfirmedSqliteTranscriptPolicyArchive({ + agentId: database.agentId, + archiveContent, + entry: { sessionId, updatedAt: Date.now() }, + sessionKey, + storePath, + }), + ).resolves.toEqual({ sessionId, sessionKey, transcriptEvents: 1 }); + expect(readAuthorizedTranscriptEventSeqs(database.db, sessionId)).toEqual(new Set([eventSeq])); + expect( + database.db + .prepare( + `SELECT session_identity_revision, subject_revision + FROM session_memory_subject_snapshots + WHERE session_id = ?`, + ) + .get(sessionId), + ).toEqual(sourceSnapshot); + }); + + it("rolls a confirmed archive import back when the retained policy is no longer current", async () => { + const sessionId = "revoked-confirmed-policy-archive-session"; + const sessionKey = "agent:main:revoked-confirmed-policy-archive"; + await replaceSessionEntry({ sessionKey, storePath }, { sessionId, updatedAt: Date.now() }); + await replaceTranscriptEvents({ sessionKey, sessionId, storePath }, [ + createTranscriptEvent(sessionId, "revoked confirmed archive restore bytes"), + ]); + const database = openLifecycleTestDatabase(storePath); + seedEnforcedArchivePolicy(database, { sessionId }); + const archive = materializeTranscriptArchiveInWorker( + planArchiveWorker(database, path.dirname(storePath), sessionId), + ); + const archiveContent = readMaterializedArchiveContent(archive.archive); + database.db + .exec(/* sqlite-allow-raw: fixture revokes after archive confirmation materializes. */ ` + DELETE FROM session_nodes WHERE session_key = '${sessionKey}'; + DELETE FROM session_windows WHERE session_id = '${sessionId}'; + DROP TRIGGER memory_resource_revisions_immutable_fields; + UPDATE memory_resource_revisions SET expires_at = 1 WHERE revision_id = 'archive-resource-revision'; + `); + + await expect( + importConfirmedSqliteTranscriptPolicyArchive({ + agentId: database.agentId, + archiveContent, + entry: { sessionId, updatedAt: Date.now() }, + sessionKey, + storePath, + }), + ).rejects.toThrow("confirmed transcript archive policy is no longer authorized"); + expect( + database.db + .prepare("SELECT COUNT(*) AS count FROM session_nodes WHERE session_key = ?") + .get(sessionKey), + ).toEqual({ count: 0 }); + expect( + database.db + .prepare("SELECT COUNT(*) AS count FROM transcript_events WHERE session_id = ?") + .get(sessionId), + ).toEqual({ count: 0 }); + }); + + it.each([ + ["missing companion", {}], + ["expired exposed resource", { expiresAt: 1 }], + ])("fails closed for an enforced archive with %s", async (_name, options) => { + const sessionId = `blocked-policy-archive-${_name.replaceAll(" ", "-")}`; + const sessionKey = `agent:main:${sessionId}`; + await replaceSessionEntry({ sessionKey, storePath }, { sessionId, updatedAt: Date.now() }); + await replaceTranscriptEvents({ sessionKey, sessionId, storePath }, [ + createTranscriptEvent(sessionId, "must remain in the source database"), + ]); + const database = openLifecycleTestDatabase(storePath); + seedEnforcedArchivePolicy(database, { + sessionId, + includeDetail: _name !== "missing companion", + ...options, + }); + + expect(() => + materializeTranscriptArchiveInWorker( + planArchiveWorker(database, path.dirname(storePath), sessionId), + ), + ).toThrow(`Unauthorized transcript policy archive event for ${sessionId}`); + }); + it("archives a logical agent transcript through the exact database's physical owner", async () => { const sharedDatabasePath = path.join(tempDir, "shared.sqlite"); const mainSessionId = "shared-physical-owner-main-session"; @@ -682,6 +901,19 @@ function readArchiveLines(archivePath: string | undefined): string[] { .split("\n"); } +function readMaterializedArchiveContent( + archive: NonNullable["archive"]>, +): string { + expect(archive).toBeTruthy(); + return decodeSessionArchiveBytes(archive.bytes, archive.encoding === "zstd"); +} + +function readMaterializedArchiveLines( + archive: NonNullable["archive"]>, +): string[] { + return readMaterializedArchiveContent(archive).trim().split("\n"); +} + function sha256(content: string): string { return createHash("sha256").update(content).digest("hex"); } @@ -697,6 +929,116 @@ function openLifecycleTestDatabase(storePath: string) { }); } +function seedEnforcedArchivePolicy( + database: ReturnType, + params: { sessionId: string; includeDetail?: boolean; expiresAt?: number }, +): { eventSeq: number } { + const session = database.db + .prepare( + `SELECT session_identity_revision, session_key, subject_revision + FROM session_memory_subject_snapshots + WHERE session_id = ?`, + ) + .get(params.sessionId) as + | { session_identity_revision: string; session_key: string; subject_revision: string } + | undefined; + if (!session) { + throw new Error(`expected session subject snapshot for ${params.sessionId}`); + } + const event = database.db + .prepare( + `SELECT seq + FROM transcript_events + WHERE session_id = ? + ORDER BY seq ASC + LIMIT 1`, + ) + .get(params.sessionId) as { seq: number } | undefined; + if (!event) { + throw new Error(`expected transcript event for ${params.sessionId}`); + } + database.db.exec(/* sqlite-allow-raw: fixture establishes one evaluable archive lineage. */ ` + INSERT INTO memory_migrations + (migration_id, source_kind, source_hash, phase, classification_json, plan_hash, + verified_at, cutover_at, updated_at) + VALUES ('archive-cutover', 'test', 'archive-source', 'cutover', '{}', 'archive-plan', 1, 1, 1); + INSERT INTO memory_policies + (policy_id, agent_id, current_revision_id, revocation_epoch, lifecycle_state, created_at, updated_at) + VALUES ('archive-policy', 'main', 'archive-policy-revision', 0, 'active', 1, 1); + INSERT INTO memory_policy_revisions + (revision_id, policy_id, revision_number, revocation_epoch, lifecycle_state, + actor_kind, actor_id, reason, created_at) + VALUES ('archive-policy-revision', 'archive-policy', 1, 0, 'active', 'human', 'alice', 'fixture', 1); + INSERT INTO memory_policy_sets + (policy_set_id, agent_id, memory_policy_revision, member_policy_set_ids_json, created_at) + VALUES ('archive-policy-set', 'main', 'archive-policy-revision', '["plugin-policy-set"]', 1); + INSERT INTO memory_policy_set_members + (policy_set_id, policy_id, expected_revision_id, expected_revocation_epoch, + audience_intersection_json, retention_state, created_at) + VALUES ('archive-policy-set', 'archive-policy', 'archive-policy-revision', 0, + '[{"id":"alice","kind":"user"}]', 'retained', 1); + INSERT INTO memory_storage_roots + (storage_root_id, agent_id, backend_kind, opaque_locator, path_key_version, path_key, + authority_kind, authority_owner_id, default_capabilities_json, lifecycle_state, created_at, updated_at) + VALUES ('archive-root', 'main', 'builtin', 'builtin:v1:archive', 1, + 's1_archive_fixture_path_key_000', 'user', 'alice', '["read"]', 'active', 1, 1); + INSERT INTO memory_stores + (store_id, agent_id, storage_root_id, policy_id, scope_kind, audience_kind, audience_id, + lifecycle_state, created_at, updated_at) + VALUES ('archive-store', 'main', 'archive-root', 'archive-policy', 'user', 'user', 'alice', 'active', 1, 1); + INSERT INTO memory_resources + (resource_id, agent_id, store_id, logical_locator, source, created_at) + VALUES ('archive-resource', 'main', 'archive-store', 'memory/archive.md', 'memory', 1); + INSERT INTO memory_resource_revisions + (revision_id, resource_id, revision_number, artifact_locator, content_hash, content_bytes, + policy_revision_id, policy_revocation_epoch, source_policy_set_id, lifecycle_state, + actor_kind, actor_id, expires_at, created_at, activated_at, retired_at) + VALUES ('archive-resource-revision', 'archive-resource', 1, 'archive.md', 'archive', 7, + 'archive-policy-revision', 0, 'plugin-policy-set', 'active', 'human', 'alice', + ${params.expiresAt ?? "NULL"}, 1, 1, NULL); + INSERT INTO memory_run_exposures + (exposure_set_id, agent_id, run_id, context_fingerprint, plan_id, revision_number, + previous_exposure_set_id, source_policy_set_ids_json, effective_source_policy_set_id, + exposed_resource_revisions_json, exposure_receipt_ids_json, egress_receipt_ids_json, + delivery_audiences_json, delivery_revision, egress_registry_revision, created_at) + VALUES ('archive-exposure-set', 'main', 'archive-run', 'archive-context', 'archive-plan', 1, + NULL, '["plugin-policy-set"]', 'archive-policy-set', + '["archive-resource-revision"]', '["archive-exposure"]', '["archive-egress"]', + '[{"id":"alice","kind":"user"}]', 'delivery-1', 'egress-1', 1); + INSERT INTO memory_run_exposure_resources + (exposure_set_id, resource_revision_id, policy_set_id, created_at) + VALUES ('archive-exposure-set', 'archive-resource-revision', 'archive-policy-set', 1); + `); + database.db + .prepare( + `INSERT INTO transcript_event_memory_policies + (session_id, event_seq, authorization_status, source_policy_set_id, run_exposure_set_id, + run_exposure_revision, delivery_audiences_json, session_identity_revision, + subject_revision, run_id, context_fingerprint, created_at) + VALUES (?, ?, 'authorized', 'archive-policy-set', 'archive-exposure-set', 1, + '[{"id":"alice","kind":"user"}]', ?, ?, 'archive-run', 'archive-context', 1)`, + ) + .run(params.sessionId, event.seq, session.session_identity_revision, session.subject_revision); + if (params.includeDetail !== false) { + database.db + .prepare( + `INSERT INTO transcript_event_memory_policy_details + (session_id, event_seq, actor_evidence_json, delegation_snapshot_json, + exposed_resource_revisions_json, exposure_receipt_ids_json, egress_receipt_ids_json, + normalized_audience_intersection_json, finalized_delivery_audiences_json, retention_state, + source_session_id, source_event_seq, created_at) + VALUES (?, ?, '{"version":1}', '{"kind":"none","version":1}', + '["archive-resource-revision"]', '["archive-exposure"]', '["archive-egress"]', + '[{"id":"alice","kind":"user"}]', '[{"id":"alice","kind":"user"}]', + 'retained', ?, ?, 1)`, + ) + .run(params.sessionId, event.seq, params.sessionId, event.seq); + } + resetMemoryIsolationCutoverForTest(); + resetTranscriptMemoryPolicyForTest(database.db); + return { eventSeq: event.seq }; +} + function planArchiveWorker( database: ReturnType, archiveDirectory: string, diff --git a/src/config/sessions/session-accessor.sqlite-archive.worker.ts b/src/config/sessions/session-accessor.sqlite-archive.worker.ts index 51b404aff0bf..aee704917c57 100644 --- a/src/config/sessions/session-accessor.sqlite-archive.worker.ts +++ b/src/config/sessions/session-accessor.sqlite-archive.worker.ts @@ -25,9 +25,56 @@ import { serializeJsonlLines } from "./transcript-jsonl.js"; type TranscriptArchiveDatabase = Pick< OpenClawAgentKyselyDatabase, - "session_transcript_archives" | "transcript_events" + | "session_transcript_archives" + | "session_memory_subject_snapshots" + | "transcript_events" + | "transcript_event_memory_policies" + | "transcript_event_memory_policy_details" + | "transcript_event_memory_policy_transitions" >; +const TRANSCRIPT_MEMORY_POLICY_ARCHIVE_RECORD_TYPE = "openclaw.memory-policy-archive-v1"; + +type TranscriptMemoryPolicyArchiveRecord = Readonly<{ + agentId: string; + type: typeof TRANSCRIPT_MEMORY_POLICY_ARCHIVE_RECORD_TYPE; + version: 1; + sessionId: string; + eventSeq: number; + subject: Readonly<{ + sessionKey: string; + sessionIdentityRevision: string; + subjectRevision: string; + }>; + policy: Readonly<{ + contextFingerprint: string; + deliveryAudiencesJson: string; + runExposureRevision: number; + runExposureSetId: string; + runId: string; + sourcePolicySetId: string; + }>; + detail: Readonly<{ + actorEvidenceJson: string; + delegationSnapshotJson: string; + egressReceiptIdsJson: string; + exposedResourceRevisionsJson: string; + exposureReceiptIdsJson: string; + finalizedDeliveryAudiencesJson: string; + normalizedAudienceIntersectionJson: string; + sourceEventSeq: number; + sourceSessionId: string; + }>; + transition?: Readonly<{ + sourceEventSeq: number; + sourceSessionId: string; + sourceSessionIdentityRevision: string; + subjectRevision: string; + targetSessionIdentityRevision: string; + kind: string; + }>; +}>; + function isSqliteTranscriptArchiveWorkerData(value: unknown): boolean { return ( Boolean(value) && @@ -137,6 +184,7 @@ function parseWorkerPlans(value: unknown): TranscriptArchiveWorkerPlan[] | undef function readTranscriptArchiveContent( database: import("node:sqlite").DatabaseSync, + agentId: string, sessionId: string, ): string { const db = getNodeSqliteKysely(database); @@ -151,11 +199,169 @@ function readTranscriptArchiveContent( // An archive is an export surface. Once cut over, a raw row without a // current companion label must not become a durable bypass of the replay fence. const authorizedSeqs = readAuthorizedTranscriptEventSeqs(database, sessionId); - return serializeJsonlLines( - (authorizedSeqs ? lines.filter((row) => authorizedSeqs.has(row.seq)) : lines).map( - (row) => row.event_json, - ), + if (!authorizedSeqs) { + return serializeJsonlLines(lines.map((row) => row.event_json)); + } + const authorizedRows = lines.filter((row) => authorizedSeqs.has(row.seq)); + if (authorizedRows.length !== lines.length) { + // Deletion must not turn a pending, stale, or revoked event into either a + // durable raw bypass or silent data loss. Keep the source rows until an + // explicit repair/confirmed import establishes their lineage. + throw new Error(`Unauthorized transcript policy archive event for ${sessionId}`); + } + const records = readTranscriptMemoryPolicyArchiveRecords( + database, + agentId, + sessionId, + authorizedSeqs, ); + if (records.size !== authorizedRows.length) { + // Policy-enforced archives are later import candidates. Refuse to emit any + // raw event whose immutable companion cannot travel with its lineage. + throw new Error(`Missing transcript policy archive companion for ${sessionId}`); + } + return serializeJsonlLines( + authorizedRows.flatMap((row) => { + const record = records.get(row.seq); + if (!record) { + throw new Error(`Missing transcript policy archive record for ${sessionId}:${row.seq}`); + } + return [row.event_json, JSON.stringify(record)]; + }), + ); +} + +function readTranscriptMemoryPolicyArchiveRecords( + database: import("node:sqlite").DatabaseSync, + agentId: string, + sessionId: string, + authorizedSeqs: ReadonlySet, +): ReadonlyMap { + const db = getNodeSqliteKysely(database); + const rows = executeSqliteQuerySync( + database, + db + .selectFrom("transcript_event_memory_policies as policy") + .innerJoin("transcript_event_memory_policy_details as detail", (join) => + join + .onRef("detail.session_id", "=", "policy.session_id") + .onRef("detail.event_seq", "=", "policy.event_seq"), + ) + .innerJoin( + "session_memory_subject_snapshots as subject", + "subject.session_id", + "policy.session_id", + ) + .leftJoin("transcript_event_memory_policy_transitions as transition", (join) => + join + .onRef("transition.session_id", "=", "policy.session_id") + .onRef("transition.event_seq", "=", "policy.event_seq"), + ) + .select([ + "policy.event_seq", + "policy.context_fingerprint", + "policy.delivery_audiences_json", + "policy.run_exposure_revision", + "policy.run_exposure_set_id", + "policy.run_id", + "policy.source_policy_set_id", + "subject.session_identity_revision", + "subject.session_key", + "subject.subject_revision", + "detail.actor_evidence_json", + "detail.delegation_snapshot_json", + "detail.egress_receipt_ids_json", + "detail.exposed_resource_revisions_json", + "detail.exposure_receipt_ids_json", + "detail.finalized_delivery_audiences_json", + "detail.normalized_audience_intersection_json", + "detail.source_event_seq", + "detail.source_session_id", + "transition.source_event_seq as transition_source_event_seq", + "transition.source_session_id as transition_source_session_id", + "transition.source_session_identity_revision as transition_source_session_identity_revision", + "transition.subject_revision as transition_subject_revision", + "transition.target_session_identity_revision as transition_target_session_identity_revision", + "transition.transition_kind", + ]) + .where("policy.session_id", "=", sessionId) + .where("policy.authorization_status", "=", "authorized") + .where("detail.retention_state", "=", "retained"), + ).rows; + const records = new Map(); + for (const row of rows) { + if ( + !authorizedSeqs.has(row.event_seq) || + row.context_fingerprint === null || + row.delivery_audiences_json === null || + row.run_exposure_revision === null || + row.run_exposure_set_id === null || + row.run_id === null || + row.source_policy_set_id === null || + row.source_event_seq === null || + row.source_session_id === null + ) { + continue; + } + const hasTransition = row.transition_source_session_id !== null; + if ( + hasTransition && + (row.transition_source_event_seq === null || + row.transition_source_session_identity_revision === null || + row.transition_subject_revision === null || + row.transition_target_session_identity_revision === null || + row.transition_kind === null) + ) { + continue; + } + records.set( + row.event_seq, + Object.freeze({ + agentId, + type: TRANSCRIPT_MEMORY_POLICY_ARCHIVE_RECORD_TYPE, + version: 1, + sessionId, + eventSeq: row.event_seq, + subject: Object.freeze({ + sessionKey: row.session_key, + sessionIdentityRevision: row.session_identity_revision, + subjectRevision: row.subject_revision, + }), + policy: Object.freeze({ + contextFingerprint: row.context_fingerprint, + deliveryAudiencesJson: row.delivery_audiences_json, + runExposureRevision: row.run_exposure_revision, + runExposureSetId: row.run_exposure_set_id, + runId: row.run_id, + sourcePolicySetId: row.source_policy_set_id, + }), + detail: Object.freeze({ + actorEvidenceJson: row.actor_evidence_json, + delegationSnapshotJson: row.delegation_snapshot_json, + egressReceiptIdsJson: row.egress_receipt_ids_json, + exposedResourceRevisionsJson: row.exposed_resource_revisions_json, + exposureReceiptIdsJson: row.exposure_receipt_ids_json, + finalizedDeliveryAudiencesJson: row.finalized_delivery_audiences_json, + normalizedAudienceIntersectionJson: row.normalized_audience_intersection_json, + sourceEventSeq: row.source_event_seq, + sourceSessionId: row.source_session_id, + }), + ...(hasTransition + ? { + transition: Object.freeze({ + sourceEventSeq: row.transition_source_event_seq!, + sourceSessionId: row.transition_source_session_id!, + sourceSessionIdentityRevision: row.transition_source_session_identity_revision!, + subjectRevision: row.transition_subject_revision!, + targetSessionIdentityRevision: row.transition_target_session_identity_revision!, + kind: row.transition_kind!, + }), + } + : {}), + }), + ); + } + return records; } export function materializeTranscriptArchiveInWorker( @@ -174,7 +380,7 @@ export function materializeTranscriptArchiveInWorker( `SQLite session state changed before archive materialization for ${plan.sessionId}`, ); } - const content = readTranscriptArchiveContent(database.db, plan.sessionId); + const content = readTranscriptArchiveContent(database.db, plan.agentId, plan.sessionId); database.db.exec("COMMIT"); // sqlite-allow-raw: closes the consistent read snapshot. transactionOpen = false; return { content, snapshot }; diff --git a/src/config/sessions/session-accessor.sqlite-checkpoint.ts b/src/config/sessions/session-accessor.sqlite-checkpoint.ts index decaa60e228b..d5f14d299681 100644 --- a/src/config/sessions/session-accessor.sqlite-checkpoint.ts +++ b/src/config/sessions/session-accessor.sqlite-checkpoint.ts @@ -26,6 +26,7 @@ import { appendTranscriptEventsInTransaction, readTranscriptIdentityByEventId, } from "./session-accessor.sqlite-transcript-store.js"; +import { preserveTranscriptMemoryPolicyTransitionInTransaction } from "./session-transcript-memory-policy.js"; import { createSessionTranscriptHeader } from "./transcript-header.js"; import { SESSION_TOTAL_TOKENS_VERSION, @@ -197,26 +198,32 @@ function branchSqliteCompactionCheckpointSessionInTransaction( if (!checkpoint) { return { status: "missing-checkpoint" }; } + let nextEntry: SessionEntry | undefined; const forked = forkSqliteCheckpointTranscriptInTransaction(database, params.resolved, { checkpoint, legacySource: params.legacySource, targetSessionKey: params.targetKey, + beforeAppend: ({ sessionId, totalTokens }) => { + const label = currentEntry.label?.trim() + ? `${currentEntry.label.trim()} (checkpoint)` + : "Checkpoint branch"; + nextEntry = cloneSqliteCheckpointSessionEntry({ + currentEntry, + label, + nextSessionId: sessionId, + parentSessionKey: params.parentSessionKey, + totalTokens, + }); + writeSessionEntry(database, params.targetKey, nextEntry); + }, }); if (forked.status !== "created") { return forked; } - const label = currentEntry.label?.trim() - ? `${currentEntry.label.trim()} (checkpoint)` - : "Checkpoint branch"; - const nextEntry = cloneSqliteCheckpointSessionEntry({ - currentEntry, - label, - nextSessionId: forked.sessionId, - parentSessionKey: params.parentSessionKey, - totalTokens: forked.totalTokens, - }); - writeSessionEntry(database, params.targetKey, nextEntry); + if (!nextEntry) { + return { status: "failed" }; + } return { status: "created", key: params.targetKey, @@ -253,22 +260,28 @@ function restoreSqliteCompactionCheckpointSessionInTransaction( if (!checkpoint) { return { status: "missing-checkpoint" }; } + let nextEntry: SessionEntry | undefined; const restored = forkSqliteCheckpointTranscriptInTransaction(database, params.resolved, { checkpoint, legacySource: params.legacySource, targetSessionKey: params.targetKey, + beforeAppend: ({ sessionId, totalTokens }) => { + nextEntry = cloneSqliteCheckpointSessionEntry({ + currentEntry, + nextSessionId: sessionId, + preserveCompactionCheckpoints: true, + totalTokens, + }); + writeSessionEntry(database, params.targetKey, nextEntry); + }, }); if (restored.status !== "created") { return restored; } - const nextEntry = cloneSqliteCheckpointSessionEntry({ - currentEntry, - nextSessionId: restored.sessionId, - preserveCompactionCheckpoints: true, - totalTokens: restored.totalTokens, - }); - writeSessionEntry(database, params.targetKey, nextEntry); + if (!nextEntry) { + return { status: "failed" }; + } return { status: "created", key: params.targetKey, @@ -282,6 +295,7 @@ function forkSqliteCheckpointTranscriptInTransaction( resolved: ResolvedSqliteScope, params: { checkpoint: SessionCompactionCheckpoint; + beforeAppend?: (transcript: { sessionId: string; totalTokens?: number }) => void; legacySource?: SqliteCompactionCheckpointLegacySource; targetSessionKey: string; }, @@ -331,6 +345,10 @@ function forkSqliteCheckpointTranscriptInTransaction( const sessionFile = formatSqliteSessionReferenceForScope(targetScope); const selectedEvents = selected?.rows ?? legacySource?.events ?? []; const totalTokens = selected?.source.totalTokens ?? legacySource?.totalTokens; + params.beforeAppend?.({ + sessionId, + ...(typeof totalTokens === "number" ? { totalTokens } : {}), + }); appendTranscriptEventsInTransaction(database, targetScope, [ createSessionTranscriptHeader({ cwd: readTranscriptHeaderCwd(selectedEvents), @@ -338,6 +356,14 @@ function forkSqliteCheckpointTranscriptInTransaction( }), ...selectedEvents.filter((event) => !isSessionTranscriptHeader(event)), ]); + if (selected) { + preserveTranscriptMemoryPolicyTransitionInTransaction({ + database, + sourceSessionId: selected.source.sessionId, + targetSessionId: sessionId, + transitionKind: "checkpoint", + }); + } return { status: "created", sessionId, diff --git a/src/config/sessions/session-accessor.sqlite-import.ts b/src/config/sessions/session-accessor.sqlite-import.ts index 4d37d89b92b6..92ea0d7a404a 100644 --- a/src/config/sessions/session-accessor.sqlite-import.ts +++ b/src/config/sessions/session-accessor.sqlite-import.ts @@ -1,7 +1,9 @@ import { executeSqliteQuerySync, executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, } from "../../infra/kysely-sync.js"; +import type { DB as OpenClawAgentDatabaseSchema } from "../../state/openclaw-agent-db.generated.js"; import { runOpenClawAgentWriteTransaction, type OpenClawAgentDatabase, @@ -26,8 +28,17 @@ import { } from "./session-accessor.sqlite-transcript-state.js"; import { appendTranscriptEventInTransaction } from "./session-accessor.sqlite-transcript-store.js"; import { reconcileSessionTranscriptIndexInTransaction } from "./session-transcript-index.js"; +import { + parseTranscriptPolicyArchive, + restoreConfirmedTranscriptPolicyArchiveInTransaction, +} from "./session-transcript-policy-archive.js"; import type { SessionEntry } from "./types.js"; +type ConfirmedTranscriptImportDatabase = Pick< + OpenClawAgentDatabaseSchema, + "session_memory_subject_snapshots" | "session_memory_subjects" | "transcript_events" +>; + /** Internal doctor/migration import target for one legacy session row. */ type SqliteSessionImportRowsParams = { allowMalformedRowRepair?: boolean; @@ -53,6 +64,150 @@ type SqliteSessionImportRowsResult = { transcriptEvents: number; }; +type ConfirmedSqliteTranscriptImportParams = { + agentId?: string; + archiveContent: string; + entry: SessionEntry; + env?: NodeJS.ProcessEnv; + sessionKey: string; + storePath?: string; +}; + +type ConfirmedSqliteTranscriptImportResult = { + sessionId: string; + sessionKey: string; + transcriptEvents: number; +}; + +/** + * Internal owner flow for an operator-confirmed archive only. Legacy Doctor + * import stays quarantined; callers must authenticate confirmation before this + * API is reached and must not expose it as a model/runtime import surface. + */ +export async function importConfirmedSqliteTranscriptPolicyArchive( + params: ConfirmedSqliteTranscriptImportParams, +): Promise { + // Archive parsing is deliberately outside the synchronous write transaction. + const archive = parseTranscriptPolicyArchive(params.archiveContent); + if (!archive) { + throw new Error("confirmed transcript archive is invalid or legacy JSONL"); + } + const resolved = resolveSqliteScope({ + ...(params.agentId ? { agentId: params.agentId } : {}), + ...(params.env ? { env: params.env } : {}), + sessionKey: params.sessionKey, + ...(params.storePath ? { storePath: params.storePath } : {}), + }); + if ( + archive.agentId !== resolved.agentId || + archive.sessionId !== params.entry.sessionId || + archive.sessionKey !== resolved.sessionKey + ) { + throw new Error("confirmed transcript archive target mismatch"); + } + return await runExclusiveSqliteSessionWrite(resolved, async () => { + let transcriptEvents = 0; + runOpenClawAgentWriteTransaction((database) => { + const db = getNodeSqliteKysely(database.db); + const currentEntry = readExactSessionEntryRowForCanonicalRepair( + database, + resolved.sessionKey, + )?.entry; + if (currentEntry && currentEntry.sessionId !== archive.sessionId) { + throw new Error("confirmed transcript archive would replace an active session"); + } + const subject = executeSqliteQueryTakeFirstSync( + database.db, + db + .selectFrom("session_memory_subjects") + .select("subject_revision") + .where("session_key", "=", resolved.sessionKey) + .limit(1), + ); + if (!subject || subject.subject_revision !== archive.subjectRevision) { + throw new Error("confirmed transcript archive subject is unavailable"); + } + const snapshotBeforeRestore = executeSqliteQueryTakeFirstSync( + database.db, + db + .selectFrom("session_memory_subject_snapshots") + .select(["session_identity_revision", "session_key", "subject_revision"]) + .where("session_id", "=", archive.sessionId) + .limit(1), + ); + if ( + snapshotBeforeRestore && + (snapshotBeforeRestore.session_key !== archive.sessionKey || + snapshotBeforeRestore.subject_revision !== archive.subjectRevision || + snapshotBeforeRestore.session_identity_revision !== archive.sessionIdentityRevision) + ) { + throw new Error("confirmed transcript archive immutable snapshot conflicts"); + } + const existingEvents = executeSqliteQuerySync( + database.db, + db + .selectFrom("transcript_events") + .select("seq") + .where("session_id", "=", archive.sessionId), + ).rows; + if (existingEvents.length > 0) { + throw new Error("confirmed transcript archive target already has transcript events"); + } + + writeSessionEntry(database, resolved.sessionKey, params.entry, { + previousEntry: currentEntry ?? null, + }); + // writeSessionEntry snapshots the surviving immutable subject. Replace only + // that just-created snapshot with the archive's proven generation identity. + if (!snapshotBeforeRestore) { + executeSqliteQuerySync( + database.db, + db + .deleteFrom("session_memory_subject_snapshots") + .where("session_id", "=", archive.sessionId), + ); + executeSqliteQuerySync( + database.db, + db.insertInto("session_memory_subject_snapshots").values({ + session_id: archive.sessionId, + session_key: archive.sessionKey, + subject_revision: archive.subjectRevision, + session_identity_revision: archive.sessionIdentityRevision, + created_at: Date.now(), + }), + ); + } + + const transcriptScope = { ...resolved, sessionId: archive.sessionId }; + for (const archived of archive.events) { + if ( + !appendTranscriptEventInTransaction(database, transcriptScope, archived.event, { + scheduleProjectionReconcile: false, + touchMutation: false, + }) + ) { + throw new Error("confirmed transcript archive event append conflict"); + } + transcriptEvents += 1; + } + restoreConfirmedTranscriptPolicyArchiveInTransaction({ + archive, + database, + sessionId: archive.sessionId, + sessionKey: archive.sessionKey, + }); + reconcileSessionTranscriptIndexInTransaction(database.db, archive.sessionId); + touchTranscriptMutationInTransaction(database, archive.sessionId); + publishSessionEntryCacheInvalidation(database); + }, toDatabaseOptions(resolved)); + return { + sessionId: archive.sessionId, + sessionKey: resolved.sessionKey, + transcriptEvents, + }; + }); +} + function prepareSqliteSessionImport(params: SqliteSessionImportRowsParams) { if (params.readExactTranscriptRows && params.readTranscriptEvents) { throw new Error("SQLite session import accepts only one transcript row source"); diff --git a/src/config/sessions/session-accessor.sqlite-message-cut.ts b/src/config/sessions/session-accessor.sqlite-message-cut.ts index 350b7bb0da7e..2311d4776c57 100644 --- a/src/config/sessions/session-accessor.sqlite-message-cut.ts +++ b/src/config/sessions/session-accessor.sqlite-message-cut.ts @@ -39,6 +39,7 @@ import type { import { buildSessionCreationStamp } from "./session-entry-provenance.js"; import { inheritSessionSelection } from "./session-entry-selection.js"; import { reconcileSessionTranscriptIndexInTransaction } from "./session-transcript-index.js"; +import { preserveTranscriptMemoryPolicyTransitionInTransaction } from "./session-transcript-memory-policy.js"; import { createSessionTranscriptHeader } from "./transcript-header.js"; import { isSessionTranscriptLeafControl, @@ -325,13 +326,6 @@ function mutateSqliteSessionAtMessageInTransaction( targetId: params.mode === "switch" ? params.entryId : (cut?.parentId ?? null), }, ]; - appendTranscriptEventsInTransaction(database, targetScope, nextEvents); - if (params.mode !== "fork") { - reconcileSessionTranscriptIndexInTransaction(database.db, nextSessionId); - } - - // Rotating transcript identity fences stale live managers: later snapshot-replace writes - // target the old session and cannot erase this leaf repoint from the active session. const nextEntry = { ...cloneMessageCutSessionEntry({ currentEntry, @@ -350,7 +344,23 @@ function mutateSqliteSessionAtMessageInTransaction( ? buildSessionCreationStamp(params.creation) : {}), }; + // The new identity snapshot must exist before copied events can receive + // transition provenance. If the write fails, the enclosing transaction leaves + // the old session untouched rather than treating raw replay as authorized. writeSessionEntry(database, params.targetKey, nextEntry); + appendTranscriptEventsInTransaction(database, targetScope, nextEvents); + preserveTranscriptMemoryPolicyTransitionInTransaction({ + database, + sourceSessionId: currentEntry.sessionId, + targetSessionId: nextSessionId, + transitionKind: params.mode, + }); + if (params.mode !== "fork") { + reconcileSessionTranscriptIndexInTransaction(database.db, nextSessionId); + } + + // Rotating transcript identity fences stale live managers: later snapshot-replace writes + // target the old session and cannot erase this leaf repoint from the active session. return { status: "created", key: params.targetKey, diff --git a/src/config/sessions/session-accessor.sqlite-parent-session.ts b/src/config/sessions/session-accessor.sqlite-parent-session.ts index 9438c8801c6b..034cf636919e 100644 --- a/src/config/sessions/session-accessor.sqlite-parent-session.ts +++ b/src/config/sessions/session-accessor.sqlite-parent-session.ts @@ -48,6 +48,7 @@ import { } from "./session-accessor.sqlite-scope.js"; import { appendTranscriptEventsInTransaction } from "./session-accessor.sqlite-transcript-store.js"; import { preserveSqliteSameKeySessionRolloverLineage } from "./session-entry-lineage.js"; +import { preserveTranscriptMemoryPolicyTransitionInTransaction } from "./session-transcript-memory-policy.js"; import type { InternalSessionEntry, SessionEntry } from "./types.js"; import { mergeSessionEntry, resolveFreshSessionTotalTokens } from "./types.js"; @@ -120,6 +121,7 @@ export async function forkSessionTranscriptFromParent( params.commitGuard?.(); writeSqliteForkedChildTranscriptInTransaction(database, targetScope, { parentSessionFile, + sourceSessionId: params.parentEntry.sessionId, source, }); }, toDatabaseOptions(target)); @@ -211,56 +213,65 @@ export async function forkSessionEntryFromParentTarget( result = { status: "missing-entry" }; return; } + let next: SessionEntry | undefined; const fork = forkSqliteParentTranscriptInTransaction(writeDatabase, resolved, { parentEntry: freshParent, parentSessionKey: parentTarget.canonicalKey, targetSessionKey: sessionTarget.canonicalKey, + beforeAppend: (transcript) => { + const patch = params.patch?.({ + decision, + entry: cloneSessionEntry(freshBase), + fork: transcript, + parentEntry: cloneSessionEntry(freshParent), + }); + const forkIdentityPatch: Partial = { + ...patch, + forkSource: { + sessionKey: parentTarget.canonicalKey, + sessionId: freshParent.sessionId, + }, + forkedFromParent: true, + lifecycleRunId: undefined, + sessionId: transcript.sessionId, + totalTokens: undefined, + totalTokensFresh: false, + totalTokensVersion: undefined, + }; + next = mergeSessionEntry(freshBase, forkIdentityPatch); + previousIdentity = readSessionIdentitySnapshot(writeDatabase, sessionTarget.storeKeys); + // Establish the target identity before copied events are appended. The + // transition helper will retain only matching current source lineage. + writeSessionEntry(writeDatabase, sessionTarget.canonicalKey, next, { + previousEntry: freshBase, + }); + rehomeSessionWindows(writeDatabase, sessionTarget.canonicalKey, sessionTarget.storeKeys); + deleteLegacySessionEntryRows( + writeDatabase, + sessionTarget.storeKeys, + sessionTarget.canonicalKey, + { rehomeMembers: freshBase.sessionId === next.sessionId }, + ); + maintenancePlans.push( + applySessionEntryMaintenance(writeDatabase, { + activeSessionKey: sessionTarget.canonicalKey, + archiveDirectory: resolveSqliteTranscriptArchiveDirectory(resolved), + skipMaintenance: true, + storePath: params.storePath, + }), + ); + currentIdentity = readSessionIdentitySnapshot(writeDatabase, sessionTarget.storeKeys); + }, }); if (fork.status !== "created") { result = fork.status === "missing-parent" ? { status: "missing-parent" } : { status: "failed" }; return; } - const patch = params.patch?.({ - decision, - entry: cloneSessionEntry(freshBase), - fork: fork.transcript, - parentEntry: cloneSessionEntry(freshParent), - }); - const forkIdentityPatch: Partial = { - ...patch, - forkSource: { - sessionKey: parentTarget.canonicalKey, - sessionId: freshParent.sessionId, - }, - forkedFromParent: true, - lifecycleRunId: undefined, - sessionId: fork.transcript.sessionId, - totalTokens: undefined, - totalTokensFresh: false, - totalTokensVersion: undefined, - }; - const next = mergeSessionEntry(freshBase, forkIdentityPatch); - previousIdentity = readSessionIdentitySnapshot(writeDatabase, sessionTarget.storeKeys); - writeSessionEntry(writeDatabase, sessionTarget.canonicalKey, next, { - previousEntry: freshBase, - }); - rehomeSessionWindows(writeDatabase, sessionTarget.canonicalKey, sessionTarget.storeKeys); - deleteLegacySessionEntryRows( - writeDatabase, - sessionTarget.storeKeys, - sessionTarget.canonicalKey, - { rehomeMembers: freshBase.sessionId === next.sessionId }, - ); - maintenancePlans.push( - applySessionEntryMaintenance(writeDatabase, { - activeSessionKey: sessionTarget.canonicalKey, - archiveDirectory: resolveSqliteTranscriptArchiveDirectory(resolved), - skipMaintenance: true, - storePath: params.storePath, - }), - ); - currentIdentity = readSessionIdentitySnapshot(writeDatabase, sessionTarget.storeKeys); + if (!next) { + result = { status: "failed" }; + return; + } result = { status: "forked", decision, @@ -291,6 +302,12 @@ async function persistSqliteParentForkSkipPatch(params: { previous: params.entry, sessionKey: params.sessionTarget.canonicalKey, }); + // A fork skipped before allocating a child has no transcript identity to + // snapshot. Keep its caller-visible patch in memory; persisting an empty + // session id would create invalid lifecycle provenance. + if (!next.sessionId?.trim()) { + return cloneSessionEntry(next); + } const maintenancePlans: SessionEntryMaintenancePlan[] = []; let previousIdentity = new Map(); let currentIdentity = new Map(); @@ -357,6 +374,7 @@ function forkSqliteParentTranscriptInTransaction( forkFrom?: "last-completed"; targetSessionId?: string; targetSessionKey: string; + beforeAppend?: (transcript: { sessionFile: string; sessionId: string }) => void; }, ): ForkSessionFromParentTranscriptResult { if (!params.parentEntry.sessionId) { @@ -385,16 +403,16 @@ function forkSqliteParentTranscriptInTransaction( sessionKey: normalizeSqliteSessionKey(params.parentSessionKey), }); const sessionFile = formatSqliteSessionReferenceForScope(targetScope); + const transcript = { sessionFile, sessionId }; + params.beforeAppend?.(transcript); writeSqliteForkedChildTranscriptInTransaction(database, targetScope, { parentSessionFile, + sourceSessionId: params.parentEntry.sessionId, source, }); return { status: "created", - transcript: { - sessionFile, - sessionId, - }, + transcript, }; } @@ -421,6 +439,7 @@ function writeSqliteForkedChildTranscriptInTransaction( targetScope: ResolvedTranscriptScope, params: { parentSessionFile: string; + sourceSessionId: string; source: ParentForkSourceTranscript; }, ): void { @@ -433,4 +452,10 @@ function writeSqliteForkedChildTranscriptInTransaction( targetSessionId: targetScope.sessionId, }), ); + preserveTranscriptMemoryPolicyTransitionInTransaction({ + database, + sourceSessionId: params.sourceSessionId, + targetSessionId: targetScope.sessionId, + transitionKind: "parent-fork", + }); } diff --git a/src/config/sessions/session-accessor.sqlite-transcript-store.ts b/src/config/sessions/session-accessor.sqlite-transcript-store.ts index 758551f15a7a..d1c305c59877 100644 --- a/src/config/sessions/session-accessor.sqlite-transcript-store.ts +++ b/src/config/sessions/session-accessor.sqlite-transcript-store.ts @@ -34,7 +34,11 @@ import { indexAppendedTranscriptEventInTransaction, reconcileSessionTranscriptIndexInTransaction, } from "./session-transcript-index.js"; -import { recordTranscriptMemoryPolicyInTransaction } from "./session-transcript-memory-policy.js"; +import { + readPreservedTranscriptMemoryPoliciesInTransaction, + recordTranscriptMemoryPolicyInTransaction, + type PreservedTranscriptMemoryPolicy, +} from "./session-transcript-memory-policy.js"; import { startSessionTranscriptIndexReconcile } from "./session-transcript-reconcile.js"; import { createSessionTranscriptHeader } from "./transcript-header.js"; import { resolveVisibleTranscriptAppendParentId } from "./transcript-visible-events.js"; @@ -191,6 +195,8 @@ function appendTranscriptEventRowInTransaction( seq: number, state: { seenEventIds: Set; seenMessageIdempotencyKeys: Set }, createdAtOverride?: number, + inheritedMemoryPolicy?: PreservedTranscriptMemoryPolicy, + replacementMemoryPolicy?: boolean, ): boolean { const persistedEvent = canonicalizeTranscriptEventMedia(event); const db = getSessionKysely(database.db); @@ -214,6 +220,8 @@ function appendTranscriptEventRowInTransaction( sessionKey: scope.sessionKey, eventSeq: seq, createdAt, + ...(inheritedMemoryPolicy ? { inherited: inheritedMemoryPolicy } : {}), + ...(replacementMemoryPolicy ? { replacement: true } : {}), }); indexAppendedTranscriptEventInTransaction(database.db, { sessionId: scope.sessionId, @@ -337,6 +345,10 @@ export function replaceSqliteTranscriptEventsInTransaction( ? readTranscriptMutationStateInTransaction(database, resolved.sessionId).updatedAt : undefined; const previousGeneration = readTranscriptGenerationInTransaction(database, resolved.sessionId); + const preservedPolicies = readPreservedTranscriptMemoryPoliciesInTransaction( + database, + resolved.sessionId, + ); const deleted = deleteTranscriptEventsInTransaction(database, resolved.sessionId); if (events.length === 0) { if (deleted || previousGeneration) { @@ -361,6 +373,12 @@ export function replaceSqliteTranscriptEventsInTransaction( const seenEventIds = new Set(); const seenMessageIdempotencyKeys = new Set(); for (const [eventIndex, event] of events.entries()) { + const persistedEvent = canonicalizeTranscriptEventMedia(event); + const preservedForEvent = preservedPolicies.get(JSON.stringify(persistedEvent)); + // Consume one matching source row per replacement row. Duplicate event + // payloads are distinguishable only by their durable source queue; a new + // duplicate has no queue entry and is intentionally pending. + const inheritedMemoryPolicy = preservedForEvent?.shift(); if ( appendTranscriptEventRowInTransaction( database, @@ -372,6 +390,8 @@ export function replaceSqliteTranscriptEventsInTransaction( seenMessageIdempotencyKeys, }, options.createdAtByIndex?.[eventIndex], + inheritedMemoryPolicy, + true, ) ) { seq += 1; diff --git a/src/config/sessions/session-transcript-memory-policy.test.ts b/src/config/sessions/session-transcript-memory-policy.test.ts index b4963df896ff..9199ef069f05 100644 --- a/src/config/sessions/session-transcript-memory-policy.test.ts +++ b/src/config/sessions/session-transcript-memory-policy.test.ts @@ -5,10 +5,13 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { runDoctorMemoryIsolation } from "../../commands/doctor-memory-isolation.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { resetMemoryIsolationCutoverForTest } from "../../plugins/memory-cutover.js"; -import { persistMemoryRunExposureBeforeContentInDatabase } from "../../plugins/memory-run-exposure-ledger.js"; +import { + persistMemoryRunExposureBeforeContentInDatabase, + readDurableMemoryRunExposure, +} from "../../plugins/memory-run-exposure-ledger.js"; import { clearMemoryRunExposureForTest, - recordMemoryRunExposure, + prepareMemoryRunExposure, } from "../../plugins/memory-run-exposure.js"; import { createCurrentMemorySessionContext } from "../../state/memory-session-subject.js"; import type { OpenClawAgentDatabase } from "../../state/openclaw-agent-db.js"; @@ -18,20 +21,23 @@ import { runOpenClawAgentWriteTransaction, } from "../../state/openclaw-agent-db.js"; import { readSessionTranscriptMessageEvents } from "./session-accessor.sqlite-active-events.js"; -import { materializeSqliteSessionStateDeletePlans } from "./session-accessor.sqlite-archive.js"; +import { materializeSessionStateDeletePlans } from "./session-accessor.sqlite-archive.js"; import { writeSessionEntry } from "./session-accessor.sqlite-entry-store.js"; -import { planSqliteSessionStateDeleteIfUnreferenced } from "./session-accessor.sqlite-lifecycle-state.js"; +import { planSessionStateDeleteIfUnreferenced } from "./session-accessor.sqlite-lifecycle-state.js"; import { - loadLatestSqliteAssistantText, - loadSqliteTranscriptEventsSync, - loadSqliteTranscriptTailEventsSync, + loadLatestAssistantText, + loadTranscriptEventsSync, + loadTranscriptTailEventsSync, } from "./session-accessor.sqlite-read.js"; import { readActiveTranscriptAppendParentId } from "./session-accessor.sqlite-transcript-store.js"; import { - appendSqliteTranscriptMessage, - trimSqliteTranscriptForManualCompact, + appendTranscriptEvent, + appendTranscriptMessage, + replaceTranscriptEvents, + trimTranscriptForManualCompact, } from "./session-accessor.sqlite-transcript-write.js"; import { + preserveTranscriptMemoryPolicyTransitionInTransaction, readAuthorizedTranscriptEventSeqs, resetTranscriptMemoryPolicyForTest, } from "./session-transcript-memory-policy.js"; @@ -80,16 +86,75 @@ function markCutOver(env: NodeJS.ProcessEnv) { VALUES (?, ?, ?, ?, 1)`, ) .run(SESSION_ID, SESSION_KEY, SUBJECT_REVISION, SESSION_IDENTITY_REVISION); + // The transcript companion resolves stable policy identity from the exposed + // resource revision. Test receipts intentionally stay opaque plugin payloads. + database.db.exec(/* sqlite-allow-raw: test fixture establishes durable policy lineage. */ ` + INSERT INTO memory_storage_roots + (storage_root_id, agent_id, backend_kind, opaque_locator, path_key_version, path_key, + authority_kind, authority_owner_id, default_capabilities_json, lifecycle_state, created_at, updated_at) + VALUES ('root-1', 'main', 'builtin', 'builtin:v1:test', 1, 's1_test_fixture_path_key_000', + 'user', 'alice', '["read"]', 'active', 1, 1); + INSERT INTO memory_policies + (policy_id, agent_id, current_revision_id, revocation_epoch, lifecycle_state, created_at, updated_at) + VALUES ('policy-1', 'main', 'policy-revision-1', 0, 'active', 1, 1); + INSERT INTO memory_policy_revisions + (revision_id, policy_id, revision_number, revocation_epoch, lifecycle_state, + actor_kind, actor_id, reason, created_at) + VALUES ('policy-revision-1', 'policy-1', 1, 0, 'active', 'human', 'alice', 'fixture', 1); + INSERT INTO memory_stores + (store_id, agent_id, storage_root_id, policy_id, scope_kind, audience_kind, audience_id, + lifecycle_state, created_at, updated_at) + VALUES ('store-1', 'main', 'root-1', 'policy-1', 'user', 'user', 'alice', 'active', 1, 1); + INSERT INTO memory_resources + (resource_id, agent_id, store_id, logical_locator, source, created_at) + VALUES ('resource-1', 'main', 'store-1', 'memory/fixture.md', 'memory', 1); + INSERT INTO memory_resource_revisions + (revision_id, resource_id, revision_number, artifact_locator, content_hash, content_bytes, + policy_revision_id, policy_revocation_epoch, source_policy_set_id, lifecycle_state, + actor_kind, actor_id, expires_at, created_at, activated_at, retired_at) + VALUES ('resource-revision-1', 'resource-1', 1, 'fixture.md', 'fixture', 7, + 'policy-revision-1', 0, 'plugin-policy-set-1', 'active', 'human', 'alice', NULL, 1, 1, NULL); + `); resetTranscriptMemoryPolicyForTest(database.db); return database; } +function seedTranscriptPolicyFixture(database: OpenClawAgentDatabase): void { + database.db.exec(/* sqlite-allow-raw: test fixture establishes durable policy lineage. */ ` + INSERT INTO memory_storage_roots + (storage_root_id, agent_id, backend_kind, opaque_locator, path_key_version, path_key, + authority_kind, authority_owner_id, default_capabilities_json, lifecycle_state, created_at, updated_at) + VALUES ('root-shadow', 'main', 'builtin', 'builtin:v1:shadow', 1, 's1_shadow_fixture_path_key_000', + 'user', 'alice', '["read"]', 'active', 1, 1); + INSERT INTO memory_policies + (policy_id, agent_id, current_revision_id, revocation_epoch, lifecycle_state, created_at, updated_at) + VALUES ('policy-shadow', 'main', 'policy-shadow-revision-1', 0, 'active', 1, 1); + INSERT INTO memory_policy_revisions + (revision_id, policy_id, revision_number, revocation_epoch, lifecycle_state, + actor_kind, actor_id, reason, created_at) + VALUES ('policy-shadow-revision-1', 'policy-shadow', 1, 0, 'active', 'human', 'alice', 'fixture', 1); + INSERT INTO memory_stores + (store_id, agent_id, storage_root_id, policy_id, scope_kind, audience_kind, audience_id, + lifecycle_state, created_at, updated_at) + VALUES ('store-shadow', 'main', 'root-shadow', 'policy-shadow', 'agent', 'agent', 'main', 'active', 1, 1); + INSERT INTO memory_resources + (resource_id, agent_id, store_id, logical_locator, source, created_at) + VALUES ('resource-shadow', 'main', 'store-shadow', 'memory/shadow.md', 'memory', 1); + INSERT INTO memory_resource_revisions + (revision_id, resource_id, revision_number, artifact_locator, content_hash, content_bytes, + policy_revision_id, policy_revocation_epoch, source_policy_set_id, lifecycle_state, + actor_kind, actor_id, expires_at, created_at, activated_at, retired_at) + VALUES ('resource-revision-1', 'resource-shadow', 1, 'shadow.md', 'shadow', 6, + 'policy-shadow-revision-1', 0, 'plugin-policy-set-1', 'active', 'human', 'alice', NULL, 1, 1, NULL); + `); +} + function recordExposure(params: { runId: string; subjectRevision?: string; sessionIdentityRevision?: string; }) { - return recordMemoryRunExposure({ + return prepareMemoryRunExposure({ agentId: AGENT_ID, sessionId: SESSION_ID, sessionKey: SESSION_KEY, @@ -106,6 +171,16 @@ function recordExposure(params: { egressRegistryRevision: "egress-registry-revision-1", sessionIdentityRevision: params.sessionIdentityRevision ?? SESSION_IDENTITY_REVISION, subjectRevision: params.subjectRevision ?? SUBJECT_REVISION, + actorEvidence: { + version: 1, + kind: "principal", + actorKind: "human", + principalId: "alice", + assurance: "gateway-profile", + evidenceRevision: "actor-revision-1", + }, + delegationSnapshot: { version: 1, kind: "none" }, + hostFactsRevision: "host-facts-1", }); } @@ -132,13 +207,60 @@ async function appendWithRun(params: { env: NodeJS.ProcessEnv; runId: string; te withTranscriptWrite: async (run) => await run(), }, async () => { - await appendSqliteTranscriptMessage(scope(params.env), { + await appendTranscriptMessage(scope(params.env), { message: { role: "assistant", content: [{ type: "text", text: params.text }] }, }); }, ); } +function copyPendingTranscriptForTransition(params: { + database: OpenClawAgentDatabase; + sourceSessionId?: string; + subjectRevision?: string; + targetSessionId: string; +}): void { + const sourceSessionId = params.sourceSessionId ?? SESSION_ID; + const targetSessionKey = `${SESSION_KEY}:transition:${params.targetSessionId}`; + const subjectRevision = params.subjectRevision ?? SUBJECT_REVISION; + params.database.db + .prepare( + `INSERT INTO session_memory_subjects + (session_key, subject_kind, binding_id, principal_id, subject_revision, created_at) + VALUES (?, 'user', 'binding-transition', 'alice', ?, 1)`, + ) + .run(targetSessionKey, subjectRevision); + writeSessionEntry(params.database, targetSessionKey, { + sessionId: params.targetSessionId, + updatedAt: 1, + }); + const sourceRows = params.database.db + .prepare( + `SELECT created_at, event_json, seq + FROM transcript_events + WHERE session_id = ? + ORDER BY seq ASC`, + ) + .all(sourceSessionId) as Array<{ created_at: number; event_json: string; seq: number }>; + for (const row of sourceRows) { + params.database.db + .prepare( + `INSERT INTO transcript_events (session_id, seq, event_json, created_at) + VALUES (?, ?, ?, ?)`, + ) + .run(params.targetSessionId, row.seq, row.event_json, row.created_at); + params.database.db + .prepare( + `INSERT INTO transcript_event_memory_policies + (session_id, event_seq, authorization_status, source_policy_set_id, run_exposure_set_id, + run_exposure_revision, delivery_audiences_json, session_identity_revision, + subject_revision, run_id, context_fingerprint, created_at) + VALUES (?, ?, 'pending', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, ?)`, + ) + .run(params.targetSessionId, row.seq, row.created_at); + } +} + afterEach(() => { clearMemoryRunExposureForTest(); resetMemoryIsolationCutoverForTest(); @@ -170,12 +292,13 @@ describe("transcript memory policy companions", () => { writeSession(alice); const database = openOpenClawAgentDatabase(options); + seedTranscriptPolicyFixture(database); const aliceContext = createCurrentMemorySessionContext({ ...alice, options }); expect(aliceContext.kind).toBe("current"); if (aliceContext.kind !== "current") { throw new Error("expected lifecycle-owned Alice subject context"); } - await appendSqliteTranscriptMessage( + await appendTranscriptMessage( { ...alice, agentId: AGENT_ID, env }, { message: { @@ -205,7 +328,7 @@ describe("transcript memory policy companions", () => { }).hits, ).toEqual([]); - const shadowExposure = recordMemoryRunExposure({ + const shadowExposure = prepareMemoryRunExposure({ agentId: AGENT_ID, sessionId: alice.sessionId, sessionKey: alice.sessionKey, @@ -222,6 +345,16 @@ describe("transcript memory policy companions", () => { egressRegistryRevision: "egress-registry-revision-1", sessionIdentityRevision: aliceContext.context.sessionIdentityRevision, subjectRevision: aliceContext.context.subjectRevision, + actorEvidence: { + version: 1, + kind: "principal", + actorKind: "human", + principalId: aliceContext.context.principalId, + assurance: "gateway-profile", + evidenceRevision: "shadow-actor-revision-1", + }, + delegationSnapshot: { version: 1, kind: "none" }, + hostFactsRevision: "shadow-host-facts-1", }); expect( persistMemoryRunExposureBeforeContentInDatabase({ database, snapshot: shadowExposure }), @@ -237,7 +370,7 @@ describe("transcript memory policy companions", () => { withTranscriptWrite: async (run) => await run(), }, async () => { - await appendSqliteTranscriptMessage( + await appendTranscriptMessage( { ...alice, agentId: AGENT_ID, env }, { message: { @@ -251,7 +384,7 @@ describe("transcript memory policy companions", () => { expect(readAuthorizedTranscriptEventSeqs(database.db, alice.sessionId)?.size).toBeGreaterThan( 0, ); - expect(loadSqliteTranscriptEventsSync({ ...alice, agentId: AGENT_ID, env })).toContainEqual( + expect(loadTranscriptEventsSync({ ...alice, agentId: AGENT_ID, env })).toContainEqual( expect.objectContaining({ message: expect.objectContaining({ content: [{ type: "text", text: "alice scoped content" }], @@ -270,31 +403,38 @@ describe("transcript memory policy companions", () => { expect(createCurrentMemorySessionContext({ ...bob, options })).toEqual({ kind: "shadow-subject-mismatch", }); - await appendSqliteTranscriptMessage( + await appendTranscriptMessage( { ...bob, agentId: AGENT_ID, env }, { message: { role: "assistant", content: [{ type: "text", text: "bob denied content" }] }, }, ); expect(readAuthorizedTranscriptEventSeqs(database.db, bob.sessionId)).toEqual(new Set()); - expect(loadSqliteTranscriptEventsSync({ ...bob, agentId: AGENT_ID, env })).toEqual([]); + expect(loadTranscriptEventsSync({ ...bob, agentId: AGENT_ID, env })).toEqual([]); }); it("fails closed for missing or stale run exposure while indexing only an authorized event", async () => { const env = createEnv(); // Establish the SQLite session before the cut-over marker is written; its old row has no // companion and must disappear as soon as the enforced policy reader is active. - await appendSqliteTranscriptMessage(scope(env), { + await appendTranscriptMessage(scope(env), { message: { role: "assistant", content: [{ type: "text", text: "legacy private content" }] }, }); const database = markCutOver(env); - await appendSqliteTranscriptMessage(scope(env), { + await appendTranscriptMessage(scope(env), { message: { role: "assistant", content: [{ type: "text", text: "missing exposure content" }] }, }); persistExposure(database, { runId: "stale-run", subjectRevision: "stale-subject-revision" }); await appendWithRun({ env, runId: "stale-run", text: "stale exposure content" }); const authorizedExposure = persistExposure(database, { runId: "authorized-run" }); + expect( + readDurableMemoryRunExposure({ + database, + sessionId: SESSION_ID, + runId: "authorized-run", + }), + ).toMatchObject({ exposureSetId: authorizedExposure.exposureSetId }); await appendWithRun({ env, runId: "authorized-run", text: "authorized exposure content" }); const policyRows = database.db @@ -320,21 +460,21 @@ describe("transcript memory policy companions", () => { ); expect(readAuthorizedTranscriptEventSeqs(database.db, SESSION_ID)).toEqual(new Set([4])); - expect(loadSqliteTranscriptEventsSync(scope(env))).toEqual([ + expect(loadTranscriptEventsSync(scope(env))).toEqual([ expect.objectContaining({ message: expect.objectContaining({ content: [{ type: "text", text: "authorized exposure content" }], }), }), ]); - expect(loadSqliteTranscriptTailEventsSync(scope(env), 2)).toEqual([ + expect(loadTranscriptTailEventsSync(scope(env), 2)).toEqual([ expect.objectContaining({ message: expect.objectContaining({ content: [{ type: "text", text: "authorized exposure content" }], }), }), ]); - expect(loadLatestSqliteAssistantText(scope(env))).toMatchObject({ + expect(loadLatestAssistantText(scope(env))).toMatchObject({ text: "authorized exposure content", }); @@ -377,13 +517,13 @@ describe("transcript memory policy companions", () => { }, async () => { authorizedMessageId = ( - await appendSqliteTranscriptMessage(scope(env), { + await appendTranscriptMessage(scope(env), { message: { role: "assistant", content: [{ type: "text", text: "authorized" }] }, }) ).messageId; }, ); - const pending = await appendSqliteTranscriptMessage(scope(env), { + const pending = await appendTranscriptMessage(scope(env), { message: { role: "assistant", content: [{ type: "text", text: "pending" }] }, }); @@ -394,14 +534,14 @@ describe("transcript memory policy companions", () => { it("does not pass a pending transcript event to manual compaction", async () => { const env = createEnv(); const database = markCutOver(env); - await appendSqliteTranscriptMessage(scope(env), { + await appendTranscriptMessage(scope(env), { message: { role: "assistant", content: [{ type: "text", text: "pending" }] }, }); const selectRetainedLines = vi.fn(() => null); - await expect( - trimSqliteTranscriptForManualCompact(scope(env), selectRetainedLines), - ).resolves.toEqual({ trimmed: false }); + await expect(trimTranscriptForManualCompact(scope(env), selectRetainedLines)).resolves.toEqual({ + trimmed: false, + }); expect(selectRetainedLines).not.toHaveBeenCalled(); expect( @@ -416,8 +556,8 @@ describe("transcript memory policy companions", () => { const database = markCutOver(env); persistExposure(database, { runId: "authorized-run" }); database.db.exec(/* sqlite-allow-raw: test-only atomicity fault injection. */ ` - CREATE TRIGGER reject_transcript_memory_policy_for_test - BEFORE INSERT ON transcript_event_memory_policies + CREATE TRIGGER reject_transcript_memory_policy_detail_for_test + BEFORE INSERT ON transcript_event_memory_policy_details BEGIN SELECT RAISE(ABORT, 'test companion persistence failure'); END; @@ -430,8 +570,11 @@ describe("transcript memory policy companions", () => { for (const table of [ "transcript_events", "transcript_event_memory_policies", + "transcript_event_memory_policy_details", "memory_policy_sets", + "memory_policy_set_members", "memory_run_exposures", + "memory_run_exposure_resources", ]) { expect(database.db.prepare(`SELECT COUNT(*) AS count FROM ${table}`).get()).toEqual({ count: 0, @@ -442,6 +585,11 @@ describe("transcript memory policy companions", () => { expect( database.db.prepare("SELECT COUNT(*) AS count FROM memory_preoutput_exposure_ledger").get(), ).toEqual({ count: 1 }); + expect( + database.db + .prepare("SELECT COUNT(*) AS count FROM memory_preoutput_exposure_authorization_facts") + .get(), + ).toEqual({ count: 1 }); }); it("replays only committed current companions after a fresh database consumer starts", async () => { @@ -480,7 +628,7 @@ describe("transcript memory policy companions", () => { expect(readAuthorizedTranscriptEventSeqs(fresh.db, SESSION_ID)?.size).toBe( committedAuthorizedCount, ); - expect(loadSqliteTranscriptEventsSync(scope(env))).toContainEqual( + expect(loadTranscriptEventsSync(scope(env))).toContainEqual( expect.objectContaining({ message: expect.objectContaining({ content: [{ type: "text", text: "committed companion content" }], @@ -490,7 +638,7 @@ describe("transcript memory policy companions", () => { // A later durable row without a companion is pending. A separate database // consumer must not infer authority from the earlier committed exposure. - await appendSqliteTranscriptMessage(scope(env), { + await appendTranscriptMessage(scope(env), { message: { role: "assistant", content: [{ type: "text", text: "missing companion" }] }, }); closeOpenClawAgentDatabasesForTest(); @@ -498,7 +646,7 @@ describe("transcript memory policy companions", () => { expect(readAuthorizedTranscriptEventSeqs(fresh.db, SESSION_ID)?.size).toBe( committedAuthorizedCount, ); - expect(loadSqliteTranscriptEventsSync(scope(env))).toContainEqual( + expect(loadTranscriptEventsSync(scope(env))).toContainEqual( expect.objectContaining({ message: expect.objectContaining({ content: [{ type: "text", text: "committed companion content" }], @@ -519,7 +667,7 @@ describe("transcript memory policy companions", () => { expect(readAuthorizedTranscriptEventSeqs(fresh.db, SESSION_ID)?.size).toBe( committedAuthorizedCount, ); - expect(loadSqliteTranscriptEventsSync(scope(env))).toContainEqual( + expect(loadTranscriptEventsSync(scope(env))).toContainEqual( expect.objectContaining({ message: expect.objectContaining({ content: [{ type: "text", text: "committed companion content" }], @@ -539,7 +687,7 @@ describe("transcript memory policy companions", () => { closeOpenClawAgentDatabasesForTest(); fresh = openOpenClawAgentDatabase({ agentId: AGENT_ID, env }); expect(readAuthorizedTranscriptEventSeqs(fresh.db, SESSION_ID)).toEqual(new Set()); - expect(loadSqliteTranscriptEventsSync(scope(env))).toEqual([]); + expect(loadTranscriptEventsSync(scope(env))).toEqual([]); }); it("removes a stale companion from replay, search, projections, compaction, and export", async () => { @@ -565,16 +713,16 @@ describe("transcript memory policy companions", () => { .run(999, SESSION_ID); expect(readAuthorizedTranscriptEventSeqs(database.db, SESSION_ID)).toEqual(new Set()); - expect(loadSqliteTranscriptEventsSync(scope(env))).toEqual([]); + expect(loadTranscriptEventsSync(scope(env))).toEqual([]); expect(search().hits).toEqual([]); expect(() => readSessionTranscriptMessageEvents(scope(env))).toThrow( /projection is rebuilding/i, ); - await expect(trimSqliteTranscriptForManualCompact(scope(env), vi.fn())).resolves.toEqual({ + await expect(trimTranscriptForManualCompact(scope(env), vi.fn())).resolves.toEqual({ trimmed: false, }); - const plan = planSqliteSessionStateDeleteIfUnreferenced({ + const plan = planSessionStateDeleteIfUnreferenced({ archiveDirectory: path.join(roots.at(-1) ?? "", "archives"), archiveTranscript: true, database, @@ -583,7 +731,317 @@ describe("transcript memory policy companions", () => { sessionId: SESSION_ID, }); expect(plan).not.toBeNull(); - const materialized = await materializeSqliteSessionStateDeletePlans([plan!]); - expect(materialized[0]?.archivedTranscript).toBeNull(); + const sourceEventCount = database.db + .prepare("SELECT COUNT(*) AS count FROM transcript_events WHERE session_id = ?") + .get(SESSION_ID); + await expect(materializeSessionStateDeletePlans([plan!])).rejects.toThrow( + `Unauthorized transcript policy archive event for ${SESSION_ID}`, + ); + // Archive failure preserves the raw source rather than leaking it or deleting it. + expect( + database.db + .prepare("SELECT COUNT(*) AS count FROM transcript_events WHERE session_id = ?") + .get(SESSION_ID), + ).toEqual(sourceEventCount); + }); + + it("records complete policy evidence for every readable transcript event class", async () => { + const env = createEnv(); + const database = markCutOver(env); + persistExposure(database, { runId: "event-classes-run" }); + + await withOwnedSessionTranscriptWrites( + { + sessionTarget: { + agentId: AGENT_ID, + expectedWriterRunId: "event-classes-run", + sessionId: SESSION_ID, + sessionKey: SESSION_KEY, + }, + withTranscriptWrite: async (run) => await run(), + }, + async () => { + await appendTranscriptMessage(scope(env), { + message: { role: "user", content: [{ type: "text", text: "user event" }] }, + }); + await appendTranscriptMessage(scope(env), { + message: { role: "assistant", content: [{ type: "text", text: "assistant event" }] }, + }); + for (const type of ["tool-result", "summary", "checkpoint", "system"] as const) { + await appendTranscriptEvent(scope(env), { type, value: `${type} event` }); + } + }, + ); + + const companions = database.db + .prepare( + `SELECT policy.authorization_status, detail.actor_evidence_json, detail.delegation_snapshot_json, + detail.exposed_resource_revisions_json, detail.normalized_audience_intersection_json, + detail.finalized_delivery_audiences_json, detail.source_session_id, detail.source_event_seq + FROM transcript_event_memory_policies AS policy + JOIN transcript_event_memory_policy_details AS detail + ON detail.session_id = policy.session_id AND detail.event_seq = policy.event_seq + WHERE policy.session_id = ? + ORDER BY policy.event_seq`, + ) + .all(SESSION_ID) as Array<{ + actor_evidence_json: string; + authorization_status: string; + delegation_snapshot_json: string; + exposed_resource_revisions_json: string; + finalized_delivery_audiences_json: string; + normalized_audience_intersection_json: string; + source_event_seq: number; + source_session_id: string; + }>; + // Header plus six requested classes all carry one atomic, evaluable row. + expect(companions).toHaveLength(7); + expect(companions.every((companion) => companion.authorization_status === "authorized")).toBe( + true, + ); + for (const companion of companions) { + expect(JSON.parse(companion.actor_evidence_json)).toMatchObject({ principalId: "alice" }); + expect(JSON.parse(companion.delegation_snapshot_json)).toMatchObject({ kind: "none" }); + expect(JSON.parse(companion.exposed_resource_revisions_json)).toEqual([ + "resource-revision-1", + ]); + expect(companion.normalized_audience_intersection_json).toBe( + companion.finalized_delivery_audiences_json, + ); + expect(companion.source_session_id).toBe(SESSION_ID); + expect(companion.source_event_seq).toBeGreaterThanOrEqual(0); + } + expect(readAuthorizedTranscriptEventSeqs(database.db, SESSION_ID)?.size).toBe(7); + }); + + it("uses the captured trusted actor and token-free delegation rather than reconstructing session facts", async () => { + const env = createEnv(); + const database = markCutOver(env); + const captured = recordExposure({ runId: "delegated-run" }); + const exposure = { + ...captured, + actorEvidence: { + version: 1, + kind: "principal", + actorKind: "agent", + principalId: "support-agent", + assurance: "service", + evidenceRevision: "actor-evidence-42", + }, + delegationSnapshot: { + version: 1, + kind: "delegated", + rootPrincipalId: "alice", + rootContextId: "root-context-42", + parentContextId: "parent-context-42", + parentMemoryPlanId: "parent-plan-42", + capabilitySnapshotId: "capability-42", + allowedOperations: ["derive", "read"], + maximumAudiences: [ + { kind: "role", id: "writer" }, + { kind: "user", id: "alice" }, + ], + depth: 1, + }, + hostFactsRevision: "host-facts-42", + } as typeof captured; + expect(persistMemoryRunExposureBeforeContentInDatabase({ database, snapshot: exposure })).toBe( + true, + ); + + await appendWithRun({ env, runId: "delegated-run", text: "delegated durable evidence" }); + + const row = database.db + .prepare( + `SELECT actor_evidence_json, delegation_snapshot_json + FROM transcript_event_memory_policy_details + WHERE session_id = ? + ORDER BY event_seq DESC + LIMIT 1`, + ) + .get(SESSION_ID) as { actor_evidence_json: string; delegation_snapshot_json: string }; + expect(JSON.parse(row.actor_evidence_json)).toEqual(exposure.actorEvidence); + expect(JSON.parse(row.delegation_snapshot_json)).toEqual(exposure.delegationSnapshot); + expect(JSON.stringify(row)).not.toContain("binding-alice"); + expect(JSON.stringify(row)).not.toContain("storeCapToken"); + }); + + it("withdraws replay eligibility when the captured stable policy revision or epoch changes", async () => { + const env = createEnv(); + const database = markCutOver(env); + persistExposure(database, { runId: "policy-revision-run" }); + await appendWithRun({ env, runId: "policy-revision-run", text: "revision-bound secret" }); + expect(readAuthorizedTranscriptEventSeqs(database.db, SESSION_ID)?.size).toBeGreaterThan(0); + + database.db.exec(/* sqlite-allow-raw: test mutates the active policy owner after capture. */ ` + UPDATE memory_policy_revisions + SET lifecycle_state = 'superseded' + WHERE revision_id = 'policy-revision-1'; + INSERT INTO memory_policy_revisions + (revision_id, policy_id, revision_number, revocation_epoch, lifecycle_state, + actor_kind, actor_id, reason, created_at) + VALUES ('policy-revision-2', 'policy-1', 2, 1, 'active', 'human', 'alice', 'revoked', 2); + UPDATE memory_policies + SET current_revision_id = 'policy-revision-2', revocation_epoch = 1, updated_at = 2 + WHERE policy_id = 'policy-1'; + `); + + expect(readAuthorizedTranscriptEventSeqs(database.db, SESSION_ID)).toEqual(new Set()); + expect(loadTranscriptEventsSync(scope(env))).toEqual([]); + expect( + searchSessionTranscripts({ agentId: AGENT_ID, env, query: "revision-bound secret" }).hits, + ).toEqual([]); + }); + + it("withdraws dependent events when an exposed resource expires", async () => { + const env = createEnv(); + const database = markCutOver(env); + persistExposure(database, { runId: "resource-expiry-run" }); + await appendWithRun({ env, runId: "resource-expiry-run", text: "resource-bound secret" }); + expect(readAuthorizedTranscriptEventSeqs(database.db, SESSION_ID)?.size).toBeGreaterThan(0); + + database.db.exec(/* sqlite-allow-raw: test advances an immutable lease past expiry. */ ` + DROP TRIGGER memory_resource_revisions_immutable_fields; + UPDATE memory_resource_revisions + SET expires_at = ${Date.now() - 1} + WHERE revision_id = 'resource-revision-1'; + `); + + expect(readAuthorizedTranscriptEventSeqs(database.db, SESSION_ID)).toEqual(new Set()); + expect(loadTranscriptEventsSync(scope(env))).toEqual([]); + }); + + it("preserves only exact same-session replacement lineage and leaves new rows pending", async () => { + const env = createEnv(); + const database = markCutOver(env); + persistExposure(database, { runId: "replace-lineage-run" }); + await appendWithRun({ env, runId: "replace-lineage-run", text: "retained source" }); + + const original = loadTranscriptEventsSync(scope(env)); + const retainedMessage = original.find( + (event) => + typeof event === "object" && + event !== null && + "message" in event && + (event as { message?: { content?: Array<{ text?: string }> } }).message?.content?.[0] + ?.text === "retained source", + ); + expect(retainedMessage).toBeDefined(); + await replaceTranscriptEvents(scope(env), [retainedMessage!]); + + expect(readAuthorizedTranscriptEventSeqs(database.db, SESSION_ID)).toEqual(new Set([0])); + expect( + database.db + .prepare( + `SELECT source_event_seq + FROM transcript_event_memory_policy_details + WHERE session_id = ? AND event_seq = 0`, + ) + .get(SESSION_ID), + ).toEqual({ source_event_seq: 1 }); + + await withOwnedSessionTranscriptWrites( + { + sessionTarget: { + agentId: AGENT_ID, + expectedWriterRunId: "replace-lineage-run", + sessionId: SESSION_ID, + sessionKey: SESSION_KEY, + }, + withTranscriptWrite: async (run) => await run(), + }, + async () => { + await replaceTranscriptEvents(scope(env), [ + { + message: { + role: "assistant", + content: [{ type: "text", text: "new derived content" }], + }, + }, + ]); + }, + ); + expect(readAuthorizedTranscriptEventSeqs(database.db, SESSION_ID)).toEqual(new Set()); + expect( + database.db + .prepare( + `SELECT authorization_status + FROM transcript_event_memory_policies + WHERE session_id = ? AND event_seq = 0`, + ) + .get(SESSION_ID), + ).toEqual({ authorization_status: "pending" }); + }); + + it("preserves a cross-session transition only with matching immutable subject provenance", async () => { + const env = createEnv(); + const database = markCutOver(env); + persistExposure(database, { runId: "transition-run" }); + await appendWithRun({ env, runId: "transition-run", text: "transition source" }); + const sourceSeqs = readAuthorizedTranscriptEventSeqs(database.db, SESSION_ID); + expect(sourceSeqs?.size).toBeGreaterThan(0); + + const targetSessionId = "transition-target"; + copyPendingTranscriptForTransition({ + database, + targetSessionId, + }); + expect( + preserveTranscriptMemoryPolicyTransitionInTransaction({ + database, + sourceSessionId: SESSION_ID, + targetSessionId, + transitionKind: "fork", + }), + ).toBe(sourceSeqs?.size); + expect(readAuthorizedTranscriptEventSeqs(database.db, targetSessionId)).toEqual(sourceSeqs); + expect( + database.db + .prepare( + `SELECT source_session_id, source_event_seq, transition_kind + FROM transcript_event_memory_policy_transitions + WHERE session_id = ? + ORDER BY event_seq ASC`, + ) + .all(targetSessionId), + ).toEqual( + [...(sourceSeqs ?? [])].map((sourceEventSeq) => ({ + source_session_id: SESSION_ID, + source_event_seq: sourceEventSeq, + transition_kind: "fork", + })), + ); + }); + + it("leaves a cross-session copy pending when its target subject differs", async () => { + const env = createEnv(); + const database = markCutOver(env); + persistExposure(database, { runId: "transition-mismatch-run" }); + await appendWithRun({ env, runId: "transition-mismatch-run", text: "isolated source" }); + + const targetSessionId = "transition-mismatched-subject"; + copyPendingTranscriptForTransition({ + database, + subjectRevision: "different-subject-revision", + targetSessionId, + }); + expect( + preserveTranscriptMemoryPolicyTransitionInTransaction({ + database, + sourceSessionId: SESSION_ID, + targetSessionId, + transitionKind: "parent-fork", + }), + ).toBe(0); + expect(readAuthorizedTranscriptEventSeqs(database.db, targetSessionId)).toEqual(new Set()); + expect( + database.db + .prepare( + `SELECT COUNT(*) AS count + FROM transcript_event_memory_policy_transitions + WHERE session_id = ?`, + ) + .get(targetSessionId), + ).toEqual({ count: 0 }); }); }); diff --git a/src/config/sessions/session-transcript-memory-policy.ts b/src/config/sessions/session-transcript-memory-policy.ts index f2305c4d5b8e..b0c0e228ece2 100644 --- a/src/config/sessions/session-transcript-memory-policy.ts +++ b/src/config/sessions/session-transcript-memory-policy.ts @@ -14,12 +14,64 @@ import { getOwnedSessionTranscriptWriterFence } from "./transcript-write-context type TranscriptMemoryPolicyDatabase = Pick< OpenClawAgentDatabaseSchema, + | "memory_policies" + | "memory_policy_revisions" + | "memory_policy_set_members" | "memory_policy_sets" + | "memory_resource_revisions" + | "memory_run_exposure_resources" | "memory_run_exposures" | "session_memory_subject_snapshots" + | "transcript_events" | "transcript_event_memory_policies" + | "transcript_event_memory_policy_details" + | "transcript_event_memory_policy_transitions" >; +type StablePolicyMember = Readonly<{ + expectedRevocationEpoch: number; + expectedRevisionId: string; + policyId: string; +}>; + +type PersistedExposure = Readonly<{ + deliveryAudiencesJson: string; + egressReceiptIdsJson: string; + exposedResourceRevisionsJson: string; + exposureReceiptIdsJson: string; + policySetId: string; +}>; + +export type PreservedTranscriptMemoryPolicy = Readonly<{ + actorEvidenceJson: string; + contextFingerprint: string; + createdAt: number; + delegationSnapshotJson: string; + deliveryAudiencesJson: string; + egressReceiptIdsJson: string; + eventSeq: number; + exposedResourceRevisionsJson: string; + exposureReceiptIdsJson: string; + finalizedDeliveryAudiencesJson: string; + normalizedAudienceIntersectionJson: string; + retentionState: "retained" | "quarantined"; + runExposureRevision: number; + runExposureSetId: string; + runId: string; + sessionIdentityRevision: string; + sourceEventSeq: number; + sourcePolicySetId: string; + sourceSessionId: string; + subjectRevision: string; +}>; + +export type TranscriptMemoryPolicyTransitionKind = + | "parent-fork" + | "fork" + | "rewind" + | "switch" + | "checkpoint"; + const enforcementByDatabase = new WeakMap(); function policyDatabase(db: DatabaseSync) { @@ -55,9 +107,18 @@ function canonicalAudiences(exposure: MemoryRunExposureSnapshot): string | undef function effectivePolicySetId( memoryPolicyRevision: string, sourcePolicySetIdsJson: string, + members: readonly StablePolicyMember[], + audienceIntersectionJson: string, ): string { - return `mpset1_${createHash("sha256") - .update(JSON.stringify({ memoryPolicyRevision, sourcePolicySetIdsJson })) + return `mpset2_${createHash("sha256") + .update( + JSON.stringify({ + audienceIntersectionJson, + members, + memoryPolicyRevision, + sourcePolicySetIdsJson, + }), + ) .digest("base64url")}`; } @@ -80,15 +141,71 @@ export function isTranscriptMemoryPolicyEnforcedInDatabase(db: DatabaseSync): bo return enforced; } +/** + * Resolve only persisted built-in revision facts. Plugin policy payload remains opaque: + * an exposure without stable resource revisions is pending instead of being reverse-engineered. + */ +function resolveStablePolicyMembersInTransaction(params: { + database: OpenClawAgentDatabase; + exposedResourceRevisions: readonly string[]; + sourcePolicySetIds: readonly string[]; +}): readonly StablePolicyMember[] | undefined { + const db = policyDatabase(params.database.db); + if (params.exposedResourceRevisions.length === 0) { + return undefined; + } + const resources = executeSqliteQuerySync( + params.database.db, + db + .selectFrom("memory_resource_revisions") + .select(["policy_revision_id", "revision_id", "source_policy_set_id"]) + .where("revision_id", "in", [...new Set(params.exposedResourceRevisions)]), + ).rows; + if (resources.length !== new Set(params.exposedResourceRevisions).size) { + return undefined; + } + if (params.sourcePolicySetIds.length === 0) { + return undefined; + } + const sourcePolicySetIds = new Set(params.sourcePolicySetIds); + if (resources.some((resource) => !sourcePolicySetIds.has(resource.source_policy_set_id))) { + return undefined; + } + const policyRevisionIds = new Set(resources.map((resource) => resource.policy_revision_id)); + const revisions = executeSqliteQuerySync( + params.database.db, + db + .selectFrom("memory_policy_revisions as revision") + .innerJoin("memory_policies as policy", "policy.policy_id", "revision.policy_id") + .select([ + "policy.policy_id as policy_id", + "revision.revision_id as revision_id", + "revision.revocation_epoch as revision_revocation_epoch", + ]) + .where("revision.revision_id", "in", [...policyRevisionIds]), + ).rows; + if (revisions.length !== policyRevisionIds.size) { + return undefined; + } + const members = revisions + .map( + (revision) => + ({ + expectedRevocationEpoch: revision.revision_revocation_epoch, + expectedRevisionId: revision.revision_id, + policyId: revision.policy_id, + }) satisfies StablePolicyMember, + ) + .toSorted((left, right) => left.policyId.localeCompare(right.policyId)); + return new Set(members.map((member) => member.policyId)).size === members.length + ? Object.freeze(members) + : undefined; +} + function persistExposureLineageInTransaction(params: { database: OpenClawAgentDatabase; current: MemoryRunExposureSnapshot; -}): - | Readonly<{ - policySetId: string; - deliveryAudiencesJson: string; - }> - | undefined { +}): PersistedExposure | undefined { const snapshots: MemoryRunExposureSnapshot[] = []; const seen = new Set(); let cursor: MemoryRunExposureSnapshot | undefined = params.current; @@ -105,7 +222,7 @@ function persistExposureLineageInTransaction(params: { cursor = cursor.previous; } const db = policyDatabase(params.database.db); - let currentResult: { policySetId: string; deliveryAudiencesJson: string } | undefined; + let currentResult: PersistedExposure | undefined; for (const snapshot of snapshots.toReversed()) { const sourcePolicySetIdsJson = canonicalStrings(snapshot.sourcePolicySetIds); const exposedResourceRevisionsJson = canonicalStrings(snapshot.exposedResourceRevisions); @@ -124,7 +241,20 @@ function persistExposureLineageInTransaction(params: { ) { return undefined; } - const policySetId = effectivePolicySetId(snapshot.memoryPolicyRevision, sourcePolicySetIdsJson); + const members = resolveStablePolicyMembersInTransaction({ + database: params.database, + exposedResourceRevisions: snapshot.exposedResourceRevisions, + sourcePolicySetIds: snapshot.sourcePolicySetIds, + }); + if (!members) { + return undefined; + } + const policySetId = effectivePolicySetId( + snapshot.memoryPolicyRevision, + sourcePolicySetIdsJson, + members, + deliveryAudiencesJson, + ); executeSqliteQuerySync( params.database.db, db @@ -138,6 +268,23 @@ function persistExposureLineageInTransaction(params: { }) .onConflict((conflict) => conflict.column("policy_set_id").doNothing()), ); + for (const member of members) { + executeSqliteQuerySync( + params.database.db, + db + .insertInto("memory_policy_set_members") + .values({ + policy_set_id: policySetId, + policy_id: member.policyId, + expected_revision_id: member.expectedRevisionId, + expected_revocation_epoch: member.expectedRevocationEpoch, + audience_intersection_json: deliveryAudiencesJson, + retention_state: "retained", + created_at: snapshot.createdAt, + }) + .onConflict((conflict) => conflict.columns(["policy_set_id", "policy_id"]).doNothing()), + ); + } executeSqliteQuerySync( params.database.db, db @@ -162,8 +309,30 @@ function persistExposureLineageInTransaction(params: { }) .onConflict((conflict) => conflict.column("exposure_set_id").doNothing()), ); + for (const resourceRevisionId of snapshot.exposedResourceRevisions) { + executeSqliteQuerySync( + params.database.db, + db + .insertInto("memory_run_exposure_resources") + .values({ + exposure_set_id: snapshot.exposureSetId, + resource_revision_id: resourceRevisionId, + policy_set_id: policySetId, + created_at: snapshot.createdAt, + }) + .onConflict((conflict) => + conflict.columns(["exposure_set_id", "resource_revision_id"]).doNothing(), + ), + ); + } if (snapshot === params.current) { - currentResult = { policySetId, deliveryAudiencesJson }; + currentResult = { + deliveryAudiencesJson, + egressReceiptIdsJson, + exposedResourceRevisionsJson, + exposureReceiptIdsJson, + policySetId, + }; } } return currentResult; @@ -196,6 +365,9 @@ export function recordTranscriptMemoryPolicyInTransaction(params: { sessionKey: string; eventSeq: number; createdAt: number; + inherited?: PreservedTranscriptMemoryPolicy; + /** Replacement rows may retain only an exact durable predecessor. */ + replacement?: boolean; }): boolean { if (!isTranscriptMemoryPolicyEnforcedInDatabase(params.database.db)) { return true; @@ -213,7 +385,21 @@ export function recordTranscriptMemoryPolicyInTransaction(params: { if (existing) { return existing.authorization_status === "authorized"; } - const runId = getOwnedSessionTranscriptWriterFence()?.expectedWriterRunId; + if ( + params.inherited && + recordInheritedTranscriptMemoryPolicyInTransaction({ + ...params, + inherited: params.inherited, + }) + ) { + return true; + } + // Rewrites must never borrow a currently active writer's exposure for new + // material. A replacement carries authorization only through exact durable + // lineage; every unmatched or invalid row remains pending. + const runId = params.replacement + ? undefined + : getOwnedSessionTranscriptWriterFence()?.expectedWriterRunId; const exposure = runId ? readDurableMemoryRunExposure({ database: params.database, @@ -226,11 +412,15 @@ export function recordTranscriptMemoryPolicyInTransaction(params: { isCurrentAuthorizedLabel({ database: params.database, sessionId: params.sessionId, exposure }) ? persistExposureLineageInTransaction({ database: params.database, current: exposure }) : undefined; - const authorized = Boolean(exposure && persisted); + // The exposure ledger carries the trusted, pre-output actor/delegation facts. + // Session subject rows only revalidate binding; they never reconstruct an actor. + const actorEvidenceJson = exposure ? JSON.stringify(exposure.actorEvidence) : undefined; + const delegationSnapshotJson = exposure ? JSON.stringify(exposure.delegationSnapshot) : undefined; + const authorized = Boolean(exposure && persisted && actorEvidenceJson && delegationSnapshotJson); executeSqliteQuerySync( params.database.db, db.insertInto("transcript_event_memory_policies").values( - authorized && exposure && persisted + authorized && exposure && persisted && actorEvidenceJson && delegationSnapshotJson ? { session_id: params.sessionId, event_seq: params.eventSeq, @@ -261,9 +451,488 @@ export function recordTranscriptMemoryPolicyInTransaction(params: { }, ), ); + if (authorized && exposure && persisted && actorEvidenceJson && delegationSnapshotJson) { + executeSqliteQuerySync( + params.database.db, + db.insertInto("transcript_event_memory_policy_details").values({ + session_id: params.sessionId, + event_seq: params.eventSeq, + actor_evidence_json: actorEvidenceJson, + delegation_snapshot_json: delegationSnapshotJson, + exposed_resource_revisions_json: persisted.exposedResourceRevisionsJson, + exposure_receipt_ids_json: persisted.exposureReceiptIdsJson, + egress_receipt_ids_json: persisted.egressReceiptIdsJson, + normalized_audience_intersection_json: persisted.deliveryAudiencesJson, + finalized_delivery_audiences_json: persisted.deliveryAudiencesJson, + retention_state: "retained", + source_session_id: params.sessionId, + source_event_seq: params.eventSeq, + created_at: params.createdAt, + }), + ); + } return authorized; } +function recordInheritedTranscriptMemoryPolicyInTransaction(params: { + database: OpenClawAgentDatabase; + sessionId: string; + sessionKey: string; + eventSeq: number; + createdAt: number; + inherited: PreservedTranscriptMemoryPolicy; +}): boolean { + const inherited = params.inherited; + // A replacement may copy only an already durable, same-session companion. + // Cross-session forks have a distinct session identity and must use the + // transition owner to establish a new authorized lineage rather than replay it. + const currentSubject = executeSqliteQueryTakeFirstSync( + params.database.db, + policyDatabase(params.database.db) + .selectFrom("session_memory_subject_snapshots") + .select(["session_identity_revision", "subject_revision"]) + .where("session_id", "=", params.sessionId) + .limit(1), + ); + if ( + inherited.retentionState !== "retained" || + inherited.sourceSessionId !== params.sessionId || + inherited.sessionIdentityRevision !== currentSubject?.session_identity_revision || + inherited.subjectRevision !== currentSubject?.subject_revision + ) { + return false; + } + const db = policyDatabase(params.database.db); + executeSqliteQuerySync( + params.database.db, + db.insertInto("transcript_event_memory_policies").values({ + session_id: params.sessionId, + event_seq: params.eventSeq, + authorization_status: "authorized", + source_policy_set_id: inherited.sourcePolicySetId, + run_exposure_set_id: inherited.runExposureSetId, + run_exposure_revision: inherited.runExposureRevision, + delivery_audiences_json: inherited.deliveryAudiencesJson, + session_identity_revision: inherited.sessionIdentityRevision, + subject_revision: inherited.subjectRevision, + run_id: inherited.runId, + context_fingerprint: inherited.contextFingerprint, + created_at: params.createdAt, + }), + ); + executeSqliteQuerySync( + params.database.db, + db.insertInto("transcript_event_memory_policy_details").values({ + session_id: params.sessionId, + event_seq: params.eventSeq, + actor_evidence_json: inherited.actorEvidenceJson, + delegation_snapshot_json: inherited.delegationSnapshotJson, + exposed_resource_revisions_json: inherited.exposedResourceRevisionsJson, + exposure_receipt_ids_json: inherited.exposureReceiptIdsJson, + egress_receipt_ids_json: inherited.egressReceiptIdsJson, + normalized_audience_intersection_json: inherited.normalizedAudienceIntersectionJson, + finalized_delivery_audiences_json: inherited.finalizedDeliveryAudiencesJson, + retention_state: inherited.retentionState, + source_session_id: inherited.sourceSessionId, + source_event_seq: inherited.sourceEventSeq, + created_at: params.createdAt, + }), + ); + return true; +} + +/** + * Capture durable companion evidence by exact stored event bytes before a + * same-session replacement deletes old rows. Replacements can reorder or drop + * events, so a sequence number is never a safe lineage key. + */ +export function readPreservedTranscriptMemoryPoliciesInTransaction( + database: OpenClawAgentDatabase, + sessionId: string, +): Map { + const rows = executeSqliteQuerySync( + database.db, + policyDatabase(database.db) + .selectFrom("transcript_event_memory_policies as policy") + .innerJoin("transcript_events as event", (join) => + join + .onRef("event.session_id", "=", "policy.session_id") + .onRef("event.seq", "=", "policy.event_seq"), + ) + .innerJoin("transcript_event_memory_policy_details as detail", (join) => + join + .onRef("detail.session_id", "=", "policy.session_id") + .onRef("detail.event_seq", "=", "policy.event_seq"), + ) + .select([ + "event.event_json", + "policy.context_fingerprint", + "policy.delivery_audiences_json", + "policy.event_seq", + "policy.run_exposure_revision", + "policy.run_exposure_set_id", + "policy.run_id", + "policy.session_identity_revision", + "policy.source_policy_set_id", + "policy.subject_revision", + "detail.actor_evidence_json", + "detail.created_at", + "detail.delegation_snapshot_json", + "detail.egress_receipt_ids_json", + "detail.exposed_resource_revisions_json", + "detail.exposure_receipt_ids_json", + "detail.finalized_delivery_audiences_json", + "detail.normalized_audience_intersection_json", + "detail.retention_state", + "detail.source_event_seq", + "detail.source_session_id", + ]) + .where("policy.session_id", "=", sessionId) + .where("policy.authorization_status", "=", "authorized") + .orderBy("policy.event_seq", "asc"), + ).rows; + const result = new Map(); + for (const row of rows) { + if ( + row.source_policy_set_id === null || + row.run_exposure_set_id === null || + row.run_exposure_revision === null || + row.delivery_audiences_json === null || + row.session_identity_revision === null || + row.subject_revision === null || + row.run_id === null || + row.context_fingerprint === null || + (row.retention_state !== "retained" && row.retention_state !== "quarantined") + ) { + continue; + } + const preserved = Object.freeze({ + actorEvidenceJson: row.actor_evidence_json, + contextFingerprint: row.context_fingerprint, + createdAt: row.created_at, + delegationSnapshotJson: row.delegation_snapshot_json, + deliveryAudiencesJson: row.delivery_audiences_json, + egressReceiptIdsJson: row.egress_receipt_ids_json, + eventSeq: row.event_seq, + exposedResourceRevisionsJson: row.exposed_resource_revisions_json, + exposureReceiptIdsJson: row.exposure_receipt_ids_json, + finalizedDeliveryAudiencesJson: row.finalized_delivery_audiences_json, + normalizedAudienceIntersectionJson: row.normalized_audience_intersection_json, + retentionState: row.retention_state, + runExposureRevision: row.run_exposure_revision, + runExposureSetId: row.run_exposure_set_id, + runId: row.run_id, + sessionIdentityRevision: row.session_identity_revision, + sourceEventSeq: row.source_event_seq, + sourcePolicySetId: row.source_policy_set_id, + sourceSessionId: row.source_session_id, + subjectRevision: row.subject_revision, + } satisfies PreservedTranscriptMemoryPolicy); + const matches = result.get(row.event_json); + if (matches) { + matches.push(preserved); + } else { + result.set(row.event_json, [preserved]); + } + } + return result; +} + +/** + * Copies only already-readable, exact source events into a new session identity. + * The target snapshot must exist first: a copied companion is transition lineage, + * never a replay of the source session's authorization context. + */ +export function preserveTranscriptMemoryPolicyTransitionInTransaction(params: { + database: OpenClawAgentDatabase; + sourceSessionId: string; + targetSessionId: string; + transitionKind: TranscriptMemoryPolicyTransitionKind; +}): number { + if ( + params.sourceSessionId === params.targetSessionId || + !isTranscriptMemoryPolicyEnforcedInDatabase(params.database.db) + ) { + return 0; + } + const db = policyDatabase(params.database.db); + const [sourceSubject, targetSubject] = [params.sourceSessionId, params.targetSessionId].map( + (sessionId) => + executeSqliteQueryTakeFirstSync( + params.database.db, + db + .selectFrom("session_memory_subject_snapshots") + .select(["session_identity_revision", "subject_revision"]) + .where("session_id", "=", sessionId) + .limit(1), + ), + ); + if ( + !sourceSubject || + !targetSubject || + sourceSubject.subject_revision !== targetSubject.subject_revision + ) { + return 0; + } + const readableSourceEventSeqs = readAuthorizedTranscriptEventSeqs( + params.database.db, + params.sourceSessionId, + ); + if (!readableSourceEventSeqs || readableSourceEventSeqs.size === 0) { + return 0; + } + const sourceRows = executeSqliteQuerySync( + params.database.db, + db + .selectFrom("transcript_events as event") + .innerJoin("transcript_event_memory_policies as policy", (join) => + join + .onRef("policy.session_id", "=", "event.session_id") + .onRef("policy.event_seq", "=", "event.seq"), + ) + .innerJoin("transcript_event_memory_policy_details as detail", (join) => + join + .onRef("detail.session_id", "=", "policy.session_id") + .onRef("detail.event_seq", "=", "policy.event_seq"), + ) + .select([ + "event.event_json", + "policy.context_fingerprint", + "policy.delivery_audiences_json", + "policy.event_seq", + "policy.run_exposure_revision", + "policy.run_exposure_set_id", + "policy.run_id", + "policy.source_policy_set_id", + "detail.actor_evidence_json", + "detail.delegation_snapshot_json", + "detail.egress_receipt_ids_json", + "detail.exposed_resource_revisions_json", + "detail.exposure_receipt_ids_json", + "detail.finalized_delivery_audiences_json", + "detail.normalized_audience_intersection_json", + "detail.retention_state", + ]) + .where("event.session_id", "=", params.sourceSessionId) + .where("policy.authorization_status", "=", "authorized") + .where("detail.retention_state", "=", "retained") + .orderBy("event.seq", "asc"), + ).rows; + const sourcesByEventJson = new Map(); + for (const row of sourceRows) { + if (!readableSourceEventSeqs.has(row.event_seq)) { + continue; + } + const matches = sourcesByEventJson.get(row.event_json); + if (matches) { + matches.push(row); + } else { + sourcesByEventJson.set(row.event_json, [row]); + } + } + const targetRows = executeSqliteQuerySync( + params.database.db, + db + .selectFrom("transcript_events") + .select(["event_json", "seq"]) + .where("session_id", "=", params.targetSessionId) + .orderBy("seq", "asc"), + ).rows; + let preserved = 0; + for (const target of targetRows) { + const source = sourcesByEventJson.get(target.event_json)?.shift(); + if ( + !source || + source.source_policy_set_id === null || + source.run_exposure_set_id === null || + source.run_exposure_revision === null || + source.delivery_audiences_json === null || + source.run_id === null || + source.context_fingerprint === null + ) { + continue; + } + const updated = executeSqliteQuerySync( + params.database.db, + db + .updateTable("transcript_event_memory_policies") + .set({ + authorization_status: "authorized", + source_policy_set_id: source.source_policy_set_id, + run_exposure_set_id: source.run_exposure_set_id, + run_exposure_revision: source.run_exposure_revision, + delivery_audiences_json: source.delivery_audiences_json, + session_identity_revision: targetSubject.session_identity_revision, + subject_revision: targetSubject.subject_revision, + run_id: source.run_id, + context_fingerprint: source.context_fingerprint, + }) + .where("session_id", "=", params.targetSessionId) + .where("event_seq", "=", target.seq) + .where("authorization_status", "=", "pending"), + ); + if (updated.numAffectedRows !== 1n) { + continue; + } + executeSqliteQuerySync( + params.database.db, + db.insertInto("transcript_event_memory_policy_details").values({ + session_id: params.targetSessionId, + event_seq: target.seq, + actor_evidence_json: source.actor_evidence_json, + delegation_snapshot_json: source.delegation_snapshot_json, + exposed_resource_revisions_json: source.exposed_resource_revisions_json, + exposure_receipt_ids_json: source.exposure_receipt_ids_json, + egress_receipt_ids_json: source.egress_receipt_ids_json, + normalized_audience_intersection_json: source.normalized_audience_intersection_json, + finalized_delivery_audiences_json: source.finalized_delivery_audiences_json, + retention_state: "retained", + source_session_id: params.sourceSessionId, + source_event_seq: source.event_seq, + created_at: Date.now(), + }), + ); + executeSqliteQuerySync( + params.database.db, + db.insertInto("transcript_event_memory_policy_transitions").values({ + session_id: params.targetSessionId, + event_seq: target.seq, + source_session_id: params.sourceSessionId, + source_event_seq: source.event_seq, + transition_kind: params.transitionKind, + source_session_identity_revision: sourceSubject.session_identity_revision, + target_session_identity_revision: targetSubject.session_identity_revision, + subject_revision: targetSubject.subject_revision, + created_at: Date.now(), + }), + ); + preserved += 1; + } + return preserved; +} + +function isCurrentPolicySetAuthorized(db: DatabaseSync, policySetId: string): boolean { + try { + const policy = policyDatabase(db); + const members = executeSqliteQuerySync( + db, + policy + .selectFrom("memory_policy_set_members") + .select([ + "expected_revocation_epoch", + "expected_revision_id", + "policy_id", + "retention_state", + ]) + .where("policy_set_id", "=", policySetId), + ).rows; + if (members.length === 0 || members.some((member) => member.retention_state !== "retained")) { + return false; + } + return members.every((member) => { + const current = executeSqliteQueryTakeFirstSync( + db, + policy + .selectFrom("memory_policies as policy") + .innerJoin( + "memory_policy_revisions as revision", + "revision.revision_id", + "policy.current_revision_id", + ) + .select([ + "policy.lifecycle_state", + "policy.revocation_epoch", + "revision.lifecycle_state as revision_state", + ]) + .where("policy.policy_id", "=", member.policy_id) + .where("policy.current_revision_id", "=", member.expected_revision_id) + .where("policy.revocation_epoch", "=", member.expected_revocation_epoch) + .limit(1), + ); + return current?.lifecycle_state === "active" && current.revision_state === "active"; + }); + } catch { + return false; + } +} + +function areExposedResourcesCurrent(db: DatabaseSync, exposureSetId: string): boolean { + try { + const policy = policyDatabase(db); + const resources = executeSqliteQuerySync( + db, + policy + .selectFrom("memory_run_exposure_resources as exposure_resource") + .innerJoin( + "memory_resource_revisions as resource", + "resource.revision_id", + "exposure_resource.resource_revision_id", + ) + .select(["resource.expires_at", "resource.lifecycle_state"]) + .where("exposure_resource.exposure_set_id", "=", exposureSetId), + ).rows; + const now = Date.now(); + return ( + resources.length > 0 && + resources.every( + (resource) => + resource.lifecycle_state === "active" && + (resource.expires_at === null || resource.expires_at > now), + ) + ); + } catch { + return false; + } +} + +function isCurrentTranscriptMemoryTransition(params: { + db: DatabaseSync; + policySessionIdentityRevision: string | null; + policySubjectRevision: string | null; + transition: { + source_event_seq: number | null; + source_session_id: string | null; + source_session_identity_revision: string | null; + subject_revision: string | null; + target_session_identity_revision: string | null; + }; +}): boolean { + const { transition } = params; + if ( + transition.source_event_seq === null || + transition.source_session_id === null || + transition.source_session_identity_revision === null || + transition.subject_revision === null || + transition.target_session_identity_revision === null || + params.policySessionIdentityRevision !== transition.target_session_identity_revision || + params.policySubjectRevision !== transition.subject_revision + ) { + return false; + } + const sourceSnapshot = executeSqliteQueryTakeFirstSync( + params.db, + policyDatabase(params.db) + .selectFrom("session_memory_subject_snapshots") + .select(["session_identity_revision", "subject_revision"]) + .where("session_id", "=", transition.source_session_id) + .limit(1), + ); + if ( + !sourceSnapshot || + sourceSnapshot.session_identity_revision !== transition.source_session_identity_revision || + sourceSnapshot.subject_revision !== transition.subject_revision + ) { + return false; + } + // A transition never grants new authority: the exact source event must still + // be readable under current policy, resource, and source-session checks. + return Boolean( + readAuthorizedTranscriptEventSeqs(params.db, transition.source_session_id)?.has( + transition.source_event_seq, + ), + ); +} + /** Legacy returns undefined; cut-over returns only companion-authorized raw event sequences. */ export function readAuthorizedTranscriptEventSeqs( db: DatabaseSync, @@ -277,6 +946,11 @@ export function readAuthorizedTranscriptEventSeqs( db, policyDatabase(db) .selectFrom("transcript_event_memory_policies as policy") + .innerJoin("transcript_event_memory_policy_details as detail", (join) => + join + .onRef("detail.session_id", "=", "policy.session_id") + .onRef("detail.event_seq", "=", "policy.event_seq"), + ) .innerJoin( "session_memory_subject_snapshots as subject", "subject.session_id", @@ -292,9 +966,28 @@ export function readAuthorizedTranscriptEventSeqs( "policy_set.policy_set_id", "policy.source_policy_set_id", ) - .select("policy.event_seq") + .leftJoin("transcript_event_memory_policy_transitions as transition", (join) => + join + .onRef("transition.session_id", "=", "policy.session_id") + .onRef("transition.event_seq", "=", "policy.event_seq"), + ) + .select([ + "policy.event_seq", + "policy.session_identity_revision as policy_session_identity_revision", + "policy.subject_revision as policy_subject_revision", + "policy.source_policy_set_id", + "policy.run_exposure_set_id", + "detail.source_event_seq", + "detail.source_session_id", + "transition.source_event_seq as transition_source_event_seq", + "transition.source_session_id as transition_source_session_id", + "transition.source_session_identity_revision", + "transition.subject_revision as transition_subject_revision", + "transition.target_session_identity_revision", + ]) .where("policy.session_id", "=", sessionId) .where("policy.authorization_status", "=", "authorized") + .where("detail.retention_state", "=", "retained") .whereRef("subject.session_identity_revision", "=", "policy.session_identity_revision") .whereRef("subject.subject_revision", "=", "policy.subject_revision") .whereRef("exposure.run_id", "=", "policy.run_id") @@ -302,9 +995,45 @@ export function readAuthorizedTranscriptEventSeqs( .whereRef("exposure.revision_number", "=", "policy.run_exposure_revision") .whereRef("exposure.effective_source_policy_set_id", "=", "policy.source_policy_set_id") .whereRef("exposure.delivery_audiences_json", "=", "policy.delivery_audiences_json") + .whereRef("detail.finalized_delivery_audiences_json", "=", "policy.delivery_audiences_json") + .whereRef( + "detail.normalized_audience_intersection_json", + "=", + "policy.delivery_audiences_json", + ) .whereRef("policy_set.policy_set_id", "=", "exposure.effective_source_policy_set_id"), ).rows; - return new Set(rows.map((row) => row.event_seq)); + return new Set( + rows.flatMap((row) => { + if ( + !row.source_policy_set_id || + !row.run_exposure_set_id || + !isCurrentPolicySetAuthorized(db, row.source_policy_set_id) || + !areExposedResourcesCurrent(db, row.run_exposure_set_id) + ) { + return []; + } + const transitioned = row.transition_source_session_id !== null; + const lineageCurrent = transitioned + ? isCurrentTranscriptMemoryTransition({ + db, + policySessionIdentityRevision: row.policy_session_identity_revision, + policySubjectRevision: row.policy_subject_revision, + transition: { + source_event_seq: row.transition_source_event_seq, + source_session_id: row.transition_source_session_id, + source_session_identity_revision: row.source_session_identity_revision, + subject_revision: row.transition_subject_revision, + target_session_identity_revision: row.target_session_identity_revision, + }, + }) + : // Exact same-session replacement retains the original source sequence + // after delete-and-reappend. No transition row means this is either the + // direct event or that already-verified replacement lineage. + row.source_session_id === sessionId; + return lineageCurrent ? [row.event_seq] : []; + }), + ); } catch { return new Set(); } diff --git a/src/config/sessions/session-transcript-policy-archive.test.ts b/src/config/sessions/session-transcript-policy-archive.test.ts new file mode 100644 index 000000000000..343e97a5f5d7 --- /dev/null +++ b/src/config/sessions/session-transcript-policy-archive.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from "vitest"; +import { parseTranscriptPolicyArchive } from "./session-transcript-policy-archive.js"; + +function archive( + params: { agentId?: string; mutate?: (record: Record) => void } = {}, +) { + const record: Record = { + agentId: params.agentId ?? "main", + type: "openclaw.memory-policy-archive-v1", + version: 1, + sessionId: "session-1", + eventSeq: 0, + subject: { + sessionKey: "agent:main:session-1", + sessionIdentityRevision: "identity-1", + subjectRevision: "subject-1", + }, + policy: { + contextFingerprint: "context-1", + deliveryAudiencesJson: '[{"id":"alice","kind":"user"}]', + runExposureRevision: 1, + runExposureSetId: "exposure-1", + runId: "run-1", + sourcePolicySetId: "policy-set-1", + }, + detail: { + actorEvidenceJson: '{"version":1}', + delegationSnapshotJson: '{"kind":"none","version":1}', + egressReceiptIdsJson: '["egress-1"]', + exposedResourceRevisionsJson: '["resource-1"]', + exposureReceiptIdsJson: '["receipt-1"]', + finalizedDeliveryAudiencesJson: '[{"id":"alice","kind":"user"}]', + normalizedAudienceIntersectionJson: '[{"id":"alice","kind":"user"}]', + sourceEventSeq: 0, + sourceSessionId: "session-1", + }, + }; + params.mutate?.(record); + return `${JSON.stringify({ id: "event-1", type: "message" })}\n${JSON.stringify(record)}\n`; +} + +describe("transcript policy archive", () => { + it("accepts canonical event and companion pairs", () => { + expect(parseTranscriptPolicyArchive(archive())).toMatchObject({ + agentId: "main", + sessionId: "session-1", + sessionKey: "agent:main:session-1", + events: [{ eventSeq: 0, eventJson: '{"id":"event-1","type":"message"}' }], + }); + }); + + it.each([ + ["legacy raw JSONL", '{"id":"event-1","type":"message"}\n'], + [ + "missing immutable subject key", + archive({ + mutate: (record) => delete (record.subject as Record).sessionKey, + }), + ], + ["cross-owner companion", `${archive()}${archive({ agentId: "other" })}`], + ])("rejects %s", (_name, content) => { + expect(parseTranscriptPolicyArchive(content)).toBeUndefined(); + }); +}); diff --git a/src/config/sessions/session-transcript-policy-archive.ts b/src/config/sessions/session-transcript-policy-archive.ts new file mode 100644 index 000000000000..db95521e396b --- /dev/null +++ b/src/config/sessions/session-transcript-policy-archive.ts @@ -0,0 +1,383 @@ +// Versioned transcript-policy archive decoding for explicit confirmed-import workflows. +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "../../infra/kysely-sync.js"; +import type { DB as OpenClawAgentDatabaseSchema } from "../../state/openclaw-agent-db.generated.js"; +import type { OpenClawAgentDatabase } from "../../state/openclaw-agent-db.js"; +import { readAuthorizedTranscriptEventSeqs } from "./session-transcript-memory-policy.js"; + +const RECORD_TYPE = "openclaw.memory-policy-archive-v1"; + +type TranscriptPolicyArchiveDatabase = Pick< + OpenClawAgentDatabaseSchema, + | "session_memory_subject_snapshots" + | "transcript_events" + | "transcript_event_memory_policies" + | "transcript_event_memory_policy_details" + | "transcript_event_memory_policy_transitions" +>; + +export type TranscriptPolicyArchiveEvent = Readonly<{ + event: unknown; + eventJson: string; + eventSeq: number; + metadata: TranscriptPolicyArchiveMetadata; +}>; + +export type TranscriptPolicyArchiveMetadata = Readonly<{ + agentId: string; + eventSeq: number; + sessionId: string; + subject: Readonly<{ + sessionIdentityRevision: string; + sessionKey: string; + subjectRevision: string; + }>; + policy: Readonly<{ + contextFingerprint: string; + deliveryAudiencesJson: string; + runExposureRevision: number; + runExposureSetId: string; + runId: string; + sourcePolicySetId: string; + }>; + detail: Readonly<{ + actorEvidenceJson: string; + delegationSnapshotJson: string; + egressReceiptIdsJson: string; + exposedResourceRevisionsJson: string; + exposureReceiptIdsJson: string; + finalizedDeliveryAudiencesJson: string; + normalizedAudienceIntersectionJson: string; + sourceEventSeq: number; + sourceSessionId: string; + }>; + transition?: Readonly<{ + kind: "parent-fork" | "fork" | "rewind" | "switch" | "checkpoint"; + sourceEventSeq: number; + sourceSessionId: string; + sourceSessionIdentityRevision: string; + subjectRevision: string; + targetSessionIdentityRevision: string; + }>; +}>; + +export type TranscriptPolicyArchive = Readonly<{ + agentId: string; + events: readonly TranscriptPolicyArchiveEvent[]; + sessionId: string; + sessionIdentityRevision: string; + sessionKey: string; + subjectRevision: string; +}>; + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function text(value: unknown): string | undefined { + return typeof value === "string" && value.trim() ? value : undefined; +} + +function integer(value: unknown): number | undefined { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : undefined; +} + +function parseJson(line: string): unknown | undefined { + try { + const parsed = JSON.parse(line) as unknown; + return JSON.stringify(parsed) === line ? parsed : undefined; + } catch { + return undefined; + } +} + +function parseMetadata(value: unknown): TranscriptPolicyArchiveMetadata | undefined { + if (!isRecord(value) || value.type !== RECORD_TYPE || value.version !== 1) { + return undefined; + } + const subject = isRecord(value.subject) ? value.subject : undefined; + const policy = isRecord(value.policy) ? value.policy : undefined; + const detail = isRecord(value.detail) ? value.detail : undefined; + const agentId = text(value.agentId); + const sessionId = text(value.sessionId); + const eventSeq = integer(value.eventSeq); + const sessionKey = text(subject?.sessionKey); + const sessionIdentityRevision = text(subject?.sessionIdentityRevision); + const subjectRevision = text(subject?.subjectRevision); + const contextFingerprint = text(policy?.contextFingerprint); + const deliveryAudiencesJson = text(policy?.deliveryAudiencesJson); + const runExposureRevision = integer(policy?.runExposureRevision); + const runExposureSetId = text(policy?.runExposureSetId); + const runId = text(policy?.runId); + const sourcePolicySetId = text(policy?.sourcePolicySetId); + const actorEvidenceJson = text(detail?.actorEvidenceJson); + const delegationSnapshotJson = text(detail?.delegationSnapshotJson); + const egressReceiptIdsJson = text(detail?.egressReceiptIdsJson); + const exposedResourceRevisionsJson = text(detail?.exposedResourceRevisionsJson); + const exposureReceiptIdsJson = text(detail?.exposureReceiptIdsJson); + const finalizedDeliveryAudiencesJson = text(detail?.finalizedDeliveryAudiencesJson); + const normalizedAudienceIntersectionJson = text(detail?.normalizedAudienceIntersectionJson); + const sourceEventSeq = integer(detail?.sourceEventSeq); + const sourceSessionId = text(detail?.sourceSessionId); + if ( + !agentId || + !sessionId || + eventSeq === undefined || + !sessionKey || + !sessionIdentityRevision || + !subjectRevision || + !contextFingerprint || + !deliveryAudiencesJson || + runExposureRevision === undefined || + !runExposureSetId || + !runId || + !sourcePolicySetId || + !actorEvidenceJson || + !delegationSnapshotJson || + !egressReceiptIdsJson || + !exposedResourceRevisionsJson || + !exposureReceiptIdsJson || + !finalizedDeliveryAudiencesJson || + !normalizedAudienceIntersectionJson || + sourceEventSeq === undefined || + !sourceSessionId + ) { + return undefined; + } + const transitionRecord = + value.transition === undefined + ? undefined + : isRecord(value.transition) + ? value.transition + : null; + if (transitionRecord === null) { + return undefined; + } + const transition = transitionRecord + ? { + kind: transitionRecord.kind, + sourceEventSeq: integer(transitionRecord.sourceEventSeq), + sourceSessionId: text(transitionRecord.sourceSessionId), + sourceSessionIdentityRevision: text(transitionRecord.sourceSessionIdentityRevision), + subjectRevision: text(transitionRecord.subjectRevision), + targetSessionIdentityRevision: text(transitionRecord.targetSessionIdentityRevision), + } + : undefined; + if ( + transition && + (!["parent-fork", "fork", "rewind", "switch", "checkpoint"].includes(String(transition.kind)) || + transition.sourceEventSeq === undefined || + !transition.sourceSessionId || + !transition.sourceSessionIdentityRevision || + !transition.subjectRevision || + !transition.targetSessionIdentityRevision) + ) { + return undefined; + } + return Object.freeze({ + agentId, + eventSeq, + sessionId, + subject: Object.freeze({ sessionIdentityRevision, sessionKey, subjectRevision }), + policy: Object.freeze({ + contextFingerprint, + deliveryAudiencesJson, + runExposureRevision, + runExposureSetId, + runId, + sourcePolicySetId, + }), + detail: Object.freeze({ + actorEvidenceJson, + delegationSnapshotJson, + egressReceiptIdsJson, + exposedResourceRevisionsJson, + exposureReceiptIdsJson, + finalizedDeliveryAudiencesJson, + normalizedAudienceIntersectionJson, + sourceEventSeq, + sourceSessionId, + }), + ...(transition + ? { + transition: Object.freeze( + transition as NonNullable, + ), + } + : {}), + }); +} + +/** Parse only canonical event/companion pairs; legacy raw JSONL is never importable here. */ +export function parseTranscriptPolicyArchive(content: string): TranscriptPolicyArchive | undefined { + if (!content.endsWith("\n")) { + return undefined; + } + const lines = content.slice(0, -1).split("\n"); + if (lines.length === 0 || lines.length % 2 !== 0) { + return undefined; + } + const events: TranscriptPolicyArchiveEvent[] = []; + for (let index = 0; index < lines.length; index += 2) { + const eventJson = lines[index]; + const metadataJson = lines[index + 1]; + if (eventJson === undefined || metadataJson === undefined) { + return undefined; + } + const event = parseJson(eventJson); + const metadata = parseMetadata(parseJson(metadataJson)); + if (event === undefined || !metadata || metadata.eventSeq !== events.length) { + return undefined; + } + events.push(Object.freeze({ event, eventJson, eventSeq: metadata.eventSeq, metadata })); + } + const first = events[0]?.metadata; + if ( + !first || + events.some( + ({ metadata, eventSeq }) => + metadata.agentId !== first.agentId || + metadata.sessionId !== first.sessionId || + metadata.subject.sessionKey !== first.subject.sessionKey || + metadata.subject.sessionIdentityRevision !== first.subject.sessionIdentityRevision || + metadata.subject.subjectRevision !== first.subject.subjectRevision || + metadata.eventSeq !== eventSeq, + ) + ) { + return undefined; + } + return Object.freeze({ + agentId: first.agentId, + events: Object.freeze(events), + sessionId: first.sessionId, + sessionIdentityRevision: first.subject.sessionIdentityRevision, + sessionKey: first.subject.sessionKey, + subjectRevision: first.subject.subjectRevision, + }); +} + +/** + * Restores only already-appended archive events. The caller owns entry creation + * and event insertion; this helper proves provenance before making companions + * readable, then reruns the canonical current-policy reader in the same txn. + */ +export function restoreConfirmedTranscriptPolicyArchiveInTransaction(params: { + archive: TranscriptPolicyArchive; + database: OpenClawAgentDatabase; + sessionId: string; + sessionKey: string; +}): void { + const { archive, database, sessionId, sessionKey } = params; + if ( + archive.agentId !== database.agentId || + archive.sessionId !== sessionId || + archive.sessionKey !== sessionKey + ) { + throw new Error("confirmed transcript archive owner mismatch"); + } + const db = getNodeSqliteKysely(database.db); + const subject = executeSqliteQueryTakeFirstSync( + database.db, + db + .selectFrom("session_memory_subject_snapshots") + .select(["session_identity_revision", "session_key", "subject_revision"]) + .where("session_id", "=", sessionId) + .limit(1), + ); + if ( + !subject || + subject.session_key !== archive.sessionKey || + subject.session_identity_revision !== archive.sessionIdentityRevision || + subject.subject_revision !== archive.subjectRevision + ) { + throw new Error("confirmed transcript archive subject mismatch"); + } + const rows = executeSqliteQuerySync( + database.db, + db + .selectFrom("transcript_events") + .select(["event_json", "seq"]) + .where("session_id", "=", sessionId) + .orderBy("seq", "asc"), + ).rows; + if ( + rows.length !== archive.events.length || + rows.some( + (row, index) => + row.seq !== archive.events[index]?.eventSeq || + row.event_json !== archive.events[index]?.eventJson, + ) + ) { + throw new Error("confirmed transcript archive event mismatch"); + } + for (const archived of archive.events) { + const { detail, policy, transition } = archived.metadata; + const updated = executeSqliteQuerySync( + database.db, + db + .updateTable("transcript_event_memory_policies") + .set({ + authorization_status: "authorized", + context_fingerprint: policy.contextFingerprint, + delivery_audiences_json: policy.deliveryAudiencesJson, + run_exposure_revision: policy.runExposureRevision, + run_exposure_set_id: policy.runExposureSetId, + run_id: policy.runId, + session_identity_revision: archive.sessionIdentityRevision, + source_policy_set_id: policy.sourcePolicySetId, + subject_revision: archive.subjectRevision, + }) + .where("session_id", "=", sessionId) + .where("event_seq", "=", archived.eventSeq) + .where("authorization_status", "=", "pending"), + ); + if (updated.numAffectedRows !== 1n) { + throw new Error("confirmed transcript archive companion conflict"); + } + executeSqliteQuerySync( + database.db, + db.insertInto("transcript_event_memory_policy_details").values({ + session_id: sessionId, + event_seq: archived.eventSeq, + actor_evidence_json: detail.actorEvidenceJson, + delegation_snapshot_json: detail.delegationSnapshotJson, + egress_receipt_ids_json: detail.egressReceiptIdsJson, + exposed_resource_revisions_json: detail.exposedResourceRevisionsJson, + exposure_receipt_ids_json: detail.exposureReceiptIdsJson, + finalized_delivery_audiences_json: detail.finalizedDeliveryAudiencesJson, + normalized_audience_intersection_json: detail.normalizedAudienceIntersectionJson, + retention_state: "retained", + source_event_seq: detail.sourceEventSeq, + source_session_id: detail.sourceSessionId, + created_at: Date.now(), + }), + ); + if (transition) { + executeSqliteQuerySync( + database.db, + db.insertInto("transcript_event_memory_policy_transitions").values({ + session_id: sessionId, + event_seq: archived.eventSeq, + source_event_seq: transition.sourceEventSeq, + source_session_id: transition.sourceSessionId, + transition_kind: transition.kind, + source_session_identity_revision: transition.sourceSessionIdentityRevision, + subject_revision: transition.subjectRevision, + target_session_identity_revision: transition.targetSessionIdentityRevision, + created_at: Date.now(), + }), + ); + } + } + const readable = readAuthorizedTranscriptEventSeqs(database.db, sessionId); + if ( + !readable || + readable.size !== archive.events.length || + archive.events.some((archived) => !readable.has(archived.eventSeq)) + ) { + throw new Error("confirmed transcript archive policy is no longer authorized"); + } +} diff --git a/src/config/sessions/session-transcript-search.ts b/src/config/sessions/session-transcript-search.ts index 3bc43c5f05dc..cb89b9b069b6 100644 --- a/src/config/sessions/session-transcript-search.ts +++ b/src/config/sessions/session-transcript-search.ts @@ -6,7 +6,10 @@ import { openOpenClawAgentDatabase } from "../../state/openclaw-agent-db.js"; import { truncateUtf16Safe } from "../../utils.js"; import { resolveSqliteTargetFromSessionStorePath } from "./session-sqlite-target.js"; import { listSessionsNeedingTranscriptIndexReconcile } from "./session-transcript-index.js"; -import { isTranscriptMemoryPolicyEnforcedInDatabase } from "./session-transcript-memory-policy.js"; +import { + isTranscriptMemoryPolicyEnforcedInDatabase, + readAuthorizedTranscriptEventSeqs, +} from "./session-transcript-memory-policy.js"; import { isSessionTranscriptIndexReconcileRunning, startSessionTranscriptIndexReconcile, @@ -112,7 +115,7 @@ export function searchSessionTranscripts(params: { // rebuilds them (indexing=true tells the caller to retry). const statement = database.db.prepare(/* sqlite-allow-raw: FTS5 MATCH/snippet/bm25 */ ` SELECT session_windows.session_key AS session_key, session_transcript_fts.session_id AS session_id, - message_id, role, timestamp, + message_id, role, timestamp, identity.seq AS event_seq, snippet(session_transcript_fts, 0, '', '', ' … ', 48) AS snippet, bm25(session_transcript_fts) AS rank FROM session_transcript_fts @@ -129,6 +132,7 @@ export function searchSessionTranscripts(params: { `); const values = [toFtsQuery(query), ...sessionKeys, limit + 1]; const rows = statement.all(...values) as Array<{ + event_seq: unknown; message_id: unknown; rank: unknown; role: unknown; @@ -137,16 +141,28 @@ export function searchSessionTranscripts(params: { snippet: unknown; timestamp: unknown; }>; + const authorizedEventSeqsBySession = new Map | undefined>(); const hits = rows.flatMap((row): SessionTranscriptSearchHit[] => { if ( typeof row.session_key !== "string" || typeof row.session_id !== "string" || typeof row.message_id !== "string" || + typeof row.event_seq !== "number" || (row.role !== "user" && row.role !== "assistant") || typeof row.snippet !== "string" ) { return []; } + if (isTranscriptMemoryPolicyEnforcedInDatabase(database.db)) { + let authorized = authorizedEventSeqsBySession.get(row.session_id); + if (authorized === undefined && !authorizedEventSeqsBySession.has(row.session_id)) { + authorized = readAuthorizedTranscriptEventSeqs(database.db, row.session_id); + authorizedEventSeqsBySession.set(row.session_id, authorized); + } + if (!authorized?.has(row.event_seq)) { + return []; + } + } const timestamp = typeof row.timestamp === "number" ? row.timestamp : Number(row.timestamp); const rank = typeof row.rank === "number" ? row.rank : Number(row.rank); return [ diff --git a/src/plugins/memory-invocation.ts b/src/plugins/memory-invocation.ts index b6b2798489df..5555dbd83136 100644 --- a/src/plugins/memory-invocation.ts +++ b/src/plugins/memory-invocation.ts @@ -29,7 +29,11 @@ import { hydrateMemoryRunExposureFromLedger, persistMemoryRunExposureBeforeContent, } from "./memory-run-exposure-ledger.js"; -import { prepareMemoryRunExposure, publishMemoryRunExposure } from "./memory-run-exposure.js"; +import { + captureDurableMemoryAuthorizationFacts, + prepareMemoryRunExposure, + publishMemoryRunExposure, +} from "./memory-run-exposure.js"; import { resolveSelectedMemoryCapabilityRegistration } from "./memory-state.js"; import type { MemoryPluginCapability, @@ -284,6 +288,7 @@ function readTranscriptExposure(params: { egressRegistryRevision: context.delivery.egressRegistryRevision, sessionIdentityRevision: context.sessionIdentityRevision, subjectRevision: context.subjectRevision, + ...captureDurableMemoryAuthorizationFacts(context), }); } diff --git a/src/plugins/memory-run-exposure-ledger.test.ts b/src/plugins/memory-run-exposure-ledger.test.ts index e8f5e134240b..5cd2b50a8939 100644 --- a/src/plugins/memory-run-exposure-ledger.test.ts +++ b/src/plugins/memory-run-exposure-ledger.test.ts @@ -1,5 +1,6 @@ import { DatabaseSync } from "node:sqlite"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { ensureMemoryPreoutputExposureLedgerSchemaInTransaction } from "../state/openclaw-agent-db-schema-helpers.js"; const mocks = vi.hoisted(() => ({ database: undefined as @@ -39,6 +40,7 @@ let database: DatabaseSync | undefined; beforeEach(() => { database = new DatabaseSync(":memory:"); + ensureMemoryPreoutputExposureLedgerSchemaInTransaction(database); mocks.database = { agentId: "main", db: database, @@ -72,6 +74,16 @@ function prepare(sessionId: string) { egressRegistryRevision: "egress-1", sessionIdentityRevision: "identity-1", subjectRevision: "subject-1", + actorEvidence: { + version: 1, + kind: "principal", + actorKind: "human", + principalId: "alice", + assurance: "gateway-profile", + evidenceRevision: "actor-revision-1", + }, + delegationSnapshot: { version: 1, kind: "none" }, + hostFactsRevision: "host-facts-1", }); } @@ -137,6 +149,116 @@ describe("memory pre-output exposure ledger", () => { ]); }); + it("persists token-free delegated facts and rehydrates them after restart", () => { + const base = prepare("session-a"); + const snapshot = { + ...base, + delegationSnapshot: { + version: 1, + kind: "delegated", + rootPrincipalId: "alice", + rootContextId: "root-context", + parentContextId: "parent-context", + parentMemoryPlanId: "parent-plan", + capabilitySnapshotId: "capability-snapshot", + allowedOperations: ["derive", "read"], + maximumAudiences: [ + { kind: "user", id: "alice" }, + { kind: "role", id: "writer" }, + ], + depth: 1, + }, + } as typeof base; + expect(persistMemoryRunExposureBeforeContent(snapshot)).toBe(true); + const canonicalDelegation = { + ...snapshot.delegationSnapshot, + maximumAudiences: [ + { kind: "role", id: "writer" }, + { kind: "user", id: "alice" }, + ], + }; + + const persisted = database + ?.prepare( + `SELECT actor_evidence_json, delegation_snapshot_json, host_facts_revision + FROM memory_preoutput_exposure_authorization_facts + WHERE exposure_set_id = ?`, + ) + .get(snapshot.exposureSetId) as { + actor_evidence_json: string; + delegation_snapshot_json: string; + host_facts_revision: string; + }; + expect(JSON.parse(persisted.actor_evidence_json)).toEqual(snapshot.actorEvidence); + expect(JSON.parse(persisted.delegation_snapshot_json)).toEqual(canonicalDelegation); + expect(persisted.host_facts_revision).toBe("host-facts-1"); + expect(JSON.stringify(persisted)).not.toContain("storeCapToken"); + + clearMemoryRunExposureForTest(); + expect( + readDurableMemoryRunExposure({ + database: mocks.database as never, + sessionId: "session-a", + runId: "shared-run-id", + }), + ).toMatchObject({ + delegationSnapshot: canonicalDelegation, + actorEvidence: snapshot.actorEvidence, + hostFactsRevision: "host-facts-1", + }); + }); + + it("fails closed rather than accepting missing, token-bearing, or partial authorization facts", () => { + const base = prepare("session-a"); + const tokenBearing = { + ...base, + delegationSnapshot: { + version: 1, + kind: "none", + storeCapToken: "must-never-persist", + } as never, + }; + expect(persistMemoryRunExposureBeforeContent(tokenBearing)).toBe(false); + expect( + database?.prepare("SELECT count(*) AS count FROM memory_preoutput_exposure_ledger").get(), + ).toEqual({ count: 0 }); + + const snapshot = prepare("session-a"); + expect(persistMemoryRunExposureBeforeContent(snapshot)).toBe(true); + database?.exec(/* sqlite-allow-raw: test corrupts immutable proof to assert fail-closed read. */ ` + DROP TRIGGER memory_preoutput_exposure_authorization_facts_no_delete; + DELETE FROM memory_preoutput_exposure_authorization_facts + WHERE exposure_set_id = '${snapshot.exposureSetId}'; + `); + expect( + readLatestDurableMemoryRunExposure({ + agentId: "main", + sessionId: "session-a", + runId: "shared-run-id", + }), + ).toEqual({ kind: "unavailable" }); + }); + + it("rolls back both ledger rows when durable authorization-fact persistence fails", () => { + database?.exec(/* sqlite-allow-raw: test-only atomicity fault injection. */ ` + CREATE TRIGGER reject_exposure_authorization_facts_for_test + BEFORE INSERT ON memory_preoutput_exposure_authorization_facts + BEGIN + SELECT RAISE(ABORT, 'test authorization facts failure'); + END; + `); + + expect(persistMemoryRunExposureBeforeContent(prepare("session-a"))).toBe(false); + for (const table of [ + "memory_preoutput_exposure_ledger", + "memory_preoutput_exposure_authorization_facts", + ]) { + expect(database?.prepare(`SELECT count(*) AS count FROM ${table}`).get()).toEqual({ + count: 0, + }); + } + }); + it("fails closed on a duplicate revision without adding a partial row", () => { const snapshot = prepare("session-a"); diff --git a/src/plugins/memory-run-exposure-ledger.ts b/src/plugins/memory-run-exposure-ledger.ts index 908d68ec3c15..bc3c4679d11c 100644 --- a/src/plugins/memory-run-exposure-ledger.ts +++ b/src/plugins/memory-run-exposure-ledger.ts @@ -1,4 +1,8 @@ -import { executeSqliteQuerySync, getNodeSqliteKysely } from "../infra/kysely-sync.js"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "../infra/kysely-sync.js"; import { runSqliteImmediateTransactionSync } from "../infra/sqlite-transaction.js"; import { logWarn } from "../logger.js"; import type { AudienceRef } from "../memory-host-sdk/host/authorization.js"; @@ -10,6 +14,8 @@ import { import { createMemoryRunExposureScopeId, reconcileMemoryRunExposureWithDurableLedger, + type DurableMemoryActorEvidence, + type DurableMemoryDelegationSnapshot, type MemoryRunExposureSnapshot, } from "./memory-run-exposure.js"; @@ -36,6 +42,13 @@ type MemoryPreoutputExposureLedgerDatabase = { subject_revision: string; created_at: number; }; + memory_preoutput_exposure_authorization_facts: { + exposure_set_id: string; + actor_evidence_json: string; + delegation_snapshot_json: string; + host_facts_revision: string; + created_at: number; + }; }; type MemoryExposureLedgerDiagnostic = "hydrate-failed" | "persist-failed"; @@ -136,6 +149,160 @@ function parseCanonicalAudiences(value: string): readonly AudienceRef[] | undefi } } +function hasExactKeys(value: object, expected: readonly string[]): boolean { + const actual = Object.keys(value).toSorted(); + return actual.length === expected.length && actual.every((key, index) => key === expected[index]); +} + +function canonicalActorEvidence(value: DurableMemoryActorEvidence): string | undefined { + if (value.kind === "principal") { + if ( + !hasExactKeys( + value, + value.expiresAt === undefined + ? ["actorKind", "assurance", "evidenceRevision", "kind", "principalId", "version"] + : [ + "actorKind", + "assurance", + "evidenceRevision", + "expiresAt", + "kind", + "principalId", + "version", + ], + ) + ) { + return undefined; + } + if ( + !["human", "agent", "service", "system"].includes(value.actorKind) || + !["gateway-profile", "adapter-attested", "oidc", "service"].includes(value.assurance) || + !value.principalId.trim() || + !value.evidenceRevision.trim() + ) { + return undefined; + } + if ( + value.expiresAt !== undefined && + (!Number.isFinite(Date.parse(value.expiresAt)) || + new Date(Date.parse(value.expiresAt)).toISOString() !== value.expiresAt) + ) { + return undefined; + } + return JSON.stringify({ + version: 1, + kind: "principal", + actorKind: value.actorKind, + principalId: value.principalId, + assurance: value.assurance, + evidenceRevision: value.evidenceRevision, + ...(value.expiresAt ? { expiresAt: value.expiresAt } : {}), + }); + } + if (!hasExactKeys(value, ["evidenceRevision", "kind", "transportAuditRef", "version"])) { + return undefined; + } + return value.transportAuditRef.trim() && value.evidenceRevision.trim() + ? JSON.stringify({ + version: 1, + kind: "unattributed", + transportAuditRef: value.transportAuditRef, + evidenceRevision: value.evidenceRevision, + }) + : undefined; +} + +function canonicalDelegationSnapshot(value: DurableMemoryDelegationSnapshot): string | undefined { + if (value.kind === "none") { + return hasExactKeys(value, ["kind", "version"]) + ? JSON.stringify({ version: 1, kind: "none" }) + : undefined; + } + if ( + !hasExactKeys(value, [ + "allowedOperations", + "capabilitySnapshotId", + "depth", + "kind", + "maximumAudiences", + "parentContextId", + "parentMemoryPlanId", + "rootContextId", + "rootPrincipalId", + "version", + ]) + ) { + return undefined; + } + const allowedOperations = [...new Set(value.allowedOperations)].toSorted(); + const maximumAudiences = canonicalAudiences({ deliveryAudiences: value.maximumAudiences }); + if ( + allowedOperations.length !== value.allowedOperations.length || + allowedOperations.some( + (operation) => + ![ + "read", + "append", + "replace", + "derive", + "deposit", + "project", + "publish", + "import", + "export", + "delete", + "sync", + "status", + "policy-admin", + ].includes(operation), + ) || + !maximumAudiences || + !value.rootPrincipalId.trim() || + !value.rootContextId.trim() || + !value.parentContextId.trim() || + !value.parentMemoryPlanId.trim() || + !value.capabilitySnapshotId.trim() || + !Number.isSafeInteger(value.depth) || + value.depth < 0 + ) { + return undefined; + } + return JSON.stringify({ + version: 1, + kind: "delegated", + rootPrincipalId: value.rootPrincipalId, + rootContextId: value.rootContextId, + parentContextId: value.parentContextId, + parentMemoryPlanId: value.parentMemoryPlanId, + capabilitySnapshotId: value.capabilitySnapshotId, + allowedOperations, + maximumAudiences: JSON.parse(maximumAudiences), + depth: value.depth, + }); +} + +function parseCanonicalActorEvidence(value: string): DurableMemoryActorEvidence | undefined { + try { + const parsed = JSON.parse(value) as DurableMemoryActorEvidence; + const canonical = canonicalActorEvidence(parsed); + return canonical === value ? Object.freeze(parsed) : undefined; + } catch { + return undefined; + } +} + +function parseCanonicalDelegationSnapshot( + value: string, +): DurableMemoryDelegationSnapshot | undefined { + try { + const parsed = JSON.parse(value) as DurableMemoryDelegationSnapshot; + const canonical = canonicalDelegationSnapshot(parsed); + return canonical === value ? Object.freeze(parsed) : undefined; + } catch { + return undefined; + } +} + function isDurableSnapshot(snapshot: MemoryRunExposureSnapshot): boolean { return Boolean( snapshot.agentId.trim() && @@ -149,6 +316,7 @@ function isDurableSnapshot(snapshot: MemoryRunExposureSnapshot): boolean { snapshot.egressRegistryRevision.trim() && snapshot.sessionIdentityRevision.trim() && snapshot.subjectRevision.trim() && + snapshot.hostFactsRevision.trim() && snapshot.revisionNumber > 0 && snapshot.revisionNumber === (snapshot.previous?.revisionNumber ?? 0) + 1 && snapshot.durableRunScopeId === createMemoryRunExposureScopeId(snapshot) && @@ -167,6 +335,8 @@ function persistMemoryRunExposureInTransaction(params: { exposureReceiptIdsJson: string; egressReceiptIdsJson: string; deliveryAudiencesJson: string; + actorEvidenceJson: string; + delegationSnapshotJson: string; }): void { const { database, snapshot } = params; ensureMemoryPreoutputExposureLedgerSchemaInTransaction(database.db); @@ -204,6 +374,22 @@ function persistMemoryRunExposureInTransaction(params: { if (inserted.numAffectedRows !== 1n) { throw new Error("memory exposure revision already has a durable ledger row"); } + const factsInserted = executeSqliteQuerySync( + database.db, + db + .insertInto("memory_preoutput_exposure_authorization_facts") + .values({ + exposure_set_id: snapshot.exposureSetId, + actor_evidence_json: params.actorEvidenceJson, + delegation_snapshot_json: params.delegationSnapshotJson, + host_facts_revision: snapshot.hostFactsRevision, + created_at: snapshot.createdAt, + }) + .onConflict((conflict) => conflict.column("exposure_set_id").doNothing()), + ); + if (factsInserted.numAffectedRows !== 1n) { + throw new Error("memory exposure revision already has durable authorization facts"); + } } /** @@ -218,13 +404,17 @@ export function persistMemoryRunExposureBeforeContent( const exposureReceiptIdsJson = canonicalStrings(snapshot.exposureReceiptIds); const egressReceiptIdsJson = canonicalStrings(snapshot.egressReceiptIds); const deliveryAudiencesJson = canonicalAudiences(snapshot); + const actorEvidenceJson = canonicalActorEvidence(snapshot.actorEvidence); + const delegationSnapshotJson = canonicalDelegationSnapshot(snapshot.delegationSnapshot); if ( !isDurableSnapshot(snapshot) || !sourcePolicySetIdsJson || !exposedResourceRevisionsJson || !exposureReceiptIdsJson || !egressReceiptIdsJson || - !deliveryAudiencesJson + !deliveryAudiencesJson || + !actorEvidenceJson || + !delegationSnapshotJson ) { return false; } @@ -250,6 +440,8 @@ export function persistMemoryRunExposureBeforeContentInDatabase(params: { const exposureReceiptIdsJson = canonicalStrings(snapshot.exposureReceiptIds); const egressReceiptIdsJson = canonicalStrings(snapshot.egressReceiptIds); const deliveryAudiencesJson = canonicalAudiences(snapshot); + const actorEvidenceJson = canonicalActorEvidence(snapshot.actorEvidence); + const delegationSnapshotJson = canonicalDelegationSnapshot(snapshot.delegationSnapshot); if ( database.agentId !== snapshot.agentId || !isDurableSnapshot(snapshot) || @@ -257,7 +449,9 @@ export function persistMemoryRunExposureBeforeContentInDatabase(params: { !exposedResourceRevisionsJson || !exposureReceiptIdsJson || !egressReceiptIdsJson || - !deliveryAudiencesJson + !deliveryAudiencesJson || + !actorEvidenceJson || + !delegationSnapshotJson ) { return false; } @@ -271,6 +465,8 @@ export function persistMemoryRunExposureBeforeContentInDatabase(params: { exposureReceiptIdsJson, egressReceiptIdsJson, deliveryAudiencesJson, + actorEvidenceJson, + delegationSnapshotJson, }); }); return true; @@ -345,12 +541,32 @@ function readDurableMemoryRunExposureOrThrow(params: { const exposureReceiptIds = parseCanonicalStrings(row.exposure_receipt_ids_json); const egressReceiptIds = parseCanonicalStrings(row.egress_receipt_ids_json); const deliveryAudiences = parseCanonicalAudiences(row.delivery_audiences_json); + const facts = executeSqliteQueryTakeFirstSync( + params.database.db, + db + .selectFrom("memory_preoutput_exposure_authorization_facts") + .select([ + "actor_evidence_json", + "delegation_snapshot_json", + "host_facts_revision", + "created_at", + ]) + .where("exposure_set_id", "=", row.exposure_set_id) + .limit(1), + ); + const actorEvidence = facts && parseCanonicalActorEvidence(facts.actor_evidence_json); + const delegationSnapshot = + facts && parseCanonicalDelegationSnapshot(facts.delegation_snapshot_json); if ( !sourcePolicySetIds || !exposedResourceRevisions || !exposureReceiptIds || !egressReceiptIds || !deliveryAudiences || + !actorEvidence || + !delegationSnapshot || + !facts?.host_facts_revision.trim() || + facts.created_at !== row.created_at || !row.session_key.trim() || !row.context_fingerprint.trim() || !row.plan_id.trim() || @@ -389,6 +605,9 @@ function readDurableMemoryRunExposureOrThrow(params: { egressRegistryRevision: row.egress_registry_revision, sessionIdentityRevision: row.session_identity_revision, subjectRevision: row.subject_revision, + actorEvidence, + delegationSnapshot, + hostFactsRevision: facts.host_facts_revision, createdAt: row.created_at, }) satisfies MemoryRunExposureSnapshot; } diff --git a/src/plugins/memory-run-exposure.ts b/src/plugins/memory-run-exposure.ts index 1105688d796c..18a54cfe7ba8 100644 --- a/src/plugins/memory-run-exposure.ts +++ b/src/plugins/memory-run-exposure.ts @@ -1,5 +1,48 @@ import { createHash, randomUUID } from "node:crypto"; -import type { AudienceRef } from "../memory-host-sdk/host/authorization.js"; +import { + MEMORY_OPERATIONS, + type AudienceRef, + type MemoryAccessContext, + type MemoryActorEvidence, + type MemoryOperation, +} from "../memory-host-sdk/host/authorization.js"; + +/** Immutable, serializable actor facts captured from the trusted access context. */ +export type DurableMemoryActorEvidence = + | Readonly<{ + version: 1; + kind: "principal"; + actorKind: "human" | "agent" | "service" | "system"; + principalId: string; + assurance: "gateway-profile" | "adapter-attested" | "oidc" | "service"; + evidenceRevision: string; + expiresAt?: string; + }> + | Readonly<{ + version: 1; + kind: "unattributed"; + transportAuditRef: string; + evidenceRevision: string; + }>; + +/** + * Delegation facts needed for an audit trail. The bearer token is intentionally + * absent: durable transcript lineage proves authority without becoming authority. + */ +export type DurableMemoryDelegationSnapshot = + | Readonly<{ version: 1; kind: "none" }> + | Readonly<{ + version: 1; + kind: "delegated"; + rootPrincipalId: string; + rootContextId: string; + parentContextId: string; + parentMemoryPlanId: string; + capabilitySnapshotId: string; + allowedOperations: readonly MemoryOperation[]; + maximumAudiences: readonly AudienceRef[]; + depth: number; + }>; export type MemoryRunExposureSnapshot = Readonly<{ exposureSetId: string; @@ -22,6 +65,9 @@ export type MemoryRunExposureSnapshot = Readonly<{ egressRegistryRevision: string; sessionIdentityRevision: string; subjectRevision: string; + actorEvidence: DurableMemoryActorEvidence; + delegationSnapshot: DurableMemoryDelegationSnapshot; + hostFactsRevision: string; createdAt: number; }>; @@ -66,6 +112,95 @@ function sortedAudiences(audiences: readonly AudienceRef[]): readonly AudienceRe ); } +function requireText(value: string, label: string): string { + if (!value.trim()) { + throw new TypeError(`${label} must be non-empty`); + } + return value; +} + +function canonicalIsoDate(value: string | undefined): string | undefined { + if (value === undefined) { + return undefined; + } + const time = Date.parse(value); + if (!Number.isFinite(time) || new Date(time).toISOString() !== value) { + throw new TypeError("actor.expiresAt must be a canonical ISO date"); + } + return value; +} + +function captureActorEvidence(actor: MemoryActorEvidence): DurableMemoryActorEvidence { + if (actor.kind === "principal") { + const expiresAt = canonicalIsoDate(actor.expiresAt); + return Object.freeze({ + version: 1, + kind: "principal", + actorKind: actor.actorKind, + principalId: requireText(actor.principalId, "actor.principalId"), + assurance: actor.assurance, + evidenceRevision: requireText(actor.evidenceRevision, "actor.evidenceRevision"), + ...(expiresAt ? { expiresAt } : {}), + }); + } + return Object.freeze({ + version: 1, + kind: "unattributed", + transportAuditRef: requireText(actor.transportAuditRef, "actor.transportAuditRef"), + evidenceRevision: requireText(actor.evidenceRevision, "actor.evidenceRevision"), + }); +} + +function captureDelegation( + delegation: MemoryAccessContext["delegation"], +): DurableMemoryDelegationSnapshot { + if (!delegation) { + return Object.freeze({ version: 1, kind: "none" }); + } + const allowedOperations = Object.freeze([...new Set(delegation.allowedOperations)].toSorted()); + if ( + allowedOperations.length !== delegation.allowedOperations.length || + allowedOperations.some((operation) => !MEMORY_OPERATIONS.includes(operation)) + ) { + throw new TypeError("delegation.allowedOperations must be canonical"); + } + const maximumAudiences = sortedAudiences(delegation.maximumAudiences); + if (maximumAudiences.length !== delegation.maximumAudiences.length) { + throw new TypeError("delegation.maximumAudiences must be canonical"); + } + if (!Number.isSafeInteger(delegation.depth) || delegation.depth < 0) { + throw new TypeError("delegation.depth must be a nonnegative integer"); + } + return Object.freeze({ + version: 1, + kind: "delegated", + rootPrincipalId: requireText(delegation.rootPrincipalId, "delegation.rootPrincipalId"), + rootContextId: requireText(delegation.rootContextId, "delegation.rootContextId"), + parentContextId: requireText(delegation.parentContextId, "delegation.parentContextId"), + parentMemoryPlanId: requireText(delegation.parentMemoryPlanId, "delegation.parentMemoryPlanId"), + capabilitySnapshotId: requireText( + delegation.capabilitySnapshotId, + "delegation.capabilitySnapshotId", + ), + allowedOperations, + maximumAudiences, + depth: delegation.depth, + }); +} + +/** Captures only the durable audit subset of a trusted access context. */ +export function captureDurableMemoryAuthorizationFacts(context: MemoryAccessContext): Readonly<{ + actorEvidence: DurableMemoryActorEvidence; + delegationSnapshot: DurableMemoryDelegationSnapshot; + hostFactsRevision: string; +}> { + return Object.freeze({ + actorEvidence: captureActorEvidence(context.actor), + delegationSnapshot: captureDelegation(context.delegation), + hostFactsRevision: requireText(context.hostFactsRevision, "hostFactsRevision"), + }); +} + /** Prepares an immutable run-exposure revision without publishing it to process state. */ export function prepareMemoryRunExposure(facts: MemoryRunExposureFacts): MemoryRunExposureSnapshot { const normalizedKey = key(facts); diff --git a/src/state/openclaw-agent-db.generated.d.ts b/src/state/openclaw-agent-db.generated.d.ts index 4f4b6edd92a4..8a9e9a1172de 100644 --- a/src/state/openclaw-agent-db.generated.d.ts +++ b/src/state/openclaw-agent-db.generated.d.ts @@ -157,6 +157,14 @@ export interface MemoryAuditOutbox { updated_at: number; } +export interface MemoryCompactionPolicies { + compaction_policy_id: string; + created_at: number; + retention_state: string; + session_id: string; + source_policy_set_id: string; +} + export interface MemoryEmbeddingCache { dims: number | null; embedding: string; @@ -275,6 +283,16 @@ export interface MemoryPolicyRevisions { revocation_epoch: number; } +export interface MemoryPolicySetMembers { + audience_intersection_json: string; + created_at: number; + expected_revision_id: string; + expected_revocation_epoch: number; + policy_id: string; + policy_set_id: string; + retention_state: string; +} + export interface MemoryPolicySets { agent_id: string; created_at: number; @@ -306,6 +324,14 @@ export interface MemoryPreoutputExposureLedger { subject_revision: string; } +export interface MemoryPreoutputExposureAuthorizationFacts { + actor_evidence_json: string; + created_at: number; + delegation_snapshot_json: string; + exposure_set_id: string; + host_facts_revision: string; +} + export interface MemoryResourceRevisions { activated_at: number | null; actor_id: string | null; @@ -343,6 +369,13 @@ export interface MemoryResources { store_id: string; } +export interface MemoryRunExposureResources { + created_at: number; + exposure_set_id: string; + policy_set_id: string; + resource_revision_id: string; +} + export interface MemoryRunExposures { agent_id: string; context_fingerprint: string; @@ -762,6 +795,34 @@ export interface TranscriptEventMemoryPolicies { subject_revision: string | null; } +export interface TranscriptEventMemoryPolicyDetails { + actor_evidence_json: string; + created_at: number; + delegation_snapshot_json: string; + egress_receipt_ids_json: string; + event_seq: number; + exposed_resource_revisions_json: string; + exposure_receipt_ids_json: string; + finalized_delivery_audiences_json: string; + normalized_audience_intersection_json: string; + retention_state: string; + session_id: string; + source_event_seq: number; + source_session_id: string; +} + +export interface TranscriptEventMemoryPolicyTransitions { + created_at: number; + event_seq: number; + session_id: string; + source_event_seq: number; + source_session_id: string; + source_session_identity_revision: string; + subject_revision: string; + target_session_identity_revision: string; + transition_kind: string; +} + export interface TranscriptEvents { created_at: number; event_json: string; @@ -787,6 +848,7 @@ export interface DB { conversations: Conversations; heartbeat_outcomes: HeartbeatOutcomes; memory_audit_outbox: MemoryAuditOutbox; + memory_compaction_policies: MemoryCompactionPolicies; memory_embedding_cache: MemoryEmbeddingCache; memory_index_chunk_provenance: MemoryIndexChunkProvenance; memory_index_chunk_recall_metadata: MemoryIndexChunkRecallMetadata; @@ -799,11 +861,14 @@ export interface DB { memory_policies: MemoryPolicies; memory_policy_entries: MemoryPolicyEntries; memory_policy_revisions: MemoryPolicyRevisions; + memory_policy_set_members: MemoryPolicySetMembers; memory_policy_sets: MemoryPolicySets; + memory_preoutput_exposure_authorization_facts: MemoryPreoutputExposureAuthorizationFacts; memory_preoutput_exposure_ledger: MemoryPreoutputExposureLedger; memory_resource_revisions: MemoryResourceRevisions; memory_resource_subjects: MemoryResourceSubjects; memory_resources: MemoryResources; + memory_run_exposure_resources: MemoryRunExposureResources; memory_run_exposures: MemoryRunExposures; memory_scoped_chunk_vectors: MemoryScopedChunkVectors; memory_scoped_chunks: MemoryScopedChunks; @@ -845,6 +910,8 @@ export interface DB { trajectory_runtime_events: TrajectoryRuntimeEvents; transcript_event_identities: TranscriptEventIdentities; transcript_event_memory_policies: TranscriptEventMemoryPolicies; + transcript_event_memory_policy_details: TranscriptEventMemoryPolicyDetails; + transcript_event_memory_policy_transitions: TranscriptEventMemoryPolicyTransitions; transcript_events: TranscriptEvents; transcript_rewrite_watermarks: TranscriptRewriteWatermarks; } diff --git a/src/state/openclaw-agent-schema.sql b/src/state/openclaw-agent-schema.sql index 3dd56d716e6e..31d041a038eb 100644 --- a/src/state/openclaw-agent-schema.sql +++ b/src/state/openclaw-agent-schema.sql @@ -973,6 +973,38 @@ BEGIN SELECT RAISE(ABORT, 'memory policy sets cannot be deleted'); END; +-- A policy-set id is an immutable retention handle, not an everlasting grant. +-- Its members retain the stable policy identity and the exact active revision +-- expected at exposure time; readers revalidate both against current policy state. +CREATE TABLE IF NOT EXISTS memory_policy_set_members ( + policy_set_id TEXT NOT NULL, + policy_id TEXT NOT NULL, + expected_revision_id TEXT NOT NULL, + expected_revocation_epoch INTEGER NOT NULL CHECK (expected_revocation_epoch >= 0), + audience_intersection_json TEXT NOT NULL, + retention_state TEXT NOT NULL CHECK (retention_state IN ('retained', 'quarantined')), + created_at INTEGER NOT NULL, + PRIMARY KEY (policy_set_id, policy_id), + FOREIGN KEY (policy_set_id) REFERENCES memory_policy_sets(policy_set_id) ON DELETE RESTRICT, + FOREIGN KEY (policy_id) REFERENCES memory_policies(policy_id) ON DELETE RESTRICT, + FOREIGN KEY (expected_revision_id) REFERENCES memory_policy_revisions(revision_id) ON DELETE RESTRICT +) STRICT; + +CREATE INDEX IF NOT EXISTS idx_memory_policy_set_members_policy + ON memory_policy_set_members(policy_id, expected_revision_id, expected_revocation_epoch); + +CREATE TRIGGER IF NOT EXISTS memory_policy_set_members_no_update +BEFORE UPDATE ON memory_policy_set_members +BEGIN + SELECT RAISE(ABORT, 'memory policy set members are immutable'); +END; + +CREATE TRIGGER IF NOT EXISTS memory_policy_set_members_no_delete +BEFORE DELETE ON memory_policy_set_members +BEGIN + SELECT RAISE(ABORT, 'memory policy set members cannot be deleted'); +END; + CREATE TABLE IF NOT EXISTS memory_run_exposures ( exposure_set_id TEXT NOT NULL PRIMARY KEY, agent_id TEXT NOT NULL, @@ -1010,6 +1042,34 @@ BEGIN SELECT RAISE(ABORT, 'memory run exposures cannot be deleted'); END; +-- Resource-to-exposure rows make revocation and expiry impact analysis a +-- durable join, rather than an opaque array scan over transcript history. +CREATE TABLE IF NOT EXISTS memory_run_exposure_resources ( + exposure_set_id TEXT NOT NULL, + resource_revision_id TEXT NOT NULL, + policy_set_id TEXT NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (exposure_set_id, resource_revision_id), + FOREIGN KEY (exposure_set_id) REFERENCES memory_run_exposures(exposure_set_id) ON DELETE RESTRICT, + FOREIGN KEY (resource_revision_id) REFERENCES memory_resource_revisions(revision_id) ON DELETE RESTRICT, + FOREIGN KEY (policy_set_id) REFERENCES memory_policy_sets(policy_set_id) ON DELETE RESTRICT +) STRICT; + +CREATE INDEX IF NOT EXISTS idx_memory_run_exposure_resources_revision + ON memory_run_exposure_resources(resource_revision_id, exposure_set_id); + +CREATE TRIGGER IF NOT EXISTS memory_run_exposure_resources_no_update +BEFORE UPDATE ON memory_run_exposure_resources +BEGIN + SELECT RAISE(ABORT, 'memory run exposure resources are immutable'); +END; + +CREATE TRIGGER IF NOT EXISTS memory_run_exposure_resources_no_delete +BEFORE DELETE ON memory_run_exposure_resources +BEGIN + SELECT RAISE(ABORT, 'memory run exposure resources cannot be deleted'); +END; + -- Selected-plugin content is never returned until this content-free ledger row commits. -- It is lazy/additive so current-version databases remain compatible until first scoped read. CREATE TABLE IF NOT EXISTS memory_preoutput_exposure_ledger ( @@ -1052,6 +1112,31 @@ BEGIN SELECT RAISE(ABORT, 'pre-output memory exposure ledger cannot be deleted'); END; +-- Trusted host facts are captured before content release. Keep their audit-only +-- projection separate from the old ledger table so existing agent databases can +-- install it lazily without a schema-version migration. +CREATE TABLE IF NOT EXISTS memory_preoutput_exposure_authorization_facts ( + exposure_set_id TEXT NOT NULL PRIMARY KEY, + actor_evidence_json TEXT NOT NULL, + delegation_snapshot_json TEXT NOT NULL, + host_facts_revision TEXT NOT NULL, + created_at INTEGER NOT NULL, + FOREIGN KEY (exposure_set_id) + REFERENCES memory_preoutput_exposure_ledger(exposure_set_id) ON DELETE RESTRICT +) STRICT; + +CREATE TRIGGER IF NOT EXISTS memory_preoutput_exposure_authorization_facts_no_update +BEFORE UPDATE ON memory_preoutput_exposure_authorization_facts +BEGIN + SELECT RAISE(ABORT, 'pre-output exposure authorization facts are immutable'); +END; + +CREATE TRIGGER IF NOT EXISTS memory_preoutput_exposure_authorization_facts_no_delete +BEFORE DELETE ON memory_preoutput_exposure_authorization_facts +BEGIN + SELECT RAISE(ABORT, 'pre-output exposure authorization facts cannot be deleted'); +END; + CREATE TABLE IF NOT EXISTS transcript_event_memory_policies ( session_id TEXT NOT NULL, event_seq INTEGER NOT NULL, @@ -1095,6 +1180,94 @@ CREATE TABLE IF NOT EXISTS transcript_event_memory_policies ( CREATE INDEX IF NOT EXISTS idx_transcript_event_memory_policies_status ON transcript_event_memory_policies(session_id, authorization_status, event_seq); +-- The narrow P1C row remains the hot replay filter. This companion holds the +-- full opaque retention evidence without making transcript JSON authoritative. +CREATE TABLE IF NOT EXISTS transcript_event_memory_policy_details ( + session_id TEXT NOT NULL, + event_seq INTEGER NOT NULL, + actor_evidence_json TEXT NOT NULL, + delegation_snapshot_json TEXT NOT NULL, + exposed_resource_revisions_json TEXT NOT NULL, + exposure_receipt_ids_json TEXT NOT NULL, + egress_receipt_ids_json TEXT NOT NULL, + normalized_audience_intersection_json TEXT NOT NULL, + finalized_delivery_audiences_json TEXT NOT NULL, + retention_state TEXT NOT NULL CHECK (retention_state IN ('retained', 'quarantined')), + source_session_id TEXT NOT NULL, + source_event_seq INTEGER NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (session_id, event_seq), + FOREIGN KEY (session_id, event_seq) + REFERENCES transcript_event_memory_policies(session_id, event_seq) ON DELETE CASCADE +) STRICT; + +CREATE INDEX IF NOT EXISTS idx_transcript_event_memory_policy_details_source + ON transcript_event_memory_policy_details(source_session_id, source_event_seq); + +CREATE TRIGGER IF NOT EXISTS transcript_event_memory_policy_details_no_update +BEFORE UPDATE ON transcript_event_memory_policy_details +BEGIN + SELECT RAISE(ABORT, 'transcript memory policy details are immutable'); +END; + +-- The immutable detail lives exactly as long as its transcript event. The +-- transcript owner deletes both through the foreign-key cascade during a +-- reset/rewind/replace; blocking that cascade would leave the session unable +-- to enforce its own retention lifecycle. +DROP TRIGGER IF EXISTS transcript_event_memory_policy_details_no_delete; + +-- A new session identity cannot reuse a parent's direct companion. Transition +-- provenance records the exact source event and both immutable identities so +-- readers can revalidate the origin without treating copied JSON as authority. +CREATE TABLE IF NOT EXISTS transcript_event_memory_policy_transitions ( + session_id TEXT NOT NULL, + event_seq INTEGER NOT NULL, + source_session_id TEXT NOT NULL, + source_event_seq INTEGER NOT NULL, + transition_kind TEXT NOT NULL CHECK (transition_kind IN ( + 'parent-fork', 'fork', 'rewind', 'switch', 'checkpoint' + )), + source_session_identity_revision TEXT NOT NULL, + target_session_identity_revision TEXT NOT NULL, + subject_revision TEXT NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (session_id, event_seq), + FOREIGN KEY (session_id, event_seq) + REFERENCES transcript_event_memory_policies(session_id, event_seq) ON DELETE CASCADE +) STRICT; + +CREATE INDEX IF NOT EXISTS idx_transcript_event_memory_policy_transitions_source + ON transcript_event_memory_policy_transitions(source_session_id, source_event_seq); + +CREATE TRIGGER IF NOT EXISTS transcript_event_memory_policy_transitions_no_update +BEFORE UPDATE ON transcript_event_memory_policy_transitions +BEGIN + SELECT RAISE(ABORT, 'transcript memory policy transitions are immutable'); +END; + +-- Compaction creates its own derived policy in Phase 2C. The table is owned +-- now so transitions can preserve the reference without inventing sidecars. +CREATE TABLE IF NOT EXISTS memory_compaction_policies ( + compaction_policy_id TEXT NOT NULL PRIMARY KEY, + session_id TEXT NOT NULL, + source_policy_set_id TEXT NOT NULL, + retention_state TEXT NOT NULL CHECK (retention_state IN ('retained', 'quarantined')), + created_at INTEGER NOT NULL, + FOREIGN KEY (source_policy_set_id) REFERENCES memory_policy_sets(policy_set_id) ON DELETE RESTRICT +) STRICT; + +CREATE TRIGGER IF NOT EXISTS memory_compaction_policies_no_update +BEFORE UPDATE ON memory_compaction_policies +BEGIN + SELECT RAISE(ABORT, 'memory compaction policies are immutable'); +END; + +CREATE TRIGGER IF NOT EXISTS memory_compaction_policies_no_delete +BEFORE DELETE ON memory_compaction_policies +BEGIN + SELECT RAISE(ABORT, 'memory compaction policies cannot be deleted'); +END; + CREATE TABLE IF NOT EXISTS standing_intents ( intent_key INTEGER PRIMARY KEY, id TEXT NOT NULL UNIQUE, diff --git a/src/state/openclaw-agent-scoped-memory-schema.ts b/src/state/openclaw-agent-scoped-memory-schema.ts index adfd487eb7a8..b9d654be1e2c 100644 --- a/src/state/openclaw-agent-scoped-memory-schema.ts +++ b/src/state/openclaw-agent-scoped-memory-schema.ts @@ -17,9 +17,15 @@ export const AGENT_SCOPED_MEMORY_TABLES = [ "memory_audit_outbox", "memory_migrations", "memory_policy_sets", + "memory_policy_set_members", "memory_run_exposures", + "memory_run_exposure_resources", "memory_preoutput_exposure_ledger", + "memory_preoutput_exposure_authorization_facts", "transcript_event_memory_policies", + "transcript_event_memory_policy_details", + "transcript_event_memory_policy_transitions", + "memory_compaction_policies", ] as const; export const AGENT_SCOPED_MEMORY_FTS_TABLE = "memory_scoped_chunks_fts";