diff --git a/docs/concepts/2026-07-29-memory-impl-plan.md b/docs/concepts/2026-07-29-memory-impl-plan.md index 271f03bc98b3..e6316f1e8b3d 100644 --- a/docs/concepts/2026-07-29-memory-impl-plan.md +++ b/docs/concepts/2026-07-29-memory-impl-plan.md @@ -791,22 +791,22 @@ groups. Add any lazily ensured columns to `allowedMissingColumns`. Phase 1B is complete only when all of the following are demonstrated: -- [ ] Builtin memory passes the full authorized-backend conformance suite for +- [x] Builtin memory passes the full authorized-backend conformance suite for store isolation, immutable revisions, policy evaluation, search, and exact reads. -- [ ] Pending, quarantined, expired, stale-hash, and tombstoned revisions cannot +- [x] Pending, quarantined, expired, stale-hash, and tombstoned revisions cannot be returned. -- [ ] Scoped resources and chunks never enter legacy +- [x] Scoped resources and chunks never enter legacy `memory_index_sources`/`memory_index_chunks` or legacy FTS/vector tables. -- [ ] Every additive per-agent table, column, and trigger group is registered +- [x] Every additive per-agent table, column, and trigger group is registered in schema compatibility so an existing current-version database opens before feature-local lazy ensure. -- [ ] A nonconforming or failed alternate backend becomes unavailable in +- [x] A nonconforming or failed alternate backend becomes unavailable in enforced mode and never falls back to broader legacy search. -- [ ] Doctor dry-run produces a deterministic, content-redacted classification, +- [x] Doctor dry-run produces a deterministic, content-redacted classification, backup, copy, reindex, verification, and cutover plan without modifying files or database state. -- [ ] Legacy single-user agents remain on the existing runtime path with no +- [x] Legacy single-user agents remain on the existing runtime path with no user-visible behavior change or runtime cutover. ### Phase 1B rollback diff --git a/docs/plugins/sdk-subpaths.md b/docs/plugins/sdk-subpaths.md index 42233339e8ba..1c4a7d9085b3 100644 --- a/docs/plugins/sdk-subpaths.md +++ b/docs/plugins/sdk-subpaths.md @@ -375,7 +375,7 @@ Use `isLoopbackHost(host)` when a plugin must accept only the local machine. It | `plugin-sdk/memory-core-host-engine-fs` | Private-local focused filesystem and user-path helpers for doctor migrations | | `plugin-sdk/memory-core-host-engine-embeddings` | Private-local after July 2026; Memory host embedding contracts, generic adapter bridging, and batch/remote helpers. Providers register through the generic embedding provider API. | | `plugin-sdk/memory-core-host-engine-sessions` | Private-local after July 2026; Memory session transcript and query helpers | - | `plugin-sdk/memory-core-host-engine-schema` | Private-local focused memory index schema and sqlite-vec helpers for doctor migrations | + | `plugin-sdk/memory-core-host-engine-schema` | Private-local focused legacy-index, scoped-memory lazy-schema, and sqlite-vec helpers for doctor migrations | | `plugin-sdk/memory-core-host-engine-storage` | Private-local after July 2026; Memory host storage engine exports | | `plugin-sdk/memory-core-host-secret` | Private-local after July 2026; Memory host secret helpers | | `plugin-sdk/memory-core-host-status` | Private-local after July 2026; Memory host status helpers | diff --git a/extensions/memory-core/doctor-contract-api.ts b/extensions/memory-core/doctor-contract-api.ts index e2e080e58099..1e577d9d24d2 100644 --- a/extensions/memory-core/doctor-contract-api.ts +++ b/extensions/memory-core/doctor-contract-api.ts @@ -6,6 +6,7 @@ import { qmdLocksStateMigration, qmdWorkspaceStateMigration, } from "./src/migration/doctor-memory-sidecar.js"; +import { scopedMemoryMigrationPreview } from "./src/migration/doctor-scoped-memory-preview.js"; import { vectorIndexProviderDiagnostic } from "./src/migration/doctor-vector-index-provider.js"; export const stateMigrations: PluginDoctorStateMigration[] = [ @@ -15,4 +16,5 @@ export const stateMigrations: PluginDoctorStateMigration[] = [ qmdWorkspaceStateMigration, qmdLocksStateMigration, vectorIndexProviderDiagnostic, + scopedMemoryMigrationPreview, ]; diff --git a/extensions/memory-core/src/memory/scoped-memory-candidates.ts b/extensions/memory-core/src/memory/scoped-memory-candidates.ts new file mode 100644 index 000000000000..84fe4204c684 --- /dev/null +++ b/extensions/memory-core/src/memory/scoped-memory-candidates.ts @@ -0,0 +1,187 @@ +import type { DatabaseSync } from "node:sqlite"; +import type { MemorySource } from "openclaw/plugin-sdk/memory-core-host-engine-storage"; +import { bm25RankToScore, buildFtsQuery } from "./hybrid.js"; + +export type ScopedMemoryCandidate = Readonly<{ + chunkId: string; + revisionId: string; + score: number; + vectorScore?: number; + textScore?: number; +}>; + +export type ScopedMemoryCandidatePageParams = Readonly<{ + database: DatabaseSync; + query: string; + queryVector?: readonly number[]; + storeIds: readonly string[]; + sources: readonly MemorySource[]; + limit: number; + offset: number; + /** Candidates are only a prefilter, but retired/expired revisions can never be returned. */ + nowMs?: number; +}>; + +export type ScopedMemoryCandidatePageReader = ( + params: ScopedMemoryCandidatePageParams, +) => readonly ScopedMemoryCandidate[]; + +function compareText(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +/** FTS candidates carry only opaque identifiers from the already-selected scoped view. */ +export const readScopedMemoryFtsCandidatePage: ScopedMemoryCandidatePageReader = (params) => { + if (params.storeIds.length === 0 || params.limit <= 0) { + return []; + } + const matchQuery = buildFtsQuery(params.query); + if (!matchQuery) { + return []; + } + const storePlaceholders = params.storeIds.map(() => "?").join(", "); + const sourcePlaceholders = params.sources.map(() => "?").join(", "); + // sqlite-allow-raw -- FTS MATCH and a view-sized IN list are SQLite primitives. + const rows = params.database + .prepare( + `SELECT chunk.chunk_id, chunk.revision_id, bm25(memory_scoped_chunks_fts) AS rank + FROM memory_scoped_chunks_fts + JOIN memory_scoped_chunks AS chunk ON chunk.chunk_key = memory_scoped_chunks_fts.rowid + JOIN memory_resource_revisions AS revision ON revision.revision_id = chunk.revision_id + JOIN memory_resources AS resource ON resource.resource_id = revision.resource_id + WHERE memory_scoped_chunks_fts MATCH ? + AND resource.store_id IN (${storePlaceholders}) + AND resource.source IN (${sourcePlaceholders}) + AND revision.lifecycle_state = 'active' + AND (revision.expires_at IS NULL OR revision.expires_at > ?) + ORDER BY rank ASC, chunk.chunk_id ASC + LIMIT ? OFFSET ?`, + ) + .all( + matchQuery, + ...params.storeIds, + ...params.sources, + params.nowMs ?? Date.now(), + params.limit, + params.offset, + ) as Array<{ + chunk_id: string; + revision_id: string; + rank: number; + }>; + return rows.map((row) => ({ + chunkId: row.chunk_id, + revisionId: row.revision_id, + score: bm25RankToScore(row.rank), + textScore: bm25RankToScore(row.rank), + })); +}; + +function cosineSimilarity(left: readonly number[], right: readonly number[]): number { + if (left.length === 0 || left.length !== right.length) { + return 0; + } + let dot = 0; + let leftNorm = 0; + let rightNorm = 0; + for (let index = 0; index < left.length; index += 1) { + const leftValue = left[index] ?? 0; + const rightValue = right[index] ?? 0; + dot += leftValue * rightValue; + leftNorm += leftValue * leftValue; + rightNorm += rightValue * rightValue; + } + return leftNorm === 0 || rightNorm === 0 ? 0 : dot / (Math.sqrt(leftNorm) * Math.sqrt(rightNorm)); +} + +/** Bounded same-store vector scan used when a scoped sqlite-vec table is unavailable. */ +export const readScopedMemoryVectorCandidatePage: ScopedMemoryCandidatePageReader = (params) => { + if (!params.queryVector?.length || params.storeIds.length === 0 || params.limit <= 0) { + return []; + } + const storePlaceholders = params.storeIds.map(() => "?").join(", "); + const sourcePlaceholders = params.sources.map(() => "?").join(", "); + // sqlite-allow-raw -- Reads only scoped vector payloads for an in-process cosine scan. + const rows = params.database + .prepare( + `SELECT chunk.chunk_id, chunk.revision_id, vector.embedding + FROM memory_scoped_chunk_vectors AS vector + JOIN memory_scoped_chunks AS chunk ON chunk.chunk_id = vector.chunk_id + JOIN memory_resource_revisions AS revision ON revision.revision_id = chunk.revision_id + JOIN memory_resources AS resource ON resource.resource_id = revision.resource_id + WHERE resource.store_id IN (${storePlaceholders}) + AND resource.source IN (${sourcePlaceholders}) + AND revision.lifecycle_state = 'active' + AND (revision.expires_at IS NULL OR revision.expires_at > ?) + ORDER BY chunk.chunk_id ASC`, + ) + .all(...params.storeIds, ...params.sources, params.nowMs ?? Date.now()) as Array<{ + chunk_id: string; + revision_id: string; + embedding: string; + }>; + return rows + .flatMap((row) => { + try { + const embedding = JSON.parse(row.embedding) as unknown; + if (!Array.isArray(embedding) || !embedding.every((value) => typeof value === "number")) { + return []; + } + const score = cosineSimilarity(params.queryVector ?? [], embedding); + return [{ chunkId: row.chunk_id, revisionId: row.revision_id, score, vectorScore: score }]; + } catch { + return []; + } + }) + .toSorted((left, right) => right.score - left.score || compareText(left.chunkId, right.chunkId)) + .slice(params.offset, params.offset + params.limit); +}; + +function vectorToBlob(vector: readonly number[]): Buffer { + return Buffer.from(new Float32Array(vector).buffer); +} + +/** sqlite-vec KNN reader; any absence falls back only to the same scoped vectors. */ +export const readScopedMemorySqliteVecCandidatePage: ScopedMemoryCandidatePageReader = (params) => { + if (!params.queryVector?.length || params.storeIds.length === 0 || params.limit <= 0) { + return []; + } + const hasVectorIndex = Boolean( + params.database + .prepare("SELECT 1 FROM main.sqlite_schema WHERE type = 'table' AND name = ?") + .get("memory_scoped_chunks_vec"), + ); + if (!hasVectorIndex) { + return readScopedMemoryVectorCandidatePage(params); + } + const storePlaceholders = params.storeIds.map(() => "?").join(", "); + const sourcePlaceholders = params.sources.map(() => "?").join(", "); + // sqlite-allow-raw -- vec0 applies k before joins, so full scoped filtering is required. + const rows = params.database + .prepare( + `SELECT chunk.chunk_id, chunk.revision_id, + vec_distance_cosine(vector.embedding, ?) AS distance + FROM memory_scoped_chunks_vec AS vector + JOIN memory_scoped_chunks AS chunk ON chunk.chunk_id = vector.chunk_id + JOIN memory_resource_revisions AS revision ON revision.revision_id = chunk.revision_id + JOIN memory_resources AS resource ON resource.resource_id = revision.resource_id + WHERE resource.store_id IN (${storePlaceholders}) + AND resource.source IN (${sourcePlaceholders}) + AND revision.lifecycle_state = 'active' + AND (revision.expires_at IS NULL OR revision.expires_at > ?) + ORDER BY distance ASC, chunk.chunk_id ASC + LIMIT ? OFFSET ?`, + ) + .all( + vectorToBlob(params.queryVector), + ...params.storeIds, + ...params.sources, + params.nowMs ?? Date.now(), + params.limit, + params.offset, + ) as Array<{ chunk_id: string; revision_id: string; distance: number }>; + return rows.map((row) => { + const score = Math.max(0, Math.min(1, 1 - row.distance)); + return { chunkId: row.chunk_id, revisionId: row.revision_id, score, vectorScore: score }; + }); +}; diff --git a/extensions/memory-core/src/memory/scoped-memory-db.ts b/extensions/memory-core/src/memory/scoped-memory-db.ts new file mode 100644 index 000000000000..b4cb35184ddc --- /dev/null +++ b/extensions/memory-core/src/memory/scoped-memory-db.ts @@ -0,0 +1,179 @@ +import path from "node:path"; +import type { DatabaseSync } from "node:sqlite"; +import type { ColumnType } from "kysely"; +import { ensureOpenClawAgentScopedMemorySchema } from "openclaw/plugin-sdk/memory-core-host-engine-schema"; +import { openOpenClawAgentDatabase } from "openclaw/plugin-sdk/sqlite-runtime"; + +export type ScopedMemoryLifecycleState = "pending" | "active" | "quarantined" | "tombstoned"; +export type ScopedMemoryScopeKind = + | "user" + | "conversation" + | "role" + | "agent-shared" + | "agent" + | "internal"; +export type ScopedMemoryActorKind = "human" | "agent" | "service" | "system" | "unattributed"; + +export type MemoryStorageRootRow = { + storage_root_id: string; + agent_id: string; + backend_kind: "builtin" | "alternate"; + opaque_locator: string; + path_key_version: number; + path_key: string | null; + authority_kind: ScopedMemoryScopeKind; + authority_owner_id: string; + default_capabilities_json: string; + lifecycle_state: ScopedMemoryLifecycleState; + created_at: number; + updated_at: number; +}; + +export type MemoryStoreRow = { + store_id: string; + agent_id: string; + storage_root_id: string; + policy_id: string; + scope_kind: ScopedMemoryScopeKind; + audience_kind: ScopedMemoryScopeKind; + audience_id: string; + lifecycle_state: ScopedMemoryLifecycleState; + created_at: number; + updated_at: number; +}; + +type MemoryPolicyRow = { + policy_id: string; + agent_id: string; + current_revision_id: string; + revocation_epoch: number; + lifecycle_state: "active" | "revoked"; + created_at: number; + updated_at: number; +}; + +type MemoryPolicyRevisionRow = { + revision_id: string; + policy_id: string; + revision_number: number; + revocation_epoch: number; + lifecycle_state: "active" | "superseded" | "revoked"; + actor_kind: ScopedMemoryActorKind; + actor_id: string | null; + reason: string; + created_at: number; +}; + +export type MemoryPolicyEntryRow = { + entry_id: string; + policy_revision_id: string; + entry_kind: "placement" | "exception" | "publish"; + effect: "allow" | "deny"; + principal_id: string; + audience_kind: ScopedMemoryScopeKind | "*"; + audience_id: string; + operation: import("openclaw/plugin-sdk/memory-authorization").MemoryOperation; + grantor_principal_id: string; + reason: string; + expires_at: number | null; + created_at: number; +}; + +export type MemoryResourceRow = { + resource_id: string; + agent_id: string; + store_id: string; + logical_locator: string; + source: "memory" | "sessions"; + created_at: number; +}; + +export type MemoryResourceRevisionRow = { + revision_id: string; + resource_id: string; + revision_number: number; + artifact_locator: string; + content_hash: string; + content_bytes: number; + policy_revision_id: string; + policy_revocation_epoch: number; + source_policy_set_id: string; + lifecycle_state: ScopedMemoryLifecycleState; + actor_kind: ScopedMemoryActorKind; + actor_id: string | null; + expires_at: number | null; + created_at: number; + activated_at: number | null; + retired_at: number | null; +}; + +export type MemoryResourceSubjectRow = { + revision_id: string; + subject_kind: "person" | "project" | "conversation" | "topic"; + subject_id: string; + evidence_revision: string; + lifecycle_state: "current" | "superseded"; + created_at: number; +}; + +export type MemoryScopedChunkRow = { + chunk_key: ColumnType; + chunk_id: string; + revision_id: string; + chunk_ordinal: number; + start_line: number; + end_line: number; + text: string; + content_hash: string; + model: string; + updated_at: number; +}; + +export type MemoryScopedChunkVectorRow = { + chunk_id: string; + model: string; + dims: number; + embedding: string; + updated_at: number; +}; + +export type MemoryMigrationRow = { + migration_id: string; + source_kind: string; + source_hash: string; + phase: "previewed" | "backed-up" | "copied" | "indexed" | "verified" | "cutover"; + classification_json: string; + plan_hash: string; + verified_at: number | null; + cutover_at: number | null; + updated_at: number; +}; + +export type ScopedMemoryDatabase = { + memory_storage_roots: MemoryStorageRootRow; + memory_stores: MemoryStoreRow; + memory_policies: MemoryPolicyRow; + memory_policy_revisions: MemoryPolicyRevisionRow; + memory_policy_entries: MemoryPolicyEntryRow; + memory_resources: MemoryResourceRow; + memory_resource_revisions: MemoryResourceRevisionRow; + memory_resource_subjects: MemoryResourceSubjectRow; + memory_scoped_chunks: MemoryScopedChunkRow; + memory_scoped_chunk_vectors: MemoryScopedChunkVectorRow; + memory_migrations: MemoryMigrationRow; +}; + +/** Open the canonical agent database and lazily add only the scoped-memory group. */ +export function withScopedMemoryDatabase( + agentId: string, + callback: (db: DatabaseSync, databasePath: string) => T, +): T { + const database = openOpenClawAgentDatabase({ agentId }); + ensureOpenClawAgentScopedMemorySchema(database.db); + return callback(database.db, database.path); +} + +/** Filesystem owner for opaque builtin memory-store directories. */ +export function resolveScopedMemoryArtifactBase(databasePath: string): string { + return path.join(path.dirname(databasePath), "memory-scopes", "v1"); +} diff --git a/extensions/memory-core/src/memory/scoped-memory-policy.test.ts b/extensions/memory-core/src/memory/scoped-memory-policy.test.ts new file mode 100644 index 000000000000..a24723aa633a --- /dev/null +++ b/extensions/memory-core/src/memory/scoped-memory-policy.test.ts @@ -0,0 +1,131 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { runMemoryAuthorizationConformanceSuite } from "openclaw/plugin-sdk/memory-authorization-conformance"; +import { closeOpenClawAgentDatabasesForTest } from "openclaw/plugin-sdk/sqlite-runtime-testing"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + builtinScopedMemoryConformanceAdapter, + evaluateBuiltinScopedMemoryPolicy, +} from "./scoped-memory-policy.js"; +import { + createBuiltinScopedMemoryStore, + reviseBuiltinScopedMemoryPolicy, +} from "./scoped-memory-store.js"; + +describe("builtin scoped memory policy conformance", () => { + let stateDir = ""; + + beforeEach(() => { + stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-scoped-memory-policy-")); + vi.stubEnv("OPENCLAW_STATE_DIR", stateDir); + }); + + afterEach(() => { + closeOpenClawAgentDatabasesForTest(); + vi.unstubAllEnvs(); + fs.rmSync(stateDir, { recursive: true, force: true }); + }); + + it("passes the full current host authorization conformance suite", async () => { + await expect( + runMemoryAuthorizationConformanceSuite(builtinScopedMemoryConformanceAdapter), + ).resolves.toEqual({ + ok: true, + failures: [], + }); + }); + + it("evaluates persisted placement, deny precedence, expiry, and operation implication", () => { + const store = createBuiltinScopedMemoryStore({ + agentId: "main", + scopeKind: "user", + audienceKind: "user", + audienceId: "alice", + authorityKind: "user", + authorityOwnerId: "alice", + defaultCapabilities: ["retrieve", "read"], + actor: { kind: "human", id: "alice" }, + reason: "private placement", + nowMs: 1_000, + }); + const evaluate = (operation: "read" | "derive", nowMs = 2_000) => + evaluateBuiltinScopedMemoryPolicy({ + agentId: "main", + storeId: store.storeId, + principalIds: ["alice"], + deliveryAudiences: [{ kind: "user", id: "alice" }], + operation, + nowMs, + }); + + expect(evaluate("read")).toMatchObject({ allowed: true, reasonCode: "allowed" }); + expect( + evaluateBuiltinScopedMemoryPolicy({ + agentId: "main", + storeId: store.storeId, + principalIds: ["bob"], + deliveryAudiences: [{ kind: "user", id: "alice" }], + operation: "read", + nowMs: 2_000, + }), + ).toMatchObject({ allowed: false, reasonCode: "outside-view" }); + expect(evaluate("derive")).toMatchObject({ allowed: false, reasonCode: "default-deny" }); + reviseBuiltinScopedMemoryPolicy({ + agentId: "main", + policyId: store.policyId, + entries: [ + { + effect: "deny", + principalId: "alice", + operation: "read", + grantorPrincipalId: "alice", + reason: "deny takes precedence", + }, + ], + actor: { kind: "human", id: "alice" }, + reason: "temporary deny", + nowMs: 3_000, + }); + expect(evaluate("read", 3_000)).toMatchObject({ + allowed: false, + reasonCode: "explicit-deny", + }); + reviseBuiltinScopedMemoryPolicy({ + agentId: "main", + policyId: store.policyId, + entries: [ + { + effect: "deny", + principalId: "alice", + operation: "read", + grantorPrincipalId: "alice", + reason: "expired deny", + expiresAt: 3_500, + }, + { + effect: "allow", + principalId: "alice", + operation: "derive", + grantorPrincipalId: "alice", + reason: "derived memory is explicit", + }, + ], + actor: { kind: "human", id: "alice" }, + reason: "allow derivation after expiry", + nowMs: 3_100, + }); + expect(evaluate("read", 3_500)).toMatchObject({ allowed: true, reasonCode: "allowed" }); + expect(evaluate("derive", 3_500)).toMatchObject({ allowed: true, reasonCode: "allowed" }); + expect( + evaluateBuiltinScopedMemoryPolicy({ + agentId: "main", + storeId: store.storeId, + principalIds: ["alice"], + deliveryAudiences: [{ kind: "conversation", id: "conversation-1" }], + operation: "read", + nowMs: 3_500, + }), + ).toMatchObject({ allowed: false, reasonCode: "outside-view" }); + }); +}); diff --git a/extensions/memory-core/src/memory/scoped-memory-policy.ts b/extensions/memory-core/src/memory/scoped-memory-policy.ts new file mode 100644 index 000000000000..32019348e5be --- /dev/null +++ b/extensions/memory-core/src/memory/scoped-memory-policy.ts @@ -0,0 +1,488 @@ +import type { + AudienceRef, + MemoryAuthorizationReasonCode, + MemoryOperation, +} from "openclaw/plugin-sdk/memory-authorization"; +import { MEMORY_AUTHORIZATION_CONTRACT_VERSION } from "openclaw/plugin-sdk/memory-authorization"; +import { + type MemoryAuthorizationConformanceAdapter, + type MemoryAuthorizationConformanceDecision, + type MemoryAuthorizationConformanceResource, + type MemoryAuthorizationConformanceScenario, +} from "openclaw/plugin-sdk/memory-authorization-conformance"; +import { normalizeAgentId } from "openclaw/plugin-sdk/routing"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "openclaw/plugin-sdk/sqlite-runtime"; +import { + type MemoryPolicyEntryRow, + type ScopedMemoryDatabase, + type ScopedMemoryScopeKind, + withScopedMemoryDatabase, +} from "./scoped-memory-db.js"; + +const OPERATION_REQUIREMENTS = { + retrieve: ["retrieve"], + read: ["retrieve", "read"], + append: ["append"], + replace: ["append", "replace"], + derive: ["retrieve", "read", "derive"], + deposit: ["deposit"], + project: ["project"], + publish: ["publish"], + import: ["import"], + export: ["export"], + delete: ["delete"], + sync: ["sync"], + status: ["status"], + "policy-admin": ["policy-admin"], +} as const satisfies Readonly>; + +export type ScopedMemoryPolicyEvaluation = Readonly<{ + allowed: boolean; + reasonCode: "allowed" | "explicit-deny" | "default-deny" | "outside-view" | "revision-stale"; + policyRevisionId?: string; + policyRevocationEpoch?: number; +}>; + +type ScopedMemoryPolicyAudience = Readonly<{ + kind: ScopedMemoryScopeKind; + id: string; +}>; + +function normalizePolicyText(value: string, label: string): string { + const normalized = value.trim(); + if (!normalized || normalized.includes("\0")) { + throw new TypeError(`${label} must not be empty`); + } + return normalized; +} + +function entryMatchesScopedPolicy(params: { + entry: MemoryPolicyEntryRow; + principalIds: ReadonlySet; + audiences: readonly ScopedMemoryPolicyAudience[]; + operation: MemoryOperation; + nowMs: number; +}): boolean { + return ( + params.entry.operation === params.operation && + (params.entry.principal_id === "*" || params.principalIds.has(params.entry.principal_id)) && + (params.entry.audience_kind === "*" || + params.audiences.some( + (audience) => + audience.kind === params.entry.audience_kind && audience.id === params.entry.audience_id, + )) && + (params.entry.expires_at === null || params.entry.expires_at > params.nowMs) + ); +} + +/** + * Evaluate one current persisted store policy. This is deliberately a pure plugin-owned policy + * operation: the core supplies only already-verified principals and delivery audiences later. + */ +export function evaluateBuiltinScopedMemoryPolicy(params: { + agentId: string; + storeId: string; + principalIds: readonly string[]; + deliveryAudiences: readonly ScopedMemoryPolicyAudience[]; + operation: MemoryOperation; + nowMs?: number; +}): ScopedMemoryPolicyEvaluation { + const agentId = normalizeAgentId(params.agentId); + const storeId = normalizePolicyText(params.storeId, "storeId"); + const principalIds = new Set( + params.principalIds.map((principalId) => normalizePolicyText(principalId, "principalId")), + ); + const audiences = params.deliveryAudiences.map((audience) => + Object.freeze({ + kind: audience.kind, + id: normalizePolicyText(audience.id, "delivery audience id"), + }), + ); + const nowMs = params.nowMs ?? Date.now(); + if (!Number.isFinite(nowMs)) { + throw new TypeError("nowMs must be finite"); + } + return withScopedMemoryDatabase(agentId, (database) => { + const db = getNodeSqliteKysely(database); + const current = executeSqliteQueryTakeFirstSync( + database, + db + .selectFrom("memory_stores as store") + .innerJoin("memory_storage_roots as root", "root.storage_root_id", "store.storage_root_id") + .innerJoin("memory_policies as policy", "policy.policy_id", "store.policy_id") + .innerJoin( + "memory_policy_revisions as revision", + "revision.revision_id", + "policy.current_revision_id", + ) + .select([ + "store.audience_kind", + "store.audience_id", + "root.authority_kind", + "root.authority_owner_id", + "root.default_capabilities_json", + "policy.current_revision_id", + "policy.revocation_epoch", + ]) + .where("store.store_id", "=", storeId) + .where("store.agent_id", "=", agentId) + .where("store.lifecycle_state", "=", "active") + .where("root.lifecycle_state", "=", "active") + .where("policy.lifecycle_state", "=", "active") + .where("revision.lifecycle_state", "=", "active"), + ); + if (!current) { + return Object.freeze({ allowed: false, reasonCode: "revision-stale" }); + } + if ( + !audiences.some( + (audience) => + audience.kind === current.audience_kind && audience.id === current.audience_id, + ) + ) { + return Object.freeze({ + allowed: false, + reasonCode: "outside-view", + policyRevisionId: current.current_revision_id, + policyRevocationEpoch: current.revocation_epoch, + }); + } + // A private user mount is self-owned at this phase. Delivery routing cannot substitute for + // the verified subject: otherwise a caller could name Alice's audience while acting as Bob. + if (current.authority_kind === "user" && !principalIds.has(current.authority_owner_id)) { + return Object.freeze({ + allowed: false, + reasonCode: "outside-view", + policyRevisionId: current.current_revision_id, + policyRevocationEpoch: current.revocation_epoch, + }); + } + let defaultCapabilities: readonly MemoryOperation[]; + try { + const parsed = JSON.parse(current.default_capabilities_json) as unknown; + if (!Array.isArray(parsed) || !parsed.every((entry) => typeof entry === "string")) { + throw new TypeError("invalid policy capabilities"); + } + defaultCapabilities = parsed.filter((entry): entry is MemoryOperation => + Object.hasOwn(OPERATION_REQUIREMENTS, entry), + ); + } catch { + return Object.freeze({ + allowed: false, + reasonCode: "revision-stale", + policyRevisionId: current.current_revision_id, + policyRevocationEpoch: current.revocation_epoch, + }); + } + const entries = executeSqliteQuerySync( + database, + db + .selectFrom("memory_policy_entries") + .selectAll() + .where("policy_revision_id", "=", current.current_revision_id), + ).rows; + const requirements = OPERATION_REQUIREMENTS[params.operation]; + for (const operation of requirements) { + if ( + entries.some( + (entry) => + entry.effect === "deny" && + entryMatchesScopedPolicy({ entry, principalIds, audiences, operation, nowMs }), + ) + ) { + return Object.freeze({ + allowed: false, + reasonCode: "explicit-deny", + policyRevisionId: current.current_revision_id, + policyRevocationEpoch: current.revocation_epoch, + }); + } + } + for (const operation of requirements) { + const allowed = + defaultCapabilities.includes(operation) || + entries.some( + (entry) => + entry.effect === "allow" && + entryMatchesScopedPolicy({ entry, principalIds, audiences, operation, nowMs }), + ); + if (!allowed) { + return Object.freeze({ + allowed: false, + reasonCode: "default-deny", + policyRevisionId: current.current_revision_id, + policyRevocationEpoch: current.revocation_epoch, + }); + } + } + return Object.freeze({ + allowed: true, + reasonCode: "allowed", + policyRevisionId: current.current_revision_id, + policyRevocationEpoch: current.revocation_epoch, + }); + }); +} + +function audienceKey(audience: AudienceRef): string { + return `${audience.kind}\0${audience.id}`; +} + +function isCurrentExpiry(expiresAt: unknown, now: string): expiresAt is string { + if (typeof expiresAt !== "string" || !expiresAt) { + return false; + } + const expiryMs = Date.parse(expiresAt); + const nowMs = Date.parse(now); + return Number.isFinite(expiryMs) && Number.isFinite(nowMs) && expiryMs > nowMs; +} + +function isExpired(expiresAt: string | undefined, now: string): boolean { + return expiresAt !== undefined && !isCurrentExpiry(expiresAt, now); +} + +function sameSet( + actual: readonly T[], + expected: readonly T[], + key: (value: T) => string, +): boolean { + const actualKeys = actual.map(key); + const expectedKeys = expected.map(key); + return ( + actualKeys.length === new Set(actualKeys).size && + expectedKeys.length === new Set(expectedKeys).size && + actualKeys.length === expectedKeys.length && + actualKeys.every((value) => expectedKeys.includes(value)) + ); +} + +function mountKey(mount: MemoryAuthorizationConformanceScenario["plan"]["mounts"][number]): string { + return JSON.stringify([ + mount.storeId, + mount.agentId, + mount.audienceRevision, + mount.capabilities.toSorted(), + ]); +} + +function planFailure( + scenario: MemoryAuthorizationConformanceScenario, +): MemoryAuthorizationReasonCode | undefined { + const { context, plan } = scenario; + if (!isCurrentExpiry(plan.expiresAt, scenario.now)) { + return "plan-expired"; + } + if ( + !plan.planId || + plan.contextFingerprint !== context.contextFingerprint || + plan.runId !== context.runId + ) { + return "invalid-context"; + } + if (plan.sessionId !== context.sessionId) { + return "session-rebound"; + } + if (plan.agentId !== context.agentId || plan.operation !== context.operation) { + return "outside-view"; + } + if (!sameSet(plan.mounts, scenario.viewMounts, mountKey)) { + return "outside-view"; + } + if (plan.mounts.some((mount) => mount.agentId !== context.agentId)) { + return "outside-view"; + } + if (!sameSet(plan.allowedEgressAudiences, context.deliveryAudiences, audienceKey)) { + return "outside-view"; + } + if ( + plan.sessionIdentityRevision !== context.sessionIdentityRevision || + plan.subjectRevision !== context.subjectRevision || + plan.policyRevision !== context.policyRevision || + plan.hostFactsRevision !== context.hostFactsRevision + ) { + return "revision-stale"; + } + return plan.deliveryRevision === context.deliveryRevision ? undefined : "delivery-rebound"; +} + +function activePrincipalIds( + scenario: MemoryAuthorizationConformanceScenario, +): ReadonlySet | undefined { + const ids = new Set(); + for (const ref of scenario.context.principalRefs) { + const facts = scenario.principals.filter((fact) => fact.principalId === ref.principalId); + const fact = facts[0]; + if ( + ids.has(ref.principalId) || + facts.length !== 1 || + !fact || + fact.status !== "active" || + fact.evidenceRevision !== ref.evidenceRevision || + !isCurrentExpiry(fact.expiresAt, scenario.now) + ) { + return undefined; + } + ids.add(ref.principalId); + } + return ids.size > 0 ? ids : undefined; +} + +function membershipFailure(params: { + scenario: MemoryAuthorizationConformanceScenario; + store: MemoryAuthorizationConformanceScenario["stores"][number]; + principalIds: ReadonlySet; +}): "membership-stale" | undefined { + const requirement = params.store.requiredMembership; + if (!requirement) { + return undefined; + } + if (!params.principalIds.has(requirement.principalId)) { + return "membership-stale"; + } + const refs = params.scenario.context.membershipRefs.filter( + (entry) => + entry.principalId === requirement.principalId && + entry.groupId === requirement.groupId && + entry.provider === requirement.provider, + ); + const facts = params.scenario.memberships.filter( + (entry) => + entry.principalId === requirement.principalId && + entry.groupId === requirement.groupId && + entry.provider === requirement.provider, + ); + const ref = refs[0]; + const fact = facts[0]; + if ( + refs.length !== 1 || + facts.length !== 1 || + !ref || + !fact || + fact.status !== "active" || + fact.evidenceRevision !== ref.evidenceRevision || + fact.hostFactsRevision !== ref.hostFactsRevision || + fact.hostFactsRevision !== params.scenario.context.hostFactsRevision || + !isCurrentExpiry(fact.expiresAt, params.scenario.now) + ) { + return "membership-stale"; + } + return undefined; +} + +function entryMatches(params: { + scenario: MemoryAuthorizationConformanceScenario; + resource: MemoryAuthorizationConformanceResource; + operation: MemoryOperation; + principalIds: ReadonlySet; + effect: "allow" | "deny"; +}): boolean { + return params.scenario.policyEntries.some( + (entry) => + entry.effect === params.effect && + entry.operation === params.operation && + (entry.resourceId === "*" || entry.resourceId === params.resource.resourceId) && + (entry.principalId === "*" || params.principalIds.has(entry.principalId)) && + !isExpired(entry.expiresAt, params.scenario.now), + ); +} + +/** + * Plugin-owned reference implementation for the reusable host fixtures. It deliberately does not + * call the host evaluator: admission compares two independent policy implementations. + */ +function evaluateBuiltinScopedMemoryConformanceScenario(params: { + scenario: MemoryAuthorizationConformanceScenario; + resource: MemoryAuthorizationConformanceResource; +}): MemoryAuthorizationConformanceDecision { + const { resource, scenario } = params; + const failedPlan = planFailure(scenario); + if (failedPlan) { + return { allowed: false, reasonCode: failedPlan }; + } + const principalIds = activePrincipalIds(scenario); + if (!principalIds) { + return { allowed: false, reasonCode: "identity-revoked" }; + } + const store = scenario.stores.find((entry) => entry.storeId === resource.storeId); + const mount = scenario.plan.mounts.find((entry) => entry.storeId === resource.storeId); + const requirements = OPERATION_REQUIREMENTS[scenario.context.operation]; + if ( + !store || + !mount || + store.agentId !== scenario.context.agentId || + resource.agentId !== scenario.context.agentId || + mount.agentId !== scenario.context.agentId || + !requirements.every((operation) => mount.capabilities.includes(operation)) + ) { + return { allowed: false, reasonCode: "outside-view" }; + } + const staleMembership = membershipFailure({ scenario, store, principalIds }); + if (staleMembership) { + return { allowed: false, reasonCode: staleMembership }; + } + if (isExpired(resource.expiresAt, scenario.now)) { + return { allowed: false, reasonCode: "revision-stale" }; + } + const audiences = new Set(resource.audiences.map(audienceKey)); + if ( + scenario.context.deliveryAudiences.some((audience) => !audiences.has(audienceKey(audience))) + ) { + return { allowed: false, reasonCode: "outside-view" }; + } + if ( + scenario.context.delegation && + (!scenario.context.delegation.allowedOperations.includes(scenario.context.operation) || + scenario.context.deliveryAudiences.some( + (audience) => + !scenario.context.delegation!.maximumAudiences.some( + (maximum) => audienceKey(maximum) === audienceKey(audience), + ), + )) + ) { + return { allowed: false, reasonCode: "default-deny" }; + } + if ( + resource.requiredLineagePolicySetIds?.some( + (policySetId) => !scenario.context.lineagePolicySetIds.includes(policySetId), + ) + ) { + return { allowed: false, reasonCode: "lineage-deny" }; + } + for (const operation of requirements) { + if (entryMatches({ scenario, resource, operation, principalIds, effect: "deny" })) { + return { allowed: false, reasonCode: "explicit-deny" }; + } + } + for (const operation of requirements) { + if ( + !store.placementCapabilities.includes(operation) && + !entryMatches({ scenario, resource, operation, principalIds, effect: "allow" }) + ) { + return { allowed: false, reasonCode: "default-deny" }; + } + } + return { + allowed: true, + reasonCode: "allowed", + handle: { + version: MEMORY_AUTHORIZATION_CONTRACT_VERSION, + handleId: "memory-core-conformance-handle", + planId: scenario.plan.planId, + contextFingerprint: scenario.plan.contextFingerprint, + resourceRevision: resource.revision, + policyRevision: scenario.plan.policyRevision, + expiresAt: scenario.plan.expiresAt, + }, + }; +} + +export const builtinScopedMemoryConformanceAdapter: MemoryAuthorizationConformanceAdapter = + Object.freeze({ + evaluate: evaluateBuiltinScopedMemoryConformanceScenario, + // The prefilter intentionally over-fetches fixture resources; the evaluator is authoritative. + prefilter: (scenario) => scenario.resources.map((resource) => resource.resourceId), + }); diff --git a/extensions/memory-core/src/memory/scoped-memory-resources.ts b/extensions/memory-core/src/memory/scoped-memory-resources.ts new file mode 100644 index 000000000000..a3162cb0f66b --- /dev/null +++ b/extensions/memory-core/src/memory/scoped-memory-resources.ts @@ -0,0 +1,522 @@ +import { createHash, randomUUID } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { normalizeAgentId } from "openclaw/plugin-sdk/routing"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, + runSqliteImmediateTransactionSync, +} from "openclaw/plugin-sdk/sqlite-runtime"; +import { + resolveScopedMemoryArtifactBase, + type ScopedMemoryDatabase, + type ScopedMemoryLifecycleState, + withScopedMemoryDatabase, +} from "./scoped-memory-db.js"; +import { + createScopedMemorySourcePolicySetId, + normalizeScopedMemoryRequiredText, + type BuiltinScopedMemoryStore, + type ScopedMemoryActor, +} from "./scoped-memory-store.js"; + +const ARTIFACT_NAME_PATTERN = + /^r1_[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.md$/u; +const LOGICAL_LOCATOR_PATTERN = /^(?!\/)(?!.*(?:^|\/)\.\.?\/)[^\0\\]+$/u; + +export type BuiltinScopedMemoryRevision = Readonly<{ + resourceId: string; + revisionId: string; + policyRevisionId: string; + policyRevocationEpoch: number; + sourcePolicySetId: string; + artifactLocator: string; +}>; + +/** Content verified against the active immutable catalog record, before any future exposure. */ +export type BuiltinScopedMemoryRevisionSnapshot = Readonly<{ + resourceId: string; + revisionId: string; + storeId: string; + logicalLocator: string; + content: string; + contentHash: string; + contentBytes: number; + policyRevisionId: string; + policyRevocationEpoch: number; +}>; + +export type ScopedMemoryChunk = Readonly<{ + ordinal: number; + startLine: number; + endLine: number; + text: string; +}>; + +function contentHash(content: string): string { + return createHash("sha256").update(content).digest("hex"); +} + +function normalizeLogicalLocator(locator: string): string { + const normalized = normalizeScopedMemoryRequiredText(locator, "logical locator").replaceAll( + "\\", + "/", + ); + if (!LOGICAL_LOCATOR_PATTERN.test(normalized)) { + throw new Error("logical locator is invalid"); + } + return normalized; +} + +function createArtifactLocator(revisionId: string): string { + return `r1_${revisionId}.md`; +} + +function resolveArtifactDirectory(params: { databasePath: string; pathKey: string }): string { + const base = path.resolve(resolveScopedMemoryArtifactBase(params.databasePath)); + if (!/^s1_[A-Za-z0-9_-]{24,}$/u.test(params.pathKey)) { + throw new Error("scoped-memory path key is invalid"); + } + const directory = path.resolve(base, params.pathKey); + if (path.dirname(directory) !== base) { + throw new Error("scoped-memory storage root is invalid"); + } + return directory; +} + +/** Resolve a canonical artifact path without allowing logical locators to affect the filesystem. */ +export function resolveBuiltinScopedMemoryArtifactPath(params: { + databasePath: string; + pathKey: string; + artifactLocator: string; +}): string { + if (!ARTIFACT_NAME_PATTERN.test(params.artifactLocator)) { + throw new Error("artifact locator is invalid"); + } + const directory = resolveArtifactDirectory(params); + const artifactPath = path.resolve(directory, params.artifactLocator); + if (path.dirname(artifactPath) !== directory) { + throw new Error("artifact locator escaped its storage root"); + } + return artifactPath; +} + +/** Deterministic Markdown chunks retained as scoped derived state beside the canonical artifact. */ +export function chunkScopedMemoryMarkdown(content: string): readonly ScopedMemoryChunk[] { + const lines = content.replaceAll("\r\n", "\n").split("\n"); + const chunks: ScopedMemoryChunk[] = []; + const chunkSize = 48; + for (let start = 0; start < lines.length; start += chunkSize) { + const selected = lines + .slice(start, start + chunkSize) + .join("\n") + .trim(); + if (!selected) { + continue; + } + chunks.push( + Object.freeze({ + ordinal: chunks.length, + startLine: start + 1, + endLine: Math.min(lines.length, start + chunkSize), + text: selected, + }), + ); + } + return Object.freeze(chunks); +} + +function writeImmutableArtifact(params: { artifactPath: string; content: string }): void { + fs.mkdirSync(path.dirname(params.artifactPath), { recursive: true, mode: 0o700 }); + fs.writeFileSync(params.artifactPath, params.content, { + encoding: "utf8", + flag: "wx", + mode: 0o600, + }); +} + +function removeArtifact(pathname: string): void { + try { + fs.unlinkSync(pathname); + } catch {} +} + +/** + * Resolve a revision only while its immutable catalog evidence is current. + * The Phase 1C runtime supplies the authorized store view; this foundation + * never treats a logical locator or filesystem path as an authorization grant. + */ +export function readBuiltinScopedMemoryRevisionSnapshot(params: { + agentId: string; + storeIds: readonly string[]; + revisionId: string; + nowMs?: number; +}): BuiltinScopedMemoryRevisionSnapshot | undefined { + const agentId = normalizeAgentId(params.agentId); + const revisionId = normalizeScopedMemoryRequiredText(params.revisionId, "revisionId"); + const storeIds = [ + ...new Set( + params.storeIds.map((storeId) => normalizeScopedMemoryRequiredText(storeId, "storeId")), + ), + ]; + if (storeIds.length === 0) { + return undefined; + } + const nowMs = params.nowMs ?? Date.now(); + return withScopedMemoryDatabase(agentId, (database, databasePath) => { + const db = getNodeSqliteKysely(database); + const revision = executeSqliteQueryTakeFirstSync( + database, + db + .selectFrom("memory_resource_revisions as revision") + .innerJoin("memory_resources as resource", "resource.resource_id", "revision.resource_id") + .innerJoin("memory_stores as store", "store.store_id", "resource.store_id") + .innerJoin("memory_storage_roots as root", "root.storage_root_id", "store.storage_root_id") + .innerJoin("memory_policies as policy", "policy.policy_id", "store.policy_id") + .innerJoin( + "memory_policy_revisions as policy_revision", + "policy_revision.revision_id", + "policy.current_revision_id", + ) + .select([ + "resource.resource_id", + "resource.store_id", + "resource.logical_locator", + "revision.revision_id", + "revision.artifact_locator", + "revision.content_hash", + "revision.content_bytes", + "revision.policy_revision_id", + "revision.policy_revocation_epoch", + "revision.source_policy_set_id", + "revision.lifecycle_state as revision_lifecycle_state", + "revision.expires_at", + "store.lifecycle_state as store_lifecycle_state", + "root.path_key", + "root.backend_kind", + "root.lifecycle_state as root_lifecycle_state", + "policy.current_revision_id", + "policy.revocation_epoch", + "policy.lifecycle_state as policy_lifecycle_state", + "policy_revision.lifecycle_state as policy_revision_lifecycle_state", + "policy_revision.revocation_epoch as current_policy_revocation_epoch", + ]) + .where("revision.revision_id", "=", revisionId), + ); + if ( + !revision?.path_key || + !storeIds.includes(revision.store_id) || + revision.revision_lifecycle_state !== "active" || + revision.store_lifecycle_state !== "active" || + revision.root_lifecycle_state !== "active" || + revision.backend_kind !== "builtin" || + revision.policy_lifecycle_state !== "active" || + revision.policy_revision_lifecycle_state !== "active" || + revision.policy_revision_id !== revision.current_revision_id || + revision.policy_revocation_epoch !== revision.revocation_epoch || + revision.current_policy_revocation_epoch !== revision.revocation_epoch || + revision.source_policy_set_id !== + createScopedMemorySourcePolicySetId(revision.current_revision_id) || + (revision.expires_at !== null && revision.expires_at <= nowMs) + ) { + return undefined; + } + const artifactPath = resolveBuiltinScopedMemoryArtifactPath({ + databasePath, + pathKey: revision.path_key, + artifactLocator: revision.artifact_locator, + }); + let content: string; + try { + // Scoped artifact roots are owner-only. Refuse a symlink rather than letting a compromised + // artifact entry make a future authorized read cross the selected store boundary. + if (fs.lstatSync(artifactPath).isSymbolicLink()) { + return undefined; + } + content = fs.readFileSync(artifactPath, "utf8"); + } catch { + return undefined; + } + if ( + Buffer.byteLength(content) !== revision.content_bytes || + contentHash(content) !== revision.content_hash + ) { + return undefined; + } + return Object.freeze({ + resourceId: revision.resource_id, + revisionId: revision.revision_id, + storeId: revision.store_id, + logicalLocator: revision.logical_locator, + content, + contentHash: revision.content_hash, + contentBytes: revision.content_bytes, + policyRevisionId: revision.policy_revision_id, + policyRevocationEpoch: revision.policy_revocation_epoch, + }); + }); +} + +function createRevision(params: { + agentId: string; + resourceId: string; + content: string; + lifecycleState: ScopedMemoryLifecycleState; + expiresAt: number | null; + actor: ScopedMemoryActor; + nowMs: number; +}): BuiltinScopedMemoryRevision { + const content = params.content; + if (!content.trim()) { + throw new Error("scoped memory content is required"); + } + if (params.lifecycleState === "tombstoned") { + throw new Error("new scoped-memory revisions cannot start tombstoned"); + } + if ( + params.expiresAt !== null && + (!Number.isSafeInteger(params.expiresAt) || params.expiresAt < 0) + ) { + throw new Error("scoped-memory expiry is invalid"); + } + const revisionId = randomUUID(); + const artifactLocator = createArtifactLocator(revisionId); + return withScopedMemoryDatabase(params.agentId, (database, databasePath) => { + const db = getNodeSqliteKysely(database); + const resource = executeSqliteQueryTakeFirstSync( + database, + db + .selectFrom("memory_resources as resource") + .innerJoin("memory_stores as store", "store.store_id", "resource.store_id") + .innerJoin("memory_storage_roots as root", "root.storage_root_id", "store.storage_root_id") + .select(["resource.resource_id", "root.path_key"]) + .where("resource.resource_id", "=", params.resourceId) + .where("resource.agent_id", "=", params.agentId) + .where("store.lifecycle_state", "=", "active") + .where("root.lifecycle_state", "=", "active"), + ); + if (!resource?.path_key) { + throw new Error("scoped-memory resource storage root is unavailable"); + } + const artifactPath = resolveBuiltinScopedMemoryArtifactPath({ + databasePath, + pathKey: resource.path_key, + artifactLocator, + }); + writeImmutableArtifact({ artifactPath, content }); + try { + let output: BuiltinScopedMemoryRevision | undefined; + runSqliteImmediateTransactionSync(database, () => { + const current = executeSqliteQueryTakeFirstSync( + database, + db + .selectFrom("memory_resources as resource") + .innerJoin("memory_stores as store", "store.store_id", "resource.store_id") + .innerJoin("memory_policies as policy", "policy.policy_id", "store.policy_id") + .innerJoin( + "memory_policy_revisions as policy_revision", + "policy_revision.revision_id", + "policy.current_revision_id", + ) + .select([ + "resource.resource_id", + "policy.current_revision_id", + "policy.revocation_epoch", + "policy_revision.revision_number", + ]) + .where("resource.resource_id", "=", params.resourceId) + .where("resource.agent_id", "=", params.agentId) + .where("store.lifecycle_state", "=", "active") + .where("policy.lifecycle_state", "=", "active") + .where("policy_revision.lifecycle_state", "=", "active"), + ); + if (!current) { + throw new Error("scoped-memory resource policy is unavailable"); + } + const previous = executeSqliteQueryTakeFirstSync( + database, + db + .selectFrom("memory_resource_revisions") + .select("revision_number") + .where("resource_id", "=", params.resourceId) + .orderBy("revision_number", "desc") + .limit(1), + ); + if (params.lifecycleState === "active") { + executeSqliteQuerySync( + database, + db + .updateTable("memory_resource_revisions") + .set({ lifecycle_state: "tombstoned", retired_at: params.nowMs }) + .where("resource_id", "=", params.resourceId) + .where("lifecycle_state", "=", "active"), + ); + } + const revisionNumber = (previous?.revision_number ?? 0) + 1; + const hash = contentHash(content); + executeSqliteQuerySync( + database, + db.insertInto("memory_resource_revisions").values({ + revision_id: revisionId, + resource_id: params.resourceId, + revision_number: revisionNumber, + artifact_locator: artifactLocator, + content_hash: hash, + content_bytes: Buffer.byteLength(content), + policy_revision_id: current.current_revision_id, + policy_revocation_epoch: current.revocation_epoch, + source_policy_set_id: createScopedMemorySourcePolicySetId(current.current_revision_id), + lifecycle_state: params.lifecycleState, + actor_kind: params.actor.kind, + actor_id: params.actor.id ?? null, + expires_at: params.expiresAt, + created_at: params.nowMs, + activated_at: params.lifecycleState === "active" ? params.nowMs : null, + retired_at: null, + }), + ); + const chunks = chunkScopedMemoryMarkdown(content); + if (chunks.length > 0) { + executeSqliteQuerySync( + database, + db.insertInto("memory_scoped_chunks").values( + chunks.map((chunk) => ({ + chunk_id: randomUUID(), + revision_id: revisionId, + chunk_ordinal: chunk.ordinal, + start_line: chunk.startLine, + end_line: chunk.endLine, + text: chunk.text, + content_hash: hash, + model: "fts-only", + updated_at: params.nowMs, + })), + ), + ); + } + output = Object.freeze({ + resourceId: params.resourceId, + revisionId, + policyRevisionId: current.current_revision_id, + policyRevocationEpoch: current.revocation_epoch, + sourcePolicySetId: createScopedMemorySourcePolicySetId(current.current_revision_id), + artifactLocator, + }); + }); + if (!output) { + throw new Error("scoped-memory revision was not created"); + } + return output; + } catch (error) { + removeArtifact(artifactPath); + throw error; + } + }); +} + +/** Create the stable resource and first immutable revision under its store policy. */ +export function createBuiltinScopedMemoryResource(params: { + agentId: string; + store: BuiltinScopedMemoryStore; + logicalLocator: string; + content: string; + lifecycleState?: Exclude; + expiresAt?: number; + actor: ScopedMemoryActor; + nowMs?: number; +}): BuiltinScopedMemoryRevision { + const agentId = normalizeAgentId(params.agentId); + const logicalLocator = normalizeLogicalLocator(params.logicalLocator); + const nowMs = params.nowMs ?? Date.now(); + const resourceId = randomUUID(); + return withScopedMemoryDatabase(agentId, (database) => { + const db = getNodeSqliteKysely(database); + runSqliteImmediateTransactionSync(database, () => { + const store = executeSqliteQueryTakeFirstSync( + database, + db + .selectFrom("memory_stores") + .select("store_id") + .where("store_id", "=", params.store.storeId) + .where("agent_id", "=", agentId) + .where("lifecycle_state", "=", "active"), + ); + if (!store) { + throw new Error("scoped-memory store is unavailable"); + } + executeSqliteQuerySync( + database, + db.insertInto("memory_resources").values({ + resource_id: resourceId, + agent_id: agentId, + store_id: store.store_id, + logical_locator: logicalLocator, + source: "memory", + created_at: nowMs, + }), + ); + }); + return createRevision({ + agentId, + resourceId, + content: params.content, + lifecycleState: params.lifecycleState ?? "active", + expiresAt: params.expiresAt ?? null, + actor: params.actor, + nowMs, + }); + }); +} + +/** Add a later immutable revision; only one active revision can exist per resource. */ +export function createBuiltinScopedMemoryResourceRevision(params: { + agentId: string; + resourceId: string; + content: string; + lifecycleState?: Exclude; + expiresAt?: number; + actor: ScopedMemoryActor; + nowMs?: number; +}): BuiltinScopedMemoryRevision { + return createRevision({ + agentId: normalizeAgentId(params.agentId), + resourceId: normalizeScopedMemoryRequiredText(params.resourceId, "resourceId"), + content: params.content, + lifecycleState: params.lifecycleState ?? "active", + expiresAt: params.expiresAt ?? null, + actor: params.actor, + nowMs: params.nowMs ?? Date.now(), + }); +} + +/** Quarantine or tombstone a revision without mutating its immutable evidence. */ +export function setBuiltinScopedMemoryRevisionLifecycle(params: { + agentId: string; + revisionId: string; + lifecycleState: "quarantined" | "tombstoned"; + nowMs?: number; +}): void { + const agentId = normalizeAgentId(params.agentId); + const revisionId = normalizeScopedMemoryRequiredText(params.revisionId, "revisionId"); + const nowMs = params.nowMs ?? Date.now(); + withScopedMemoryDatabase(agentId, (database) => { + const db = getNodeSqliteKysely(database); + const updated = executeSqliteQuerySync( + database, + db + .updateTable("memory_resource_revisions as revision") + .set({ lifecycle_state: params.lifecycleState, retired_at: nowMs }) + .where("revision.revision_id", "=", revisionId) + .where( + "revision.resource_id", + "in", + db.selectFrom("memory_resources").select("resource_id").where("agent_id", "=", agentId), + ) + .where("revision.lifecycle_state", "in", ["pending", "active", "quarantined"]), + ); + if (updated.numAffectedRows !== 1n) { + throw new Error("invalid scoped-memory revision lifecycle transition"); + } + }); +} diff --git a/extensions/memory-core/src/memory/scoped-memory-store.test.ts b/extensions/memory-core/src/memory/scoped-memory-store.test.ts new file mode 100644 index 000000000000..a3cc09826062 --- /dev/null +++ b/extensions/memory-core/src/memory/scoped-memory-store.test.ts @@ -0,0 +1,471 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { closeOpenClawAgentDatabasesForTest } from "openclaw/plugin-sdk/sqlite-runtime-testing"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + readScopedMemoryFtsCandidatePage, + readScopedMemorySqliteVecCandidatePage, + readScopedMemoryVectorCandidatePage, +} from "./scoped-memory-candidates.js"; +import { withScopedMemoryDatabase } from "./scoped-memory-db.js"; +import { + createBuiltinScopedMemoryResource, + createBuiltinScopedMemoryResourceRevision, + readBuiltinScopedMemoryRevisionSnapshot, + resolveBuiltinScopedMemoryArtifactPath, + setBuiltinScopedMemoryRevisionLifecycle, +} from "./scoped-memory-resources.js"; +import { + createBuiltinScopedMemoryStore, + createOpaqueScopedMemoryDirectory, + reviseBuiltinScopedMemoryPolicy, +} from "./scoped-memory-store.js"; + +describe("builtin scoped memory store", () => { + let stateDir = ""; + + beforeEach(() => { + stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-scoped-memory-store-")); + vi.stubEnv("OPENCLAW_STATE_DIR", stateDir); + }); + + afterEach(() => { + closeOpenClawAgentDatabasesForTest(); + vi.unstubAllEnvs(); + fs.rmSync(stateDir, { recursive: true, force: true }); + }); + + function storeFor(agentId = "main", audienceId = "principal-owner") { + return createBuiltinScopedMemoryStore({ + agentId, + scopeKind: "user", + audienceKind: "user", + audienceId, + authorityKind: "user", + authorityOwnerId: audienceId, + defaultCapabilities: ["retrieve", "read"], + actor: { kind: "human", id: audienceId }, + reason: "test placement", + nowMs: 1_000, + }); + } + + it("retries opaque directory collisions and rejects traversal", () => { + const baseDir = path.join(stateDir, "opaque"); + const first = `s1_${"a".repeat(32)}`; + const second = `s1_${"b".repeat(32)}`; + fs.mkdirSync(path.join(baseDir, first), { recursive: true }); + const generated = [first, second]; + + const allocated = createOpaqueScopedMemoryDirectory(baseDir, { + generatePathKey: () => generated.shift() ?? second, + }); + + expect(allocated.pathKey).toBe(second); + expect(fs.statSync(allocated.directoryPath).isDirectory()).toBe(true); + expect(() => + createOpaqueScopedMemoryDirectory(baseDir, { generatePathKey: () => "../principal-owner" }), + ).toThrow("path key is invalid"); + expect(() => + resolveBuiltinScopedMemoryArtifactPath({ + databasePath: path.join(stateDir, "agent.sqlite"), + pathKey: second, + artifactLocator: "../private.md", + }), + ).toThrow("artifact locator is invalid"); + }); + + it("stores opaque roots, immutable resource revisions, and scoped chunks only", () => { + const store = storeFor(); + const first = createBuiltinScopedMemoryResource({ + agentId: "main", + store, + logicalLocator: "MEMORY.md", + content: "first immutable revision", + actor: { kind: "human", id: "principal-owner" }, + nowMs: 2_000, + }); + const second = createBuiltinScopedMemoryResourceRevision({ + agentId: "main", + resourceId: first.resourceId, + content: "second immutable revision", + actor: { kind: "human", id: "principal-owner" }, + nowMs: 3_000, + }); + + withScopedMemoryDatabase("main", (database, databasePath) => { + const root = database + .prepare("SELECT path_key, authority_owner_id FROM memory_storage_roots") + .get() as { path_key: string; authority_owner_id: string }; + expect(root.path_key).not.toContain("principal-owner"); + expect(databasePath).not.toContain("principal-owner"); + expect( + database + .prepare( + "SELECT revision_id, revision_number, lifecycle_state FROM memory_resource_revisions ORDER BY revision_number", + ) + .all(), + ).toEqual([ + { revision_id: first.revisionId, revision_number: 1, lifecycle_state: "tombstoned" }, + { revision_id: second.revisionId, revision_number: 2, lifecycle_state: "active" }, + ]); + expect(() => + database + .prepare("UPDATE memory_resource_revisions SET content_hash = ? WHERE revision_id = ?") + .run("rewritten", second.revisionId), + ).toThrow("immutable"); + expect(database.prepare("SELECT * FROM memory_index_sources").all()).toEqual([]); + expect(database.prepare("SELECT * FROM memory_index_chunks").all()).toEqual([]); + expect(database.prepare("SELECT * FROM memory_scoped_chunks").all()).toHaveLength(2); + expect(root.authority_owner_id).toBe("principal-owner"); + }); + }); + + it("never writes scoped resources or chunks into legacy index, FTS, or vector tables", () => { + const store = storeFor(); + const legacy = withScopedMemoryDatabase("main", (database) => { + database.exec(` + CREATE VIRTUAL TABLE IF NOT EXISTS memory_index_chunks_fts USING fts5(text); + CREATE VIRTUAL TABLE IF NOT EXISTS memory_index_paths_fts USING fts5(path); + CREATE TABLE IF NOT EXISTS memory_index_chunks_vec (id TEXT PRIMARY KEY, embedding BLOB); + `); + database + .prepare( + "INSERT INTO memory_index_sources(path, source, hash, mtime, size) VALUES (?, ?, ?, ?, ?)", + ) + .run("legacy.md", "memory", "legacy-source", 1, 1); + database + .prepare( + "INSERT INTO memory_index_chunks(id, path, source, start_line, end_line, hash, model, text, embedding, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ) + .run("legacy-chunk", "legacy.md", "memory", 1, 1, "legacy", "legacy", "legacy", "[]", 1); + database.prepare("INSERT INTO memory_index_chunks_fts(text) VALUES (?)").run("legacy fts"); + database.prepare("INSERT INTO memory_index_paths_fts(path) VALUES (?)").run("legacy.md"); + database + .prepare("INSERT INTO memory_index_chunks_vec(id, embedding) VALUES (?, ?)") + .run("legacy-vector", "legacy"); + return { + sources: database.prepare("SELECT * FROM memory_index_sources").all(), + chunks: database.prepare("SELECT * FROM memory_index_chunks").all(), + chunkFts: database.prepare("SELECT * FROM memory_index_chunks_fts").all(), + pathFts: database.prepare("SELECT * FROM memory_index_paths_fts").all(), + vectors: database.prepare("SELECT * FROM memory_index_chunks_vec").all(), + }; + }); + + createBuiltinScopedMemoryResource({ + agentId: "main", + store, + logicalLocator: "private.md", + content: "scoped only", + actor: { kind: "human", id: "principal-owner" }, + }); + + withScopedMemoryDatabase("main", (database) => { + expect(database.prepare("SELECT * FROM memory_index_sources").all()).toEqual(legacy.sources); + expect(database.prepare("SELECT * FROM memory_index_chunks").all()).toEqual(legacy.chunks); + expect(database.prepare("SELECT * FROM memory_index_chunks_fts").all()).toEqual( + legacy.chunkFts, + ); + expect(database.prepare("SELECT * FROM memory_index_paths_fts").all()).toEqual( + legacy.pathFts, + ); + expect(database.prepare("SELECT * FROM memory_index_chunks_vec").all()).toEqual( + legacy.vectors, + ); + }); + }); + + it("returns FTS candidate identifiers only from the requested scoped store", () => { + const owner = storeFor("main", "owner"); + const other = storeFor("main", "other"); + const ownerRevision = createBuiltinScopedMemoryResource({ + agentId: "main", + store: owner, + logicalLocator: "MEMORY.md", + content: "owner-only recall needle", + actor: { kind: "human", id: "owner" }, + }); + createBuiltinScopedMemoryResource({ + agentId: "main", + store: other, + logicalLocator: "MEMORY.md", + content: "other-only recall needle", + actor: { kind: "human", id: "other" }, + }); + + withScopedMemoryDatabase("main", (database) => { + const candidates = readScopedMemoryFtsCandidatePage({ + database, + query: "recall needle", + storeIds: [owner.storeId], + sources: ["memory"], + limit: 10, + offset: 0, + }); + expect(candidates).toHaveLength(1); + expect(candidates[0]?.revisionId).toBe(ownerRevision.revisionId); + }); + }); + + it("does not prefilter pending, quarantined, tombstoned, or expired revisions as candidates", () => { + const store = storeFor(); + const active = createBuiltinScopedMemoryResource({ + agentId: "main", + store, + logicalLocator: "active.md", + content: "candidate lifecycle sentinel", + actor: { kind: "human", id: "principal-owner" }, + nowMs: 2_000, + }); + const pending = createBuiltinScopedMemoryResource({ + agentId: "main", + store, + logicalLocator: "pending.md", + content: "candidate lifecycle sentinel", + lifecycleState: "pending", + actor: { kind: "human", id: "principal-owner" }, + nowMs: 2_000, + }); + const expired = createBuiltinScopedMemoryResource({ + agentId: "main", + store, + logicalLocator: "expired.md", + content: "candidate lifecycle sentinel", + expiresAt: 2_500, + actor: { kind: "human", id: "principal-owner" }, + nowMs: 2_000, + }); + const tombstoned = createBuiltinScopedMemoryResource({ + agentId: "main", + store, + logicalLocator: "tombstoned.md", + content: "candidate lifecycle sentinel", + actor: { kind: "human", id: "principal-owner" }, + nowMs: 2_000, + }); + setBuiltinScopedMemoryRevisionLifecycle({ + agentId: "main", + revisionId: tombstoned.revisionId, + lifecycleState: "tombstoned", + nowMs: 2_100, + }); + + withScopedMemoryDatabase("main", (database) => { + for (const revisionId of [ + active.revisionId, + pending.revisionId, + expired.revisionId, + tombstoned.revisionId, + ]) { + const chunk = database + .prepare("SELECT chunk_id FROM memory_scoped_chunks WHERE revision_id = ?") + .get(revisionId) as { chunk_id: string }; + database + .prepare( + "INSERT INTO memory_scoped_chunk_vectors(chunk_id, model, dims, embedding, updated_at) VALUES (?, ?, ?, ?, ?)", + ) + .run(chunk.chunk_id, "fixture", 2, "[1,0]", 3_000); + } + const candidates = readScopedMemoryFtsCandidatePage({ + database, + query: "candidate lifecycle", + storeIds: [store.storeId], + sources: ["memory"], + limit: 10, + offset: 0, + nowMs: 3_000, + }); + expect(candidates.map((candidate) => candidate.revisionId)).toEqual([active.revisionId]); + expect(candidates.map((candidate) => candidate.revisionId)).not.toContain(pending.revisionId); + expect(candidates.map((candidate) => candidate.revisionId)).not.toContain(expired.revisionId); + + const params = { + database, + query: "ignored", + queryVector: [1, 0], + storeIds: [store.storeId], + sources: ["memory"] as const, + limit: 10, + offset: 0, + nowMs: 3_000, + }; + expect( + readScopedMemoryVectorCandidatePage(params).map((candidate) => candidate.revisionId), + ).toEqual([active.revisionId]); + // No sqlite-vec table is installed in this fixture, so the same scoped scan is the exact + // fallback path rather than a broader legacy search. + expect( + readScopedMemorySqliteVecCandidatePage(params).map((candidate) => candidate.revisionId), + ).toEqual([active.revisionId]); + }); + }); + + it("rejects inactive, expired, stale-policy, and stale-hash exact revision reads", () => { + const store = storeFor(); + const revision = createBuiltinScopedMemoryResource({ + agentId: "main", + store, + logicalLocator: "exact.md", + content: "exact read sentinel", + actor: { kind: "human", id: "principal-owner" }, + nowMs: 2_000, + }); + const read = () => + readBuiltinScopedMemoryRevisionSnapshot({ + agentId: "main", + storeIds: [store.storeId], + revisionId: revision.revisionId, + nowMs: 3_000, + }); + + expect(read()).toMatchObject({ content: "exact read sentinel" }); + expect( + readBuiltinScopedMemoryRevisionSnapshot({ + agentId: "main", + storeIds: ["unrelated-store"], + revisionId: revision.revisionId, + nowMs: 3_000, + }), + ).toBeUndefined(); + + withScopedMemoryDatabase("main", (database, databasePath) => { + const artifact = database + .prepare("SELECT artifact_locator FROM memory_resource_revisions WHERE revision_id = ?") + .get(revision.revisionId) as { artifact_locator: string }; + const pathKey = database + .prepare("SELECT path_key FROM memory_storage_roots WHERE storage_root_id = ?") + .get(store.storageRootId) as { path_key: string }; + const artifactPath = resolveBuiltinScopedMemoryArtifactPath({ + databasePath, + pathKey: pathKey.path_key, + artifactLocator: artifact.artifact_locator, + }); + fs.writeFileSync(artifactPath, "tampered bytes"); + }); + expect(read()).toBeUndefined(); + + const clean = createBuiltinScopedMemoryResourceRevision({ + agentId: "main", + resourceId: revision.resourceId, + content: "clean exact read sentinel", + actor: { kind: "human", id: "principal-owner" }, + nowMs: 4_000, + }); + expect( + readBuiltinScopedMemoryRevisionSnapshot({ + agentId: "main", + storeIds: [store.storeId], + revisionId: clean.revisionId, + nowMs: 5_000, + }), + ).toMatchObject({ content: "clean exact read sentinel" }); + reviseBuiltinScopedMemoryPolicy({ + agentId: "main", + policyId: store.policyId, + entries: [], + actor: { kind: "human", id: "principal-owner" }, + reason: "invalidate old resource policy", + nowMs: 5_500, + }); + expect( + readBuiltinScopedMemoryRevisionSnapshot({ + agentId: "main", + storeIds: [store.storeId], + revisionId: clean.revisionId, + nowMs: 5_500, + }), + ).toBeUndefined(); + + const expired = createBuiltinScopedMemoryResource({ + agentId: "main", + store, + logicalLocator: "expired-exact.md", + content: "expired exact read sentinel", + expiresAt: 6_000, + actor: { kind: "human", id: "principal-owner" }, + nowMs: 5_600, + }); + expect( + readBuiltinScopedMemoryRevisionSnapshot({ + agentId: "main", + storeIds: [store.storeId], + revisionId: expired.revisionId, + nowMs: 6_000, + }), + ).toBeUndefined(); + setBuiltinScopedMemoryRevisionLifecycle({ + agentId: "main", + revisionId: clean.revisionId, + lifecycleState: "quarantined", + nowMs: 6_000, + }); + expect( + readBuiltinScopedMemoryRevisionSnapshot({ + agentId: "main", + storeIds: [store.storeId], + revisionId: clean.revisionId, + nowMs: 6_000, + }), + ).toBeUndefined(); + }); + + it("rejects direct private user-to-user allows during store and policy creation", () => { + expect(() => + createBuiltinScopedMemoryStore({ + agentId: "main", + scopeKind: "user", + audienceKind: "user", + audienceId: "bob", + authorityKind: "user", + authorityOwnerId: "alice", + defaultCapabilities: ["retrieve"], + actor: { kind: "human", id: "alice" }, + reason: "private publish", + }), + ).toThrow("private user scoped memory"); + + const store = storeFor(); + expect(() => + reviseBuiltinScopedMemoryPolicy({ + agentId: "main", + policyId: store.policyId, + entries: [ + { + effect: "allow", + principalId: "bob", + operation: "read", + grantorPrincipalId: "principal-owner", + reason: "private publish", + }, + ], + actor: { kind: "human", id: "principal-owner" }, + reason: "private publish", + }), + ).toThrow("direct private user-to-user"); + }); + + it("never revives a terminal revision", () => { + const store = storeFor(); + const revision = createBuiltinScopedMemoryResource({ + agentId: "main", + store, + logicalLocator: "MEMORY.md", + content: "temporary text", + actor: { kind: "human", id: "principal-owner" }, + }); + setBuiltinScopedMemoryRevisionLifecycle({ + agentId: "main", + revisionId: revision.revisionId, + lifecycleState: "tombstoned", + }); + expect(() => + setBuiltinScopedMemoryRevisionLifecycle({ + agentId: "main", + revisionId: revision.revisionId, + lifecycleState: "quarantined", + }), + ).toThrow("invalid scoped-memory revision lifecycle transition"); + }); +}); diff --git a/extensions/memory-core/src/memory/scoped-memory-store.ts b/extensions/memory-core/src/memory/scoped-memory-store.ts new file mode 100644 index 000000000000..66c0f0155ad4 --- /dev/null +++ b/extensions/memory-core/src/memory/scoped-memory-store.ts @@ -0,0 +1,430 @@ +import { createHash, randomBytes, randomUUID } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import type { MemoryOperation } from "openclaw/plugin-sdk/memory-authorization"; +import { normalizeAgentId } from "openclaw/plugin-sdk/routing"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, + runSqliteImmediateTransactionSync, +} from "openclaw/plugin-sdk/sqlite-runtime"; +import { + resolveScopedMemoryArtifactBase, + type MemoryPolicyEntryRow, + type ScopedMemoryActorKind, + type ScopedMemoryDatabase, + type ScopedMemoryScopeKind, + withScopedMemoryDatabase, +} from "./scoped-memory-db.js"; + +const OPAQUE_PATH_KEY_VERSION = 1; +const OPAQUE_PATH_ATTEMPTS = 8; +const OPAQUE_PATH_KEY_PATTERN = /^s1_[A-Za-z0-9_-]{24,}$/u; + +export type ScopedMemoryActor = Readonly<{ kind: ScopedMemoryActorKind; id?: string }>; +export type ScopedMemoryPolicyEntryInput = Readonly<{ + kind?: MemoryPolicyEntryRow["entry_kind"]; + effect: MemoryPolicyEntryRow["effect"]; + principalId: string; + audienceKind?: MemoryPolicyEntryRow["audience_kind"]; + audienceId?: string; + operation: MemoryOperation; + grantorPrincipalId: string; + reason: string; + expiresAt?: number; +}>; + +export type BuiltinScopedMemoryStore = Readonly<{ + storageRootId: string; + storeId: string; + policyId: string; + policyRevisionId: string; + policyRevocationEpoch: number; + sourcePolicySetId: string; +}>; + +type OpaqueDirectoryDependencies = Readonly<{ + generatePathKey?: () => string; + mkdir?: typeof fs.mkdirSync; +}>; + +function hashText(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} + +function createOpaquePathKey(): string { + return `s1_${randomBytes(24).toString("base64url")}`; +} + +function assertOpaquePathKey(pathKey: string): void { + if (!OPAQUE_PATH_KEY_PATTERN.test(pathKey)) { + throw new Error("generated scoped-memory path key is invalid"); + } +} + +function resolveChildPath(base: string, child: string): string { + const resolvedBase = path.resolve(base); + const resolved = path.resolve(resolvedBase, child); + if (path.dirname(resolved) !== resolvedBase || path.basename(resolved) !== child) { + throw new Error("scoped-memory locator escaped its storage root"); + } + return resolved; +} + +/** Create one exclusive CSPRNG directory; collisions retry with a fresh opaque key. */ +export function createOpaqueScopedMemoryDirectory( + baseDir: string, + dependencies: OpaqueDirectoryDependencies = {}, +): { directoryPath: string; pathKey: string } { + const mkdir = dependencies.mkdir ?? fs.mkdirSync; + mkdir(baseDir, { recursive: true, mode: 0o700 }); + for (let attempt = 0; attempt < OPAQUE_PATH_ATTEMPTS; attempt += 1) { + const pathKey = dependencies.generatePathKey?.() ?? createOpaquePathKey(); + assertOpaquePathKey(pathKey); + const directoryPath = resolveChildPath(baseDir, pathKey); + try { + mkdir(directoryPath, { recursive: false, mode: 0o700 }); + return { directoryPath, pathKey }; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") { + throw error; + } + } + } + throw new Error("could not allocate an opaque scoped-memory directory"); +} + +export function normalizeScopedMemoryRequiredText(value: string, label: string): string { + const normalized = value.trim(); + if (!normalized || normalized.includes("\0")) { + throw new Error(`${label} is required`); + } + return normalized; +} + +function normalizeCapabilities(capabilities: readonly MemoryOperation[]): MemoryOperation[] { + return [...new Set(capabilities)].toSorted(); +} + +export function createScopedMemorySourcePolicySetId(policyRevisionId: string): string { + return `mps1_${hashText(`v1\0${policyRevisionId}`)}`; +} + +function removeEmptyDirectory(directoryPath: string): void { + try { + fs.rmdirSync(directoryPath); + } catch {} +} + +function assertNoDirectPrivateUserPublish(params: { + scopeKind: ScopedMemoryScopeKind; + audienceKind: ScopedMemoryScopeKind; + audienceId: string; + authorityKind: ScopedMemoryScopeKind; + authorityOwnerId: string; + entries: readonly ScopedMemoryPolicyEntryInput[]; +}): void { + // Private user stores are self-owned in the first product. Allowing a different user here + // would turn policy mutation into unreviewed Alice-to-Bob private-memory publication. + if (params.scopeKind === "user") { + if ( + params.audienceKind !== "user" || + params.authorityKind !== "user" || + params.audienceId !== params.authorityOwnerId + ) { + throw new Error("private user scoped memory must be owned and addressed by the same user"); + } + if ( + params.entries.some( + (entry) => entry.effect === "allow" && entry.principalId !== params.authorityOwnerId, + ) + ) { + throw new Error("direct private user-to-user memory publishing is unavailable"); + } + } +} + +function normalizeEntries(params: { + entries: readonly ScopedMemoryPolicyEntryInput[]; + policyRevisionId: string; + defaultAudienceKind: ScopedMemoryScopeKind | "*"; + defaultAudienceId: string; + nowMs: number; +}) { + return params.entries.map((entry) => ({ + entry_id: randomUUID(), + policy_revision_id: params.policyRevisionId, + entry_kind: entry.kind ?? "exception", + effect: entry.effect, + principal_id: normalizeScopedMemoryRequiredText(entry.principalId, "policy principalId"), + audience_kind: entry.audienceKind ?? params.defaultAudienceKind, + audience_id: normalizeScopedMemoryRequiredText( + entry.audienceId ?? params.defaultAudienceId, + "policy audienceId", + ), + operation: entry.operation, + grantor_principal_id: normalizeScopedMemoryRequiredText( + entry.grantorPrincipalId, + "policy grantorPrincipalId", + ), + reason: normalizeScopedMemoryRequiredText(entry.reason, "policy reason"), + expires_at: entry.expiresAt ?? null, + created_at: params.nowMs, + })); +} + +/** Register one builtin logical store without putting authority identities in its path. */ +export function createBuiltinScopedMemoryStore(params: { + agentId: string; + scopeKind: ScopedMemoryScopeKind; + audienceKind: ScopedMemoryScopeKind; + audienceId: string; + authorityKind: ScopedMemoryScopeKind; + authorityOwnerId: string; + defaultCapabilities: readonly MemoryOperation[]; + policyEntries?: readonly ScopedMemoryPolicyEntryInput[]; + actor: ScopedMemoryActor; + reason: string; + nowMs?: number; +}): BuiltinScopedMemoryStore { + const agentId = normalizeAgentId(params.agentId); + const audienceId = normalizeScopedMemoryRequiredText(params.audienceId, "audienceId"); + const authorityOwnerId = normalizeScopedMemoryRequiredText( + params.authorityOwnerId, + "authorityOwnerId", + ); + const entries = params.policyEntries ?? []; + assertNoDirectPrivateUserPublish({ + scopeKind: params.scopeKind, + audienceKind: params.audienceKind, + audienceId, + authorityKind: params.authorityKind, + authorityOwnerId, + entries, + }); + const reason = normalizeScopedMemoryRequiredText(params.reason, "reason"); + const nowMs = params.nowMs ?? Date.now(); + const storageRootId = randomUUID(); + const storeId = randomUUID(); + const policyId = randomUUID(); + const policyRevisionId = randomUUID(); + const policyRevocationEpoch = 0; + const policySetId = createScopedMemorySourcePolicySetId(policyRevisionId); + + return withScopedMemoryDatabase(agentId, (database, databasePath) => { + const allocated = createOpaqueScopedMemoryDirectory( + resolveScopedMemoryArtifactBase(databasePath), + ); + const db = getNodeSqliteKysely(database); + try { + runSqliteImmediateTransactionSync(database, () => { + executeSqliteQuerySync( + database, + db.insertInto("memory_storage_roots").values({ + storage_root_id: storageRootId, + agent_id: agentId, + backend_kind: "builtin", + opaque_locator: `builtin:v1:${allocated.pathKey}`, + path_key_version: OPAQUE_PATH_KEY_VERSION, + path_key: allocated.pathKey, + authority_kind: params.authorityKind, + authority_owner_id: authorityOwnerId, + default_capabilities_json: JSON.stringify( + normalizeCapabilities(params.defaultCapabilities), + ), + lifecycle_state: "active", + created_at: nowMs, + updated_at: nowMs, + }), + ); + executeSqliteQuerySync( + database, + db.insertInto("memory_policies").values({ + policy_id: policyId, + agent_id: agentId, + current_revision_id: policyRevisionId, + revocation_epoch: policyRevocationEpoch, + lifecycle_state: "active", + created_at: nowMs, + updated_at: nowMs, + }), + ); + executeSqliteQuerySync( + database, + db.insertInto("memory_policy_revisions").values({ + revision_id: policyRevisionId, + policy_id: policyId, + revision_number: 1, + revocation_epoch: policyRevocationEpoch, + lifecycle_state: "active", + actor_kind: params.actor.kind, + actor_id: params.actor.id ?? null, + reason, + created_at: nowMs, + }), + ); + const rows = normalizeEntries({ + entries, + policyRevisionId, + defaultAudienceKind: params.audienceKind, + defaultAudienceId: audienceId, + nowMs, + }); + if (rows.length > 0) { + executeSqliteQuerySync(database, db.insertInto("memory_policy_entries").values(rows)); + } + executeSqliteQuerySync( + database, + db.insertInto("memory_stores").values({ + store_id: storeId, + agent_id: agentId, + storage_root_id: storageRootId, + policy_id: policyId, + scope_kind: params.scopeKind, + audience_kind: params.audienceKind, + audience_id: audienceId, + lifecycle_state: "active", + created_at: nowMs, + updated_at: nowMs, + }), + ); + }); + } catch (error) { + removeEmptyDirectory(allocated.directoryPath); + throw error; + } + return Object.freeze({ + storageRootId, + storeId, + policyId, + policyRevisionId, + policyRevocationEpoch, + sourcePolicySetId: policySetId, + }); + }); +} + +/** Replace one policy with a new immutable revision and revocation epoch. */ +export function reviseBuiltinScopedMemoryPolicy(params: { + agentId: string; + policyId: string; + entries: readonly ScopedMemoryPolicyEntryInput[]; + actor: ScopedMemoryActor; + reason: string; + nowMs?: number; +}): { + policyId: string; + policyRevisionId: string; + policyRevocationEpoch: number; + sourcePolicySetId: string; +} { + const agentId = normalizeAgentId(params.agentId); + const policyId = normalizeScopedMemoryRequiredText(params.policyId, "policyId"); + const reason = normalizeScopedMemoryRequiredText(params.reason, "reason"); + const policyRevisionId = randomUUID(); + const nowMs = params.nowMs ?? Date.now(); + return withScopedMemoryDatabase(agentId, (database) => { + const db = getNodeSqliteKysely(database); + let policyRevocationEpoch = 0; + runSqliteImmediateTransactionSync(database, () => { + const current = executeSqliteQueryTakeFirstSync( + database, + db + .selectFrom("memory_policies as policy") + .innerJoin( + "memory_policy_revisions as revision", + "revision.revision_id", + "policy.current_revision_id", + ) + .innerJoin("memory_stores as store", "store.policy_id", "policy.policy_id") + .innerJoin( + "memory_storage_roots as root", + "root.storage_root_id", + "store.storage_root_id", + ) + .select([ + "policy.current_revision_id", + "policy.revocation_epoch", + "revision.revision_number", + "store.scope_kind", + "store.audience_kind", + "store.audience_id", + "root.authority_kind", + "root.authority_owner_id", + ]) + .where("policy.policy_id", "=", policyId) + .where("policy.agent_id", "=", agentId) + .where("policy.lifecycle_state", "=", "active") + .where("revision.lifecycle_state", "=", "active") + .where("store.lifecycle_state", "=", "active") + .where("root.lifecycle_state", "=", "active"), + ); + if (!current) { + throw new Error("active scoped-memory policy is unavailable"); + } + assertNoDirectPrivateUserPublish({ + scopeKind: current.scope_kind, + audienceKind: current.audience_kind, + audienceId: current.audience_id, + authorityKind: current.authority_kind, + authorityOwnerId: current.authority_owner_id, + entries: params.entries, + }); + policyRevocationEpoch = current.revocation_epoch + 1; + executeSqliteQuerySync( + database, + db + .updateTable("memory_policy_revisions") + .set({ lifecycle_state: "superseded" }) + .where("revision_id", "=", current.current_revision_id) + .where("lifecycle_state", "=", "active"), + ); + executeSqliteQuerySync( + database, + db.insertInto("memory_policy_revisions").values({ + revision_id: policyRevisionId, + policy_id: policyId, + revision_number: current.revision_number + 1, + revocation_epoch: policyRevocationEpoch, + lifecycle_state: "active", + actor_kind: params.actor.kind, + actor_id: params.actor.id ?? null, + reason, + created_at: nowMs, + }), + ); + const rows = normalizeEntries({ + entries: params.entries, + policyRevisionId, + defaultAudienceKind: current.audience_kind, + defaultAudienceId: current.audience_id, + nowMs, + }); + if (rows.length > 0) { + executeSqliteQuerySync(database, db.insertInto("memory_policy_entries").values(rows)); + } + const updated = executeSqliteQuerySync( + database, + db + .updateTable("memory_policies") + .set({ + current_revision_id: policyRevisionId, + revocation_epoch: policyRevocationEpoch, + updated_at: nowMs, + }) + .where("policy_id", "=", policyId) + .where("current_revision_id", "=", current.current_revision_id), + ); + if (updated.numAffectedRows !== 1n) { + throw new Error("scoped-memory policy changed during revision"); + } + }); + return Object.freeze({ + policyId, + policyRevisionId, + policyRevocationEpoch, + sourcePolicySetId: createScopedMemorySourcePolicySetId(policyRevisionId), + }); + }); +} diff --git a/extensions/memory-core/src/migration/doctor-scoped-memory-preview.test.ts b/extensions/memory-core/src/migration/doctor-scoped-memory-preview.test.ts new file mode 100644 index 000000000000..670f78f4ba95 --- /dev/null +++ b/extensions/memory-core/src/migration/doctor-scoped-memory-preview.test.ts @@ -0,0 +1,109 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; +import type { PluginDoctorStateMigrationContext } from "openclaw/plugin-sdk/runtime-doctor-migrations"; +import { afterEach, describe, expect, it } from "vitest"; +import { scopedMemoryMigrationPreview } from "./doctor-scoped-memory-preview.js"; + +describe("scoped memory doctor dry-run", () => { + const roots = new Set(); + + afterEach(() => { + for (const root of roots) { + fs.rmSync(root, { recursive: true, force: true }); + } + roots.clear(); + }); + + function root(): string { + const value = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-scoped-memory-doctor-")); + roots.add(value); + return value; + } + + function params( + config: OpenClawConfig, + stateDir: string, + ): Parameters[0] { + return { + config, + stateDir, + oauthDir: path.join(stateDir, "oauth"), + env: {}, + context: {} as PluginDoctorStateMigrationContext, + }; + } + + it("is deterministic, redacted, and leaves legacy sources untouched", async () => { + const fixture = root(); + const stateDir = path.join(fixture, "state"); + const workspace = path.join(fixture, "workspace"); + const sessions = path.join(stateDir, "agents", "main", "sessions"); + fs.mkdirSync(path.join(workspace, "memory"), { recursive: true }); + fs.mkdirSync(sessions, { recursive: true }); + fs.writeFileSync(path.join(workspace, "MEMORY.md"), "private curated content"); + fs.writeFileSync(path.join(workspace, "memory", "private-memory.md"), "private memory content"); + fs.writeFileSync(path.join(sessions, "turn.jsonl"), '{"content":"private transcript"}\n'); + fs.writeFileSync(path.join(sessions, "sessions.json"), '{"private":"metadata"}\n'); + const config = { + session: { dmScope: "main" }, + agents: { list: [{ id: "main", workspace, sandbox: { mode: "all" } }] }, + } as OpenClawConfig; + const input = params(config, stateDir); + const before = fs.readdirSync(fixture, { recursive: true }).sort(); + + const first = await scopedMemoryMigrationPreview.detectLegacyState(input); + const second = await scopedMemoryMigrationPreview.detectLegacyState(input); + const apply = await scopedMemoryMigrationPreview.migrateLegacyState(input); + + expect(second).toEqual(first); + expect(apply.changes).toEqual([]); + expect(apply.warnings).toEqual([]); + expect(apply.notices).toEqual(first?.preview); + const report = JSON.stringify({ first, apply }); + expect(report).toContain("curated=1, memory=1, transcripts=1"); + expect(report).toContain("dmScope=1, backend=1, filesystem=0, sandbox=1"); + expect(report).toContain("classify -> backup -> copy -> reindex -> verify -> cutover"); + expect(report).not.toContain("private curated content"); + expect(report).not.toContain("private-memory.md"); + expect(report).not.toContain("private transcript"); + expect(fs.readdirSync(fixture, { recursive: true }).sort()).toEqual(before); + }); + + it("reports invalid agent identity without touching a traversal-shaped path", async () => { + const fixture = root(); + const stateDir = path.join(fixture, "state"); + const config = { agents: { list: [{ id: ".." }] } } as OpenClawConfig; + + const preview = await scopedMemoryMigrationPreview.detectLegacyState(params(config, stateDir)); + + expect(preview?.preview.join("\n")).toContain("invalidAgent=1"); + expect(fs.existsSync(path.join(fixture, "sessions"))).toBe(false); + }); + + it("uses canonical entries precedence and counts symlinked legacy files as blockers", async () => { + const fixture = root(); + const stateDir = path.join(fixture, "state"); + const workspace = path.join(fixture, "workspace"); + fs.mkdirSync(path.join(workspace, "memory"), { recursive: true }); + fs.writeFileSync(path.join(workspace, "MEMORY.md"), "canonical workspace content"); + fs.symlinkSync(path.join(workspace, "MEMORY.md"), path.join(workspace, "memory", "link.md")); + const config = { + agents: { + // `entries` intentionally wins over a malformed legacy list, matching the core roster reader. + entries: { main: { workspace } }, + list: [{ id: "..", workspace: path.join(fixture, "ignored") }], + }, + } as OpenClawConfig; + + const preview = await scopedMemoryMigrationPreview.detectLegacyState(params(config, stateDir)); + const report = preview?.preview.join("\n") ?? ""; + + expect(report).toContain("curated=1, memory=0, transcripts=0"); + expect(report).toContain("filesystem=1"); + expect(report).toContain("invalidAgent=0"); + expect(report).not.toContain("canonical workspace content"); + expect(fs.existsSync(path.join(fixture, "ignored"))).toBe(false); + }); +}); diff --git a/extensions/memory-core/src/migration/doctor-scoped-memory-preview.ts b/extensions/memory-core/src/migration/doctor-scoped-memory-preview.ts new file mode 100644 index 000000000000..9db7ce723de4 --- /dev/null +++ b/extensions/memory-core/src/migration/doctor-scoped-memory-preview.ts @@ -0,0 +1,186 @@ +import { createHash } from "node:crypto"; +import type { Dirent } from "node:fs"; +import fs from "node:fs/promises"; +import path from "node:path"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; +import { isValidAgentId } from "openclaw/plugin-sdk/routing"; +import type { PluginDoctorStateMigration } from "openclaw/plugin-sdk/runtime-doctor-migrations"; + +type PreviewKind = "curated" | "memory" | "transcript" | "quarantine"; +type PreviewItem = Readonly<{ id: string; kind: PreviewKind; bytes: number }>; + +function opaqueId(value: string): string { + return createHash("sha256").update(value).digest("hex").slice(0, 16); +} + +function isReadableMarkdownName(name: string): boolean { + return name.toLowerCase().endsWith(".md") && name !== "." && name !== ".."; +} + +async function scanRegularFiles(params: { + directory: string; + agentId: string; + kind: PreviewKind; + extension?: string; +}): Promise<{ items: PreviewItem[]; filesystemBlockers: number }> { + let entries: Dirent[]; + try { + entries = await fs.readdir(params.directory, { withFileTypes: true }); + } catch (error) { + return (error as NodeJS.ErrnoException).code === "ENOENT" + ? { items: [], filesystemBlockers: 0 } + : { items: [], filesystemBlockers: 1 }; + } + const items: PreviewItem[] = []; + let filesystemBlockers = 0; + for (const entry of entries.toSorted((left, right) => left.name.localeCompare(right.name))) { + if (!entry.isFile() || entry.isSymbolicLink()) { + if (entry.isSymbolicLink()) { + filesystemBlockers += 1; + } + continue; + } + if (params.extension && !entry.name.endsWith(params.extension)) { + continue; + } + if (params.kind === "memory" && !isReadableMarkdownName(entry.name)) { + continue; + } + try { + const stat = await fs.stat(path.join(params.directory, entry.name)); + if (!stat.isFile()) { + continue; + } + items.push( + Object.freeze({ + id: opaqueId(`${params.agentId}\0${params.kind}\0${entry.name}`), + kind: params.kind, + bytes: stat.size, + }), + ); + } catch { + filesystemBlockers += 1; + } + } + return { items, filesystemBlockers }; +} + +function previewLines(params: { + items: readonly PreviewItem[]; + dmScope: number; + backend: number; + filesystem: number; + sandbox: number; + invalidAgent: number; +}): string[] { + const counts = Object.fromEntries( + (["curated", "memory", "transcript", "quarantine"] as const).map((kind) => [ + kind, + params.items.filter((item) => item.kind === kind).length, + ]), + ) as Record; + const bytes = params.items.reduce((total, item) => total + item.bytes, 0); + const entries = params.items + .toSorted((left, right) => left.id.localeCompare(right.id)) + .map((item) => `${item.kind}:${item.id}:${item.bytes}`); + const planHash = opaqueId(entries.join("\n")); + return [ + `Scoped memory dry-run: curated=${counts.curated}, memory=${counts.memory}, transcripts=${counts.transcript}, quarantine=${counts.quarantine}; classify -> backup -> copy -> reindex -> verify -> cutover.`, + `Scoped memory dry-run blockers: dmScope=${params.dmScope}, backend=${params.backend}, filesystem=${params.filesystem}, sandbox=${params.sandbox}, invalidAgent=${params.invalidAgent}.`, + `Scoped memory dry-run estimates: backup=${bytes}B, copy=${bytes}B, reindex=${params.items.length} item(s), verify=${params.items.length} hash check(s), cutover=0; plan=${planHash}.`, + ...entries.map((entry) => `Scoped memory dry-run item: ${entry}.`), + ]; +} + +async function buildPreview(params: { + config: OpenClawConfig; + stateDir: string; +}): Promise { + const { listAgentIds, readAgentRosterProperty, resolveAgentWorkspaceDir } = + await import("openclaw/plugin-sdk/memory-host-core"); + const roster = readAgentRosterProperty(params.config); + const rawAgentIds = + roster?.kind === "entries" && roster.value && typeof roster.value === "object" + ? Object.keys(roster.value) + : roster?.kind === "list" && Array.isArray(roster.value) + ? roster.value.flatMap((entry) => { + const id = + entry && typeof entry === "object" ? (entry as { id?: unknown }).id : undefined; + return typeof id === "string" ? [id] : []; + }) + : []; + if (rawAgentIds.some((agentId) => !isValidAgentId(agentId))) { + return previewLines({ + items: [], + dmScope: 0, + backend: 1, + filesystem: 0, + sandbox: 0, + invalidAgent: 1, + }); + } + let agentIds: readonly string[]; + try { + agentIds = listAgentIds(params.config); + } catch { + return previewLines({ + items: [], + dmScope: 0, + backend: 0, + filesystem: 0, + sandbox: 0, + invalidAgent: 1, + }); + } + const items: PreviewItem[] = []; + let filesystem = 0; + let sandbox = 0; + for (const agentId of agentIds.toSorted()) { + const workspace = resolveAgentWorkspaceDir(params.config, agentId); + const curated = await scanRegularFiles({ + directory: workspace, + agentId, + kind: "curated", + extension: ".md", + }); + const memory = await scanRegularFiles({ + directory: path.join(workspace, "memory"), + agentId, + kind: "memory", + }); + // Transcripts are direct JSONL files. sessions.json is metadata, never memory content. + const transcripts = await scanRegularFiles({ + directory: path.join(params.stateDir, "agents", agentId, "sessions"), + agentId, + kind: "transcript", + extension: ".jsonl", + }); + items.push(...curated.items, ...memory.items, ...transcripts.items); + filesystem += + curated.filesystemBlockers + memory.filesystemBlockers + transcripts.filesystemBlockers; + const entry = + params.config.agents?.entries?.[agentId] ?? + params.config.agents?.list?.find((candidate) => candidate?.id === agentId); + if (entry?.sandbox?.mode === "all") { + sandbox += 1; + } + } + const dmScope = params.config.session?.dmScope === "main" ? 1 : 0; + const backend = 1; // Current memory backend is builtin-only; no alternate configuration is revived. + return previewLines({ items, dmScope, backend, filesystem, sandbox, invalidAgent: 0 }); +} + +/** Preview only: this phase deliberately makes no state, file, database, or config mutation. */ +export const scopedMemoryMigrationPreview: PluginDoctorStateMigration = { + id: "memory-core-scoped-memory-dry-run", + label: "Preview Memory Core scoped-memory migration", + doctorOnly: true, + async detectLegacyState({ config, stateDir }) { + const preview = await buildPreview({ config, stateDir }); + return preview ? { preview } : null; + }, + async migrateLegacyState({ config, stateDir }) { + const notices = (await buildPreview({ config, stateDir })) ?? []; + return { changes: [], warnings: [], notices }; + }, +}; diff --git a/src/memory-host-sdk/host/authorization-conformance.ts b/src/memory-host-sdk/host/authorization-conformance.ts new file mode 100644 index 000000000000..0125171a028a --- /dev/null +++ b/src/memory-host-sdk/host/authorization-conformance.ts @@ -0,0 +1,2 @@ +/** Core facade for the independently specified memory authorization conformance suite. */ +export * from "../../../packages/memory-host-sdk/src/host/authorization-conformance.js"; diff --git a/src/plugin-sdk/memory-core-host-engine-schema.ts b/src/plugin-sdk/memory-core-host-engine-schema.ts index 2611e386ce6e..ee547d33ef52 100644 --- a/src/plugin-sdk/memory-core-host-engine-schema.ts +++ b/src/plugin-sdk/memory-core-host-engine-schema.ts @@ -1,4 +1,5 @@ // Focused memory host schema helpers for doctor and migration control-plane paths. +export { ensureOpenClawAgentScopedMemorySchema } from "../state/openclaw-agent-scoped-memory-schema.js"; export { ensureMemoryIndexSchema, MEMORY_EMBEDDING_CACHE_TABLE, diff --git a/src/plugin-sdk/memory-host-core.ts b/src/plugin-sdk/memory-host-core.ts index 64cfda4064f7..ceb107392a09 100644 --- a/src/plugin-sdk/memory-host-core.ts +++ b/src/plugin-sdk/memory-host-core.ts @@ -221,6 +221,11 @@ export type { MemoryPromptSectionBuilder, } from "../plugins/memory-state.js"; export { resolveDefaultAgentId } from "../agents/agent-scope-config.js"; +export { + listAgentIds, + readAgentRosterProperty, + resolveAgentWorkspaceDir, +} from "../agents/agent-scope-config.js"; export { resolveSessionAgentId } from "../agents/agent-scope.js"; export { resolveSessionTranscriptsDirForAgent } from "../config/sessions/paths.js"; diff --git a/src/plugin-sdk/routing.ts b/src/plugin-sdk/routing.ts index a7e4d765e7c0..ac67e24c5606 100644 --- a/src/plugin-sdk/routing.ts +++ b/src/plugin-sdk/routing.ts @@ -19,6 +19,7 @@ export { isAcpSessionKey, isIncognitoSessionKey, isSubagentSessionKey, + isValidAgentId, normalizeAccountId, normalizeAgentId, normalizeAgentIdStrict, diff --git a/src/plugins/memory-authorization-runtime.test.ts b/src/plugins/memory-authorization-runtime.test.ts index 9796d90babca..e36d09b61703 100644 --- a/src/plugins/memory-authorization-runtime.test.ts +++ b/src/plugins/memory-authorization-runtime.test.ts @@ -1,10 +1,14 @@ import { describe, expect, it, vi } from "vitest"; +import { referenceMemoryAuthorizationConformanceAdapter } from "../memory-host-sdk/host/authorization-conformance.js"; import { COMPLETE_MEMORY_AUTHORIZATION_CAPABILITIES, LEGACY_MEMORY_AUTHORIZATION_CAPABILITIES, MEMORY_AUTHORIZATION_CAPABILITY_NAMES, } from "../memory-host-sdk/host/authorization.js"; -import { inspectMemoryAuthorizationCapability } from "./memory-authorization-runtime.js"; +import { + admitMemoryAuthorizationReadRuntime, + inspectMemoryAuthorizationCapability, +} from "./memory-authorization-runtime.js"; import { observeMemoryAuthorizationShadowSurface } from "./memory-authorization-shadow.js"; import { createEmptyPluginRegistry } from "./registry-empty.js"; @@ -99,6 +103,24 @@ describe("memory authorization capability inspection", () => { expect(runtime.legacyManager.search).not.toHaveBeenCalled(); }); + it("fails closed for an enforced nonconforming alternate without calling legacy search", async () => { + const legacySearch = vi.fn(); + const rejected = await admitMemoryAuthorizationReadRuntime({ + authorization: LEGACY_MEMORY_AUTHORIZATION_CAPABILITIES, + authorizationConformance: referenceMemoryAuthorizationConformanceAdapter, + runtime: { ...createRuntime(), legacySearch }, + }); + expect(rejected).toEqual({ ok: false, reasonCode: "backend-nonconforming" }); + expect(legacySearch).not.toHaveBeenCalled(); + + const admitted = await admitMemoryAuthorizationReadRuntime({ + authorization: COMPLETE_MEMORY_AUTHORIZATION_CAPABILITIES, + authorizationConformance: referenceMemoryAuthorizationConformanceAdapter, + runtime: createRuntime(), + }); + expect(admitted.ok).toBe(true); + }); + it("reports all-false and incomplete declarations as nonconforming", () => { const legacy = inspectMemoryAuthorizationCapability({ authorization: LEGACY_MEMORY_AUTHORIZATION_CAPABILITIES, diff --git a/src/plugins/memory-authorization-runtime.ts b/src/plugins/memory-authorization-runtime.ts index 9d9419ec37a6..bb2f16d32b3e 100644 --- a/src/plugins/memory-authorization-runtime.ts +++ b/src/plugins/memory-authorization-runtime.ts @@ -1,3 +1,7 @@ +import { + runMemoryAuthorizationConformanceSuite, + type MemoryAuthorizationConformanceAdapter, +} from "../memory-host-sdk/host/authorization-conformance.js"; import { MEMORY_AUTHORIZATION_CAPABILITY_NAMES, isMemoryAuthorizationCapabilities, @@ -19,6 +23,20 @@ const AUTHORIZED_MEMORY_RUNTIME_METHODS = [ type AuthorizedMemoryRuntimeMethodName = (typeof AUTHORIZED_MEMORY_RUNTIME_METHODS)[number]; +const AUTHORIZED_MEMORY_READ_METHODS = ["authorize", "searchAuthorized", "readAuthorized"] as const; +const AUTHORIZED_MEMORY_READ_CAPABILITIES = [ + "scopedCandidates", + "exactReadByAuthorizedHandle", +] as const satisfies readonly MemoryAuthorizationCapabilityName[]; + +export type AdmittedAuthorizedMemoryReadRuntime = Readonly< + Pick +>; + +export type MemoryAuthorizationReadAdmission = + | Readonly<{ ok: true; runtime: AdmittedAuthorizedMemoryReadRuntime }> + | Readonly<{ ok: false; reasonCode: "backend-nonconforming" }>; + type MemoryAuthorizationCapabilityInspection = Readonly<{ version: 1; capabilityDeclaration: "missing" | "malformed" | "partial" | "complete"; @@ -157,3 +175,69 @@ export function inspectMemoryAuthorizationCapability( reasonCode: surfaceComplete ? "surface-complete" : "backend-nonconforming", }); } + +function isConformanceAdapter(value: unknown): value is MemoryAuthorizationConformanceAdapter { + const evaluate = readCallable(value, "evaluate"); + const prefilter = readCallable(value, "prefilter"); + return ( + isObjectReference(value) && typeof evaluate === "function" && typeof prefilter === "function" + ); +} + +function readCallable(value: unknown, key: string): ((...args: never[]) => unknown) | undefined { + const property = readDataProperty(value, key); + return property.kind === "data" && typeof property.value === "function" + ? (property.value as (...args: never[]) => unknown) + : undefined; +} + +/** + * Enforced callers use this admission result directly. A failed alternate has no legacy runtime + * in the result, so it cannot silently broaden a scoped read through the old search manager. + */ +export async function admitMemoryAuthorizationReadRuntime( + capability: unknown, +): Promise { + const authorization = readDataProperty(capability, "authorization"); + const runtime = readDataProperty(capability, "runtime"); + const conformance = readDataProperty(capability, "authorizationConformance"); + const authorizationCapabilities = + authorization.kind === "data" && isMemoryAuthorizationCapabilities(authorization.value) + ? authorization.value + : undefined; + if ( + runtime.kind !== "data" || + conformance.kind !== "data" || + !authorizationCapabilities || + AUTHORIZED_MEMORY_READ_CAPABILITIES.some((name) => !authorizationCapabilities[name]) || + !isConformanceAdapter(conformance.value) + ) { + return Object.freeze({ ok: false, reasonCode: "backend-nonconforming" }); + } + const authorize = readCallable(runtime.value, "authorize"); + const searchAuthorized = readCallable(runtime.value, "searchAuthorized"); + const readAuthorized = readCallable(runtime.value, "readAuthorized"); + if (!authorize || !searchAuthorized || !readAuthorized) { + return Object.freeze({ ok: false, reasonCode: "backend-nonconforming" }); + } + try { + const report = await runMemoryAuthorizationConformanceSuite(conformance.value); + if (!report.ok) { + return Object.freeze({ ok: false, reasonCode: "backend-nonconforming" }); + } + } catch { + return Object.freeze({ ok: false, reasonCode: "backend-nonconforming" }); + } + return Object.freeze({ + ok: true, + runtime: Object.freeze({ + authorize: (authorize as AuthorizedMemoryRuntime["authorize"]).bind(runtime.value), + searchAuthorized: (searchAuthorized as AuthorizedMemoryRuntime["searchAuthorized"]).bind( + runtime.value, + ), + readAuthorized: (readAuthorized as AuthorizedMemoryRuntime["readAuthorized"]).bind( + runtime.value, + ), + }), + }); +} diff --git a/src/plugins/registry-contribution-types.ts b/src/plugins/registry-contribution-types.ts index c3d7b59a2241..a9121ba2aeed 100644 --- a/src/plugins/registry-contribution-types.ts +++ b/src/plugins/registry-contribution-types.ts @@ -3,6 +3,7 @@ import type { EmbeddingInput } from "../../packages/memory-host-sdk/src/engine-e import type { MemoryCitationsMode } from "../config/types.memory.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import type { ContextEngine } from "../context-engine/types.js"; +import type { MemoryAuthorizationConformanceAdapter } from "../memory-host-sdk/host/authorization-conformance.js"; import type { AuthorizedMemoryRuntime, MemoryAuthorizationCapabilities, @@ -311,6 +312,8 @@ export type MemoryPluginPublicArtifactsProvider = { export type MemoryPluginCapability = { /** Declares the selected backend's authorization support even when it has no runtime. */ authorization?: MemoryAuthorizationCapabilities; + /** Plugin-owned pure evaluator; core verifies it before an enforced read admission. */ + authorizationConformance?: MemoryAuthorizationConformanceAdapter; promptBuilder?: MemoryPromptSectionBuilder; flushPlanResolver?: MemoryFlushPlanResolver; runtime?: MemoryPluginRuntime; diff --git a/src/state/openclaw-agent-db-schema-helpers.ts b/src/state/openclaw-agent-db-schema-helpers.ts index 83b3f4fc3446..0e0fcfe578a7 100644 --- a/src/state/openclaw-agent-db-schema-helpers.ts +++ b/src/state/openclaw-agent-db-schema-helpers.ts @@ -35,6 +35,12 @@ import { } from "./openclaw-agent-progress-card-schema.js"; import { OPENCLAW_AGENT_SCHEMA_SQL } from "./openclaw-agent-schema.js"; import { SESSION_PARTICIPANTS_TABLE } from "./openclaw-agent-session-participants-schema.js"; +import { + AGENT_SCOPED_MEMORY_FTS_SHADOW_TABLES, + AGENT_SCOPED_MEMORY_FTS_TABLE, + AGENT_SCOPED_MEMORY_FTS_TRIGGER_DEFINITIONS, + AGENT_SCOPED_MEMORY_TABLES, +} from "./openclaw-agent-scoped-memory-schema.js"; import { AGENT_V14_ADDITIVE_SCHEMA_SQL, AGENT_V14_CORE_SCHEMA_SQL, @@ -65,6 +71,9 @@ const AGENT_SCHEMA_COMPATIBILITY = { SESSION_TRANSCRIPT_ARCHIVES_TABLE, "session_memory_subjects", "session_memory_subject_snapshots", + ...AGENT_SCOPED_MEMORY_TABLES, + AGENT_SCOPED_MEMORY_FTS_TABLE, + ...AGENT_SCOPED_MEMORY_FTS_SHADOW_TABLES, STANDING_INTENTS_TABLE, STANDING_INTENTS_FTS_TABLE, ...STANDING_INTENTS_FTS_SHADOW_TABLES, @@ -84,6 +93,11 @@ const AGENT_SCHEMA_COMPATIBILITY = { tableName: MEMORY_INDEX_SOURCES_TABLE, triggers: MEMORY_PATH_FTS_TRIGGER_DEFINITIONS, }, + { + optionalWhenTableMissing: AGENT_SCOPED_MEMORY_FTS_TABLE, + tableName: "memory_scoped_chunks", + triggers: AGENT_SCOPED_MEMORY_FTS_TRIGGER_DEFINITIONS, + }, ], } satisfies SqliteSchemaCompatibility; diff --git a/src/state/openclaw-agent-db.generated.d.ts b/src/state/openclaw-agent-db.generated.d.ts index da9e2882f43c..2e5ce9ab89cb 100644 --- a/src/state/openclaw-agent-db.generated.d.ts +++ b/src/state/openclaw-agent-db.generated.d.ts @@ -206,6 +206,179 @@ export interface MessageToolRunOutcomes { session_key: string; } +export interface MemoryMigrations { + classification_json: string; + cutover_at: number | null; + migration_id: string; + phase: string; + plan_hash: string; + source_hash: string; + source_kind: string; + updated_at: number; + verified_at: number | null; +} + +export interface MemoryPolicies { + agent_id: string; + created_at: number; + current_revision_id: string; + lifecycle_state: string; + policy_id: string; + revocation_epoch: Generated; + updated_at: number; +} + +export interface MemoryPolicyEntries { + audience_id: string; + audience_kind: string; + created_at: number; + effect: string; + entry_id: string; + entry_kind: string; + expires_at: number | null; + grantor_principal_id: string; + operation: string; + policy_revision_id: string; + principal_id: string; + reason: string; +} + +export interface MemoryPolicyRevisions { + actor_id: string | null; + actor_kind: string; + created_at: number; + lifecycle_state: string; + policy_id: string; + reason: string; + revision_id: string; + revision_number: number; + revocation_epoch: number; +} + +export interface MemoryResourceRevisions { + activated_at: number | null; + actor_id: string | null; + actor_kind: string; + artifact_locator: string; + content_bytes: number; + content_hash: string; + created_at: number; + expires_at: number | null; + lifecycle_state: string; + policy_revision_id: string; + policy_revocation_epoch: number; + resource_id: string; + retired_at: number | null; + revision_id: string; + revision_number: number; + source_policy_set_id: string; +} + +export interface MemoryResourceSubjects { + created_at: number; + evidence_revision: string; + lifecycle_state: string; + revision_id: string; + subject_id: string; + subject_kind: string; +} + +export interface MemoryResources { + agent_id: string; + created_at: number; + logical_locator: string; + resource_id: string; + source: Generated; + store_id: string; +} + +export interface MemoryScopedChunkVectors { + chunk_id: string; + dims: number; + embedding: string; + model: string; + updated_at: number; +} + +export interface MemoryScopedChunks { + chunk_id: string; + chunk_key: Generated; + chunk_ordinal: number; + content_hash: string; + end_line: number; + model: string; + revision_id: string; + start_line: number; + text: string; + updated_at: number; +} + +export interface MemoryScopedChunksFts { + chunk_id: string | null; + end_line: string | null; + revision_id: string | null; + start_line: string | null; + text: string | null; +} + +export interface MemoryScopedChunksFtsConfig { + k: string; + v: string | null; +} + +export interface MemoryScopedChunksFtsContent { + c0: string | null; + c1: string | null; + c2: string | null; + c3: string | null; + c4: string | null; + id: Generated; +} + +export interface MemoryScopedChunksFtsData { + block: Uint8Array | null; + id: Generated; +} + +export interface MemoryScopedChunksFtsDocsize { + id: Generated; + sz: Uint8Array | null; +} + +export interface MemoryScopedChunksFtsIdx { + pgno: string | null; + segid: string; + term: string; +} + +export interface MemoryStorageRoots { + agent_id: string; + authority_kind: string; + authority_owner_id: string; + backend_kind: string; + created_at: number; + default_capabilities_json: string; + lifecycle_state: string; + opaque_locator: string; + path_key: string | null; + path_key_version: number; + storage_root_id: string; + updated_at: number; +} + +export interface MemoryStores { + agent_id: string; + audience_id: string; + audience_kind: string; + created_at: number; + lifecycle_state: string; + policy_id: string; + scope_kind: string; + storage_root_id: string; + store_id: string; + updated_at: number; +} + export interface SchemaMeta { agent_id: string | null; app_version: string | null; @@ -237,6 +410,23 @@ export interface SessionMembers { session_key: string; } +export interface SessionMemorySubjectSnapshots { + created_at: number; + session_id: string; + session_identity_revision: string; + session_key: string; + subject_revision: string; +} + +export interface SessionMemorySubjects { + binding_id: string | null; + created_at: number; + principal_id: string | null; + session_key: string; + subject_kind: string; + subject_revision: string; +} + export interface SessionNodes { archived_at: number | null; category: string | null; @@ -495,10 +685,29 @@ export interface DB { memory_index_sources: MemoryIndexSources; memory_index_state: MemoryIndexState; message_tool_run_outcomes: MessageToolRunOutcomes; + memory_migrations: MemoryMigrations; + memory_policies: MemoryPolicies; + memory_policy_entries: MemoryPolicyEntries; + memory_policy_revisions: MemoryPolicyRevisions; + memory_resource_revisions: MemoryResourceRevisions; + memory_resource_subjects: MemoryResourceSubjects; + memory_resources: MemoryResources; + memory_scoped_chunk_vectors: MemoryScopedChunkVectors; + memory_scoped_chunks: MemoryScopedChunks; + memory_scoped_chunks_fts: MemoryScopedChunksFts; + memory_scoped_chunks_fts_config: MemoryScopedChunksFtsConfig; + memory_scoped_chunks_fts_content: MemoryScopedChunksFtsContent; + memory_scoped_chunks_fts_data: MemoryScopedChunksFtsData; + memory_scoped_chunks_fts_docsize: MemoryScopedChunksFtsDocsize; + memory_scoped_chunks_fts_idx: MemoryScopedChunksFtsIdx; + memory_storage_roots: MemoryStorageRoots; + memory_stores: MemoryStores; schema_meta: SchemaMeta; session_conversations: SessionConversations; session_key_contract: SessionKeyContract; session_members: SessionMembers; + session_memory_subject_snapshots: SessionMemorySubjectSnapshots; + session_memory_subjects: SessionMemorySubjects; session_nodes: SessionNodes; session_participants: SessionParticipants; session_progress_cards: SessionProgressCards; diff --git a/src/state/openclaw-agent-schema.sql b/src/state/openclaw-agent-schema.sql index 4e532ce6f811..f995c3f1f836 100644 --- a/src/state/openclaw-agent-schema.sql +++ b/src/state/openclaw-agent-schema.sql @@ -599,6 +599,281 @@ CREATE TABLE IF NOT EXISTS memory_index_state ( revision INTEGER NOT NULL ) STRICT; +-- Scoped memory is additive and feature-local. Existing agent databases do +-- not create this group until the scoped backend is selected. +CREATE TABLE IF NOT EXISTS memory_storage_roots ( + storage_root_id TEXT NOT NULL PRIMARY KEY, + agent_id TEXT NOT NULL, + backend_kind TEXT NOT NULL CHECK (backend_kind IN ('builtin', 'alternate')), + opaque_locator TEXT NOT NULL, + path_key_version INTEGER NOT NULL CHECK (path_key_version > 0), + path_key TEXT, + authority_kind TEXT NOT NULL CHECK (authority_kind IN ('user', 'conversation', 'role', 'agent-shared', 'agent', 'internal')), + authority_owner_id TEXT NOT NULL, + default_capabilities_json TEXT NOT NULL, + lifecycle_state TEXT NOT NULL CHECK (lifecycle_state IN ('pending', 'active', 'quarantined', 'tombstoned')), + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + CHECK ((backend_kind = 'builtin' AND path_key IS NOT NULL) OR backend_kind <> 'builtin'), + UNIQUE (agent_id, opaque_locator), + UNIQUE (agent_id, path_key) +) STRICT; + +CREATE INDEX IF NOT EXISTS idx_memory_storage_roots_agent_state + ON memory_storage_roots(agent_id, lifecycle_state, storage_root_id); + +CREATE TABLE IF NOT EXISTS memory_stores ( + store_id TEXT NOT NULL PRIMARY KEY, + agent_id TEXT NOT NULL, + storage_root_id TEXT NOT NULL, + policy_id TEXT NOT NULL, + scope_kind TEXT NOT NULL CHECK (scope_kind IN ('user', 'conversation', 'role', 'agent-shared', 'agent', 'internal')), + audience_kind TEXT NOT NULL CHECK (audience_kind IN ('user', 'conversation', 'role', 'agent-shared', 'agent', 'internal')), + audience_id TEXT NOT NULL, + lifecycle_state TEXT NOT NULL CHECK (lifecycle_state IN ('pending', 'active', 'quarantined', 'tombstoned')), + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + FOREIGN KEY (storage_root_id) REFERENCES memory_storage_roots(storage_root_id) ON DELETE RESTRICT, + FOREIGN KEY (policy_id) REFERENCES memory_policies(policy_id) ON DELETE RESTRICT, + UNIQUE (agent_id, storage_root_id, scope_kind, audience_kind, audience_id) +) STRICT; + +CREATE INDEX IF NOT EXISTS idx_memory_stores_agent_scope + ON memory_stores(agent_id, scope_kind, audience_kind, audience_id, lifecycle_state); + +CREATE INDEX IF NOT EXISTS idx_memory_stores_policy + ON memory_stores(agent_id, policy_id, lifecycle_state, store_id); + +CREATE TABLE IF NOT EXISTS memory_policies ( + policy_id TEXT NOT NULL PRIMARY KEY, + agent_id TEXT NOT NULL, + current_revision_id TEXT NOT NULL, + revocation_epoch INTEGER NOT NULL DEFAULT 0 CHECK (revocation_epoch >= 0), + lifecycle_state TEXT NOT NULL CHECK (lifecycle_state IN ('active', 'revoked')), + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL +) STRICT; + +CREATE INDEX IF NOT EXISTS idx_memory_policies_agent_state + ON memory_policies(agent_id, lifecycle_state, policy_id); + +CREATE TABLE IF NOT EXISTS memory_policy_revisions ( + revision_id TEXT NOT NULL PRIMARY KEY, + policy_id TEXT NOT NULL, + revision_number INTEGER NOT NULL CHECK (revision_number > 0), + revocation_epoch INTEGER NOT NULL CHECK (revocation_epoch >= 0), + lifecycle_state TEXT NOT NULL CHECK (lifecycle_state IN ('active', 'superseded', 'revoked')), + actor_kind TEXT NOT NULL CHECK (actor_kind IN ('human', 'agent', 'service', 'system', 'unattributed')), + actor_id TEXT, + reason TEXT NOT NULL, + created_at INTEGER NOT NULL, + FOREIGN KEY (policy_id) REFERENCES memory_policies(policy_id) ON DELETE RESTRICT, + UNIQUE (policy_id, revision_number) +) STRICT; + +CREATE UNIQUE INDEX IF NOT EXISTS idx_memory_policy_revisions_one_active + ON memory_policy_revisions(policy_id) + WHERE lifecycle_state = 'active'; + +CREATE TRIGGER IF NOT EXISTS memory_policy_revisions_immutable_fields +BEFORE UPDATE OF policy_id, revision_number, revocation_epoch, actor_kind, actor_id, reason, created_at +ON memory_policy_revisions +BEGIN + SELECT RAISE(ABORT, 'memory policy revision fields are immutable'); +END; + +CREATE TRIGGER IF NOT EXISTS memory_policy_revisions_no_delete +BEFORE DELETE ON memory_policy_revisions +BEGIN + SELECT RAISE(ABORT, 'memory policy revisions cannot be deleted'); +END; + +CREATE TRIGGER IF NOT EXISTS memory_policy_revisions_terminal_lifecycle +BEFORE UPDATE OF lifecycle_state ON memory_policy_revisions +WHEN old.lifecycle_state <> 'active' AND new.lifecycle_state <> old.lifecycle_state +BEGIN + SELECT RAISE(ABORT, 'retired memory policy revisions cannot be reactivated'); +END; + +CREATE TABLE IF NOT EXISTS memory_policy_entries ( + entry_id TEXT NOT NULL PRIMARY KEY, + policy_revision_id TEXT NOT NULL, + entry_kind TEXT NOT NULL CHECK (entry_kind IN ('placement', 'exception', 'publish')), + effect TEXT NOT NULL CHECK (effect IN ('allow', 'deny')), + principal_id TEXT NOT NULL, + audience_kind TEXT NOT NULL CHECK (audience_kind IN ('user', 'conversation', 'role', 'agent-shared', 'agent', 'internal', '*')), + audience_id TEXT NOT NULL, + operation TEXT NOT NULL CHECK (operation IN ('retrieve', 'read', 'append', 'replace', 'derive', 'deposit', 'project', 'publish', 'import', 'export', 'delete', 'sync', 'status', 'policy-admin')), + grantor_principal_id TEXT NOT NULL, + reason TEXT NOT NULL, + expires_at INTEGER, + created_at INTEGER NOT NULL, + FOREIGN KEY (policy_revision_id) REFERENCES memory_policy_revisions(revision_id) ON DELETE RESTRICT +) STRICT; + +CREATE INDEX IF NOT EXISTS idx_memory_policy_entries_revision_operation + ON memory_policy_entries(policy_revision_id, operation, effect, principal_id); + +CREATE TRIGGER IF NOT EXISTS memory_policy_entries_no_update +BEFORE UPDATE ON memory_policy_entries +BEGIN + SELECT RAISE(ABORT, 'memory policy entries are immutable'); +END; + +CREATE TRIGGER IF NOT EXISTS memory_policy_entries_no_delete +BEFORE DELETE ON memory_policy_entries +BEGIN + SELECT RAISE(ABORT, 'memory policy entries cannot be deleted'); +END; + +CREATE TABLE IF NOT EXISTS memory_resources ( + resource_id TEXT NOT NULL PRIMARY KEY, + agent_id TEXT NOT NULL, + store_id TEXT NOT NULL, + logical_locator TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'memory' CHECK (source IN ('memory', 'sessions')), + created_at INTEGER NOT NULL, + FOREIGN KEY (store_id) REFERENCES memory_stores(store_id) ON DELETE RESTRICT, + UNIQUE (agent_id, store_id, logical_locator) +) STRICT; + +CREATE INDEX IF NOT EXISTS idx_memory_resources_agent_store + ON memory_resources(agent_id, store_id, resource_id); + +CREATE TABLE IF NOT EXISTS memory_resource_revisions ( + revision_id TEXT NOT NULL PRIMARY KEY, + resource_id TEXT NOT NULL, + revision_number INTEGER NOT NULL CHECK (revision_number > 0), + artifact_locator TEXT NOT NULL, + content_hash TEXT NOT NULL, + content_bytes INTEGER NOT NULL CHECK (content_bytes >= 0), + policy_revision_id TEXT NOT NULL, + policy_revocation_epoch INTEGER NOT NULL CHECK (policy_revocation_epoch >= 0), + source_policy_set_id TEXT NOT NULL, + lifecycle_state TEXT NOT NULL CHECK (lifecycle_state IN ('pending', 'active', 'quarantined', 'tombstoned')), + actor_kind TEXT NOT NULL CHECK (actor_kind IN ('human', 'agent', 'service', 'system', 'unattributed')), + actor_id TEXT, + expires_at INTEGER, + created_at INTEGER NOT NULL, + activated_at INTEGER, + retired_at INTEGER, + FOREIGN KEY (resource_id) REFERENCES memory_resources(resource_id) ON DELETE RESTRICT, + FOREIGN KEY (policy_revision_id) REFERENCES memory_policy_revisions(revision_id) ON DELETE RESTRICT, + UNIQUE (resource_id, revision_number), + UNIQUE (resource_id, artifact_locator) +) STRICT; + +CREATE UNIQUE INDEX IF NOT EXISTS idx_memory_resource_revisions_one_active + ON memory_resource_revisions(resource_id) + WHERE lifecycle_state = 'active'; + +CREATE INDEX IF NOT EXISTS idx_memory_resource_revisions_policy + ON memory_resource_revisions(policy_revision_id, lifecycle_state, revision_id); + +CREATE TRIGGER IF NOT EXISTS memory_resource_revisions_immutable_fields +BEFORE UPDATE OF resource_id, revision_number, artifact_locator, content_hash, content_bytes, policy_revision_id, policy_revocation_epoch, source_policy_set_id, actor_kind, actor_id, expires_at, created_at +ON memory_resource_revisions +BEGIN + SELECT RAISE(ABORT, 'memory resource revision fields are immutable'); +END; + +CREATE TRIGGER IF NOT EXISTS memory_resource_revisions_no_delete +BEFORE DELETE ON memory_resource_revisions +BEGIN + SELECT RAISE(ABORT, 'memory resource revisions cannot be deleted'); +END; + +CREATE TRIGGER IF NOT EXISTS memory_resource_revisions_terminal_lifecycle +BEFORE UPDATE OF lifecycle_state ON memory_resource_revisions +WHEN old.lifecycle_state = 'tombstoned' AND new.lifecycle_state <> old.lifecycle_state +BEGIN + SELECT RAISE(ABORT, 'tombstoned memory resource revisions cannot be reactivated'); +END; + +CREATE TABLE IF NOT EXISTS memory_resource_subjects ( + revision_id TEXT NOT NULL, + subject_kind TEXT NOT NULL CHECK (subject_kind IN ('person', 'project', 'conversation', 'topic')), + subject_id TEXT NOT NULL, + evidence_revision TEXT NOT NULL, + lifecycle_state TEXT NOT NULL CHECK (lifecycle_state IN ('current', 'superseded')), + created_at INTEGER NOT NULL, + PRIMARY KEY (revision_id, subject_kind, subject_id), + FOREIGN KEY (revision_id) REFERENCES memory_resource_revisions(revision_id) ON DELETE RESTRICT +) STRICT; + +CREATE INDEX IF NOT EXISTS idx_memory_resource_subjects_lookup + ON memory_resource_subjects(subject_kind, subject_id, lifecycle_state, revision_id); + +CREATE TABLE IF NOT EXISTS memory_scoped_chunks ( + chunk_key INTEGER PRIMARY KEY, + chunk_id TEXT NOT NULL UNIQUE, + revision_id TEXT NOT NULL, + chunk_ordinal INTEGER NOT NULL CHECK (chunk_ordinal >= 0), + start_line INTEGER NOT NULL CHECK (start_line > 0), + end_line INTEGER NOT NULL CHECK (end_line >= start_line), + text TEXT NOT NULL, + content_hash TEXT NOT NULL, + model TEXT NOT NULL, + updated_at INTEGER NOT NULL, + FOREIGN KEY (revision_id) REFERENCES memory_resource_revisions(revision_id) ON DELETE RESTRICT, + UNIQUE (revision_id, chunk_ordinal) +) STRICT; + +CREATE INDEX IF NOT EXISTS idx_memory_scoped_chunks_revision + ON memory_scoped_chunks(revision_id, chunk_ordinal); + +CREATE TABLE IF NOT EXISTS memory_scoped_chunk_vectors ( + chunk_id TEXT NOT NULL PRIMARY KEY, + model TEXT NOT NULL, + dims INTEGER NOT NULL CHECK (dims > 0), + embedding TEXT NOT NULL, + updated_at INTEGER NOT NULL, + FOREIGN KEY (chunk_id) REFERENCES memory_scoped_chunks(chunk_id) ON DELETE CASCADE +) STRICT; + +CREATE VIRTUAL TABLE IF NOT EXISTS memory_scoped_chunks_fts USING fts5( + text, + chunk_id UNINDEXED, + revision_id UNINDEXED, + start_line UNINDEXED, + end_line UNINDEXED, + tokenize = 'unicode61 remove_diacritics 2' +); + +CREATE TRIGGER IF NOT EXISTS memory_scoped_chunks_fts_after_insert +AFTER INSERT ON memory_scoped_chunks +BEGIN + INSERT INTO memory_scoped_chunks_fts(rowid, text, chunk_id, revision_id, start_line, end_line) + VALUES (new.chunk_key, new.text, new.chunk_id, new.revision_id, new.start_line, new.end_line); +END; + +CREATE TRIGGER IF NOT EXISTS memory_scoped_chunks_fts_after_delete +AFTER DELETE ON memory_scoped_chunks +BEGIN + DELETE FROM memory_scoped_chunks_fts WHERE rowid = old.chunk_key; +END; + +CREATE TRIGGER IF NOT EXISTS memory_scoped_chunks_fts_after_update +AFTER UPDATE OF text, chunk_id, revision_id, start_line, end_line ON memory_scoped_chunks +BEGIN + DELETE FROM memory_scoped_chunks_fts WHERE rowid = old.chunk_key; + INSERT INTO memory_scoped_chunks_fts(rowid, text, chunk_id, revision_id, start_line, end_line) + VALUES (new.chunk_key, new.text, new.chunk_id, new.revision_id, new.start_line, new.end_line); +END; + +CREATE TABLE IF NOT EXISTS memory_migrations ( + migration_id TEXT NOT NULL PRIMARY KEY, + source_kind TEXT NOT NULL, + source_hash TEXT NOT NULL, + phase TEXT NOT NULL CHECK (phase IN ('previewed', 'backed-up', 'copied', 'indexed', 'verified', 'cutover')), + classification_json TEXT NOT NULL, + plan_hash TEXT NOT NULL, + verified_at INTEGER, + cutover_at INTEGER, + updated_at INTEGER NOT NULL, + UNIQUE (source_kind, source_hash) +) STRICT; + CREATE TABLE IF NOT EXISTS standing_intents ( intent_key INTEGER PRIMARY KEY, id TEXT NOT NULL UNIQUE, diff --git a/src/state/openclaw-agent-scoped-memory-schema.test.ts b/src/state/openclaw-agent-scoped-memory-schema.test.ts new file mode 100644 index 000000000000..f6597661121c --- /dev/null +++ b/src/state/openclaw-agent-scoped-memory-schema.test.ts @@ -0,0 +1,114 @@ +import { DatabaseSync } from "node:sqlite"; +import { afterEach, describe, expect, it } from "vitest"; +import { assertOpenClawAgentSchemaContains } from "./openclaw-agent-db-schema-helpers.js"; +import { OPENCLAW_AGENT_SCHEMA_SQL } from "./openclaw-agent-schema.js"; +import { + AGENT_SCOPED_MEMORY_FTS_SHADOW_TABLES, + AGENT_SCOPED_MEMORY_FTS_TABLE, + AGENT_SCOPED_MEMORY_SCHEMA_SQL, + AGENT_SCOPED_MEMORY_TABLES, + ensureOpenClawAgentScopedMemorySchema, +} from "./openclaw-agent-scoped-memory-schema.js"; + +describe("scoped memory additive agent schema", () => { + const databases: DatabaseSync[] = []; + + afterEach(() => { + for (const database of databases.splice(0)) { + database.close(); + } + }); + + function createDatabase(): DatabaseSync { + const database = new DatabaseSync(":memory:"); + databases.push(database); + return database; + } + + function tableNames(database: DatabaseSync): string[] { + return ( + database + .prepare("SELECT name FROM sqlite_schema WHERE type = 'table' ORDER BY name") + .all() as Array<{ name: string }> + ).map((row) => row.name); + } + + function expectScopedGroup(database: DatabaseSync): void { + expect(tableNames(database)).toEqual( + expect.arrayContaining([ + ...AGENT_SCOPED_MEMORY_TABLES, + AGENT_SCOPED_MEMORY_FTS_TABLE, + ...AGENT_SCOPED_MEMORY_FTS_SHADOW_TABLES, + ]), + ); + } + + it("leaves a current database compatible until first feature use", () => { + const database = createDatabase(); + database.exec(OPENCLAW_AGENT_SCHEMA_SQL.replace(AGENT_SCOPED_MEMORY_SCHEMA_SQL, "")); + + expect(tableNames(database)).not.toContain("memory_storage_roots"); + expect(() => + assertOpenClawAgentSchemaContains(database, ":memory:", OPENCLAW_AGENT_SCHEMA_SQL), + ).not.toThrow(); + + ensureOpenClawAgentScopedMemorySchema(database); + expectScopedGroup(database); + }); + + it("converges a partially interrupted group idempotently", () => { + const database = createDatabase(); + const firstTableOnly = AGENT_SCOPED_MEMORY_SCHEMA_SQL.slice( + 0, + AGENT_SCOPED_MEMORY_SCHEMA_SQL.indexOf("CREATE INDEX IF NOT EXISTS idx_memory_storage_roots"), + ); + database.exec(firstTableOnly); + expect(tableNames(database)).toContain("memory_storage_roots"); + expect(tableNames(database)).not.toContain("memory_stores"); + + ensureOpenClawAgentScopedMemorySchema(database); + ensureOpenClawAgentScopedMemorySchema(database); + expectScopedGroup(database); + }); + + it("does not cache a rolled back transaction-local ensure", () => { + const database = createDatabase(); + + database.exec("BEGIN"); + ensureOpenClawAgentScopedMemorySchema(database); + expect(tableNames(database)).toContain("memory_storage_roots"); + database.exec("ROLLBACK"); + expect(tableNames(database)).not.toContain("memory_storage_roots"); + + ensureOpenClawAgentScopedMemorySchema(database); + expectScopedGroup(database); + }); + + it("synchronizes scoped FTS without touching the legacy index tables", () => { + const database = createDatabase(); + ensureOpenClawAgentScopedMemorySchema(database); + database.exec("PRAGMA foreign_keys = OFF"); + database + .prepare( + `INSERT INTO memory_scoped_chunks + (chunk_id, revision_id, chunk_ordinal, start_line, end_line, text, content_hash, model, updated_at) + VALUES (?, ?, 0, 1, 1, ?, ?, 'test', 1)`, + ) + .run("chunk-1", "revision-1", "alpha token", "hash-1"); + + expect( + database + .prepare( + "SELECT chunk_id FROM memory_scoped_chunks_fts WHERE memory_scoped_chunks_fts MATCH ?", + ) + .all('"alpha"'), + ).toEqual([{ chunk_id: "chunk-1" }]); + expect( + database + .prepare( + "SELECT name FROM sqlite_schema WHERE type = 'table' AND name = 'memory_index_chunks'", + ) + .get(), + ).toBeUndefined(); + }); +}); diff --git a/src/state/openclaw-agent-scoped-memory-schema.ts b/src/state/openclaw-agent-scoped-memory-schema.ts new file mode 100644 index 000000000000..b167b219eff9 --- /dev/null +++ b/src/state/openclaw-agent-scoped-memory-schema.ts @@ -0,0 +1,90 @@ +import type { DatabaseSync } from "node:sqlite"; +import { runSqliteImmediateTransactionSync } from "../infra/sqlite-transaction.js"; +import { OPENCLAW_AGENT_SCHEMA_SQL } from "./openclaw-agent-schema.js"; + +export const AGENT_SCOPED_MEMORY_TABLES = [ + "memory_storage_roots", + "memory_stores", + "memory_policies", + "memory_policy_revisions", + "memory_policy_entries", + "memory_resources", + "memory_resource_revisions", + "memory_resource_subjects", + "memory_scoped_chunks", + "memory_scoped_chunk_vectors", + "memory_migrations", +] as const; + +export const AGENT_SCOPED_MEMORY_FTS_TABLE = "memory_scoped_chunks_fts"; +export const AGENT_SCOPED_MEMORY_FTS_SHADOW_TABLES = [ + "memory_scoped_chunks_fts_config", + "memory_scoped_chunks_fts_content", + "memory_scoped_chunks_fts_data", + "memory_scoped_chunks_fts_docsize", + "memory_scoped_chunks_fts_idx", +] as const; + +export const AGENT_SCOPED_MEMORY_FTS_TRIGGER_DEFINITIONS = [ + { + name: "memory_scoped_chunks_fts_after_insert", + sql: ` + CREATE TRIGGER IF NOT EXISTS memory_scoped_chunks_fts_after_insert + AFTER INSERT ON memory_scoped_chunks + BEGIN + INSERT INTO memory_scoped_chunks_fts(rowid, text, chunk_id, revision_id, start_line, end_line) + VALUES (new.chunk_key, new.text, new.chunk_id, new.revision_id, new.start_line, new.end_line); + END; + `, + }, + { + name: "memory_scoped_chunks_fts_after_delete", + sql: ` + CREATE TRIGGER IF NOT EXISTS memory_scoped_chunks_fts_after_delete + AFTER DELETE ON memory_scoped_chunks + BEGIN + DELETE FROM memory_scoped_chunks_fts WHERE rowid = old.chunk_key; + END; + `, + }, + { + name: "memory_scoped_chunks_fts_after_update", + sql: ` + CREATE TRIGGER IF NOT EXISTS memory_scoped_chunks_fts_after_update + AFTER UPDATE OF text, chunk_id, revision_id, start_line, end_line ON memory_scoped_chunks + BEGIN + DELETE FROM memory_scoped_chunks_fts WHERE rowid = old.chunk_key; + INSERT INTO memory_scoped_chunks_fts(rowid, text, chunk_id, revision_id, start_line, end_line) + VALUES (new.chunk_key, new.text, new.chunk_id, new.revision_id, new.start_line, new.end_line); + END; + `, + }, +] as const; + +const SCOPED_MEMORY_SCHEMA_START = "CREATE TABLE IF NOT EXISTS memory_storage_roots ("; +const SCOPED_MEMORY_SCHEMA_END = "CREATE TABLE IF NOT EXISTS standing_intents ("; + +function extractScopedMemorySchema(): string { + const start = OPENCLAW_AGENT_SCHEMA_SQL.indexOf(SCOPED_MEMORY_SCHEMA_START); + const end = OPENCLAW_AGENT_SCHEMA_SQL.indexOf(SCOPED_MEMORY_SCHEMA_END, start); + if (start < 0 || end <= start) { + throw new Error("canonical scoped memory schema markers are missing"); + } + return OPENCLAW_AGENT_SCHEMA_SQL.slice(start, end).trim(); +} + +/** Canonical additive schema for scoped resources, policy, indexes, and receipts. */ +export const AGENT_SCOPED_MEMORY_SCHEMA_SQL = extractScopedMemorySchema(); + +/** Lazily install the full idempotent group; do not cache transaction-local success. */ +export function ensureOpenClawAgentScopedMemorySchema(db: DatabaseSync): void { + const ensure = () => { + // A partially applied group is completed rather than inferred from one marker table. + db.exec(AGENT_SCOPED_MEMORY_SCHEMA_SQL); // sqlite-allow-raw -- Canonical additive DDL only. + }; + if (db.isTransaction) { + ensure(); + return; + } + runSqliteImmediateTransactionSync(db, ensure); +} diff --git a/src/state/openclaw-state-db.generated.d.ts b/src/state/openclaw-state-db.generated.d.ts index 2ca21f968d33..6bbb19ec5d7e 100644 --- a/src/state/openclaw-state-db.generated.d.ts +++ b/src/state/openclaw-state-db.generated.d.ts @@ -904,6 +904,50 @@ export interface MeetingTranscriptUtterances { utterance_id: string | null; } +export interface MemoryIdentityBindings { + account_id: string; + adapter_id: string; + assurance: string; + binding_id: string; + channel: string; + created_at: number; + created_by_profile_id: string; + evidence_revision: string; + expires_at: number | null; + principal_id: string; + revision: string; + revoked_at: number | null; + sender_lookup_hmac: string; + verification_method: string; +} + +export interface MemoryPairingIdentityReceipts { + account_id: string; + adapter_id: string; + assurance: string; + binding_id: string | null; + channel: string; + consumed_at: number | null; + created_at: number; + evidence_revision: string; + expires_at: number; + receipt_id: string; + request_identity_hmac: string; + sender_lookup_hmac: string; + verification_method: string; +} + +export interface MemoryPrincipals { + created_at: number; + principal_id: string; + principal_kind: string; + principal_lookup_hmac: string | null; + revision: string; + revoked_at: number | null; + state: string; + user_profile_id: string | null; +} + export interface MigrationRuns { finished_at: number | null; id: string; @@ -1838,6 +1882,9 @@ export interface DB { meeting_transcript_sessions: MeetingTranscriptSessions; meeting_transcript_summaries: MeetingTranscriptSummaries; meeting_transcript_utterances: MeetingTranscriptUtterances; + memory_identity_bindings: MemoryIdentityBindings; + memory_pairing_identity_receipts: MemoryPairingIdentityReceipts; + memory_principals: MemoryPrincipals; migration_runs: MigrationRuns; migration_sources: MigrationSources; model_capability_cache: ModelCapabilityCache;