From bc552109369335abf01fd5461fe8a777ec523fac Mon Sep 17 00:00:00 2001 From: Patrick Buckley Date: Fri, 14 Aug 2026 20:54:47 -0700 Subject: [PATCH] Add immutable memory index snapshots (#1022) * feat: add immutable memory index snapshots Capture the visible memory metadata index at first model admission, preserve it as immutable system-prefix context, and emit relevance pointers without rewriting cached history. Align project authorization, MCP actor refresh ordering, storage APIs, SDKs, console surfaces, and regression coverage with the snapshot lifecycle. * fix: stabilize memory index for release candidate * fix(sdk): avoid polynomial description trim * chore: split memory index documentation --- .github/workflows/ci.yml | 10 +- scripts/recovery_e2e.py | 1 + sdk/typescript/openapi-console.json | 684 ++++- sdk/typescript/openapi-server.json | 225 +- sdk/typescript/src/console.ts | 17 + sdk/typescript/src/index.ts | 5 + sdk/typescript/src/memory_description.ts | 27 + sdk/typescript/src/server.ts | 25 +- sdk/typescript/src/types.ts | 36 +- sdk/typescript/tests/console.test.ts | 55 + .../tests/memory_description.test.ts | 64 + sdk/typescript/tests/server.test.ts | 97 + tests/_helpers.py | 7 +- tests/_parity_832.py | 4 +- tests/_session_helpers.py | 80 +- tests/data/memory_description_parity.json | 39 + tests/test_832_parity.py | 8 +- tests/test_app_js.py | 117 +- tests/test_bm25.py | 2 +- tests/test_cancel.py | 274 +- tests/test_console_db_config.py | 144 + tests/test_console_routing_proxy.py | 125 + tests/test_coordinator_endpoints.py | 196 +- tests/test_eval_nudges.py | 13 +- tests/test_governance_endpoints.py | 29 +- tests/test_governance_storage.py | 8 + tests/test_history_commit_handoff.py | 27 +- tests/test_history_total_prefix.py | 17 +- tests/test_idle_nudge_wake_integration.py | 45 +- tests/test_judge_storage.py | 18 + tests/test_mcp_client.py | 508 ++++ tests/test_memory_api.py | 263 +- tests/test_memory_index.py | 813 ++++++ tests/test_memory_relevance.py | 668 +---- tests/test_metacognition.py | 16 - tests/test_migration_072.py | 186 ++ tests/test_model_turn.py | 206 +- tests/test_oidc_credential_storage.py | 3 +- tests/test_oidc_storage.py | 46 + tests/test_openapi.py | 61 + tests/test_per_user_message_context.py | 8 +- tests/test_perception.py | 4 +- tests/test_persona_guards.py | 37 +- tests/test_project_api.py | 19 +- tests/test_project_memory_scope.py | 123 +- tests/test_project_storage.py | 31 +- tests/test_project_workstream_visibility.py | 175 +- tests/test_reasoning_audit_log_discipline.py | 5 +- tests/test_recall_compaction_scope.py | 33 +- tests/test_require_project.py | 5 +- tests/test_schedule_api.py | 12 +- tests/test_sdk_console.py | 55 + tests/test_sdk_server.py | 123 + tests/test_sdk_sync.py | 48 + tests/test_search_history_visibility.py | 51 +- tests/test_server_authz.py | 147 +- tests/test_session.py | 1709 ++++++++---- tests/test_session_backend_error_format.py | 19 +- tests/test_session_chat_reasoning_replay.py | 5 +- tests/test_session_helpers.py | 126 + tests/test_session_lifecycle_commands.py | 101 +- tests/test_session_manager.py | 38 +- tests/test_session_manager_lifecycle_races.py | 9 +- tests/test_session_mcp_dispatch_error.py | 6 + tests/test_session_replay_reasoning.py | 40 +- tests/test_session_synth_reasoning_block.py | 11 +- tests/test_session_ui_base.py | 158 +- tests/test_sessions.py | 45 +- tests/test_settings_registry.py | 15 + tests/test_skill_substitution_unification.py | 7 +- tests/test_skills_tool.py | 90 +- tests/test_sse_cursor_resume.py | 4 +- tests/test_storage_deferred_create.py | 77 + tests/test_storage_fork_clone.py | 34 +- tests/test_storage_sqlite.py | 67 - tests/test_structured_memory.py | 104 +- tests/test_structured_memory_storage.py | 1254 ++++++++- tests/test_think_tag_split.py | 11 +- tests/test_turn_provenance.py | 30 +- tests/test_user_storage.py | 21 + tests/test_watch_integration.py | 19 +- tests/test_workstream_endpoints.py | 59 +- turnstone.example.toml | 13 +- turnstone/api/console_schemas.py | 44 +- turnstone/api/console_spec.py | 30 +- turnstone/api/openapi.py | 97 +- turnstone/api/server_schemas.py | 56 +- turnstone/api/server_spec.py | 32 +- turnstone/console/server.py | 289 +- turnstone/console/session_factory.py | 12 - turnstone/console/static/admin.js | 7 +- turnstone/console/static/governance.js | 99 + turnstone/console/static/index.html | 13 +- turnstone/core/auth.py | 178 +- turnstone/core/config.py | 2 +- turnstone/core/memory.py | 583 +++- turnstone/core/memory_index.py | 224 ++ turnstone/core/memory_relevance.py | 72 +- turnstone/core/metacognition.py | 28 +- turnstone/core/model_turn.py | 125 +- turnstone/core/personas.py | 5 +- turnstone/core/project_access.py | 87 + turnstone/core/session.py | 2373 +++++++++++------ turnstone/core/session_manager.py | 181 +- turnstone/core/session_routes.py | 251 +- turnstone/core/session_ui_base.py | 229 +- turnstone/core/settings_registry.py | 62 +- turnstone/core/storage/_postgresql.py | 638 ++++- turnstone/core/storage/_protocol.py | 167 +- turnstone/core/storage/_schema.py | 43 +- turnstone/core/storage/_sqlite.py | 574 +++- turnstone/core/storage/_utils.py | 494 +++- .../versions/072_memory_index_snapshots.py | 71 + turnstone/core/tool_advisory.py | 2 +- turnstone/eval/cli.py | 7 - turnstone/optimizer.py | 6 - turnstone/sdk/console.py | 33 + turnstone/sdk/server.py | 43 +- turnstone/server.py | 114 +- turnstone/shared_static/utils.js | 1 + turnstone/tools/memory.json | 18 +- 121 files changed, 13744 insertions(+), 3663 deletions(-) create mode 100644 sdk/typescript/src/memory_description.ts create mode 100644 sdk/typescript/tests/memory_description.test.ts create mode 100644 tests/data/memory_description_parity.json create mode 100644 tests/test_console_db_config.py create mode 100644 tests/test_memory_index.py create mode 100644 tests/test_migration_072.py create mode 100644 tests/test_session_helpers.py create mode 100644 turnstone/core/memory_index.py create mode 100644 turnstone/core/project_access.py create mode 100644 turnstone/core/storage/migrations/versions/072_memory_index_snapshots.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3cbd31f..f7209454 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,11 +35,11 @@ jobs: test: runs-on: ubuntu-latest - # Cap a hung run at 30 min instead of riding GitHub's 6-hour default + # Cap a hung run at 45 min instead of riding GitHub's 6-hour default # (a flaky-hang run otherwise streams -v output for hours). Was 20; - # the suite's growth (~9.7k tests, coverage-instrumented, 3-version - # matrix) started brushing the old cap on healthy runs. - timeout-minutes: 30 + # the suite's growth (~12.3k tests, coverage-instrumented, 3-version + # matrix) started brushing the 30-minute cap on healthy runs. + timeout-minutes: 45 strategy: matrix: python-version: ["3.11", "3.12", "3.13"] @@ -68,7 +68,7 @@ jobs: test-postgres: runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 45 services: postgres: image: postgres:18 diff --git a/scripts/recovery_e2e.py b/scripts/recovery_e2e.py index 605e0124..7e3b0448 100644 --- a/scripts/recovery_e2e.py +++ b/scripts/recovery_e2e.py @@ -1598,6 +1598,7 @@ def _launch_chrome(chrome: str, profile: Path) -> tuple[subprocess.Popen[bytes], "--disable-gpu", "--no-sandbox", "--no-first-run", + "--password-store=basic", "--disable-extensions", "--disable-background-timer-throttling", f"--remote-debugging-port={cdp_port}", diff --git a/sdk/typescript/openapi-console.json b/sdk/typescript/openapi-console.json index acf7babe..d89ca824 100644 --- a/sdk/typescript/openapi-console.json +++ b/sdk/typescript/openapi-console.json @@ -3053,6 +3053,105 @@ } } } + }, + "500": { + "description": "Error 500", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "503": { + "description": "Error 503", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + } + } + }, + "patch": { + "summary": "Update a memory's authored index description", + "operationId": "v1_api_admin_memories_{memory_id}_patch", + "tags": [ + "Admin" + ], + "parameters": [ + { + "name": "memory_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UpdateMemoryDescriptionRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AdminMemorySummary" + } + } + } + }, + "400": { + "description": "Error 400", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "404": { + "description": "Error 404", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "500": { + "description": "Error 500", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "503": { + "description": "Error 503", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } } } }, @@ -3096,6 +3195,47 @@ } } }, + "/v1/api/admin/memories/index-health": { + "get": { + "summary": "Get derived live memory-index budget and legacy-hook health", + "operationId": "v1_api_admin_memories_index-health_get", + "tags": [ + "Admin" + ], + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MemoryIndexHealthResponse" + } + } + } + }, + "500": { + "description": "Error 500", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "503": { + "description": "Error 503", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + } + } + } + }, "/v1/api/admin/settings": { "get": { "summary": "List all settings with effective values", @@ -6484,7 +6624,7 @@ "tags": [ "Coordinator" ], - "description": "Approves or denies the pending tool call(s). Set ``always`` to True to also add the pending tool name(s) to the session's auto-approve set so subsequent calls of the same tool skip the prompt.", + "description": "Approves or denies the pending tool call(s). An authorized peer may make a binary decision, but only the initiating execution principal may add feedback or set ``always``. Always grants are scoped to that execution principal and tool.", "parameters": [ { "name": "ws_id", @@ -8770,12 +8910,12 @@ } ], "default": null, - "description": "Optional human feedback string forwarded to the model.", + "description": "Optional feedback forwarded under the initiating execution principal; authorized peer resolvers must omit it.", "title": "Feedback" }, "always": { "default": false, - "description": "When approved=True, also adds the pending tool name(s) to the session's auto-approve set so subsequent calls of the same tool skip the prompt.", + "description": "For a same-principal approval, adds the pending tool name(s) to that execution principal's auto-approve set. Authorized peers cannot set it.", "title": "Always", "type": "boolean" }, @@ -8829,12 +8969,12 @@ } ], "default": null, - "description": "Optional denial reason", + "description": "Optional feedback forwarded under the initiating execution principal; authorized peer resolvers must omit it.", "title": "Feedback" }, "always": { "default": false, - "description": "Auto-approve the tools in this batch going forward", + "description": "For a same-principal approval, auto-approve these tools for future calls executing as that principal. Authorized peers cannot set this.", "title": "Always", "type": "boolean" }, @@ -10843,6 +10983,16 @@ "title": "User Decision", "type": "string" }, + "resolver_principal_id": { + "default": "", + "title": "Resolver Principal Id", + "type": "string" + }, + "execution_principal_id": { + "default": "", + "title": "Execution Principal Id", + "type": "string" + }, "latency_ms": { "default": 0, "title": "Latency Ms", @@ -11025,9 +11175,99 @@ "title": "Scope Id", "type": "string" }, + "scope_label": { + "default": "", + "title": "Scope Label", + "type": "string" + }, + "created": { + "title": "Created", + "type": "string" + }, + "updated": { + "title": "Updated", + "type": "string" + }, + "last_accessed": { + "default": "", + "title": "Last Accessed", + "type": "string" + }, + "access_count": { + "default": 0, + "title": "Access Count", + "type": "integer" + }, "content": { "title": "Content", "type": "string" + } + }, + "required": [ + "memory_id", + "name", + "type", + "scope", + "created", + "updated", + "content" + ], + "title": "AdminMemoryInfo", + "type": "object" + }, + "ListAdminMemoriesResponse": { + "properties": { + "memories": { + "items": { + "$ref": "#/components/schemas/AdminMemorySummary" + }, + "title": "Memories", + "type": "array" + }, + "total": { + "default": 0, + "title": "Total", + "type": "integer" + } + }, + "required": [ + "memories" + ], + "title": "ListAdminMemoriesResponse", + "type": "object" + }, + "AdminMemorySummary": { + "properties": { + "memory_id": { + "title": "Memory Id", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "description": { + "default": "", + "title": "Description", + "type": "string" + }, + "type": { + "title": "Type", + "type": "string" + }, + "scope": { + "title": "Scope", + "type": "string" + }, + "scope_id": { + "default": "", + "title": "Scope Id", + "type": "string" + }, + "scope_label": { + "default": "", + "title": "Scope Label", + "type": "string" }, "created": { "title": "Created", @@ -11053,32 +11293,10 @@ "name", "type", "scope", - "content", "created", "updated" ], - "title": "AdminMemoryInfo", - "type": "object" - }, - "ListAdminMemoriesResponse": { - "properties": { - "memories": { - "items": { - "$ref": "#/components/schemas/AdminMemoryInfo" - }, - "title": "Memories", - "type": "array" - }, - "total": { - "default": 0, - "title": "Total", - "type": "integer" - } - }, - "required": [ - "memories" - ], - "title": "ListAdminMemoriesResponse", + "title": "AdminMemorySummary", "type": "object" }, "SettingInfo": { @@ -15312,6 +15530,416 @@ ], "title": "WorkstreamHistoryResponse", "type": "object" + }, + "AuthWhoamiResponse": { + "description": "GET /v1/api/auth/whoami response.", + "properties": { + "user_id": { + "title": "User Id", + "type": "string" + }, + "permissions": { + "default": "", + "title": "Permissions", + "type": "string" + } + }, + "required": [ + "user_id" + ], + "title": "AuthWhoamiResponse", + "type": "object" + }, + "RoleEffectiveResponse": { + "properties": { + "baseline": { + "items": { + "type": "string" + }, + "title": "Baseline", + "type": "array" + }, + "grants": { + "items": { + "type": "string" + }, + "title": "Grants", + "type": "array" + }, + "revokes": { + "items": { + "type": "string" + }, + "title": "Revokes", + "type": "array" + }, + "effective": { + "items": { + "type": "string" + }, + "title": "Effective", + "type": "array" + } + }, + "required": [ + "baseline", + "grants", + "revokes", + "effective" + ], + "title": "RoleEffectiveResponse", + "type": "object" + }, + "RoleOverridesRequest": { + "properties": { + "grant": { + "default": [], + "items": { + "type": "string" + }, + "title": "Grant", + "type": "array" + }, + "revoke": { + "default": [], + "items": { + "type": "string" + }, + "title": "Revoke", + "type": "array" + } + }, + "title": "RoleOverridesRequest", + "type": "object" + }, + "UpdateMemoryDescriptionRequest": { + "properties": { + "description": { + "maxLength": 512, + "minLength": 1, + "title": "Description", + "type": "string" + } + }, + "required": [ + "description" + ], + "title": "UpdateMemoryDescriptionRequest", + "type": "object" + }, + "MemoryIndexHealthResponse": { + "properties": { + "budget_chars": { + "title": "Budget Chars", + "type": "integer" + }, + "over_budget": { + "title": "Over Budget", + "type": "boolean" + }, + "max_char_count": { + "title": "Max Char Count", + "type": "integer" + }, + "max_entry_count": { + "title": "Max Entry Count", + "type": "integer" + }, + "over_by_chars": { + "title": "Over By Chars", + "type": "integer" + }, + "invalid_description_count": { + "title": "Invalid Description Count", + "type": "integer" + }, + "envelope_count": { + "title": "Envelope Count", + "type": "integer" + } + }, + "required": [ + "budget_chars", + "over_budget", + "max_char_count", + "max_entry_count", + "over_by_chars", + "invalid_description_count", + "envelope_count" + ], + "title": "MemoryIndexHealthResponse", + "type": "object" + }, + "NodeMetadataResponse": { + "properties": { + "node_id": { + "title": "Node Id", + "type": "string" + }, + "metadata": { + "items": { + "$ref": "#/components/schemas/NodeMetadataEntry" + }, + "title": "Metadata", + "type": "array" + } + }, + "required": [ + "node_id" + ], + "title": "NodeMetadataResponse", + "type": "object" + }, + "BulkSetNodeMetadataRequest": { + "properties": { + "entries": { + "items": { + "$ref": "#/components/schemas/SetNodeMetadataRequest" + }, + "title": "Entries", + "type": "array" + } + }, + "title": "BulkSetNodeMetadataRequest", + "type": "object" + }, + "SetNodeMetadataRequest": { + "description": "Single entry in a bulk metadata set.", + "properties": { + "key": { + "title": "Key", + "type": "string" + }, + "value": { + "title": "Value" + } + }, + "required": [ + "key", + "value" + ], + "title": "SetNodeMetadataRequest", + "type": "object" + }, + "SetNodeMetadataValueRequest": { + "description": "Request body for PUT /admin/nodes/{node_id}/metadata/{key}.", + "properties": { + "value": { + "title": "Value" + } + }, + "required": [ + "value" + ], + "title": "SetNodeMetadataValueRequest", + "type": "object" + }, + "RewindRequest": { + "properties": { + "turns": { + "description": "Number of conversation turns (user message + its responses) to drop from the end. Clamped to the available turn count.", + "minimum": 1, + "title": "Turns", + "type": "integer" + } + }, + "required": [ + "turns" + ], + "title": "RewindRequest", + "type": "object" + }, + "ListWorkstreamsResponse": { + "description": "Response body for ``GET /v1/api/workstreams`` on either kind.\n\nTop-level key is ``workstreams`` regardless of the kind serving\nthe request \u2014 pre-lift coord returned ``{\"coordinators\": [...]}``;\nconvergence lifted both kinds onto the same shape. Coord SDK /\nfrontend consumers branching on ``data.coordinators`` swap to\n``data.workstreams``.", + "properties": { + "workstreams": { + "items": { + "$ref": "#/components/schemas/WorkstreamInfo" + }, + "title": "Workstreams", + "type": "array" + } + }, + "required": [ + "workstreams" + ], + "title": "ListWorkstreamsResponse", + "type": "object" + }, + "WorkstreamInfo": { + "description": "Active-list row shape, shared across both kinds.\n\nRenamed ``id`` \u2192 ``ws_id`` and added ``user_id`` in the Stage 2\n``list``/``saved`` verb lift so the active-list response shape\nmatches the rest of the v1 surface (every other shared verb's\npayload uses ``ws_id``). ``user_id`` was previously coord-only;\ninteractive now populates it too. SDK consumers reading\n``row.id`` should swap to ``row.ws_id``.", + "properties": { + "ws_id": { + "title": "Ws Id", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "state": { + "title": "State", + "type": "string" + }, + "kind": { + "$ref": "#/components/schemas/WorkstreamKind", + "default": "interactive" + }, + "parent_ws_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Parent Ws Id" + }, + "user_id": { + "default": "", + "title": "User Id", + "type": "string" + }, + "project_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Project Id" + }, + "persistence_state": { + "default": "healthy", + "description": "Sanitized durable-history status for the loaded workstream: healthy, pending its first save, retrying automatically, or blocked by a permanent commit conflict. Older servers and unloaded rows default to healthy.", + "enum": [ + "healthy", + "pending", + "retrying", + "conflict" + ], + "title": "Persistence State", + "type": "string" + } + }, + "required": [ + "ws_id", + "name", + "state" + ], + "title": "WorkstreamInfo", + "type": "object" + }, + "UploadAttachmentResponse": { + "description": "Returned after a successful upload.", + "properties": { + "attachment_id": { + "description": "Opaque id for this attachment", + "title": "Attachment Id", + "type": "string" + }, + "filename": { + "description": "Original upload filename", + "title": "Filename", + "type": "string" + }, + "mime_type": { + "description": "Canonicalized MIME type", + "title": "Mime Type", + "type": "string" + }, + "size_bytes": { + "description": "Payload size in bytes", + "title": "Size Bytes", + "type": "integer" + }, + "kind": { + "description": "'image', 'text', 'pdf', or 'audio'", + "examples": [ + "image", + "text", + "pdf", + "audio" + ], + "title": "Kind", + "type": "string" + } + }, + "required": [ + "attachment_id", + "filename", + "mime_type", + "size_bytes", + "kind" + ], + "title": "UploadAttachmentResponse", + "type": "object" + }, + "ListAttachmentsResponse": { + "properties": { + "attachments": { + "description": "Pending (unconsumed) attachments for caller+workstream", + "items": { + "$ref": "#/components/schemas/AttachmentInfo" + }, + "title": "Attachments", + "type": "array" + } + }, + "required": [ + "attachments" + ], + "title": "ListAttachmentsResponse", + "type": "object" + }, + "AttachmentInfo": { + "properties": { + "attachment_id": { + "description": "Opaque id for this attachment", + "title": "Attachment Id", + "type": "string" + }, + "filename": { + "description": "Original upload filename", + "title": "Filename", + "type": "string" + }, + "mime_type": { + "description": "Canonicalized MIME type", + "title": "Mime Type", + "type": "string" + }, + "size_bytes": { + "description": "Payload size in bytes", + "title": "Size Bytes", + "type": "integer" + }, + "kind": { + "description": "'image', 'text', 'pdf', or 'audio'", + "examples": [ + "image", + "text", + "pdf", + "audio" + ], + "title": "Kind", + "type": "string" + } + }, + "required": [ + "attachment_id", + "filename", + "mime_type", + "size_bytes", + "kind" + ], + "title": "AttachmentInfo", + "type": "object" } } } diff --git a/sdk/typescript/openapi-server.json b/sdk/typescript/openapi-server.json index d2aa0f42..fda27ca1 100644 --- a/sdk/typescript/openapi-server.json +++ b/sdk/typescript/openapi-server.json @@ -401,6 +401,16 @@ } } }, + "400": { + "description": "Error 400", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, "404": { "description": "Error 404", "content": { @@ -1988,7 +1998,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/MemoryInfo" + "$ref": "#/components/schemas/MemorySummary" } } } @@ -2108,6 +2118,94 @@ } }, "/v1/api/memories/{name}": { + "get": { + "summary": "Fetch a structured memory body by exact name and scope", + "operationId": "v1_api_memories_{name}_get", + "tags": [ + "Memories" + ], + "parameters": [ + { + "name": "name", + "in": "path", + "required": true, + "schema": { + "type": "string" + }, + "description": "Memory identifier. Raw aliases may contain supported Latin letters that fold to ASCII, ASCII digits, Unicode space separators, Unicode hyphens, and single underscores. The server normalizes them to a lowercase ASCII snake_case key of at most 256 characters. Other characters and leading, trailing, or repeated underscores are rejected." + }, + { + "name": "scope", + "in": "query", + "required": false, + "schema": { + "type": "string" + }, + "description": "Scope (default: global)" + }, + { + "name": "scope_id", + "in": "query", + "required": false, + "schema": { + "type": "string" + }, + "description": "Scope identifier" + } + ], + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/MemoryInfo" + } + } + } + }, + "400": { + "description": "Error 400", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "403": { + "description": "Error 403", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "404": { + "description": "Error 404", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + }, + "500": { + "description": "Error 500", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + } + } + } + }, "delete": { "summary": "Delete a structured memory by name and scope", "operationId": "v1_api_memories_{name}_delete", @@ -2121,7 +2219,8 @@ "required": true, "schema": { "type": "string" - } + }, + "description": "Memory identifier. Raw aliases may contain supported Latin letters that fold to ASCII, ASCII digits, Unicode space separators, Unicode hyphens, and single underscores. The server normalizes them to a lowercase ASCII snake_case key of at most 256 characters. Other characters and leading, trailing, or repeated underscores are rejected." }, { "name": "scope", @@ -2673,12 +2772,12 @@ } ], "default": null, - "description": "Optional denial reason", + "description": "Optional feedback forwarded under the initiating execution principal; authorized peer resolvers must omit it.", "title": "Feedback" }, "always": { "default": false, - "description": "Auto-approve the tools in this batch going forward", + "description": "For a same-principal approval, auto-approve these tools for future calls executing as that principal. Authorized peers cannot set this.", "title": "Always", "type": "boolean" }, @@ -3993,8 +4092,7 @@ "SaveMemoryRequest": { "properties": { "name": { - "description": "Memory identifier (normalized to snake_case)", - "maxLength": 256, + "description": "Memory identifier. Raw aliases may contain supported Latin letters that fold to ASCII, ASCII digits, Unicode space separators, Unicode hyphens, and single underscores. The server normalizes them to a lowercase ASCII snake_case key of at most 256 characters. Other characters and leading, trailing, or repeated underscores are rejected.", "minLength": 1, "title": "Name", "type": "string" @@ -4007,7 +4105,8 @@ "type": "string" }, "description": { - "description": "Required non-empty description used for relevance matching", + "description": "Required authored one-line memory-index hook", + "maxLength": 512, "minLength": 1, "title": "Description", "type": "string" @@ -4096,10 +4195,6 @@ "title": "Scope Id", "type": "string" }, - "content": { - "title": "Content", - "type": "string" - }, "created": { "title": "Created", "type": "string" @@ -4107,6 +4202,20 @@ "updated": { "title": "Updated", "type": "string" + }, + "last_accessed": { + "default": "", + "title": "Last Accessed", + "type": "string" + }, + "access_count": { + "default": 0, + "title": "Access Count", + "type": "integer" + }, + "content": { + "title": "Content", + "type": "string" } }, "required": [ @@ -4114,9 +4223,9 @@ "name", "type", "scope", - "content", "created", - "updated" + "updated", + "content" ], "title": "MemoryInfo", "type": "object" @@ -4125,7 +4234,7 @@ "properties": { "memories": { "items": { - "$ref": "#/components/schemas/MemoryInfo" + "$ref": "#/components/schemas/MemorySummary" }, "title": "Memories", "type": "array" @@ -4142,6 +4251,75 @@ "title": "ListMemoriesResponse", "type": "object" }, + "MemorySummary": { + "properties": { + "memory_id": { + "title": "Memory Id", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "description": { + "default": "", + "title": "Description", + "type": "string" + }, + "type": { + "enum": [ + "user", + "general", + "feedback", + "reference" + ], + "title": "Type", + "type": "string" + }, + "scope": { + "enum": [ + "global", + "workstream", + "user" + ], + "title": "Scope", + "type": "string" + }, + "scope_id": { + "default": "", + "title": "Scope Id", + "type": "string" + }, + "created": { + "title": "Created", + "type": "string" + }, + "updated": { + "title": "Updated", + "type": "string" + }, + "last_accessed": { + "default": "", + "title": "Last Accessed", + "type": "string" + }, + "access_count": { + "default": 0, + "title": "Access Count", + "type": "integer" + } + }, + "required": [ + "memory_id", + "name", + "type", + "scope", + "created", + "updated" + ], + "title": "MemorySummary", + "type": "object" + }, "SearchMemoriesRequest": { "properties": { "query": { @@ -4403,6 +4581,25 @@ }, "title": "ListAvailableModelsResponse", "type": "object" + }, + "AuthWhoamiResponse": { + "description": "GET /v1/api/auth/whoami response.", + "properties": { + "user_id": { + "title": "User Id", + "type": "string" + }, + "permissions": { + "default": "", + "title": "Permissions", + "type": "string" + } + }, + "required": [ + "user_id" + ], + "title": "AuthWhoamiResponse", + "type": "object" } } } diff --git a/sdk/typescript/src/console.ts b/sdk/typescript/src/console.ts index 9124c8a0..035a100e 100644 --- a/sdk/typescript/src/console.ts +++ b/sdk/typescript/src/console.ts @@ -1,8 +1,10 @@ import { BaseClient, type ClientOptions } from "./base.js"; import type { ClusterEvent } from "./events.js"; +import { normalizeMemoryDescription } from "./memory_description.js"; import type { AdminListMemoriesOptions, AdminMemoryInfo, + AdminMemorySummary, AdminSearchMemoriesOptions, AttachmentContent, AttachmentUpload, @@ -34,6 +36,7 @@ import type { ListSettingsResponse, ListSkillResourcesResponse, ListSkillsResponse, + MemoryIndexHealthResponse, McpServerDetail, RegistryInstallRequest, RegistrySearchResponse, @@ -494,6 +497,20 @@ export class TurnstoneConsole extends BaseClient { return this.request("GET", `/v1/api/admin/memories/${memoryId}`); } + async updateMemoryDescription( + memoryId: string, + description: string, + ): Promise { + const normalized = normalizeMemoryDescription(description); + return this.request("PATCH", `/v1/api/admin/memories/${memoryId}`, { + json: { description: normalized }, + }); + } + + async memoryIndexHealth(): Promise { + return this.request("GET", "/v1/api/admin/memories/index-health"); + } + async deleteMemory(memoryId: string): Promise { return this.request("DELETE", `/v1/api/admin/memories/${memoryId}`); } diff --git a/sdk/typescript/src/index.ts b/sdk/typescript/src/index.ts index 863231c0..7d0cd5f6 100644 --- a/sdk/typescript/src/index.ts +++ b/sdk/typescript/src/index.ts @@ -159,15 +159,20 @@ export type { WorkstreamsOptions, // Memory types SaveMemoryRequest, + MemorySummary, MemoryInfo, ListMemoriesResponse, SearchMemoriesRequest, ListMemoriesOptions, + MemoryScopeOptions, + GetMemoryOptions, DeleteMemoryOptions, + AdminMemorySummary, AdminMemoryInfo, ListAdminMemoriesResponse, AdminListMemoriesOptions, AdminSearchMemoriesOptions, + MemoryIndexHealthResponse, // Settings types SettingInfo, ListSettingsResponse, diff --git a/sdk/typescript/src/memory_description.ts b/sdk/typescript/src/memory_description.ts new file mode 100644 index 00000000..7ebb8a4c --- /dev/null +++ b/sdk/typescript/src/memory_description.ts @@ -0,0 +1,27 @@ +const DESCRIPTION_WHITESPACE = + /[\u0009-\u000d\u0020\u0085\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]+/g; + +/** Internal wire-boundary normalizer shared by both SDK clients. */ +export function normalizeMemoryDescription(description: unknown): string { + if (typeof description !== "string") { + throw new TypeError( + "memory description is required and must be non-empty", + ); + } + let normalized = description.replace(DESCRIPTION_WHITESPACE, " "); + if (normalized.startsWith(" ")) { + normalized = normalized.slice(1); + } + if (normalized.endsWith(" ")) { + normalized = normalized.slice(0, -1); + } + if (!normalized) { + throw new TypeError( + "memory description is required and must be non-empty", + ); + } + if (Array.from(normalized).length > 512) { + throw new TypeError("memory description exceeds 512 characters"); + } + return normalized; +} diff --git a/sdk/typescript/src/server.ts b/sdk/typescript/src/server.ts index 18bc759b..4fc6d92b 100644 --- a/sdk/typescript/src/server.ts +++ b/sdk/typescript/src/server.ts @@ -1,5 +1,6 @@ import { BaseClient, type ClientOptions } from "./base.js"; import type { ServerEvent } from "./events.js"; +import { normalizeMemoryDescription } from "./memory_description.js"; import type { AttachmentContent, AttachmentUpload, @@ -12,6 +13,7 @@ import type { CreateWorkstreamResponse, DashboardResponse, DeleteMemoryOptions, + GetMemoryOptions, HealthResponse, ListAttachmentsResponse, ListMemoriesOptions, @@ -19,6 +21,7 @@ import type { ListSavedWorkstreamsResponse, ListWorkstreamsResponse, MemoryInfo, + MemorySummary, SaveMemoryRequest, SearchMemoriesRequest, SendAndWaitOptions, @@ -393,14 +396,10 @@ export class TurnstoneServer extends BaseClient { return this.request("GET", "/v1/api/memories", { params }); } - async saveMemory(opts: SaveMemoryRequest): Promise { - if (typeof opts.description !== "string" || !opts.description.trim()) { - throw new TypeError( - "memory description is required and must be non-empty", - ); - } + async saveMemory(opts: SaveMemoryRequest): Promise { + const description = normalizeMemoryDescription(opts.description); return this.request("POST", "/v1/api/memories", { - json: { ...opts, description: opts.description.trim() }, + json: { ...opts, description }, }); } @@ -410,6 +409,16 @@ export class TurnstoneServer extends BaseClient { return this.request("POST", "/v1/api/memories/search", { json: opts }); } + async getMemory( + name: string, + opts?: GetMemoryOptions, + ): Promise { + const params: Record = {}; + if (opts?.scope) params.scope = opts.scope; + if (opts?.scope_id) params.scope_id = opts.scope_id; + return this.request("GET", `/v1/api/memories/${encodeURIComponent(name)}`, { params }); + } + async deleteMemory( name: string, opts?: DeleteMemoryOptions, @@ -417,7 +426,7 @@ export class TurnstoneServer extends BaseClient { const params: Record = {}; if (opts?.scope) params.scope = opts.scope; if (opts?.scope_id) params.scope_id = opts.scope_id; - return this.request("DELETE", `/v1/api/memories/${name}`, { params }); + return this.request("DELETE", `/v1/api/memories/${encodeURIComponent(name)}`, { params }); } // -- Auth ----------------------------------------------------------------- diff --git a/sdk/typescript/src/types.ts b/sdk/typescript/src/types.ts index a12a4c7b..841d255b 100644 --- a/sdk/typescript/src/types.ts +++ b/sdk/typescript/src/types.ts @@ -900,20 +900,25 @@ export interface SaveMemoryRequest { scope_id?: string; } -export interface MemoryInfo { +export interface MemorySummary { memory_id: string; name: string; description: string; type: string; scope: string; scope_id: string; - content: string; created: string; updated: string; + last_accessed: string; + access_count: number; +} + +export interface MemoryInfo extends MemorySummary { + content: string; } export interface ListMemoriesResponse { - memories: MemoryInfo[]; + memories: MemorySummary[]; total: number; } @@ -932,29 +937,36 @@ export interface ListMemoriesOptions { limit?: number; } -export interface DeleteMemoryOptions { +export interface MemoryScopeOptions { scope?: string; scope_id?: string; } +export type GetMemoryOptions = MemoryScopeOptions; +export type DeleteMemoryOptions = MemoryScopeOptions; + // -- Console API: Admin Memories -------------------------------------------- -export interface AdminMemoryInfo { +export interface AdminMemorySummary { memory_id: string; name: string; description: string; type: string; scope: string; scope_id: string; - content: string; + scope_label: string; created: string; updated: string; last_accessed: string; access_count: number; } +export interface AdminMemoryInfo extends AdminMemorySummary { + content: string; +} + export interface ListAdminMemoriesResponse { - memories: AdminMemoryInfo[]; + memories: AdminMemorySummary[]; total: number; } @@ -973,6 +985,16 @@ export interface AdminSearchMemoriesOptions { limit?: number; } +export interface MemoryIndexHealthResponse { + budget_chars: number; + over_budget: boolean; + max_char_count: number; + max_entry_count: number; + over_by_chars: number; + invalid_description_count: number; + envelope_count: number; +} + // -- Console API: MCP Servers ----------------------------------------------- export interface McpServerStatus { diff --git a/sdk/typescript/tests/console.test.ts b/sdk/typescript/tests/console.test.ts index 846e4bdd..e1f37003 100644 --- a/sdk/typescript/tests/console.test.ts +++ b/sdk/typescript/tests/console.test.ts @@ -11,6 +11,61 @@ function mockFetch(response: object): typeof globalThis.fetch { } describe("TurnstoneConsole", () => { + it("updates memory hooks and reads index health", async () => { + const fetchFn = vi + .fn() + .mockResolvedValueOnce( + new Response( + JSON.stringify({ + memory_id: "m1", + name: "deployment_process", + description: "Production deployment workflow", + type: "general", + scope: "global", + scope_id: "", + content: "Deploy from main", + created: "2026-08-11T00:00:00", + updated: "2026-08-11T00:00:00", + last_accessed: "", + access_count: 0, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ), + ) + .mockResolvedValueOnce( + new Response( + JSON.stringify({ + budget_chars: 65536, + over_budget: false, + max_char_count: 120, + max_entry_count: 2, + over_by_chars: 0, + invalid_description_count: 0, + envelope_count: 1, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ), + ); + const client = new TurnstoneConsole({ + baseUrl: "http://test", + fetch: fetchFn, + }); + + await client.updateMemoryDescription( + "m1", + " Production\n deployment workflow ", + ); + const health = await client.memoryIndexHealth(); + + const [url, init] = (fetchFn as ReturnType).mock.calls[0]; + expect(url).toBe("http://test/v1/api/admin/memories/m1"); + expect(init.method).toBe("PATCH"); + expect(JSON.parse(init.body)).toEqual({ + description: "Production deployment workflow", + }); + expect(health.budget_chars).toBe(65536); + }); + it("overview returns parsed response", async () => { const fetchFn = mockFetch({ nodes: 2, diff --git a/sdk/typescript/tests/memory_description.test.ts b/sdk/typescript/tests/memory_description.test.ts new file mode 100644 index 00000000..e773f53a --- /dev/null +++ b/sdk/typescript/tests/memory_description.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from "vitest"; +import { readFileSync } from "node:fs"; +import { normalizeMemoryDescription } from "../src/memory_description.js"; + +interface DescriptionParityCorpus { + whitespace_code_points: number[]; + preserved_code_points: number[]; + empty_inputs: string[]; + non_string_inputs: unknown[]; + boundaries: Array<{ + label: string; + character: string; + count: number; + valid: boolean; + }>; +} + +const CORPUS = JSON.parse( + readFileSync( + new URL("../../../tests/data/memory_description_parity.json", import.meta.url), + "utf8", + ), +) as DescriptionParityCorpus; + +describe("memory description normalization", () => { + it.each(CORPUS.whitespace_code_points)("folds U+%s", (codePoint) => { + const space = String.fromCodePoint(codePoint); + expect(normalizeMemoryDescription(`${space}alpha${space}${space}beta${space}`)) + .toBe("alpha beta"); + }); + + it("preserves characters outside the explicit whitespace set", () => { + const preserved = String.fromCodePoint(...CORPUS.preserved_code_points); + expect(normalizeMemoryDescription(`${preserved}alpha${preserved}`)).toBe( + `${preserved}alpha${preserved}`, + ); + }); + + it.each(CORPUS.empty_inputs)( + "rejects empty-after-normalization input", + (description) => { + expect(() => normalizeMemoryDescription(description)).toThrow( + "description is required", + ); + }, + ); + + it.each([...CORPUS.non_string_inputs, undefined])( + "rejects non-string input %#", + (description) => { + expect(() => normalizeMemoryDescription(description)).toThrow(TypeError); + }, + ); + + it.each(CORPUS.boundaries)("enforces the code-point cap for $label", (boundary) => { + const value = boundary.character.repeat(boundary.count); + const valid = boundary.valid; + if (valid) { + expect(normalizeMemoryDescription(value)).toBe(value); + } else { + expect(() => normalizeMemoryDescription(value)).toThrow("512"); + } + }); +}); diff --git a/sdk/typescript/tests/server.test.ts b/sdk/typescript/tests/server.test.ts index 7686d088..863e5132 100644 --- a/sdk/typescript/tests/server.test.ts +++ b/sdk/typescript/tests/server.test.ts @@ -109,7 +109,104 @@ describe("TurnstoneServer", () => { description: " ", }), ).rejects.toThrow("description is required"); + await expect( + client.saveMemory({ + name: "deployment_process", + content: "Deploy from main", + description: "\u0085".repeat(4), + }), + ).rejects.toThrow("description is required"); + await expect( + client.saveMemory({ + name: "deployment_process", + content: "Deploy from main", + description: "x".repeat(513), + }), + ).rejects.toThrow("512"); + const unicodeFetch = mockFetch({}); + const unicodeClient = new TurnstoneServer({ + baseUrl: "http://test", + fetch: unicodeFetch, + }); + await unicodeClient.saveMemory({ + name: "unicode_hook", + content: "body", + description: "πŸ™‚".repeat(512), + }); expect(fetchFn).toHaveBeenCalledTimes(1); + expect(unicodeFetch).toHaveBeenCalledTimes(1); + }); + + it("getMemory fetches one exact body with scope", async () => { + const fetchFn = mockFetch({ + memory_id: "m1", + name: "deployment_process", + description: "Production deployment workflow", + type: "general", + scope: "workstream", + scope_id: "ws1", + content: "Deploy from main", + created: "2026-08-11T00:00:00", + updated: "2026-08-11T00:00:00", + last_accessed: "", + access_count: 0, + }); + const client = new TurnstoneServer({ + baseUrl: "http://test", + fetch: fetchFn, + }); + + const memory = await client.getMemory("deployment_process", { + scope: "workstream", + scope_id: "ws1", + }); + + expect(memory.content).toBe("Deploy from main"); + const [url, init] = (fetchFn as ReturnType).mock.calls[0]; + expect(url).toContain("/v1/api/memories/deployment_process"); + expect(url).toContain("scope=workstream"); + expect(url).toContain("scope_id=ws1"); + expect(init.method).toBe("GET"); + }); + + it("percent-encodes memory names as one path segment", async () => { + const responseBody = { + memory_id: "m1", + name: "reserved_name", + description: "Reserved-name probe", + type: "general", + scope: "global", + scope_id: "", + content: "body", + created: "2026-08-11T00:00:00", + updated: "2026-08-11T00:00:00", + last_accessed: "", + access_count: 0, + status: "ok", + }; + const fetchFn = vi.fn().mockImplementation(() => + Promise.resolve( + new Response(JSON.stringify(responseBody), { + status: 200, + headers: { "content-type": "application/json" }, + }), + ), + ) as typeof globalThis.fetch; + const client = new TurnstoneServer({ + baseUrl: "http://test", + fetch: fetchFn, + }); + + await client.getMemory("cafΓ©/name?#"); + await client.deleteMemory("cafΓ©/name?#"); + + const urls = (fetchFn as ReturnType).mock.calls.map( + ([url]) => url, + ); + expect(urls).toEqual([ + "http://test/v1/api/memories/caf%C3%A9%2Fname%3F%23", + "http://test/v1/api/memories/caf%C3%A9%2Fname%3F%23", + ]); }); it("send posts correct payload", async () => { diff --git a/tests/_helpers.py b/tests/_helpers.py index 9064da23..872780d1 100644 --- a/tests/_helpers.py +++ b/tests/_helpers.py @@ -39,7 +39,7 @@ def make_chat_session(**overrides: Any) -> Any: """Build a minimal ``ChatSession`` with sane test defaults. Caller passes any constructor arg as a kwarg to override the default β€” - e.g. ``make_chat_session(memory_config=MemoryConfig(fetch_limit=5))``. + e.g. ``make_chat_session(memory_config=MemoryConfig(relevance_k=5))``. """ from turnstone.core.session import ChatSession @@ -62,10 +62,7 @@ def patch_session_storage( active: bool = True, raise_on_is_active: bool = False, ) -> list[str]: - """Patch ``session.get_storage`` to a stub whose ``is_watch_active`` - returns *active* (or raises if *raise_on_is_active*). Returns the - list of ``watch_id``s the predicate was called with. - """ + """Patch session storage for watch predicate tests.""" from turnstone.core import session as session_mod calls: list[str] = [] diff --git a/tests/_parity_832.py b/tests/_parity_832.py index 2625e4e4..a0b0743e 100644 --- a/tests/_parity_832.py +++ b/tests/_parity_832.py @@ -36,7 +36,7 @@ from typing import Any from tests._session_helpers import ( RecordingUI, - make_session, + make_registered_session, replace_session_lane, scripted_provider, ) @@ -170,7 +170,7 @@ def run_scenario(name: str) -> dict[str, Any]: behavior β€” ``write_fixture`` refuses one. """ ui = RecordingUI() - session = make_session(ui=ui) + session = make_registered_session(ui=ui) # Zero the ladder backoff: a scenario that reaches the mid-stream # re-issue ladder (no_finish_clean_exhaust) must not sleep real # exponential delays in a unit run. The retry-notice transform in diff --git a/tests/_session_helpers.py b/tests/_session_helpers.py index 939c4ec6..0dfa741a 100644 --- a/tests/_session_helpers.py +++ b/tests/_session_helpers.py @@ -26,6 +26,7 @@ from turnstone.core.providers import ModelCapabilities, StreamChunk, ToolCallDel from turnstone.core.session import ChatSession from turnstone.core.session_ui_base import SessionUIBase from turnstone.core.trajectory import ProviderNative, ToolCall, Turn +from turnstone.core.workstream import WorkstreamKind class NullUI(SessionUIBase): @@ -78,7 +79,15 @@ def replace_session_lane( def make_session(**kwargs: Any) -> ChatSession: """Build a ChatSession with minimal defaults; tests override - individual fields via kwargs.""" + individual fields via kwargs. + + This is the ordinary factory. It never publishes a durable workstream; + tests that exercise first-provider-request admission opt in through + :func:`make_registered_session` after initializing a test storage backend. + Storage selection remains ChatSession's normal process-global contract, + including its file-backed SQLite fallback when the host has not initialized + another backend. + """ defaults: dict[str, Any] = { "client": MagicMock(), "model": "test-model", @@ -106,6 +115,75 @@ def make_session(**kwargs: Any) -> ChatSession: return ChatSession(**defaults) +def make_registered_session(**kwargs: Any) -> ChatSession: + """Build a session backed by an explicitly initialized storage backend. + + The helper never invokes ``get_storage`` until the singleton has already + been initialized, preventing an unrelated test from creating + ``.turnstone.db`` in its ambient cwd. A repeated id is accepted only when + the durable identity metadata is exactly the identity this session asks + for; collisions are surfaced instead of quietly borrowing another row. + """ + import uuid + + from turnstone.core.storage import get_storage, is_storage_initialized + + if not is_storage_initialized(): + raise RuntimeError("make_registered_session requires initialized test storage") + storage = get_storage() + ws_id = str(kwargs.get("ws_id") or uuid.uuid4().hex) + user_id = str(kwargs.get("user_id") or "") or None + raw_kind = kwargs.get("kind", WorkstreamKind.INTERACTIVE) + kind = raw_kind if isinstance(raw_kind, WorkstreamKind) else WorkstreamKind(str(raw_kind)) + if kind == WorkstreamKind.COORDINATOR and user_id is None: + raise ValueError( + "coordinator sessions require an authenticated user_id; " + f"refusing to construct an anonymous coordinator (ws_id={ws_id!r})" + ) + project_id = str(kwargs.get("project_id") or "").strip() or None + persona_snapshot = kwargs.get("persona_snapshot") + persona = ( + str(getattr(persona_snapshot, "name", "") or "").strip() or None + if persona_snapshot is not None + else None + ) + expected = { + "user_id": user_id, + "kind": kind.value, + "project_id": project_id, + "persona": persona, + } + existing = storage.get_workstream(ws_id) + if existing is None: + inserted = storage.register_workstream( + ws_id, + user_id=expected["user_id"], + kind=kind, + project_id=expected["project_id"], + persona=expected["persona"], + ) + existing = storage.get_workstream(ws_id) + if inserted is False and existing is None: + raise RuntimeError(f"workstream {ws_id!r} registration lost its durable row") + actual = ( + { + "user_id": existing.get("user_id") or None, + "kind": str(existing.get("kind") or ""), + "project_id": existing.get("project_id") or None, + "persona": existing.get("persona") or None, + } + if existing is not None + else None + ) + if actual != expected: + raise RuntimeError( + f"workstream {ws_id!r} is already registered with different metadata: " + f"expected {expected!r}, found {actual!r}" + ) + kwargs["ws_id"] = ws_id + return make_session(**kwargs) + + def mock_completion_result( content: str = "", tool_calls: list[dict[str, Any]] | None = None, diff --git a/tests/data/memory_description_parity.json b/tests/data/memory_description_parity.json new file mode 100644 index 00000000..882d3419 --- /dev/null +++ b/tests/data/memory_description_parity.json @@ -0,0 +1,39 @@ +{ + "whitespace_code_points": [ + 9, + 10, + 11, + 12, + 13, + 32, + 133, + 160, + 5760, + 8192, + 8193, + 8194, + 8195, + 8196, + 8197, + 8198, + 8199, + 8200, + 8201, + 8202, + 8232, + 8233, + 8239, + 8287, + 12288, + 65279 + ], + "preserved_code_points": [6158, 8203], + "empty_inputs": ["", " \n\u3000\ufeff "], + "non_string_inputs": [null, 7, {}, []], + "boundaries": [ + {"label": "BMP 512", "character": "x", "count": 512, "valid": true}, + {"label": "BMP 513", "character": "x", "count": 513, "valid": false}, + {"label": "astral 512", "character": "πŸ™‚", "count": 512, "valid": true}, + {"label": "astral 513", "character": "πŸ™‚", "count": 513, "valid": false} + ] +} diff --git a/tests/test_832_parity.py b/tests/test_832_parity.py index da5d5b1e..cf2dd8fe 100644 --- a/tests/test_832_parity.py +++ b/tests/test_832_parity.py @@ -32,7 +32,7 @@ from tests._parity_832 import ( ) from tests._session_helpers import ( RecordingUI, - make_session, + make_registered_session, replace_session_lane, scripted_provider, ) @@ -102,7 +102,7 @@ def _apply_ruled_deltas(name: str, baseline: dict[str, Any]) -> dict[str, Any]: @pytest.mark.parametrize("name", sorted(SCENARIOS)) -def test_parity(name: str) -> None: +def test_parity(name: str, tmp_db: str) -> None: record = run_scenario(name) if UPDATE: write_fixture(name, record) @@ -132,7 +132,7 @@ class TestDisplayCommitMirror: def _mirror(self, chunks: list[StreamChunk]) -> tuple[str, str]: ui = RecordingUI() - session = make_session(ui=ui) + session = make_registered_session(ui=ui) session._RETRY_BASE_DELAY = 0 replace_session_lane(session, provider=scripted_provider(chunks)) session.messages.append(Turn.user("hi")) @@ -254,7 +254,7 @@ class TestDisplayCommitMirror: ), ], ) - def test_mirror(self, name: str, chunks: list[StreamChunk]) -> None: + def test_mirror(self, name: str, chunks: list[StreamChunk], tmp_db: str) -> None: stamped = [*chunks] # Ride usage on the finish chunk so the strict gate passes. for i, c in enumerate(stamped): diff --git a/tests/test_app_js.py b/tests/test_app_js.py index 36a710e8..9b725c83 100644 --- a/tests/test_app_js.py +++ b/tests/test_app_js.py @@ -538,7 +538,7 @@ def test_retry_walk_skips_operator_context_cards() -> None: def test_operator_nudge_labels_use_shared_helper() -> None: """Operator-context nudge bubbles collapse the metacognition nudge types - (start / resume / correction / denial / completion / repeat) to one + (including legacy persisted start turns) to one 'metacognition' category via the shared ``utils.js`` ``operatorSourceLabel`` helper rather than leaking the raw ``_source`` (the 'operator Β· start' regression). Both panes call the one helper so they can't drift.""" @@ -3696,6 +3696,121 @@ def test_every_system_turn_source_has_a_fallback_label() -> None: assert not missing, f"system turn sources with no operator label: {sorted(missing)}" +def test_memory_description_editor_defers_normalization_to_server() -> None: + root = Path(__file__).resolve().parent.parent + governance = (root / "turnstone/console/static/governance.js").read_text(encoding="utf-8") + + editor = governance.split("function editMemoryDescription(memoryId) {", 1)[1].split( + "\nfunction showMemoryDetailModal", 1 + )[0] + assert "JSON.stringify({ description: value })" in editor + assert ".replace(" not in editor + assert "Array.from(" not in editor + + +def test_memory_health_refresh_lifecycle() -> None: + import tempfile + + governance = _CONSOLE_GOVERNANCE_JS.read_text(encoding="utf-8") + admin = _CONSOLE_ADMIN_JS.read_text(encoding="utf-8") + load_memories = _slice_function_body(governance, "loadAdminMemories") + load_health = _slice_function_body(governance, "loadMemoryIndexHealth") + edit_memory = _slice_function_body(governance, "editMemoryDescription") + delete_memory = _slice_function_body(governance, "deleteAdminMemory") + assert load_memories and load_health and edit_memory and delete_memory + + # Activation owns the ordinary refresh; list/search/filter work never does. + memories_tab = re.search(r'if \(tab === "memories"\) \{(?P.*?)\n\s*\}', admin, re.S) + assert memories_tab is not None + assert memories_tab.group("body").count("loadAdminMemories();") == 1 + assert memories_tab.group("body").count("loadMemoryIndexHealth();") == 1 + assert "loadMemoryIndexHealth" not in load_memories + # Each successful mutation forces exactly one new health generation. + assert edit_memory.count("loadMemoryIndexHealth(true);") == 1 + assert delete_memory.count("loadMemoryIndexHealth(true);") == 1 + + script = f""" +let _memoryHealthRequest = null; +let _memoryHealthGeneration = 0; +let _memoryHealthHasValid = false; +const banner = {{ textContent: "", style: {{ display: "none" }} }}; +const document = {{ + getElementById: function (id) {{ + if (id !== "memory-index-warning") throw new Error("unexpected element " + id); + return banner; + }}, +}}; +const pending = []; +function authFetch(url, options) {{ + if (url !== "/v1/api/admin/memories/index-health") throw new Error(url); + return new Promise(function (resolve, reject) {{ + pending.push({{ resolve: resolve, reject: reject, options: options }}); + }}); +}} +function response(health) {{ + return {{ ok: true, json: function () {{ return Promise.resolve(health); }} }}; +}} +function loadMemoryIndexHealth(force) {load_health} + +(async function () {{ + const first = loadMemoryIndexHealth(); + const coalesced = loadMemoryIndexHealth(); + if (first !== coalesced || pending.length !== 1) throw new Error("not single flight"); + pending[0].resolve(response({{ + over_budget: true, over_by_chars: 7, budget_chars: 65536, + invalid_description_count: 0, + }})); + await first; + if (!banner.textContent.includes("7") || banner.style.display !== "block") + throw new Error("first health did not render"); + + const stale = loadMemoryIndexHealth(); + const newer = loadMemoryIndexHealth(true); + if (pending.length !== 3) throw new Error("forced refresh did not start"); + if (!pending[1].options.signal.aborted) throw new Error("old request was not aborted"); + pending[2].resolve(response({{ + over_budget: true, over_by_chars: 2, budget_chars: 65536, + invalid_description_count: 0, + }})); + await newer; + const newestBanner = banner.textContent; + pending[1].resolve(response({{ + over_budget: true, over_by_chars: 999, budget_chars: 65536, + invalid_description_count: 9, + }})); + await stale; + if (banner.textContent !== newestBanner || !banner.textContent.includes("2")) + throw new Error("stale response won"); + + const failed = loadMemoryIndexHealth(); + pending[3].reject(new Error("offline")); + await failed; + if (banner.textContent !== newestBanner) throw new Error("valid banner was erased"); + const retry = loadMemoryIndexHealth(); + if (pending.length !== 5) throw new Error("failed request blocked retry"); + pending[4].resolve(response({{ + over_budget: false, over_by_chars: 0, budget_chars: 65536, + invalid_description_count: 0, + }})); + await retry; + if (banner.style.display !== "none") throw new Error("retry did not publish"); +}})().catch(function (error) {{ + console.error(error.stack || error); + process.exitCode = 1; +}}); +""" + with tempfile.NamedTemporaryFile(mode="w", suffix=".mjs", delete=False) as handle: + handle.write(script) + path = handle.name + try: + proc = subprocess.run(["node", path], capture_output=True, text=True, timeout=15) + except FileNotFoundError: + pytest.skip("node binary not available on PATH") + finally: + os.unlink(path) + assert proc.returncode == 0, proc.stderr + + def test_copy_button_survives_retry_teardown_in_both_clients() -> None: """Every assistant bubble carries a persistent copy button in its ``.msg-actions`` bar; the retry-holder teardown in BOTH clients must diff --git a/tests/test_bm25.py b/tests/test_bm25.py index 2d7ed837..d91dc9a5 100644 --- a/tests/test_bm25.py +++ b/tests/test_bm25.py @@ -125,7 +125,7 @@ class TestBM25Reranking: # an endpoint failure, not a floor verdict -> BM25 fallback, NOT empty. # This is the parse-failure-vs-floor distinction at the seam: the # _bm25_reranker closure raises on an unparseable/empty response so - # memory composition can't be silently suppressed by a broken endpoint. + # memory-pointer relevance filtering can't be silently suppressed by a broken endpoint. def boom(q, d): raise RuntimeError("rerank endpoint down") diff --git a/tests/test_cancel.py b/tests/test_cancel.py index c240fd91..d2624267 100644 --- a/tests/test_cancel.py +++ b/tests/test_cancel.py @@ -11,6 +11,7 @@ import pytest from tests._session_helpers import ( arm_session, + make_registered_session, make_session, provider_shell, replace_session_lane, @@ -43,6 +44,15 @@ from turnstone.core.trajectory import ( from turnstone.core.workstream import WorkstreamKind, WorkstreamState +def _bind_storage_mock() -> MagicMock: + """Replace the process-global backend for one storage-boundary test.""" + from turnstone.core.storage import _registry + + storage = MagicMock() + _registry._storage = storage + return storage + + class NullUI: """UI adapter that records state changes and discards other output.""" @@ -172,14 +182,12 @@ def _make_session(ui=None, **kwargs): recording NullUI. The defaults live in tests/_session_helpers.make_session β€” duplicating them here is exactly the drift its docstring warns about.""" - session = make_session(ui=ui or NullUI(), **kwargs) - # Keyed conversation commits refuse orphan writes by design; production's - # manager creates the parent workstream row before constructing a live - # session, so direct-session tests mirror that prerequisite. - from turnstone.core.memory import register_workstream + return make_session(ui=ui or NullUI(), **kwargs) - register_workstream(session.ws_id, user_id=kwargs.get("user_id")) - return session + +def _make_registered_session(ui=None, **kwargs): + """Build the durable variant for tests that reach model admission.""" + return make_registered_session(ui=ui or NullUI(), **kwargs) class _BlockingAgentStream: @@ -290,7 +298,7 @@ class TestCancelEvent: def test_cancel_event_cleared_on_send_start(self, tmp_db): """send() clears a stale cancel flag before starting.""" ui = NullUI() - session = _make_session(ui=ui) + session = _make_registered_session(ui=ui) session.cancel() # Set stale flag fake_stream = iter([StreamChunk(content_delta="Hello", finish_reason="stop")]) @@ -409,7 +417,7 @@ class TestCancelDuringStreaming: def test_preserves_partial_content(self, tmp_db): """Partial content already streamed should be preserved in messages.""" ui = NullUI() - session = _make_session(ui=ui) + session = _make_registered_session(ui=ui) def cancelling_stream(): """Yield a few chunks then cancel.""" @@ -447,7 +455,7 @@ class TestCancelDuringToolExecution: def test_rollback_incomplete_tool_results(self, tmp_db): """When cancelled during tool execution, synthesized results replace missing tool outputs.""" ui = NullUI() - session = _make_session(ui=ui) + session = _make_registered_session(ui=ui) # First call: return content with a tool call def stream_with_tool(): @@ -497,7 +505,7 @@ class TestCancelWhenIdle: """Cancelling when no generation is active is harmless.""" def test_cancel_when_idle_is_noop(self, tmp_db): - session = _make_session() + session = _make_registered_session() session.cancel() # Next send should work normally (cancel cleared at start) @@ -516,7 +524,7 @@ class TestCancelThreadSafety: def test_cancel_from_another_thread(self, tmp_db): ui = NullUI() - session = _make_session(ui=ui) + session = _make_registered_session(ui=ui) barrier = threading.Event() @@ -579,7 +587,7 @@ class TestStreamFlushBeforeToolCalls: super().on_stream_end() ui = TrackingUI() - session = _make_session(ui=ui) + session = _make_registered_session(ui=ui) def stream_content_then_tool(): # Content long enough to leave chars in the tag-scan carry @@ -653,7 +661,7 @@ class TestStreamAbort: ``cancel()`` closes to unblock a stuck read β€” and send()'s finally clears it.""" ui = NullUI() - session = _make_session(ui=ui) + session = _make_registered_session(ui=ui) seen: dict = {} @@ -675,7 +683,7 @@ class TestStreamAbort: """When cancel() closes the stream, the resulting transport error is converted to GenerationCancelled.""" ui = NullUI() - session = _make_session(ui=ui) + session = _make_registered_session(ui=ui) def stream_that_errors(): yield StreamChunk(content_delta="Hello") @@ -700,7 +708,7 @@ class TestStreamAbort: """Exceptions during streaming that aren't caused by cancel should propagate normally.""" ui = NullUI() - session = _make_session(ui=ui) + session = _make_registered_session(ui=ui) def stream_that_errors(): yield StreamChunk(content_delta="Hello") @@ -2177,7 +2185,7 @@ class TestCancelRef: linger into tool execution, where cancel() would close a dead handle instead of nothing).""" ui = NullUI() - session = _make_session(ui=ui) + session = _make_registered_session(ui=ui) arm_session(session, iter([StreamChunk(content_delta="hi", finish_reason="stop")])) session.send("test") @@ -2236,7 +2244,7 @@ class TestCancelRef: tmp_db, ) -> None: """Close aborts the foreground SDK read and latches future arrivals.""" - session = _make_session() + session = _make_registered_session() blocking_stream = _BlockingAgentStream() provider = provider_shell() @@ -2396,7 +2404,7 @@ class TestForceCancelOrphanNoReissue: def test_orphan_death_not_reissued_no_ui_finalize(self, tmp_db): ui = NullUI() - session = _make_session(ui=ui) + session = _make_registered_session(ui=ui) def dying_orphan_stream(): yield StreamChunk(content_delta="old ") @@ -2452,7 +2460,7 @@ class TestForceCancelGeneration: def test_new_cancel_event_per_generation_in_send(self, tmp_db): """send() replaces _cancel_event with a fresh Event each generation.""" ui = NullUI() - session = _make_session(ui=ui) + session = _make_registered_session(ui=ui) original_event = session._cancel_event @@ -2469,18 +2477,18 @@ class TestSendGenerationInitializationPublication: """The claimed generation owns every pre-stream send mutation.""" @pytest.mark.parametrize("takeover", ["successor", "close"]) - def test_owner_lost_during_memory_count_cannot_consume_nudge_cooldown( + def test_owner_lost_during_memory_pointer_plan_cannot_publish( self, tmp_db, takeover: str, ) -> None: - """Storage-backed nudge planning is inert until its owner commits.""" + """Storage-backed pointer planning is inert until its owner commits.""" session = _make_session() + _bind_storage_mock() session._title_generated = True - session._system_composed_with_context = True generation = session._claim_generation() - count_started = threading.Event() - release_count = threading.Event() + planning_started = threading.Event() + release_planning = threading.Event() errors: list[BaseException] = [] session._metacog_state["reflection"] = 123.0 @@ -2488,11 +2496,11 @@ class TestSendGenerationInitializationPublication: prior_metacog = dict(session._metacog_state) prior_nudges = tuple(session._nudge_queue.pending()) - def blocked_memory_count() -> int: - count_started.set() - if not release_count.wait(2): - raise RuntimeError("test memory count was not released") - return 1 + def blocked_pointer_plan(*_args: Any, **_kwargs: Any) -> str: + planning_started.set() + if not release_planning.wait(2): + raise RuntimeError("test memory pointer plan was not released") + return "stale private pointer" def initialize() -> None: try: @@ -2510,8 +2518,7 @@ class TestSendGenerationInitializationPublication: worker = threading.Thread(target=initialize) with ( - patch.object(session, "_nudges_enabled", return_value=True), - patch.object(session, "_visible_memory_count", side_effect=blocked_memory_count), + patch.object(session, "_plan_memory_pointer", side_effect=blocked_pointer_plan), patch.object( session, "_plan_metacognitive_nudge", @@ -2521,13 +2528,13 @@ class TestSendGenerationInitializationPublication: ): worker.start() try: - assert count_started.wait(2) + assert planning_started.wait(2) if takeover == "successor": assert session._claim_generation() == generation + 1 else: session.close() finally: - release_count.set() + release_planning.set() worker.join(2) assert not worker.is_alive() @@ -2541,8 +2548,8 @@ class TestSendGenerationInitializationPublication: def test_stop_does_not_wait_for_blocked_user_turn_storage(self, tmp_db) -> None: """Durable opening-turn storage cannot delay provider cancellation.""" session = _make_session() + storage = _bind_storage_mock() session._title_generated = True - session._system_composed_with_context = True generation = session._claim_generation() storage_started = threading.Event() release_storage = threading.Event() @@ -2596,8 +2603,9 @@ class TestSendGenerationInitializationPublication: child_scope.cancel_ref.append(child_handle) _CancelRef(session, generation).append(main_handle) with ( - patch( - "turnstone.core.session.save_message", + patch.object( + storage, + "save_message", side_effect=blocked_save_message, ) as save, patch.object(session, "_check_metacognitive_nudge", return_value=None), @@ -2634,6 +2642,7 @@ class TestSendGenerationInitializationPublication: takeover: str, ) -> None: session = _make_session() + _bind_storage_mock() origin_generation = session._claim_generation() if takeover == "successor": @@ -2692,104 +2701,84 @@ class TestSendGenerationInitializationPublication: check_metacog.assert_not_called() init_system.assert_not_called() - def test_stale_system_composition_cannot_publish_private_memory_plan( + def test_stale_admission_cannot_commit_or_publish_private_memory_index( self, tmp_db, ) -> None: - """A superseded memory search cannot leak its cache or touch plan.""" - from turnstone.core.memory_relevance import MemoryConfig - - session = _make_session( - memory_config=MemoryConfig(fetch_limit=1, relevance_k=1), - ) - session._invalidate_memory_cache() - session.messages = [ - turn_from_dict({"role": "user", "content": "old private query"}), - ] + """A superseded capture rolls back before its durable commit.""" + session = _make_session() + storage = _bind_storage_mock() + storage.get_memory_index_snapshot.return_value = None old_generation = session._claim_generation() - old_search_started = threading.Event() - release_old_search = threading.Event() - old_results: list[bool] = [] + session._memory_index_admission_generation = old_generation + old_capture_started = threading.Event() + release_old_capture = threading.Event() + committed_principals: list[str] = [] errors: list[BaseException] = [] - touch_calls: list[list[tuple[str, str, str]]] = [] - old_row = { - "memory_id": "old-private-id", - "name": "old_private_memory", - "description": "old generation only", - "content": "old private query details", - "type": "general", - "scope": "user", - "scope_id": "old-private-user", - } - successor_row = { - "memory_id": "successor-id", - "name": "successor_memory", - "description": "successor generation only", - "content": "successor query details", - "type": "general", - "scope": "user", - "scope_id": "successor-user", - } + def capture_snapshot( + _ws_id: str, + principal_id: str, + *, + commit_context: Any, + ) -> dict[str, Any]: + if principal_id == "old-private-user": + old_capture_started.set() + if not release_old_capture.wait(2): + raise RuntimeError("test old index capture was not released") + content = "old_private_memory" + else: + assert principal_id == "successor-user" + content = "successor_memory" + candidate = { + "content": content, + "principal_id": principal_id, + "entry_count": 1, + "char_count": len(content), + "invalid_description_count": 0, + "project_id": "", + "project_name": "", + } + with commit_context(candidate): + committed_principals.append(principal_id) + return candidate - def searched_memories( - query: str, - *_args: Any, - **_kwargs: Any, - ) -> list[dict[str, str]]: - if query == "old private query": - old_search_started.set() - if not release_old_search.wait(2): - raise RuntimeError("test old memory search was not released") - return [old_row] - assert query == "successor query" - return [successor_row] - - def compose_old() -> None: + def admit_old() -> None: try: - old_results.append( - session._init_system_messages(origin_generation=old_generation), + session._admit_memory_index_request( + session._primary_lane(), + my_generation=old_generation, + principal_id="old-private-user", ) except BaseException as exc: errors.append(exc) - worker = threading.Thread(target=compose_old) - with ( - patch( - "turnstone.core.session.search_visible_structured_memories", - side_effect=searched_memories, - ), - patch( - "turnstone.core.session.score_memories", - side_effect=lambda rows, _query, **_kwargs: list(rows), - ), - patch( - "turnstone.core.session.touch_structured_memories", - side_effect=lambda keys: touch_calls.append(list(keys)), - ), + worker = threading.Thread(target=admit_old) + with patch.object( + storage, + "acquire_memory_index_snapshot", + side_effect=capture_snapshot, ): worker.start() try: - assert old_search_started.wait(2) + assert old_capture_started.wait(2) successor_generation = session._claim_generation() - session.messages = [ - turn_from_dict({"role": "user", "content": "successor query"}), - ] - session._invalidate_memory_cache() - assert session._init_system_messages(origin_generation=successor_generation) is True + session._memory_index_admission_generation = successor_generation + session._admit_memory_index_request( + session._primary_lane(), + my_generation=successor_generation, + principal_id="successor-user", + ) + successor_wire = list(session.system_messages) finally: - release_old_search.set() + release_old_capture.set() worker.join(2) assert not worker.is_alive() - assert errors == [] - assert old_results == [False] - cached_names = {row["name"] for rows in session._mem_search_cache.values() for row in rows} - assert cached_names == {"successor_memory"} - assert session._touched_memory_keys == { - ("successor_memory", "user", "successor-user"), - } - assert touch_calls == [[("successor_memory", "user", "successor-user")]] + assert len(errors) == 1 + assert isinstance(errors[0], GenerationCancelled) + assert committed_principals == ["successor-user"] + assert "successor_memory" in str(successor_wire) rendered = "\n".join(str(message.get("content", "")) for message in session.system_messages) assert "successor_memory" in rendered assert "old_private_memory" not in rendered @@ -2799,8 +2788,8 @@ class TestSendGenerationInitializationPublication: tmp_db, ) -> None: """A resume during the user save cannot retarget deferred title work.""" - session = _make_session(ws_id="opening-ws") - session._system_composed_with_context = True + session = _make_session(ws_id="opening-ws", user_id="opening-principal") + _bind_storage_mock() generation = session._claim_generation() successor_turn = turn_from_dict( {"role": "user", "content": "successor workstream history"}, @@ -2816,13 +2805,10 @@ class TestSendGenerationInitializationPublication: patch.object( session, "_plan_shared_state", - return_value=("opening-ws", set(), True), + return_value=("opening-ws", {"opening-principal"}, True), ), patch.object(session, "_init_system_messages") as init_system, - patch( - "turnstone.core.session.load_message_turns", - return_value=[successor_turn], - ), + patch("turnstone.core.session.load_message_turns", return_value=[successor_turn]), patch("turnstone.core.session.load_workstream_config", return_value={}), patch("turnstone.core.session.save_message", side_effect=save_then_resume), patch("turnstone.core.session.threading.Thread") as title_thread, @@ -2856,8 +2842,8 @@ class TestSendGenerationInitializationPublication: tmp_db, ) -> None: """A durable user row cannot launch auxiliary work past close.""" - session = _make_session(ws_id="opening-ws") - session._system_composed_with_context = True + session = _make_session(ws_id="opening-ws", user_id="opening-principal") + _bind_storage_mock() generation = session._claim_generation() save_started = threading.Event() release_save = threading.Event() @@ -2907,13 +2893,12 @@ class TestSendGenerationInitializationPublication: tmp_db, ) -> None: """A failed sender seed remains retryable, but not in this commit.""" - session = _make_session() + session = _make_session(user_id="principal") session._title_generated = True - session._system_composed_with_context = True session._db_senders_loaded = False session._senders_dirty = True generation = session._claim_generation() - storage = MagicMock() + storage = _bind_storage_mock() lock_owned_during_reads: list[bool] = [] def fail_sender_read(_ws_id: str) -> list[str]: @@ -2923,7 +2908,6 @@ class TestSendGenerationInitializationPublication: storage.list_message_senders.side_effect = fail_sender_read with ( - patch("turnstone.core.session.get_storage", return_value=storage), patch.object(session, "_visible_memory_count", return_value=0), patch("turnstone.core.session.save_message", return_value=1), ): @@ -3133,6 +3117,7 @@ class TestMainToolCancellationDisposition: """ ui = _ToolResultTrackingUI() session = _make_session(ui=ui) + _bind_storage_mock() generation = session._claim_generation() call_ids = ("call-a", "call-b") detail = "Cancelled before tool execution; no side effects." @@ -3223,6 +3208,7 @@ class TestMainToolCancellationDisposition: """ ui = _ToolResultTrackingUI() session = _make_session(ui=ui) + _bind_storage_mock() generation = session._claim_generation() status_label = effect_status.value if effect_status is not None else "unclassified" call_id = f"call-{report_order}-{status_label}" @@ -3384,6 +3370,7 @@ class TestGenerationDurabilityFIFO: """A superseded recovery cannot consume the durable-error latch.""" ui = NullUI() session = _make_session(ui=ui) + storage = _bind_storage_mock() session._has_persisted_error = True session._persisted_error_revision = 1 old_generation = session._claim_generation() @@ -3412,9 +3399,10 @@ class TestGenerationDurabilityFIFO: predecessor = threading.Thread(target=run_old) successor: threading.Thread | None = None - with patch( - "turnstone.core.memory.clear_last_error", - side_effect=lambda ws_id: clear_calls.append(ws_id), + with patch.object( + storage, + "save_workstream_config", + side_effect=lambda ws_id, _config: clear_calls.append(ws_id), ): predecessor.start() try: @@ -3960,9 +3948,8 @@ class TestCancelledSendCleanupOwnership: def test_successor_waits_for_complete_cancel_cleanup_transaction(self, tmp_db): """A claim already waiting on the lock observes every cleanup effect.""" ui = NullUI() - session = _make_session(ui=ui) + session = _make_registered_session(ui=ui) session._title_generated = True - session._system_composed_with_context = True observed_lock = _ObservedRLock() session._generation_lock = observed_lock # This test replaces the generation lock to observe ownership. The @@ -4057,9 +4044,8 @@ class TestCancelledSendCleanupOwnership: def test_successor_claim_before_cleanup_refuses_entire_transaction(self, tmp_db): """Once a successor owns the session, no old cleanup action starts.""" ui = NullUI() - session = _make_session(ui=ui) + session = _make_registered_session(ui=ui) session._title_generated = True - session._system_composed_with_context = True publish_entered = threading.Event() release_publish = threading.Event() send_errors: list[BaseException] = [] @@ -4153,7 +4139,7 @@ class TestForceCancelThreaded: """After force cancel + new send(), the orphaned thread must not append stale content to session.messages.""" ui = NullUI() - session = _make_session(ui=ui) + session = _make_registered_session(ui=ui) barrier = threading.Event() old_done = threading.Event() @@ -4195,7 +4181,7 @@ class TestForceCancelThreaded: def test_force_cancel_then_new_send_succeeds(self, tmp_db): """A new send() after force cancel works cleanly.""" ui = NullUI() - session = _make_session(ui=ui) + session = _make_registered_session(ui=ui) barrier = threading.Event() @@ -4289,6 +4275,7 @@ class TestSynthesizeCancelledResults: """ ui = self._ui_with_tool_result_tracking() session = _make_session(ui=ui) + _bind_storage_mock() call_id = "reused-call" disposition = "Completed before cancel: read_file. Task was interrupted." session.messages.append( @@ -4419,7 +4406,12 @@ class TestTimeoutDisposition: session._mcp_client = MagicMock() session._mcp_client.call_tool_sync.side_effect = TimeoutError() call_id, result = session._exec_mcp_tool( - {"call_id": "c1", "mcp_func_name": "send_email", "mcp_args": {}} + { + "call_id": "c1", + "mcp_func_name": "send_email", + "mcp_args": {}, + "_principal_id": "", + } ) assert call_id == "c1" assert "timed out" in result.lower() @@ -4434,7 +4426,7 @@ class TestTimeoutDisposition: session._mcp_client = MagicMock() session._mcp_client.read_resource_sync.side_effect = TimeoutError() call_id, result = session._exec_read_resource( - {"call_id": "c1", "resource_uri": "file:///doc"} + {"call_id": "c1", "resource_uri": "file:///doc", "_principal_id": ""} ) assert call_id == "c1" assert "timed out" in result.lower() @@ -4745,7 +4737,7 @@ class TestNeverArmedStopLeavesNoRow: via record_cancelled_partial β€” TestCancelDuringStreaming pins that side.)""" ui = NullUI() - session = _make_session(ui=ui) + session = _make_registered_session(ui=ui) provider = arm_session(session) # provider shell; create scripted below def create_cancel_then_fail(**kwargs): @@ -4873,7 +4865,7 @@ class TestSupersessionVerdictAgreement: finalizing on one path and not the other.""" def _session_at_generation(self, gen, ui): - session = _make_session(ui=ui) + session = _make_registered_session(ui=ui) session._generation = gen session.messages.append(Turn.user("hi")) return session diff --git a/tests/test_console_db_config.py b/tests/test_console_db_config.py new file mode 100644 index 00000000..9ef4bce2 --- /dev/null +++ b/tests/test_console_db_config.py @@ -0,0 +1,144 @@ +"""Console database bootstrap configuration precedence.""" + +from __future__ import annotations + +import argparse +from typing import TYPE_CHECKING +from unittest.mock import patch + +import pytest + +import turnstone.core.config as config_mod +from turnstone.console.server import _get_console_storage + +if TYPE_CHECKING: + from collections.abc import Iterator + from pathlib import Path + + +_DB_ENV_VARS = ( + "TURNSTONE_DB_BACKEND", + "TURNSTONE_DB_URL", + "TURNSTONE_DB_PATH", + "TURNSTONE_DB_POOL_SIZE", + "TURNSTONE_DB_SSLMODE", + "TURNSTONE_DB_SSLROOTCERT", + "TURNSTONE_DB_SSLCERT", + "TURNSTONE_DB_SSLKEY", + "TURNSTONE_DB_LISTEN_URL", + "TURNSTONE_CONFIG", +) + + +def _reset_config_cache() -> None: + config_mod._cache = None + config_mod._config_path = None + + +def _build_args(config_path: str | None) -> argparse.Namespace: + config_mod.set_config_path(config_path or "/nonexistent/turnstone-console-test.toml") + parser = argparse.ArgumentParser() + config_mod.apply_config(parser, ["database"]) + return parser.parse_args([]) + + +@pytest.fixture(autouse=True) +def _clean_database_configuration(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + for variable in _DB_ENV_VARS: + monkeypatch.delenv(variable, raising=False) + _reset_config_cache() + yield + _reset_config_cache() + + +def test_config_toml_database_section_drives_console_storage(tmp_path: Path) -> None: + config = tmp_path / "config.toml" + config.write_text( + "[database]\n" + 'backend = "postgresql"\n' + 'url = "postgresql+psycopg://from-config/db"\n' + 'path = "/ignored-for-postgresql"\n' + "pool_size = 7\n" + 'sslmode = "verify-full"\n' + 'sslrootcert = "/certs/root.pem"\n' + 'sslcert = "/certs/client.pem"\n' + 'sslkey = "/certs/client.key"\n' + 'listen_url = "postgresql+psycopg://listener/db"\n' + ) + + with patch("turnstone.core.storage.init_storage") as init_storage: + storage = _get_console_storage(_build_args(str(config))) + + assert storage is init_storage.return_value + assert init_storage.call_args.args == ("postgresql",) + assert init_storage.call_args.kwargs == { + "path": "/ignored-for-postgresql", + "url": "postgresql+psycopg://from-config/db", + "pool_size": 7, + "sslmode": "verify-full", + "sslrootcert": "/certs/root.pem", + "sslcert": "/certs/client.pem", + "sslkey": "/certs/client.key", + "listen_url": "postgresql+psycopg://listener/db", + } + + +def test_environment_drives_console_storage_when_config_is_absent( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("TURNSTONE_DB_BACKEND", "postgresql") + monkeypatch.setenv("TURNSTONE_DB_URL", "postgresql+psycopg://from-env/db") + monkeypatch.setenv("TURNSTONE_DB_POOL_SIZE", "9") + monkeypatch.setenv("TURNSTONE_DB_SSLMODE", "require") + monkeypatch.setenv("TURNSTONE_DB_LISTEN_URL", "postgresql+psycopg://listener-env/db") + + with patch("turnstone.core.storage.init_storage") as init_storage: + _get_console_storage(_build_args(None)) + + assert init_storage.call_args.args == ("postgresql",) + assert init_storage.call_args.kwargs["url"] == "postgresql+psycopg://from-env/db" + assert init_storage.call_args.kwargs["pool_size"] == 9 + assert init_storage.call_args.kwargs["sslmode"] == "require" + assert init_storage.call_args.kwargs["listen_url"] == "postgresql+psycopg://listener-env/db" + + +def test_config_values_win_over_environment_per_key( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + monkeypatch.setenv("TURNSTONE_DB_BACKEND", "sqlite") + monkeypatch.setenv("TURNSTONE_DB_URL", "postgresql+psycopg://from-env/db") + monkeypatch.setenv("TURNSTONE_DB_POOL_SIZE", "11") + monkeypatch.setenv("TURNSTONE_DB_SSLMODE", "require") + config = tmp_path / "config.toml" + config.write_text( + "[database]\n" + 'backend = "postgresql"\n' + 'url = "postgresql+psycopg://from-config/db"\n' + "pool_size = 5\n" + 'sslmode = "verify-full"\n' + ) + + with patch("turnstone.core.storage.init_storage") as init_storage: + _get_console_storage(_build_args(str(config))) + + assert init_storage.call_args.args == ("postgresql",) + assert init_storage.call_args.kwargs["url"] == "postgresql+psycopg://from-config/db" + assert init_storage.call_args.kwargs["pool_size"] == 5 + assert init_storage.call_args.kwargs["sslmode"] == "verify-full" + + +def test_explicit_empty_config_value_beats_environment( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + monkeypatch.setenv("TURNSTONE_DB_URL", "postgresql+psycopg://from-env/db") + monkeypatch.setenv("TURNSTONE_DB_LISTEN_URL", "postgresql+psycopg://listener-env/db") + config = tmp_path / "config.toml" + config.write_text('[database]\nbackend = "sqlite"\nurl = ""\nlisten_url = ""\n') + + with patch("turnstone.core.storage.init_storage") as init_storage: + _get_console_storage(_build_args(str(config))) + + assert init_storage.call_args.kwargs["url"] == "" + assert init_storage.call_args.kwargs["listen_url"] == "" diff --git a/tests/test_console_routing_proxy.py b/tests/test_console_routing_proxy.py index 6b39aa95..6acb2df2 100644 --- a/tests/test_console_routing_proxy.py +++ b/tests/test_console_routing_proxy.py @@ -556,6 +556,131 @@ class TestRouteCreate503Retry: router.route.assert_called_once_with(_DEST_WS_ID) +class TestRouteCreate409Retry: + """Generated destination ids retry live collisions at the router.""" + + def test_generated_ws_id_collision_draws_another_id(self, monkeypatch): + first_id = "1" * 32 + second_id = "2" * 32 + generated = MagicMock(side_effect=[first_id, second_id]) + monkeypatch.setattr("turnstone.console.server.secrets.token_hex", generated) + router = _make_mock_router() + app = _make_app(router=router) + posted_ids: list[str] = [] + + async def _mock_post(*args: Any, **kwargs: Any) -> httpx.Response: + posted_ids.append(kwargs["json"]["ws_id"]) + status = 409 if len(posted_ids) == 1 else 200 + payload = ( + {"error": "Workstream already exists"} + if status == 409 + else {"ws_id": second_id, "name": "retry"} + ) + return httpx.Response( + status, + json=payload, + request=httpx.Request("POST", args[0]), + ) + + _wire_proxy(app, MagicMock(side_effect=_mock_post)) + client = TestClient(app, raise_server_exceptions=False) + resp = client.post( + "/v1/api/route/workstreams/new", + json={"name": "generated"}, + headers=_TEST_AUTH_HEADERS, + ) + client.close() + + assert resp.status_code == 200 + assert resp.json()["ws_id"] == second_id + assert posted_ids == [first_id, second_id] + assert generated.call_count == 2 + + def test_target_node_collision_retries_with_targeted_generator(self): + first_id = "1" * 32 + second_id = "2" * 32 + router = _make_mock_router() + router.generate_ws_id_for_node.side_effect = [first_id, second_id] + router.route.return_value = NodeRef("node-c", "http://c:8080") + app = _make_app(router=router) + posted_ids: list[str] = [] + + async def _mock_post(*args: Any, **kwargs: Any) -> httpx.Response: + posted_ids.append(kwargs["json"]["ws_id"]) + status = 409 if len(posted_ids) == 1 else 200 + payload = ( + {"error": "Workstream already exists"} + if status == 409 + else {"ws_id": second_id, "name": "targeted-retry"} + ) + return httpx.Response( + status, + json=payload, + request=httpx.Request("POST", args[0]), + ) + + _wire_proxy(app, MagicMock(side_effect=_mock_post)) + client = TestClient(app, raise_server_exceptions=False) + resp = client.post( + "/v1/api/route/workstreams/new", + json={"target_node": "node-c"}, + headers=_TEST_AUTH_HEADERS, + ) + client.close() + + assert resp.status_code == 200 + assert resp.json()["node_id"] == "node-c" + assert posted_ids == [first_id, second_id] + assert router.generate_ws_id_for_node.call_count == 2 + assert router.generate_ws_id_for_node.call_args_list[0].args == ("node-c",) + assert router.generate_ws_id_for_node.call_args_list[1].args == ("node-c",) + + def test_explicit_ws_id_collision_is_not_retried(self): + router = _make_mock_router() + app = _make_app(router=router) + post = _make_proxy_post( + status_code=409, + json_data={"error": "Workstream already exists"}, + ) + _wire_proxy(app, post) + client = TestClient(app, raise_server_exceptions=False) + + resp = client.post( + "/v1/api/route/workstreams/new", + json={"ws_id": _DEST_WS_ID}, + headers=_TEST_AUTH_HEADERS, + ) + client.close() + + assert resp.status_code == 409 + assert post.call_count == 1 + assert post.call_args.kwargs["json"]["ws_id"] == _DEST_WS_ID + + def test_generated_ws_id_collision_retry_is_bounded(self, monkeypatch): + generated_ids = [f"{value:x}" * 32 for value in range(1, 5)] + generated = MagicMock(side_effect=generated_ids) + monkeypatch.setattr("turnstone.console.server.secrets.token_hex", generated) + router = _make_mock_router() + app = _make_app(router=router) + post = _make_proxy_post( + status_code=409, + json_data={"error": "Workstream already exists"}, + ) + _wire_proxy(app, post) + client = TestClient(app, raise_server_exceptions=False) + + resp = client.post( + "/v1/api/route/workstreams/new", + json={"name": "generated"}, + headers=_TEST_AUTH_HEADERS, + ) + client.close() + + assert resp.status_code == 409 + assert post.call_count == 4 + assert generated.call_count == 4 + + # --------------------------------------------------------------------------- # Tests β€” cluster create (capacity-routed proxy) # --------------------------------------------------------------------------- diff --git a/tests/test_coordinator_endpoints.py b/tests/test_coordinator_endpoints.py index 1a6e9dcb..8c0307d2 100644 --- a/tests/test_coordinator_endpoints.py +++ b/tests/test_coordinator_endpoints.py @@ -12,6 +12,7 @@ the lifted ``approve`` and ``close`` handlers from from __future__ import annotations import hashlib +from types import SimpleNamespace from typing import TYPE_CHECKING, Any, cast from unittest.mock import MagicMock @@ -163,6 +164,7 @@ def _make_client( coord_mgr=None, alias="my-model", registry=None, + raise_server_exceptions: bool = True, ) -> TestClient: """Build a TestClient exposing just the coordinator routes.""" coord_attachments = make_attachment_handlers(_coord_endpoint_config) @@ -295,7 +297,7 @@ def _make_client( app.state.coord_registry_error = "" if coord_mgr else "registry missing" app.state.auth_storage = storage app.state.jwt_secret = "x" * 64 - return TestClient(app) + return TestClient(app, raise_server_exceptions=raise_server_exceptions) # --------------------------------------------------------------------------- @@ -573,6 +575,29 @@ def test_create_returns_ws_id_and_records_audit(storage): assert "coordinator.create" in actions +def test_create_unreadable_project_refuses_without_partial_create(storage): + storage.create_project( + "public-without-read", + "Public Without Read", + "project-owner", + visibility="public", + ) + mgr = _build_mgr(storage) + client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry()) + + resp = client.post( + "/v1/api/workstreams/new", + json={"name": "must-not-exist", "project_id": "public-without-read"}, + headers=_COORD_HEADERS, + ) + + assert resp.status_code == 403 + assert resp.json() == {"error": "project is not available for workstream attachment"} + assert mgr.list_all() == [] + assert storage.list_workstreams() == [] + assert storage.list_audit_events(action="coordinator.create") == [] + + def _capture_factory_pair(): """Return ``(factory, captured)`` β€” factory records model_alias + judge_model into the captured dict on every call so tests can assert @@ -1144,7 +1169,42 @@ def test_approve_resolves_ui_event(storage): assert resp.json()["cycle_id"] == cycle.cycle_id assert cycle.event.is_set() assert cycle.result == (True, None) - assert "spawn_workstream" in ws.ui.auto_approve_tools + assert ws.ui._always_approve_tools_by_principal["user-1"] == {"spawn_workstream"} + + +def test_peer_approval_is_binary_only_and_keeps_execution_principal(storage): + mgr = _build_mgr(storage) + ws = mgr.create(user_id="user-1") + client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry()) + peer_headers = {"X-Test-User": "user-2", "X-Test-Perms": "admin.coordinator"} + + feedback_cycle = _seed_pending(ws, "c-feedback") + response = client.post( + f"/v1/api/workstreams/{ws.id}/approve", + json={"approved": False, "feedback": "change this", "call_id": "c-feedback"}, + headers=peer_headers, + ) + assert response.status_code == 409 + assert not feedback_cycle.resolved + + always_cycle = _seed_pending(ws, "c-always") + response = client.post( + f"/v1/api/workstreams/{ws.id}/approve", + json={"approved": True, "always": True, "call_id": "c-always"}, + headers=peer_headers, + ) + assert response.status_code == 409 + assert not always_cycle.resolved + + response = client.post( + f"/v1/api/workstreams/{ws.id}/approve", + json={"approved": True, "call_id": "c-feedback"}, + headers=peer_headers, + ) + assert response.status_code == 200 + assert feedback_cycle.resolver_principal_id == "user-2" + assert feedback_cycle.execution_principal_id == "user-1" + assert feedback_cycle.result == (True, None) def _seed_pending(ws, *call_ids: str, func_name: str = "spawn_workstream"): @@ -1160,6 +1220,7 @@ def _seed_pending(ws, *call_ids: str, func_name: str = "spawn_workstream"): "func_name": func_name, "approval_label": func_name, "needs_approval": True, + "_principal_id": ws.user_id, } for cid in call_ids ] @@ -1268,6 +1329,114 @@ def test_approve_call_id_matches_any_item_in_multi_envelope(storage): assert cycle.event.is_set() +def test_approve_invokes_modern_handler_once_with_pinned_identity(storage): + mgr = _build_mgr(storage) + ws = mgr.create(user_id="user-1") + cycle = _seed_pending(ws, "c-modern") + real_find = ws.ui.find_approval_cycle + real_resolve = ws.ui.resolve_approval + ws.ui.find_approval_cycle = MagicMock(wraps=real_find) + ws.ui.resolve_approval = MagicMock(wraps=real_resolve) + client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry()) + + resp = client.post( + f"/v1/api/workstreams/{ws.id}/approve", + json={"approved": True, "feedback": "ship it", "call_id": "c-modern"}, + headers=_COORD_HEADERS, + ) + + assert resp.status_code == 200 + ws.ui.find_approval_cycle.assert_called_once_with(cycle_id=None, call_id="c-modern") + ws.ui.resolve_approval.assert_called_once_with( + True, + "ship it", + always=False, + cycle_id=cycle.cycle_id, + resolver_principal_id="user-1", + ) + + +@pytest.mark.parametrize( + ("body", "field"), + [ + ({}, "approved"), + ({"approved": "false"}, "approved"), + ({"approved": True, "always": "false"}, "always"), + ({"approved": True, "feedback": ["no"]}, "feedback"), + ({"approved": True, "call_id": 123}, "call_id"), + ({"approved": True, "cycle_id": ["cycle"]}, "cycle_id"), + ], +) +def test_approve_rejects_malformed_fields_without_resolving(storage, body, field): + mgr = _build_mgr(storage) + ws = mgr.create(user_id="user-1") + cycle = _seed_pending(ws, "c-malformed") + ws.ui.find_approval_cycle = MagicMock(wraps=ws.ui.find_approval_cycle) + ws.ui.resolve_approval = MagicMock(wraps=ws.ui.resolve_approval) + client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry()) + + resp = client.post( + f"/v1/api/workstreams/{ws.id}/approve", + json=body, + headers=_COORD_HEADERS, + ) + + assert resp.status_code == 400 + assert field in resp.json()["error"] + ws.ui.find_approval_cycle.assert_not_called() + ws.ui.resolve_approval.assert_not_called() + assert not cycle.event.is_set() + + +def test_approve_rejects_ui_without_cycle_routing(storage): + class _LegacyApprovalUI: + def resolve_approval(self, *_args, **_kwargs): + raise AssertionError("legacy resolver must not be called") + + mgr = _build_mgr(storage) + ws = mgr.create(user_id="user-1") + ws.ui = _LegacyApprovalUI() + client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry()) + + resp = client.post( + f"/v1/api/workstreams/{ws.id}/approve", + json={"approved": True}, + headers=_COORD_HEADERS, + ) + + assert resp.status_code == 409 + assert resp.json() == {"error": "session UI does not support principal-aware approval"} + + +def test_approve_callback_type_error_is_not_retried(storage): + mgr = _build_mgr(storage) + ws = mgr.create(user_id="user-1") + cycle = _seed_pending(ws, "c-bug") + ws.ui.resolve_approval = MagicMock(side_effect=TypeError("callback implementation bug")) + client = _make_client( + storage, + coord_mgr=mgr, + registry=_fake_registry(), + raise_server_exceptions=False, + ) + + resp = client.post( + f"/v1/api/workstreams/{ws.id}/approve", + json={"approved": False, "call_id": "c-bug"}, + headers=_COORD_HEADERS, + ) + + assert resp.status_code == 500 + ws.ui.resolve_approval.assert_called_once_with( + False, + None, + always=False, + cycle_id=cycle.cycle_id, + resolver_principal_id="user-1", + ) + assert not cycle.event.is_set() + + def test_selectorless_always_whitelists_only_the_resolved_oldest_cycle(storage): """sweep-3 regression: with several live cycles, a selector-less "Approve + Always" must whitelist the tools of the cycle it @@ -1286,8 +1455,9 @@ def test_selectorless_always_whitelists_only_the_resolved_oldest_cycle(storage): assert resp.json()["cycle_id"] == oldest.cycle_id assert oldest.event.is_set() assert not newer.event.is_set() - assert "spawn_workstream" in ws.ui.auto_approve_tools - assert "send_message" not in ws.ui.auto_approve_tools + grants = ws.ui._always_approve_tools_by_principal["user-1"] + assert "spawn_workstream" in grants + assert "send_message" not in grants def test_approve_always_skips_whitelist_when_pinned_cycle_lost_the_race(storage): @@ -1319,7 +1489,7 @@ def test_approve_always_skips_whitelist_when_pinned_cycle_lost_the_race(storage) ) assert resp.status_code == 200 assert resp.json()["cycle_id"] is None - assert "spawn_workstream" not in ws.ui.auto_approve_tools + assert "spawn_workstream" not in ws.ui._always_approve_tools_by_principal.get("user-1", set()) # --------------------------------------------------------------------------- @@ -1779,6 +1949,22 @@ def test_cancel_resolves_pending_approval(storage): assert first.event.is_set() assert second.event.is_set() assert first.result == (False, "Cancelled by user") + assert first.resolver_principal_id == "user-1" + assert second.resolver_principal_id == "user-1" + + +def test_cancel_does_not_fallback_to_single_cycle_approval_api(storage): + """An incompatible UI cannot bypass the attributed all-cycle sweep.""" + mgr = _build_mgr(storage) + ws = mgr.create(user_id="user-1") + single_cycle_resolver = MagicMock() + ws.ui = SimpleNamespace(resolve_approval=single_cycle_resolver) + client = _make_client(storage, coord_mgr=mgr, registry=_fake_registry()) + + resp = client.post(f"/v1/api/workstreams/{ws.id}/cancel", headers=_COORD_HEADERS) + + assert resp.status_code == 200 + single_cycle_resolver.assert_not_called() def test_cancel_response_always_includes_dropped_key(storage): diff --git a/tests/test_eval_nudges.py b/tests/test_eval_nudges.py index a991ec9a..aecee4eb 100644 --- a/tests/test_eval_nudges.py +++ b/tests/test_eval_nudges.py @@ -266,15 +266,22 @@ class TestWorldSeeding: } def test_memory_rows_read_back_through_the_production_listing(self, eval_storage): - from turnstone.core.memory import list_structured_memories + from turnstone.core.memory import ( + get_structured_memory_by_name, + list_structured_memories, + ) _seed_world(eval_storage, self._WORLD_CELL) rows = list_structured_memories(scope="global") by_name = {r["name"]: r for r in rows} # The production writer normalizes names (normalize_key), so the - # seeded row reads back exactly as a model-saved one would. + # metadata listing names the seeded row exactly as a model-saved one + # would. The body remains behind the explicit get boundary. assert "proj_context" in by_name - assert by_name["proj_context"]["content"] == "acme-api: staging tracks main." + assert "content" not in by_name["proj_context"] + full = get_structured_memory_by_name("proj_context", "global", "") + assert full is not None + assert full["content"] == "acme-api: staging tracks main." def test_nodes_read_back_through_the_real_list_nodes(self, eval_storage): _seed_world(eval_storage, self._WORLD_CELL) diff --git a/tests/test_governance_endpoints.py b/tests/test_governance_endpoints.py index edd54928..6fc62fa8 100644 --- a/tests/test_governance_endpoints.py +++ b/tests/test_governance_endpoints.py @@ -255,7 +255,8 @@ class TestRoles: from turnstone.console.server import _VALID_PERMISSIONS - src = Path("turnstone/console/static/governance.js").read_text() + root = Path(__file__).resolve().parents[1] + src = (root / "turnstone/console/static/governance.js").read_text() # _PERMISSION_SECTIONS is a `const X = [...]` containing nested # `permissions: ["a", "b", ...]` arrays. Pull every quoted # string out of every permissions: [...] block; we don't need @@ -713,6 +714,32 @@ class TestRoleAssignments: roles = list_resp.json()["roles"] assert len(roles) >= 1 + def test_assign_role_user_deleted_after_precheck_returns_404( + self, + client, + storage, + monkeypatch, + ): + create_resp = client.post("/v1/api/admin/roles", json=_role_payload()) + role_id = create_resp.json()["role_id"] + assign_role = storage.assign_role + + def delete_then_assign(user_id, target_role_id, assigned_by=""): + assert storage.delete_user(user_id) + return assign_role(user_id, target_role_id, assigned_by) + + monkeypatch.setattr(storage, "assign_role", delete_then_assign) + + resp = client.post( + "/v1/api/admin/users/user-1/roles", + json={"role_id": role_id}, + ) + + assert resp.status_code == 404 + assert resp.json() == {"error": "User not found"} + assert storage.list_user_roles("user-1") == [] + assert storage.list_audit_events(action="role.assign") == [] + def test_assign_role_missing_role_id(self, client): resp = client.post( "/v1/api/admin/users/user-1/roles", diff --git a/tests/test_governance_storage.py b/tests/test_governance_storage.py index cea1c068..ced5a683 100644 --- a/tests/test_governance_storage.py +++ b/tests/test_governance_storage.py @@ -102,6 +102,14 @@ class TestRoleCRUD: assert roles[0]["role_id"] == "r1" assert roles[0]["assigned_by"] == "admin" + def test_assign_role_rejects_missing_user(self, db): + db.create_role("r1", "editor", "Editor", "read,write", builtin=False, org_id="") + + with pytest.raises(ValueError, match="user 'missing' does not exist"): + db.assign_role("missing", "r1", assigned_by="admin") + + assert db.list_user_roles("missing") == [] + def test_assign_role_idempotent(self, db): db.create_role("r1", "editor", "Editor", "read,write", builtin=False, org_id="") db.create_user("u1", "alice", "Alice", "$2b$hash") diff --git a/tests/test_history_commit_handoff.py b/tests/test_history_commit_handoff.py index 5bcddd6e..1bb044f1 100644 --- a/tests/test_history_commit_handoff.py +++ b/tests/test_history_commit_handoff.py @@ -17,7 +17,7 @@ from unittest.mock import MagicMock, patch import pytest -from tests._session_helpers import make_result, make_session +from tests._session_helpers import make_registered_session, make_result, make_session from tests.test_session_manager import _make_manager from turnstone.core import session as session_module from turnstone.core.attachments import Attachment @@ -197,15 +197,12 @@ def _send_environment( def _ready_session(**kwargs: Any) -> Any: - session = make_session(**kwargs) - # Keyed conversation commits intentionally refuse to resurrect a missing - # workstream after hard delete. Direct-session tests therefore install the - # parent row that production's manager/create path establishes first. - from turnstone.core.memory import register_workstream + from turnstone.core.storage import is_storage_initialized - register_workstream(session.ws_id, user_id=kwargs.get("user_id", "")) + session = ( + make_registered_session(**kwargs) if is_storage_initialized() else make_session(**kwargs) + ) session._title_generated = True - session._system_composed_with_context = True return session @@ -1870,7 +1867,7 @@ def test_history_load_failure_rejects_pending_only_handoff_until_durable_prefix_ storage = get_storage() mgr = _build_mgr(storage) ws = mgr.create(user_id="user-1") - session = _ready_session(ws_id=ws.id, user_id="user-1") + session = _ready_session(ws_id=ws.id, user_id="user-1", kind="coordinator") ws.session = session ws.ui = session.ui store = _ConversationStore(ambiguous_assistant_ack=True) @@ -2119,7 +2116,9 @@ def test_conflicted_pending_row_renders_in_place_inside_widened_window() -> None assert conflict_key in session._pending_conversation_commits -def test_capture_never_runs_the_loader_under_the_handoff_lock() -> None: +def test_capture_never_runs_the_loader_under_the_handoff_lock( + monkeypatch: pytest.MonkeyPatch, +) -> None: """Structural pin for the deleted in-lock storage probe. The loader is the only storage touchpoint in a capture; running it under @@ -2127,7 +2126,13 @@ def test_capture_never_runs_the_loader_under_the_handoff_lock() -> None: and SSE registration behind a slow database. The overscan is sampled first, the load runs unlocked, and the merge is pure in-memory work. """ - session = _ready_session() + storage = MagicMock() + storage.save_message.return_value = 1 + from turnstone.core.storage import _registry + + monkeypatch.setattr(_registry, "_storage", storage) + session = make_session() + session._title_generated = True session._append_system_turn("correction", "pending row") lock_free_during_load: list[bool] = [] diff --git a/tests/test_history_total_prefix.py b/tests/test_history_total_prefix.py index 1ed4742b..8cf1e0b1 100644 --- a/tests/test_history_total_prefix.py +++ b/tests/test_history_total_prefix.py @@ -22,7 +22,7 @@ from unittest.mock import MagicMock, patch import pytest -from tests._session_helpers import make_result, make_session +from tests._session_helpers import make_registered_session, make_result from tests.test_history_commit_handoff import _send_environment, _start_send from tests.test_session_manager import _make_manager from turnstone.core import session as session_module @@ -136,9 +136,8 @@ class _PrefixStore: def _ready_session(**kwargs: Any) -> Any: - session = make_session(**kwargs) + session = make_registered_session(**kwargs) session._title_generated = True - session._system_composed_with_context = True return session @@ -226,8 +225,8 @@ def test_tool_system_user_fold_is_one_visible_causal_prefix(tmp_db: Any) -> None assert all(row.get("_commit_key") for row in rows_during[-3:]) -def test_initial_user_and_nudge_are_visible_in_append_order(tmp_db: Any) -> None: - """The initialization batch cannot expose USER without its accepted nudge.""" +def test_initial_user_and_correction_are_visible_in_append_order(tmp_db: Any) -> None: + """The initialization batch cannot expose USER without its accepted correction.""" session = _ready_session() store = _PrefixStore() @@ -236,8 +235,8 @@ def test_initial_user_and_nudge_are_visible_in_append_order(tmp_db: Any) -> None def _emit_init_nudge(*, deferred_persistence: list[Callable[[], None]] | None = None) -> None: session._append_system_turn( - "start", - "initial metacognitive nudge", + "correction", + "initial metacognitive correction", deferred_persistence=deferred_persistence, ) @@ -274,7 +273,7 @@ def test_initial_user_and_nudge_are_visible_in_append_order(tmp_db: Any) -> None assert send_errors == [] assert _roles_and_content(rows_during)[-2:] == [ ("user", "opening user"), - ("system", "initial metacognitive nudge"), + ("system", "initial metacognitive correction"), ] assert all(row.get("_commit_key") for row in rows_during[-2:]) @@ -797,7 +796,7 @@ def test_soft_close_retries_the_latched_pending_prefix( assert session._publication_shutdown is False -def test_soft_close_terminal_latch_refuses_a_fresh_worker_claim() -> None: +def test_soft_close_terminal_latch_refuses_a_fresh_worker_claim(tmp_db: Any) -> None: """No POST-equivalent dispatch may be acknowledged inside close's latch gap.""" ws_id = "ws-soft-close-dispatch-gap" diff --git a/tests/test_idle_nudge_wake_integration.py b/tests/test_idle_nudge_wake_integration.py index 7578e03d..8c19db44 100644 --- a/tests/test_idle_nudge_wake_integration.py +++ b/tests/test_idle_nudge_wake_integration.py @@ -30,7 +30,6 @@ import pytest from tests._helpers import wait_until as _wait_until from tests._session_helpers import make_result -from tests.test_session_manager import FakeStorage from turnstone.core import session_worker from turnstone.core.idle_nudge_watcher import IdleNudgeWatcher, wake_workstream_if_pending from turnstone.core.metacognition import ( @@ -43,9 +42,8 @@ from turnstone.core.trajectory import dicts_from_turns, turn_from_dict from turnstone.core.workstream import Workstream, WorkstreamKind, WorkstreamState # --------------------------------------------------------------------------- -# Minimal fake adapter / UI for this integration test. Storage reuses -# the canonical FakeStorage from test_session_manager.py to avoid the -# drift risk of a parallel fake. +# Minimal fake adapter / UI for this integration test. The session and +# manager share the disposable backend supplied by the storage fixture. # --------------------------------------------------------------------------- @@ -116,7 +114,8 @@ class _BuildRealSessionAdapter: that production ``WebUI`` / coord adapters expose. """ - def __init__(self, kind: WorkstreamKind = WorkstreamKind.INTERACTIVE) -> None: + def __init__(self, storage: Any, kind: WorkstreamKind = WorkstreamKind.INTERACTIVE) -> None: + self.storage = storage self.kind = kind self.events: list[str] = [] self.cleaned_up: list[str] = [] @@ -165,6 +164,11 @@ class _BuildRealSessionAdapter: temperature=0.5, max_tokens=4096, tool_timeout=30, + ws_id=ws.id, + user_id=ws.user_id, + kind=self.kind, + parent_ws_id=ws.parent_ws_id, + project_id=ws.project_id, ) @@ -174,15 +178,17 @@ class _BuildRealSessionAdapter: @pytest.fixture -def real_mgr() -> tuple[SessionManager, _BuildRealSessionAdapter]: +def real_mgr(tmp_db: str) -> tuple[SessionManager, _BuildRealSessionAdapter]: """Real SessionManager wired to an adapter that builds real ChatSessions. No StateWriter is wired so ``set_state`` writes directly to storage on the calling thread (we want subscriber dispatch to fire in the same thread the test invokes ``set_state`` on). """ - adapter = _BuildRealSessionAdapter() - storage = FakeStorage() + from turnstone.core.storage import get_storage + + storage = get_storage() + adapter = _BuildRealSessionAdapter(storage) mgr = SessionManager( adapter, storage=storage, @@ -239,7 +245,6 @@ def test_idle_event_through_real_session_manager_drives_wake_send(real_mgr, tmp_ patch.object(ws.session, "_update_token_table"), patch.object(ws.session, "_print_status_line"), patch.object(ws.session, "_visible_memory_count", return_value=0), - patch("turnstone.core.session.save_message"), ): # Suppress the auto-title side-thread; orthogonal to wake. ws.session._title_generated = True @@ -341,7 +346,6 @@ def test_watch_fire_on_already_idle_session_drives_wake_send(real_mgr, tmp_db): patch.object(ws.session, "_update_token_table"), patch.object(ws.session, "_print_status_line"), patch.object(ws.session, "_visible_memory_count", return_value=0), - patch("turnstone.core.session.save_message"), ): ws.session._title_generated = True # Idle all along β€” no worker, and no state transition coming. @@ -368,14 +372,16 @@ def test_watch_fire_on_already_idle_session_drives_wake_send(real_mgr, tmp_db): @pytest.fixture -def coord_mgr() -> tuple[SessionManager, _BuildRealSessionAdapter, FakeStorage]: +def coord_mgr(tmp_db: str) -> tuple[SessionManager, _BuildRealSessionAdapter, Any]: """Real coord-side SessionManager with the adapter's kind set to COORDINATOR. Same shape as ``real_mgr`` but for the coord half of the lifespan. No StateWriter wired so subscriber dispatch fires synchronously on the test thread. """ - adapter = _BuildRealSessionAdapter(kind=WorkstreamKind.COORDINATOR) - storage = FakeStorage() + from turnstone.core.storage import get_storage + + storage = get_storage() + adapter = _BuildRealSessionAdapter(storage, kind=WorkstreamKind.COORDINATOR) mgr = SessionManager( adapter, storage=storage, @@ -448,7 +454,6 @@ def test_coord_idle_with_active_children_emits_envelope_via_real_managers(coord_ patch.object(coord.session, "_update_token_table"), patch.object(coord.session, "_print_status_line"), patch.object(coord.session, "_visible_memory_count", return_value=0), - patch("turnstone.core.session.save_message"), ): coord.session._title_generated = True mgr.set_state(coord.id, WorkstreamState.IDLE) @@ -537,7 +542,6 @@ def test_coord_idle_with_children_and_open_tasks_delivers_both(coord_mgr, tmp_db patch.object(coord.session, "_update_token_table"), patch.object(coord.session, "_print_status_line"), patch.object(coord.session, "_visible_memory_count", return_value=0), - patch("turnstone.core.session.save_message"), ): coord.session._title_generated = True mgr.set_state(coord.id, WorkstreamState.IDLE) @@ -639,7 +643,6 @@ def test_coord_idle_with_open_tasks_and_no_children_omits_children_content(coord patch.object(coord.session, "_update_token_table"), patch.object(coord.session, "_print_status_line"), patch.object(coord.session, "_visible_memory_count", return_value=0), - patch("turnstone.core.session.save_message"), ): coord.session._title_generated = True mgr.set_state(coord.id, WorkstreamState.IDLE) @@ -746,7 +749,6 @@ def test_stop_latch_survives_the_liveness_wake(coord_mgr, tmp_db): patch.object(coord.session, "_update_token_table"), patch.object(coord.session, "_print_status_line"), patch.object(coord.session, "_visible_memory_count", return_value=0), - patch("turnstone.core.session.save_message"), ): coord.session._title_generated = True @@ -846,7 +848,6 @@ def test_coord_idle_emitted_from_worker_thread_still_wakes(coord_mgr, tmp_db): patch.object(coord.session, "_update_token_table"), patch.object(coord.session, "_print_status_line"), patch.object(coord.session, "_visible_memory_count", return_value=0), - patch("turnstone.core.session.save_message"), ): coord.session._title_generated = True @@ -923,7 +924,6 @@ def _patch_llm_surface(session: Any) -> tuple[Any, ...]: patch.object(session, "_update_token_table"), patch.object(session, "_print_status_line"), patch.object(session, "_visible_memory_count", return_value=0), - patch("turnstone.core.session.save_message"), ) @@ -940,7 +940,7 @@ def test_wake_channel_survives_real_seam_drains_and_delivers_via_wake(tmp_db): session._nudge_queue.enqueue("idle_children", "kids waiting", "wake") p = _patch_llm_surface(session) - with p[0], p[1], p[2], p[3], p[4]: + with p[0], p[1], p[2], p[3]: # Real user-seam drain: appends any drained entry as a system # turn β€” a wake-channel entry must neither drain nor render. session._emit_pending_user_nudges() @@ -1026,7 +1026,7 @@ def test_quiet_ride_along_still_delivers_when_wake_proceeds(tmp_db): session._nudge_queue.enqueue("idle_children", "kids waiting", "wake") p = _patch_llm_surface(session) - with p[0], p[1], p[2], p[3], p[4]: + with p[0], p[1], p[2], p[3]: session.deliver_wake_nudge_from_queue() msgs = dicts_from_turns(session.messages) @@ -1062,7 +1062,7 @@ def test_interjection_handoff_delivers_externals_and_drops_only_idle_nudges(tmp_ session.queue_message("pivot: focus on the flaky login test") p = _patch_llm_surface(session) - with p[0], p[1], p[2], p[3], p[4]: + with p[0], p[1], p[2], p[3]: session.deliver_wake_nudge_from_queue() msgs = dicts_from_turns(session.messages) @@ -1140,7 +1140,6 @@ def test_queued_interjection_owns_the_idle_seam(coord_mgr, tmp_db): patch.object(coord.session, "_update_token_table"), patch.object(coord.session, "_print_status_line"), patch.object(coord.session, "_visible_memory_count", return_value=0), - patch("turnstone.core.session.save_message"), ): coord.session._title_generated = True diff --git a/tests/test_judge_storage.py b/tests/test_judge_storage.py index a333e392..334f1f4f 100644 --- a/tests/test_judge_storage.py +++ b/tests/test_judge_storage.py @@ -56,8 +56,22 @@ class TestIntentVerdictCRUD: # from pre-convention legacy rows that carry the column's # server_default of ``""``. assert v["user_decision"] == "pending" + assert v["resolver_principal_id"] == "" + assert v["execution_principal_id"] == "" assert "created" in v + def test_create_records_resolver_and_execution_principals(self, db): + db.create_intent_verdict( + **_make_verdict_kwargs( + resolver_principal_id="reviewer", + execution_principal_id="executor", + ) + ) + verdict = db.get_intent_verdict("v_001") + assert verdict is not None + assert verdict["resolver_principal_id"] == "reviewer" + assert verdict["execution_principal_id"] == "executor" + def test_get_nonexistent(self, db): assert db.get_intent_verdict("nonexistent") is None @@ -82,6 +96,8 @@ class TestIntentVerdictCRUD: tier="llm", judge_model="gpt-5", latency_ms=500, + resolver_principal_id="reviewer", + execution_principal_id="executor", ) assert ok is True v = db.get_intent_verdict("v_001") @@ -95,6 +111,8 @@ class TestIntentVerdictCRUD: assert v["tier"] == "llm" assert v["judge_model"] == "gpt-5" assert v["latency_ms"] == 500 + assert v["resolver_principal_id"] == "reviewer" + assert v["execution_principal_id"] == "executor" def test_update_rejects_immutable_fields(self, db): """Non-mutable fields like ws_id, call_id, func_name are rejected.""" diff --git a/tests/test_mcp_client.py b/tests/test_mcp_client.py index 758bcef6..dbe90798 100644 --- a/tests/test_mcp_client.py +++ b/tests/test_mcp_client.py @@ -822,6 +822,7 @@ class TestSessionIntegration: "call_id": "call_789", "mcp_func_name": "mcp__test__search", "mcp_args": {"query": "hello"}, + "_principal_id": "", } call_id, output = session._exec_mcp_tool(item) assert call_id == "call_789" @@ -845,6 +846,7 @@ class TestSessionIntegration: "call_id": "call_err", "mcp_func_name": "mcp__test__search", "mcp_args": {"query": "hello"}, + "_principal_id": "", } call_id, output = session._exec_mcp_tool(item) assert call_id == "call_err" @@ -1442,6 +1444,32 @@ class TestSessionRefresh: defaults.update(kwargs) return ChatSession(**defaults) + @staticmethod + def _actor_catalog(actor: str, count: int = 25) -> list[dict[str, Any]]: + token = {"actor-a": "alphacatalogtoken", "actor-b": "bravocatalogtoken"}[actor] + tools = [_fake_openai_tool(f"mcp__{actor}__tool{i}") for i in range(count)] + for tool in tools: + tool["function"]["description"] = f"{token} tool" + return tools + + @staticmethod + def _mcp_names(tools: list[dict[str, Any]]) -> set[str]: + return { + str(tool.get("function", {}).get("name", "")) + for tool in tools + if str(tool.get("function", {}).get("name", "")).startswith("mcp__") + } + + def _assert_actor_projection(self, session, actor: str, *, coordinator: bool) -> None: + expected = {f"mcp__{actor}__tool{i}" for i in range(25)} + assert self._mcp_names(session._tools) == expected + assert self._mcp_names(session._task_tools) == (set() if coordinator else expected) + assert session._tool_search is not None + token = "alphacatalogtoken" if actor == "actor-a" else "bravocatalogtoken" + results = session._tool_search.search(token) + assert results + assert self._mcp_names(results) <= expected + def test_listener_registered_on_init(self, tmp_db): mock_mcp = MagicMock() mock_mcp.get_tools.return_value = [] @@ -1482,6 +1510,445 @@ class TestSessionRefresh: ] session._on_mcp_tools_changed() assert len(session._tools) == initial_count + 1 + task_names = {tool["function"]["name"] for tool in session._task_tools} + assert {"mcp__test__a", "mcp__test__b"} <= task_names + assert "memory" not in task_names + + @pytest.mark.parametrize("kind", ["interactive", "coordinator"]) + def test_actor_handoff_discards_stalled_prior_catalog(self, tmp_db, kind): + started = threading.Event() + release = threading.Event() + catalogs = { + "actor-a": self._actor_catalog("actor-a"), + "actor-b": self._actor_catalog("actor-b"), + } + + def get_tools(*, user_id=None): + actor = user_id or "actor-a" + if threading.current_thread().name == "stale-actor-a": + started.set() + assert release.wait(timeout=5) + return catalogs[actor] + + manager = MagicMock() + manager.get_tools.side_effect = get_tools + with patch("turnstone.core.session.try_prime_user_pools"): + session = self._make_session( + mcp_client=manager, + user_id="actor-a", + kind=kind, + tool_search="on", + ) + stale = threading.Thread( + target=session._on_mcp_tools_changed, + name="stale-actor-a", + ) + stale.start() + assert started.wait(timeout=5) + session.bind_acting_user("actor-b") + self._assert_actor_projection( + session, + "actor-b", + coordinator=kind == "coordinator", + ) + release.set() + stale.join(timeout=5) + + assert not stale.is_alive() + self._assert_actor_projection( + session, + "actor-b", + coordinator=kind == "coordinator", + ) + + def test_actor_handoff_aba_discards_first_actor_epoch(self, tmp_db): + started = threading.Event() + release = threading.Event() + catalogs = { + "actor-a": self._actor_catalog("actor-a"), + "actor-b": self._actor_catalog("actor-b"), + } + first_a_catalog = self._actor_catalog("actor-a") + for tool in first_a_catalog: + tool["function"]["name"] = tool["function"]["name"].replace( + "mcp__actor-a__", "mcp__stale-a__" + ) + + def get_tools(*, user_id=None): + if threading.current_thread().name == "stale-actor-a": + started.set() + assert release.wait(timeout=5) + return first_a_catalog + return catalogs[user_id or "actor-a"] + + manager = MagicMock() + manager.get_tools.side_effect = get_tools + with patch("turnstone.core.session.try_prime_user_pools"): + session = self._make_session( + mcp_client=manager, + user_id="actor-a", + tool_search="on", + ) + stale = threading.Thread( + target=session._on_mcp_tools_changed, + name="stale-actor-a", + ) + stale.start() + assert started.wait(timeout=5) + session.bind_acting_user("actor-b") + session.bind_acting_user("actor-a") + release.set() + stale.join(timeout=5) + + assert not stale.is_alive() + self._assert_actor_projection(session, "actor-a", coordinator=False) + assert not self._mcp_names(session._tools) & {f"mcp__stale-a__tool{i}" for i in range(25)} + + def test_same_actor_epoch_refresh_publishes(self, tmp_db): + manager = MagicMock() + manager.get_tools.return_value = self._actor_catalog("actor-a") + session = self._make_session( + mcp_client=manager, + user_id="actor-a", + tool_search="on", + ) + manager.get_tools.return_value = self._actor_catalog("actor-b") + + session._on_mcp_tools_changed() + + self._assert_actor_projection(session, "actor-b", coordinator=False) + + def test_same_actor_callbacks_publish_in_start_order(self, tmp_db): + """A slower older callback cannot overwrite a newer same-actor read.""" + older_started = threading.Event() + release_older = threading.Event() + initial_catalog = self._actor_catalog("actor-a") + older_catalog = self._actor_catalog("actor-a") + for tool in older_catalog: + tool["function"]["name"] = tool["function"]["name"].replace( + "mcp__actor-a__", "mcp__stale__" + ) + latest_catalog = self._actor_catalog("actor-b") + + def get_tools(*, user_id=None): + assert user_id == "actor-a" + if threading.current_thread().name == "older-same-actor-refresh": + older_started.set() + assert release_older.wait(timeout=5) + return older_catalog + if older_started.is_set(): + return latest_catalog + return initial_catalog + + manager = MagicMock() + manager.get_tools.side_effect = get_tools + session = self._make_session( + mcp_client=manager, + user_id="actor-a", + tool_search="on", + ) + older = threading.Thread( + target=session._on_mcp_tools_changed, + name="older-same-actor-refresh", + ) + older.start() + assert older_started.wait(timeout=5) + + assert session._on_mcp_tools_changed() is True + self._assert_actor_projection(session, "actor-b", coordinator=False) + + release_older.set() + older.join(timeout=5) + assert not older.is_alive() + self._assert_actor_projection(session, "actor-b", coordinator=False) + assert not self._mcp_names(session._tools) & {f"mcp__stale__tool{i}" for i in range(25)} + + def test_stalled_refresh_cannot_republish_after_surface_drop(self, tmp_db): + started = threading.Event() + release = threading.Event() + catalog = self._actor_catalog("actor-a") + + def get_tools(*, user_id=None): + if threading.current_thread().name == "stale-drop": + started.set() + assert release.wait(timeout=5) + return catalog + + manager = MagicMock() + manager.get_tools.side_effect = get_tools + session = self._make_session( + mcp_client=manager, + user_id="actor-a", + tool_search="on", + ) + stale = threading.Thread(target=session._on_mcp_tools_changed, name="stale-drop") + stale.start() + assert started.wait(timeout=5) + session._drop_mcp_surface() + session._rebuild_tool_search() + release.set() + stale.join(timeout=5) + + assert not stale.is_alive() + assert self._mcp_names(session._tools) == set() + assert self._mcp_names(session._task_tools) == set() + assert session._tool_search is not None + assert session._tool_search.search("alphacatalogtoken") == [] + + def test_stalled_refresh_cannot_republish_after_surface_replacement(self, tmp_db): + started = threading.Event() + release = threading.Event() + old_manager = MagicMock() + + def old_get_tools(*, user_id=None): + if threading.current_thread().name == "stale-replacement": + started.set() + assert release.wait(timeout=5) + return self._actor_catalog("actor-a") + + old_manager.get_tools.side_effect = old_get_tools + session = self._make_session( + mcp_client=old_manager, + user_id="actor-a", + tool_search="on", + ) + stale = threading.Thread( + target=session._on_mcp_tools_changed, + name="stale-replacement", + ) + stale.start() + assert started.wait(timeout=5) + + new_manager = MagicMock() + new_manager.get_tools.return_value = self._actor_catalog("actor-b") + with session._acting_user_bind_lock: + session._mcp_client = new_manager + session._mcp_projection_epoch += 1 + session._on_mcp_tools_changed() + release.set() + stale.join(timeout=5) + + assert not stale.is_alive() + self._assert_actor_projection(session, "actor-b", coordinator=False) + + def test_stalled_refresh_cannot_publish_after_close(self, tmp_db): + started = threading.Event() + release = threading.Event() + live_catalog = self._actor_catalog("actor-a") + stale_catalog = self._actor_catalog("actor-b") + + def get_tools(*, user_id=None): + if threading.current_thread().name == "stale-close": + started.set() + assert release.wait(timeout=5) + return stale_catalog + return live_catalog + + manager = MagicMock() + manager.get_tools.side_effect = get_tools + session = self._make_session( + mcp_client=manager, + user_id="actor-a", + tool_search="on", + ) + stale = threading.Thread(target=session._on_mcp_tools_changed, name="stale-close") + stale.start() + assert started.wait(timeout=5) + session.close() + release.set() + stale.join(timeout=5) + + assert not stale.is_alive() + self._assert_actor_projection(session, "actor-a", coordinator=False) + + @pytest.mark.parametrize( + ("callback_name", "catalog_method", "stale_row"), + [ + ( + "_on_mcp_resources_changed", + "get_resources", + {"uri": "stale://resource", "description": "stale resource"}, + ), + ( + "_on_mcp_prompts_changed", + "get_prompts", + {"name": "stale_prompt", "description": "stale prompt", "arguments": []}, + ), + ], + ) + def test_stalled_catalog_prefix_refresh_cannot_publish_after_close( + self, + tmp_db, + callback_name, + catalog_method, + stale_row, + ): + """Resource/prompt recomposition shares the terminal publish latch.""" + started = threading.Event() + release = threading.Event() + manager = MagicMock() + manager.get_tools.return_value = [] + manager.get_resources.return_value = [] + manager.get_prompts.return_value = [] + session = self._make_session(mcp_client=manager, user_id="actor-a") + before = list(session.system_messages) + + def stalled_catalog(*, user_id=None): + assert user_id == "actor-a" + started.set() + assert release.wait(timeout=5) + return [stale_row] + + getattr(manager, catalog_method).side_effect = stalled_catalog + stale = threading.Thread( + target=getattr(session, callback_name), + name=f"stale-{catalog_method}", + ) + stale.start() + assert started.wait(timeout=5) + session.close() + release.set() + stale.join(timeout=5) + + assert not stale.is_alive() + assert session.system_messages == before + assert "stale" not in str(session.system_messages) + + def test_failed_soft_close_reconciles_suppressed_tool_notification(self, tmp_db): + """A catalog edge suppressed by soft close is refreshed after rollback.""" + from turnstone.core.session import ConversationPersistenceError + + manager = MagicMock() + manager.get_tools.return_value = self._actor_catalog("actor-a") + session = self._make_session( + mcp_client=manager, + user_id="actor-a", + tool_search="on", + ) + manager.get_tools.return_value = self._actor_catalog("actor-b") + + def fail_reconciliation(**_kwargs): + assert session._publication_shutdown is True + assert session._on_mcp_tools_changed() is False + assert session._mcp_projection_dirty is True + raise ConversationPersistenceError("durability still unavailable") + + with patch.object( + session, + "_reconcile_pending_conversation_commits", + side_effect=fail_reconciliation, + ): + assert session.prepare_soft_close() is False + + assert session._publication_shutdown is False + assert session._mcp_projection_dirty is False + self._assert_actor_projection(session, "actor-b", coordinator=False) + + def test_failed_soft_close_refresh_failure_retries_at_next_admission(self, tmp_db): + """A failed rollback refresh remains dirty until the admission fence retries.""" + from turnstone.core.session import ConversationPersistenceError + + initial_catalog = self._actor_catalog("actor-a") + recovered_catalog = self._actor_catalog("actor-b") + reads = 0 + + def get_tools(*, user_id=None): + nonlocal reads + assert user_id == "actor-a" + reads += 1 + if reads == 1: + return initial_catalog + if reads == 2: + raise RuntimeError("catalog temporarily unavailable") + return recovered_catalog + + manager = MagicMock() + manager.get_tools.side_effect = get_tools + session = self._make_session( + mcp_client=manager, + user_id="actor-a", + tool_search="on", + ) + + def fail_reconciliation(**_kwargs): + assert session._on_mcp_tools_changed() is False + raise ConversationPersistenceError("durability still unavailable") + + with patch.object( + session, + "_reconcile_pending_conversation_commits", + side_effect=fail_reconciliation, + ): + assert session.prepare_soft_close() is False + + assert reads == 2 + assert session._mcp_projection_dirty is True + self._assert_actor_projection(session, "actor-a", coordinator=False) + + # This is the first operation in the provider-attempt admission path; + # it must converge before active tools are derived for the wire. + session._ensure_mcp_projection_current() + + assert reads == 3 + assert session._mcp_projection_dirty is False + self._assert_actor_projection(session, "actor-b", coordinator=False) + + @pytest.mark.parametrize( + ("callback_name", "catalog_method", "fresh_row", "marker"), + [ + ( + "_on_mcp_resources_changed", + "get_resources", + {"uri": "fresh://resource", "description": "fresh resource"}, + "fresh://resource", + ), + ( + "_on_mcp_prompts_changed", + "get_prompts", + {"name": "fresh_prompt", "description": "fresh prompt", "arguments": []}, + "fresh_prompt", + ), + ], + ) + def test_failed_soft_close_keeps_catalog_prefix_dirty_for_admission( + self, + tmp_db, + callback_name, + catalog_method, + fresh_row, + marker, + ): + """Resource/prompt notifications suppressed by rollback remain observable.""" + from turnstone.core.session import ConversationPersistenceError + + manager = MagicMock() + manager.get_tools.return_value = [] + manager.get_resources.return_value = [] + manager.get_prompts.return_value = [] + session = self._make_session(mcp_client=manager, user_id="actor-a") + before = list(session.system_messages) + getattr(manager, catalog_method).return_value = [fresh_row] + + def fail_reconciliation(**_kwargs): + assert session._publication_shutdown is True + getattr(session, callback_name)() + assert session._system_prefix_dirty is True + raise ConversationPersistenceError("durability still unavailable") + + with patch.object( + session, + "_reconcile_pending_conversation_commits", + side_effect=fail_reconciliation, + ): + assert session.prepare_soft_close() is False + + assert session.system_messages == before + assert session._system_prefix_dirty is True + + session._ensure_system_prefix_fresh(principal_id="actor-a") + + assert session._system_prefix_dirty is False + assert marker in str(session.system_messages) def test_tool_search_preserved_across_refresh(self, tmp_db): # Create enough MCP tools to trigger tool search @@ -1504,6 +1971,47 @@ class TestSessionRefresh: assert session._tool_search is not None assert "mcp__srv__tool0" in session._tool_search.get_expanded_names() + def test_live_tool_search_expansion_survives_stalled_refresh(self, tmp_db): + """The real expansion commit and refresh publication share one witness.""" + refresh_started = threading.Event() + release_refresh = threading.Event() + target = "mcp__srv__tool24" + catalog = [_fake_openai_tool(f"mcp__srv__tool{i}") for i in range(25)] + for tool in catalog: + if tool["function"]["name"] == target: + tool["function"]["description"] = "liveexpansionuniquetoken" + + def get_tools(*, user_id=None): + if threading.current_thread().name == "stalled-expansion-refresh": + refresh_started.set() + assert release_refresh.wait(timeout=5) + return catalog + + manager = MagicMock() + manager.get_tools.side_effect = get_tools + session = self._make_session( + mcp_client=manager, + tool_search="on", + ) + refresh = threading.Thread( + target=session._on_mcp_tools_changed, + name="stalled-expansion-refresh", + ) + refresh.start() + assert refresh_started.wait(timeout=5) + + call_id, output = session._exec_tool_search( + {"call_id": "call-expand", "query": "liveexpansionuniquetoken"} + ) + assert call_id == "call-expand" + assert target in output + assert target in session._tool_search.get_expanded_names() + + release_refresh.set() + refresh.join(timeout=5) + assert not refresh.is_alive() + assert target in session._tool_search.get_expanded_names() + def test_tool_search_prunes_removed_from_expanded(self, tmp_db): mcp_tools = [_fake_openai_tool(f"mcp__srv__tool{i}") for i in range(25)] mock_mcp = MagicMock() diff --git a/tests/test_memory_api.py b/tests/test_memory_api.py index 7fd99194..fbb82815 100644 --- a/tests/test_memory_api.py +++ b/tests/test_memory_api.py @@ -19,12 +19,15 @@ from turnstone.console.server import ( admin_delete_memory, admin_get_memory, admin_list_memories, + admin_memory_index_health, admin_search_memories, + admin_update_memory_description, ) from turnstone.core.auth import AuthResult from turnstone.core.storage._sqlite import SQLiteBackend from turnstone.server import ( delete_memory_endpoint, + get_memory_endpoint, list_memories, save_memory, search_memories, @@ -78,6 +81,7 @@ def server_client(storage): Route("/api/memories", list_memories), Route("/api/memories", save_memory, methods=["POST"]), Route("/api/memories/search", search_memories, methods=["POST"]), + Route("/api/memories/{name}", get_memory_endpoint, methods=["GET"]), Route("/api/memories/{name}", delete_memory_endpoint, methods=["DELETE"]), ], ), @@ -98,7 +102,13 @@ def admin_client(storage): routes=[ Route("/api/admin/memories", admin_list_memories), Route("/api/admin/memories/search", admin_search_memories), + Route("/api/admin/memories/index-health", admin_memory_index_health), Route("/api/admin/memories/{memory_id}", admin_get_memory), + Route( + "/api/admin/memories/{memory_id}", + admin_update_memory_description, + methods=["PATCH"], + ), Route( "/api/admin/memories/{memory_id}", admin_delete_memory, @@ -163,6 +173,7 @@ class TestServerListMemories: r = server_client.get("/v1/api/memories") assert r.status_code == 200 assert r.json()["total"] == 2 + assert all("content" not in row for row in r.json()["memories"]) def test_filter_by_type(self, server_client, storage): _seed_memory(storage, "a", "x", mem_type="user") @@ -224,8 +235,20 @@ class TestServerSaveMemory: ) assert r.status_code == 201 data = r.json() + assert set(data) == { + "memory_id", + "name", + "description", + "type", + "scope", + "scope_id", + "created", + "updated", + "last_accessed", + "access_count", + } assert data["name"] == "my_key" - assert data["content"] == "my content" + assert "content" not in data assert data["type"] == "general" assert data["scope"] == "global" @@ -239,7 +262,60 @@ class TestServerSaveMemory: json=_save_body("key", "v2", description="Updated key description"), ) assert r.status_code == 200 - assert r.json()["content"] == "v2" + assert "content" not in r.json() + fetched = server_client.get("/v1/api/memories/key") + assert fetched.status_code == 200 + assert fetched.json()["content"] == "v2" + + @pytest.mark.anyio + async def test_python_sdk_omission_preserves_type_through_server( + self, + server_client: TestClient, + ) -> None: + import httpx + + from turnstone.sdk.server import AsyncTurnstoneServer + + def forward(request: httpx.Request) -> httpx.Response: + response = server_client.request( + request.method, + request.url.raw_path.decode(), + content=request.content, + headers={"content-type": request.headers.get("content-type", "")}, + ) + return httpx.Response( + response.status_code, + content=response.content, + headers={"content-type": response.headers.get("content-type", "")}, + ) + + transport = httpx.MockTransport(forward) + async with httpx.AsyncClient(transport=transport, base_url="http://test") as http: + sdk = AsyncTurnstoneServer(httpx_client=http) + created = await sdk.save_memory( + "typed_note", + "v1", + description="Typed note", + mem_type="feedback", + ) + preserved = await sdk.save_memory( + "typed_note", + "v2", + description="Updated typed note", + ) + fetched = await sdk.get_memory("typed_note") + reclassified = await sdk.save_memory( + "typed_note", + "v3", + description="Reclassified typed note", + mem_type="general", + ) + + assert created.type == "feedback" + assert preserved.type == "feedback" + assert fetched.type == "feedback" + assert fetched.content == "v2" + assert reclassified.type == "general" def test_with_type_and_scope(self, server_client, storage): _seed_workstream(storage) @@ -279,6 +355,32 @@ class TestServerSaveMemory: assert r.status_code == 400 assert "description is required" in r.json()["error"] + def test_description_is_normalized_and_bounded(self, server_client): + normalized = server_client.post( + "/v1/api/memories", + json=_save_body("hook", "body", description=" alpha\n beta\t gamma "), + ) + assert normalized.status_code == 201 + assert normalized.json()["description"] == "alpha beta gamma" + + raw_over_limit = server_client.post( + "/v1/api/memories", + json=_save_body( + "collapsed_hook", + "body", + description="alpha" + " " * 600 + "beta", + ), + ) + assert raw_over_limit.status_code == 201 + assert raw_over_limit.json()["description"] == "alpha beta" + + too_long = server_client.post( + "/v1/api/memories", + json=_save_body("long_hook", "body", description="x" * 513), + ) + assert too_long.status_code == 400 + assert "512" in too_long.json()["error"] + def test_invalid_type(self, server_client): r = server_client.post( "/v1/api/memories", @@ -311,6 +413,37 @@ class TestServerSaveMemory: assert r.status_code == 201 assert r.json()["name"] == "my_key_name" + def test_normalized_latin_name_round_trips_all_public_surfaces(self, server_client): + created = server_client.post( + "/v1/api/memories", + json=_save_body("CafΓ© Notes", "native body"), + ) + assert created.status_code == 201 + assert created.json()["name"] == "cafe_notes" + + listed = server_client.get("/v1/api/memories") + assert [row["name"] for row in listed.json()["memories"]] == ["cafe_notes"] + + fetched = server_client.get("/v1/api/memories/Caf%C3%A9%20Notes") + assert fetched.status_code == 200 + assert fetched.json()["content"] == "native body" + + deleted = server_client.delete("/v1/api/memories/Caf%C3%A9%20Notes") + assert deleted.status_code == 200 + assert deleted.json()["name"] == "cafe_notes" + + @pytest.mark.parametrize( + "name", + ["bad/name", "bad?name", "bad#name", "bad__name", "部署手順"], + ) + def test_invalid_name_is_rejected_before_storage(self, server_client, name): + response = server_client.post( + "/v1/api/memories", + json=_save_body(name, "body"), + ) + assert response.status_code == 400 + assert "memory name" in response.json()["error"] + def test_create_and_update_are_audited(self, server_client, storage): first = server_client.post( "/v1/api/memories", @@ -437,6 +570,7 @@ class TestServerSearchMemories: assert r.status_code == 200 assert r.json()["total"] == 1 assert r.json()["memories"][0]["name"] == "db_config" + assert "content" not in r.json()["memories"][0] def test_no_results(self, server_client, storage): _seed_memory(storage, "a", "b") @@ -452,9 +586,30 @@ class TestServerSearchMemories: assert r.status_code == 400 def test_unscoped_search_is_caller_bound(self, server_client, storage): - _seed_memory(storage, "own", "needle", scope="user", scope_id="test-user") - _seed_memory(storage, "victim", "needle", scope="user", scope_id="victim") - _seed_memory(storage, "project", "needle", scope="project", scope_id="p1") + _seed_memory( + storage, + "own", + "body", + description="needle", + scope="user", + scope_id="test-user", + ) + _seed_memory( + storage, + "victim", + "body", + description="needle", + scope="user", + scope_id="victim", + ) + _seed_memory( + storage, + "project", + "body", + description="needle", + scope="project", + scope_id="p1", + ) r = server_client.post("/v1/api/memories/search", json={"query": "needle"}) assert r.status_code == 200 assert {row["name"] for row in r.json()["memories"]} == {"own"} @@ -467,6 +622,35 @@ class TestServerSearchMemories: assert r.status_code == 400 +class TestServerGetMemory: + def test_get_is_the_only_read_that_touches_access(self, server_client, storage): + _seed_memory(storage, "live_body", "secret body", memory_id="m-live") + + listed = server_client.get("/v1/api/memories") + searched = server_client.post( + "/v1/api/memories/search", + json={"query": "secret"}, + ) + before = storage.get_structured_memory("m-live") + assert listed.status_code == searched.status_code == 200 + assert before["access_count"] == 0 + assert before["last_accessed"] == "" + + fetched = server_client.get("/v1/api/memories/live_body") + after = storage.get_structured_memory("m-live") + assert fetched.status_code == 200 + assert fetched.json()["content"] == "secret body" + assert after["access_count"] == 1 + assert after["last_accessed"] + + def test_not_found_and_internal_scope(self, server_client): + assert server_client.get("/v1/api/memories/missing").status_code == 404 + assert ( + server_client.get("/v1/api/memories/missing?scope=project&scope_id=private").status_code + == 400 + ) + + class TestServerDeleteMemory: def test_delete(self, server_client, storage): _seed_memory(storage, "doomed") @@ -517,6 +701,8 @@ class TestAdminListMemories: _seed_memory(storage, "b", "2") r = admin_client.get("/v1/api/admin/memories") assert r.json()["total"] == 2 + assert all("content" not in row for row in r.json()["memories"]) + assert all("scope_label" in row for row in r.json()["memories"]) def test_filter(self, admin_client, storage): _seed_memory(storage, "a", "1", mem_type="user") @@ -556,6 +742,7 @@ class TestAdminSearchMemories: r = admin_client.get("/v1/api/admin/memories/search?q=database") assert r.status_code == 200 assert r.json()["total"] == 1 + assert "content" not in r.json()["memories"][0] def test_missing_query(self, admin_client): r = admin_client.get("/v1/api/admin/memories/search") @@ -568,12 +755,78 @@ class TestAdminGetMemory: r = admin_client.get(f"/v1/api/admin/memories/{mid}") assert r.status_code == 200 assert r.json()["name"] == "k" + assert r.json()["content"] == "content" + assert r.json()["scope_label"] == "" + assert r.json()["access_count"] == 1 + assert storage.get_structured_memory(mid)["access_count"] == 1 def test_not_found(self, admin_client): r = admin_client.get("/v1/api/admin/memories/nonexistent-id") assert r.status_code == 404 +class TestAdminMemoryIndexMaintenance: + def test_update_description_normalizes_and_audits(self, admin_client, storage): + mid = _seed_memory(storage, "legacy", "body") + response = admin_client.patch( + f"/v1/api/admin/memories/{mid}", + json={"description": " useful\n hook "}, + ) + assert response.status_code == 200 + assert response.json()["description"] == "useful hook" + assert response.json()["scope_label"] == "" + assert "content" not in response.json() + assert storage.get_structured_memory(mid)["description"] == "useful hook" + events = storage.list_audit_events(action="memory.description_update") + assert len(events) == 1 + assert events[0]["resource_id"] == mid + + def test_update_description_applies_limit_after_normalization( + self, + admin_client, + storage, + ): + mid = _seed_memory(storage, "legacy", "body") + response = admin_client.patch( + f"/v1/api/admin/memories/{mid}", + json={"description": "alpha" + " " * 600 + "beta"}, + ) + assert response.status_code == 200 + assert response.json()["description"] == "alpha beta" + + @pytest.mark.parametrize("description", [None, "", " ", "x" * 513]) + def test_update_description_rejects_invalid_hooks( + self, + admin_client, + storage, + description, + ): + mid = _seed_memory(storage, "legacy", "body") + response = admin_client.patch( + f"/v1/api/admin/memories/{mid}", + json={"description": description}, + ) + assert response.status_code == 400 + + def test_health_includes_project_envelope_and_budget(self, admin_client, storage): + storage.create_project("project-1", "Project One", "u1") + storage.register_workstream("ws-health", user_id="u1", project_id="project-1") + _seed_memory( + storage, + "project_memory", + "body", + scope="project", + scope_id="project-1", + description="project hook", + ) + response = admin_client.get("/v1/api/admin/memories/index-health") + + assert response.status_code == 200 + assert response.json()["budget_chars"] == 65_536 + assert response.json()["envelope_count"] == 3 + assert response.json()["max_entry_count"] == 1 + + class TestAdminDeleteMemory: def test_delete(self, admin_client, storage): mid = _seed_memory(storage, "doomed", "data") diff --git a/tests/test_memory_index.py b/tests/test_memory_index.py new file mode 100644 index 00000000..01a5dd75 --- /dev/null +++ b/tests/test_memory_index.py @@ -0,0 +1,813 @@ +"""Durable complete memory-index rendering and storage semantics.""" + +import contextlib +import json +import random +from pathlib import Path + +import pytest + +from turnstone.core.memory import memory_index_health +from turnstone.core.memory_index import ( + MEMORY_INDEX_DEFAULT_BUDGET_CHARS, + memory_index_base_char_count, + memory_index_entry_metrics, + memory_visibility_key, + normalize_memory_description, + parse_memory_visibility_key, + render_memory_index, + render_memory_pointer, +) +from turnstone.core.project_access import decide_project_access, fold_role_permissions + +_DESCRIPTION_PARITY = json.loads( + (Path(__file__).parent / "data" / "memory_description_parity.json").read_text() +) + + +def test_description_is_one_line_required_and_bounded() -> None: + for codepoint in _DESCRIPTION_PARITY["whitespace_code_points"]: + whitespace = chr(codepoint) + assert ( + normalize_memory_description( + f"{whitespace}alpha{whitespace}{whitespace}beta{whitespace}" + ) + == "alpha beta" + ) + preserved = "".join( + chr(codepoint) for codepoint in _DESCRIPTION_PARITY["preserved_code_points"] + ) + assert normalize_memory_description(f"{preserved}alpha{preserved}") == ( + f"{preserved}alpha{preserved}" + ) + for invalid in [ + *_DESCRIPTION_PARITY["empty_inputs"], + *_DESCRIPTION_PARITY["non_string_inputs"], + ]: + with pytest.raises(ValueError, match="required"): + normalize_memory_description(invalid) + for boundary in _DESCRIPTION_PARITY["boundaries"]: + value = boundary["character"] * boundary["count"] + if boundary["valid"]: + assert normalize_memory_description(value) == value + else: + with pytest.raises(ValueError, match="512"): + normalize_memory_description(value) + + +def test_visibility_key_is_deterministic_and_round_trips() -> None: + scopes = [("user", "u1"), ("global", ""), ("global", "")] + key = memory_visibility_key(scopes) + assert parse_memory_visibility_key(key) == [("global", ""), ("user", "u1")] + + +@pytest.mark.parametrize( + ( + "principal_id", + "owner_id", + "visibility", + "state", + "member", + "permissions", + "expected", + ), + [ + ("owner", "owner", "private", "active", False, set(), (True, True)), + ("member", "owner", "private", "active", True, {"project.read"}, (True, False)), + ("member", "owner", "private", "active", True, set(), (False, False)), + ("reader", "owner", "public", "active", False, {"project.read"}, (True, False)), + ("reader", "owner", "public", "active", False, set(), (False, False)), + ( + "writer", + "owner", + "private", + "active", + True, + {"project.read", "project.write"}, + (True, True), + ), + ("writer", "owner", "public", "active", False, {"project.write"}, (False, False)), + ("owner", "owner", "public", "archived", True, {"project.read"}, (False, False)), + ("owner", "owner", "public", "missing", True, {"project.read"}, (False, False)), + ], +) +def test_project_access_policy_matrix( + principal_id: str, + owner_id: str, + visibility: str, + state: str, + member: bool, + permissions: set[str], + expected: tuple[bool, bool], +) -> None: + decision = decide_project_access( + principal_id=principal_id, + owner_id=owner_id, + visibility=visibility, + state=state, + is_member=member, + permissions=permissions, + ) + assert (decision.can_read, decision.can_write) == expected + + +def test_builtin_grants_and_revokes_fold_before_project_policy() -> None: + assert fold_role_permissions("project.read", revokes={"project.read"}) == set() + assert fold_role_permissions("", grants={"project.read"}) == {"project.read"} + + +def test_complete_index_is_deterministic_escaped_and_body_free() -> None: + rows = [ + { + "memory_id": "2", + "name": "later